Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

### 🚀 Features

- Add optional `process` field to upload metadata. When `false`, create
the item without conversion/OCR/analysis; reprocess later if needed
(#3344). The upload form exposes this as "Process files".
- Upload endpoints now return `fileKeys` and `jobIds` so clients can track
submitted files and processing jobs immediately.
- Add secured endpoint to download files by file key for retrieving uploaded
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ object NewFile {
tags = None,
reprocess = false,
attachmentsOnly = attachmentsOnly,
customData = customData
customData = customData,
process = None
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ object NewItem {
tags = tags,
reprocess = false,
attachmentsOnly = attachmentsOnly,
customData = customData
customData = customData,
process = None
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ object OUpload {
attachmentsOnly: Option[Boolean],
flattenArchives: Option[Boolean],
customData: Option[Json],
priority: Option[Priority]
priority: Option[Priority],
process: Option[Boolean]
)

case class UploadData[F[_]](
Expand Down Expand Up @@ -163,7 +164,8 @@ object OUpload {
data.meta.tags.some,
reprocess = false,
data.meta.attachmentsOnly,
data.meta.customData
data.meta.customData,
data.meta.process
)
args = ProcessItemArgs(meta, files.toList)
jobs <- right(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ class OUploadTest extends DatabaseTest {
attachmentsOnly = None,
flattenArchives = None,
customData = None,
priority = None
priority = None,
process = None
),
files = Vector(
OUpload.File(
Expand All @@ -64,6 +65,60 @@ class OUploadTest extends DatabaseTest {
}
}

test("submit with process=false stores flag on process-item job args") {
val store = h2Store()
val content = "store-only".getBytes("UTF-8")

OUpload[IO](store, JobStoreImpl(store)).use { upload =>
for {
cid <- prepareCollective(store)
data = OUpload.UploadData(
multiple = true,
meta = OUpload.UploadMeta(
direction = None,
sourceAbbrev = "webapp",
folderId = None,
validFileTypes = Seq.empty,
skipDuplicates = false,
fileFilter = Glob.all,
tags = Nil,
language = Some(Language.English),
attachmentsOnly = None,
flattenArchives = None,
customData = None,
priority = None,
process = Some(false)
),
files = Vector(
OUpload.File(
Some("big.xlsx"),
None,
Stream.emits(content).covary[IO]
)
),
priority = Priority.Low,
tracker = None
)
result <- upload.submit(data, cid, None, None)
jobId <- result match {
case OUpload.UploadResult.Success(_, jobs) =>
jobs.headOption match {
case Some(id) => IO.pure(id)
case None => IO.raiseError(new Exception("expected a job id"))
}
case other =>
IO.raiseError(new Exception(s"expected Success, got $other"))
}
job <- store.transact(docspell.store.records.RJob.findById(jobId))
j <- IO.fromOption(job)(new Exception("job missing"))
args <- IO.fromEither(ProcessItemArgs.parse(j.args))
} yield {
assertEquals(args.meta.process, Some(false))
assert(!args.isProcessingEnabled)
}
}
}

test("submitted file can be loaded by returned file key") {
val store = h2Store()
val content = "retrieve-me".getBytes("UTF-8")
Expand All @@ -86,7 +141,8 @@ class OUploadTest extends DatabaseTest {
attachmentsOnly = None,
flattenArchives = None,
customData = None,
priority = None
priority = None,
process = None
),
files = Vector(
OUpload.File(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) extends TaskArg

def isNormalProcessing: Boolean =
!meta.reprocess

def isProcessingEnabled: Boolean =
meta.process.getOrElse(true)
}

object ProcessItemArgs {
Expand All @@ -55,7 +58,8 @@ object ProcessItemArgs {
tags: Option[List[String]],
reprocess: Boolean,
attachmentsOnly: Option[Boolean],
customData: Option[Json]
customData: Option[Json],
process: Option[Boolean]
)

object ProcessMeta {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

package docspell.common

import io.circe.syntax._
import munit.FunSuite

class ProcessItemArgsTest extends FunSuite {

private def meta(process: Option[Boolean]): ProcessItemArgs.ProcessMeta =
ProcessItemArgs.ProcessMeta(
collective = CollectiveId(1),
itemId = None,
language = Language.English,
direction = None,
sourceAbbrev = "webapp",
folderId = None,
validFileTypes = Seq.empty,
skipDuplicate = false,
fileFilter = None,
tags = None,
reprocess = false,
attachmentsOnly = None,
customData = None,
process = process
)

test("isProcessingEnabled defaults to true") {
val args = ProcessItemArgs(meta(None), Nil)
assert(args.isProcessingEnabled)
}

test("isProcessingEnabled respects process=false") {
val args = ProcessItemArgs(meta(Some(false)), Nil)
assert(!args.isProcessingEnabled)
}

test("decode ProcessMeta without process field") {
val json =
"""{
| "collective": 1,
| "itemId": null,
| "language": "eng",
| "direction": null,
| "sourceAbbrev": "webapp",
| "folderId": null,
| "validFileTypes": [],
| "skipDuplicate": false,
| "fileFilter": null,
| "tags": null,
| "reprocess": false,
| "attachmentsOnly": null,
| "customData": null
|}""".stripMargin

val decoded = io.circe.parser.decode[ProcessItemArgs.ProcessMeta](json).toOption.get
assertEquals(decoded.process, None)
assert(ProcessItemArgs(decoded, Nil).isProcessingEnabled)
}

test("roundtrip process=false") {
val m = meta(Some(false))
assertEquals(m.asJson.as[ProcessItemArgs.ProcessMeta], Right(m))
}
}
38 changes: 26 additions & 12 deletions modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import docspell.scheduler.Task
import docspell.store.Store

object ProcessItem {
type Args = ProcessItemArgs

def apply[F[_]: Async: Files](
cfg: Config,
Expand All @@ -31,31 +32,38 @@ object ProcessItem {
regexNer: RegexNerFile[F],
addonOps: AddonOps[F],
store: Store[F]
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
ExtractArchive(store)(item)
.flatMap(Task.setProgress(20))
.flatMap(processAttachments0(cfg, fts, analyser, regexNer, store, (40, 60, 80)))
.flatMap(LinkProposal.onlyNew[F](store))
.flatMap(SetGivenData.onlyNew[F](itemOps))
.flatMap(Task.setProgress(99))
.flatMap(RemoveEmptyItem(itemOps))
.flatMap(RunAddons(addonOps, store, AddonTriggerType.FinalProcessItem))
)(item: ItemData): Task[F, Args, ItemData] =
isProcessingEnabled[F].flatMap {
case true =>
ExtractArchive(store)(item)
.flatMap(Task.setProgress(20))
.flatMap(processAttachments0(cfg, fts, analyser, regexNer, store, (40, 60, 80)))
.flatMap(LinkProposal.onlyNew[F](store))
.flatMap(SetGivenData.onlyNew[F](itemOps))
.flatMap(Task.setProgress(99))
.flatMap(RemoveEmptyItem(itemOps))
.flatMap(RunAddons(addonOps, store, AddonTriggerType.FinalProcessItem))
case false =>
logStoreOnly[F]
.flatMap(_ => SetGivenData.onlyNew[F](itemOps)(item))
.flatMap(Task.setProgress(99))
}

def processAttachments[F[_]: Async: Files](
cfg: Config,
fts: FtsClient[F],
analyser: TextAnalyser[F],
regexNer: RegexNerFile[F],
store: Store[F]
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
)(item: ItemData): Task[F, Args, ItemData] =
processAttachments0[F](cfg, fts, analyser, regexNer, store, (30, 60, 90))(item)

def analysisOnly[F[_]: Async: Files](
cfg: Config,
analyser: TextAnalyser[F],
regexNer: RegexNerFile[F],
store: Store[F]
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
)(item: ItemData): Task[F, Args, ItemData] =
TextAnalysis[F](cfg.textAnalysis, analyser, regexNer, store)(item)
.flatMap(FindProposal[F](cfg.textAnalysis, store))
.flatMap(EvalProposals[F](store))
Expand All @@ -69,7 +77,7 @@ object ProcessItem {
regexNer: RegexNerFile[F],
store: Store[F],
progress: (Int, Int, Int)
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
)(item: ItemData): Task[F, Args, ItemData] =
ConvertPdf(cfg.convert, store, item)
.flatMap(Task.setProgress(progress._1))
.flatMap(TextExtraction(cfg.extraction, fts, store))
Expand All @@ -78,4 +86,10 @@ object ProcessItem {
.flatMap(Task.setProgress(progress._2))
.flatMap(analysisOnly[F](cfg, analyser, regexNer, store))
.flatMap(Task.setProgress(progress._3))

private def isProcessingEnabled[F[_]: Sync]: Task[F, Args, Boolean] =
Task(ctx => ctx.args.isProcessingEnabled.pure[F])

private def logStoreOnly[F[_]]: Task[F, Args, Unit] =
Task.log(_.info("Not processing files. Only storing the item."))
}
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ object ReProcessItem {
None,
reprocess = true,
None, // attachOnly (not used when reprocessing attachments)
None // cannot retain customData from an already existing item
None, // cannot retain customData from an already existing item
None
),
Nil
).pure[F]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ object ScanMailboxTask {
args.attachmentsOnly,
None,
None,
None,
None
)
data = OUpload.UploadData(
Expand Down
9 changes: 9 additions & 0 deletions modules/restapi/src/main/resources/docspell-openapi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8476,6 +8476,15 @@ components:
Processing priority for the upload jobs. If omitted, the endpoint
default applies (high for secured upload, source priority for open
upload, server config for integration).
process:
type: boolean
default: true
description: |
Whether to run the processing pipeline (conversion, text
extraction/OCR, preview, analysis). Defaults to `true`. If
`false`, an item is still created and given metadata is
applied, but these stages are skipped. Processing can be
started later via the reprocess endpoints.

Collective:
description: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,8 @@ trait Conversions {
m.attachmentsOnly,
m.flattenArchives,
m.customData,
m.priority
m.priority,
m.process
)
)
)
Expand All @@ -341,6 +342,7 @@ trait Conversions {
None,
None,
None,
None,
None
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,8 @@ object MigrateCollectiveIdTaskArgs extends TransactorSupport {
tags = oldArgs.meta.tags,
reprocess = oldArgs.meta.reprocess,
attachmentsOnly = oldArgs.meta.attachmentsOnly,
customData = None
customData = None,
process = None
),
oldArgs.files.map(f =>
ProcessItemArgs
Expand Down
Loading