updates for OCRWorker (#565)

.dockerignore

@@ -17,3 +17,4 @@ ui2/.vscode
 ui2/.pnp.cjs
 ui2/.pnp.loader.mjs
 ui2/.env.development.local
+ui2/public/papermerge-runtime-config.js

.gitignore

@@ -38,3 +38,4 @@ simplest.yml
 ui/public/runtime/config.js
 .ruff_cache/
 .pytest_cache/
+.enwardrc

docker/standard/Dockerfile

@@ -14,11 +14,14 @@ ENV CORE_APP=/core_app
 ENV PAPERMERGE__DATABASE__URL=sqlite:////db/db.sqlite3
 ENV PAPERMERGE__AUTH__USERNAME=admin
 ENV [email protected]
-ENV PAPERMERGE__OCR__DEFAULT_LANGUAGE=deu
 ENV PAPERMERGE__MAIN__API_PREFIX=""
+ENV PAPERMERGE__OCR__LANG_CODES="deu,eng,ron"
+ENV PAPERMERGE__OCR__DEFAULT_LANG_CODE="deu"
+ENV PAPERMERGE__OCR__AUTOMATIC="false"
 
 RUN apk update && apk add linux-headers python3-dev \
     gcc \
+    curl \
     libc-dev \
     supervisor \
     imagemagick \
@@ -27,6 +30,7 @@ RUN apk update && apk add linux-headers python3-dev \
     poppler-utils
 
 RUN pip install --upgrade poetry roco==0.4.2
+RUN curl -L -o /bin/env2js https://github.com/papermerge/env2js/releases/download/0.2/env2js.amd64
 
 COPY poetry.lock pyproject.toml README.md LICENSE ${CORE_APP}/
 
@@ -37,6 +41,7 @@ COPY docker/standard/entrypoint.sh /entrypoint.sh
 COPY docker/standard/bundles/supervisor/* /etc/papermerge/
 COPY docker/standard/bundles/nginx/* /etc/nginx/
 COPY docker/standard/logging.yaml /etc/papermerge/
+COPY docker/standard/core.js.tmpl /${CORE_APP}/core.js.tmpl
 COPY ./papermerge ${CORE_APP}/papermerge/
 COPY alembic.ini ${CORE_APP}/
 

docker/standard/core.js.tmpl

@@ -0,0 +1,5 @@
+window.__PAPERMERGE_RUNTIME_CONFIG__ = {
+  ocr__lang_codes: "{{ .PAPERMERGE__OCR__LANG_CODES }}",
+  ocr__default_lang_code: "{{ .PAPERMERGE__OCR__DEFAULT_LANG_CODE }}",
+  ocr__automatic: {{ .PAPERMERGE__OCR__AUTOMATIC }}
+}

docker/standard/entrypoint.sh

@@ -55,8 +55,15 @@ case $CMD in
     ;;
   server)
     exec_init
+    # TODO: replace roco with env2js
     roco > /usr/share/nginx/html/auth_server/papermerge-runtime-config.js
-    roco > /usr/share/nginx/html/ui/papermerge-runtime-config.js
+    /bin/env2js -f /core_app/core.js.tmpl > /usr/share/nginx/html/ui/papermerge-runtime-config.js
+    exec /usr/bin/supervisord -c /etc/papermerge/supervisord.conf
+    ;;
+  server_without_init)
+    # TODO: replace roco with env2js
+    roco > /usr/share/nginx/html/auth_server/papermerge-runtime-config.js
+    /bin/env2js -f /core_app/core.js.tmpl > /usr/share/nginx/html/ui/papermerge-runtime-config.js
     exec /usr/bin/supervisord -c /etc/papermerge/supervisord.conf
     ;;
   create_token.sh)

papermerge/core/config.py

@@ -19,7 +19,15 @@ class Settings(BaseSettings):
     papermerge__main__cf_domain: str | None = None
     papermerge__database__url: str = "sqlite:////db/db.sqlite3"
     papermerge__redis__url: str | None = None
-    papermerge__ocr__default_language: str = 'deu'
+    papermerge__ocr__default_lang_code: str = 'deu'
+    # When is OCR triggered ?
+    # `ocr__automatic` = True means that OCR will be performed without
+    #   end user intervention i.e. via background scheduler like celery scheduler
+    # `ocr__automatic` = False means that OCR will be performed only
+    #   if requested by end user. In this case user can choose to
+    #   start schedule OCR on upload; also in this case use can choose to
+    #   scheduler OCR later on any document.
+    papermerge__ocr__automatic: bool = False
     papermerge__search__url: str | None = None
 
 

papermerge/core/features/document/cli/cli.py

@@ -0,0 +1,32 @@
+import uuid
+import typer
+
+from papermerge.core.tasks import send_task
+from papermerge.core.db.engine import Session
+
+from papermerge.core import dbapi, constants, types
+
+
+app = typer.Typer(help="OCR tasks")
+
+
+@app.command()
+def schedule_ocr(node_id: uuid.UUID, force: bool = False, lang: str | None = None):
+    """Schedules OCR for given node ID"""
+    with Session() as db_session:
+        node_type: types.CType = dbapi.get_node_type(db_session, node_id)
+
+        if node_type == "document":
+            if lang is None:
+                lang = dbapi.get_document_lang(db_session, node_id)
+            send_task(
+                constants.WORKER_OCR_DOCUMENT,
+                kwargs={
+                    "document_id": str(node_id),
+                    "lang": lang,
+                },
+                route_name="ocr",
+            )
+        else:
+            # get all descendants of node_id
+            pass

papermerge/core/features/document/db/api.py

@@ -27,10 +27,11 @@
 )
 from papermerge.core.features.document.schema import DocumentCFVRow
 from papermerge.core.features.document.ordered_document_cfv import OrderedDocumentCFV
+from papermerge.core import config
 
 from .selectors import select_doc_cfv, select_docs_by_type
 
-
+settings = config.get_settings()
 
 logger = logging.getLogger(__name__)
 
@@ -634,6 +635,17 @@ def upload(
         route_name="s3",
     )
 
+    if not settings.papermerge__ocr__automatic:
+        if doc.ocr is True:
+            # user chose "schedule OCR" when uploading document
+            tasks.send_task(
+                constants.WORKER_OCR_DOCUMENT,
+                kwargs={
+                    "document_id": str(doc.id),
+                    "lang": doc.lang,
+                },
+                route_name="ocr",
+            )
 
     return validated_model, None
 

ui2/index.html

@@ -9,6 +9,7 @@
   <body>
     <div id="root"></div>
     <div id="modals"></div>
+    <script type="module" src="/papermerge-runtime-config.js"></script>
     <script type="module" src="/src/main.tsx"></script>
   </body>
 </html>

ui2/papermerge-runtime-config.js

@@ -0,0 +1,5 @@
+window.__PAPERMERGE_RUNTIME_CONFIG__ = {
+  ocr__lang_codes: "deu,eng,ron, spa, ita, fra",
+  ocr__default_lang_code: "eng",
+  ocr__automatic: false
+}

ui2/src/components/ScheduleOCRProcessCheckbox/ScheduleOCRProcessCheckbox.tsx

@@ -0,0 +1,85 @@
+import {OCR_LANG} from "@/cconstants"
+import {useRuntimeConfig} from "@/hooks/runtime_config"
+import {OCRCode} from "@/types/ocr"
+import {Checkbox, ComboboxData, Select, Stack} from "@mantine/core"
+import {useState} from "react"
+
+interface Args {
+  initialCheckboxValue: boolean
+  defaultLang: OCRCode
+  onLangChange: (newLang: OCRCode) => void
+  onCheckboxChange: (newValue: boolean) => void
+}
+
+export default function ScheduleOCRProcessCheckbox({
+  initialCheckboxValue,
+  defaultLang,
+  onCheckboxChange,
+  onLangChange
+}: Args) {
+  const runtimeConfig = useRuntimeConfig()
+  const langData = langCodes2ComboboxData(runtimeConfig.ocr__lang_codes)
+  const [checked, setChecked] = useState<boolean>(initialCheckboxValue)
+  const [lang, setLang] = useState<OCRCode>(defaultLang)
+
+  const onCheckboxChangeLocal = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const newValue = e.currentTarget.checked
+    setChecked(newValue)
+    onCheckboxChange(newValue)
+  }
+  const onLangChangeLocal = (value: string | null) => {
+    if (value) {
+      setLang(value as OCRCode)
+      onLangChange(value as OCRCode)
+    }
+  }
+
+  return (
+    <Stack my={"md"}>
+      <Checkbox
+        mt="md"
+        mb="md"
+        label="Schedule OCR processing"
+        checked={checked}
+        onChange={onCheckboxChangeLocal}
+      />
+      {checked && (
+        <Select
+          searchable
+          defaultValue={defaultLang}
+          onChange={onLangChangeLocal}
+          data={langData}
+          value={lang}
+        />
+      )}
+    </Stack>
+  )
+}
+
+function langCodes2ComboboxData(langCodes: string): ComboboxData {
+  /*
+  Input/Output examples:
+  example 1:
+
+      input:  "deu,eng,ron"
+      output: [
+        {value: "deu", label: "Deutsch"},
+        {value: "eng", label: "English"},
+        {value: "ron", label: "Română"}
+      ]
+
+  example 2:
+
+    input:  "fra,spa"
+    output: [
+      {value: "fra", label: "Français"},
+      {value: "spa", label: "Español"},
+    ]
+  */
+  return langCodes
+    .split(",")
+    .map(v => v.trim())
+    .map(v => {
+      return {value: v, label: OCR_LANG[v] || "Unknown Code"}
+    })
+}

ui2/src/features/nodes/components/Commander/NodesCommander/DropFiles.tsx

@@ -1,20 +1,14 @@
-import {
-  Button,
-  Checkbox,
-  Container,
-  Group,
-  Loader,
-  Modal,
-  Text
-} from "@mantine/core"
+import {Button, Container, Group, Loader, Modal, Text} from "@mantine/core"
 import {useState} from "react"
 
 import {useAppDispatch} from "@/app/hooks"
 import {apiSlice} from "@/features/api/slice"
 import {uploadFile} from "@/features/nodes/uploadFile"
 
 import Error from "@/components/Error"
-import type {FolderType} from "@/types"
+import ScheduleOCRProcessCheckbox from "@/components/ScheduleOCRProcessCheckbox/ScheduleOCRProcessCheckbox"
+import {useRuntimeConfig} from "@/hooks/runtime_config"
+import type {FolderType, OCRCode} from "@/types"
 
 type Args = {
   opened: boolean
@@ -34,18 +28,30 @@ export const DropFilesModal = ({
   if (!source_files) {
     return
   }
+  const runtimeConfig = useRuntimeConfig()
   const dispatch = useAppDispatch()
   const [error, setError] = useState("")
+  const [scheduleOCR, setScheduleOCR] = useState<boolean>(false)
+  const [lang, setLang] = useState<OCRCode>("deu")
   const source_titles = [...source_files].map(n => n.name).join(", ")
   const target_title = target.title
 
+  const onLangChange = (newLang: OCRCode) => {
+    setLang(newLang)
+  }
+
+  const onCheckboxChange = (newValue: boolean) => {
+    setScheduleOCR(newValue)
+  }
+
   const localSubmit = async () => {
     for (let i = 0; i < source_files.length; i++) {
       dispatch(
         uploadFile({
           file: source_files[i],
           refreshTarget: true,
-          skipOCR: false,
+          ocr: scheduleOCR,
+          lang: lang,
           target
         })
       ).then(() => {
@@ -74,7 +80,14 @@ export const DropFilesModal = ({
           {` ${target_title}`}
         </Text>
         ?
-        <Checkbox mt="md" mb="md" label="Skip OCR" />
+        {!runtimeConfig.ocr__automatic && (
+          <ScheduleOCRProcessCheckbox
+            initialCheckboxValue={false}
+            defaultLang={runtimeConfig.ocr__default_lang_code}
+            onCheckboxChange={onCheckboxChange}
+            onLangChange={onLangChange}
+          />
+        )}
         {error && <Error message={error} />}
         <Group gap="lg" justify="space-between">
           <Button variant="default" onClick={localSubmit}>

ui2/src/features/nodes/uploadFile.ts

@@ -1,21 +1,23 @@
 import {createAsyncThunk} from "@reduxjs/toolkit"
 
 import {uploaderFileItemUpdated} from "@/features/ui/uiSlice"
-import type {FolderType, NodeType} from "@/types"
+import type {FolderType, NodeType, OCRCode} from "@/types"
 import {getBaseURL, getDefaultHeaders} from "@/utils"
 import axios from "axios"
 
 type UploadFileInput = {
   file: File
   refreshTarget: boolean
   target: FolderType
-  skipOCR: boolean
+  ocr: boolean
+  lang: OCRCode
 }
 
 type CreateDocumentType = {
   title: string
   parent_id: string
   ctype: "document"
+  lang: string
   ocr: boolean
 }
 
@@ -38,7 +40,8 @@ export const uploadFile = createAsyncThunk<UploadFileOutput, UploadFileInput>(
       title: args.file.name,
       parent_id: args.target.id,
       ctype: "document",
-      ocr: !args.skipOCR
+      lang: args.lang,
+      ocr: args.ocr
     }
 
     thunkApi.dispatch(

ui2/src/hooks/runtime_config.tsx

@@ -0,0 +1,20 @@
+import {RuntimeConfig} from "@/types/runtime_config"
+import {useEffect, useState} from "react"
+
+const RUNTIME_CONFIG_DEFAULT: RuntimeConfig = {
+  ocr__lang_codes: "deu,eng,ron",
+  ocr__default_lang_code: "deu",
+  ocr__automatic: false
+}
+
+export function useRuntimeConfig(): RuntimeConfig {
+  const [config, setConfig] = useState<RuntimeConfig>(RUNTIME_CONFIG_DEFAULT)
+
+  useEffect(() => {
+    if (window.hasOwnProperty("__PAPERMERGE_RUNTIME_CONFIG__")) {
+      setConfig(window.__PAPERMERGE_RUNTIME_CONFIG__)
+    }
+  }, [JSON.stringify(window.__PAPERMERGE_RUNTIME_CONFIG__)])
+
+  return config
+}

ui2/src/types/runtime_config.ts

@@ -0,0 +1,13 @@
+import {OCRCode} from "@/types"
+
+export type RuntimeConfig = {
+  ocr__lang_codes: string
+  ocr__default_lang_code: OCRCode
+  ocr__automatic: boolean
+}
+
+declare global {
+  interface Window {
+    __PAPERMERGE_RUNTIME_CONFIG__: RuntimeConfig
+  }
+}