diff --git a/datashare-python/datashare_python/utils.py b/datashare-python/datashare_python/utils.py index 0fef1948..5cfb7de7 100644 --- a/datashare-python/datashare_python/utils.py +++ b/datashare-python/datashare_python/utils.py @@ -1111,7 +1111,7 @@ async def run_with_concurrency( ) -> AsyncGenerator: max_concurrency = asyncio.Semaphore(max_concurrency) aws = (_run_with_semaphore(aw, max_concurrency) for aw in aws) - async for res in asyncio.as_completed(aws): + for res in asyncio.as_completed(aws): yield await res diff --git a/workers/passport-worker/passport_worker/activities.py b/workers/passport-worker/passport_worker/activities.py index c08ca16b..ee776f1e 100644 --- a/workers/passport-worker/passport_worker/activities.py +++ b/workers/passport-worker/passport_worker/activities.py @@ -41,12 +41,14 @@ PassportDetectionResponse, PassportExportArgs, PassportExportResponse, + PDFPreprocessorConfig, PreprocessingBatches, PreprocessingConfig, ) from .preprocessing import ( ImagePreprocessor, PDFConverter, + PDFPreprocessor, convert_to_pdfs_act, preprocess_images_act, preprocess_pdfs_act, @@ -177,7 +179,7 @@ async def convert_to_pdfs( ] = None, ) -> tuple[Path, Path]: worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) - config = worker_config.preprocessing.pdfs + config = worker_config.preprocessing.pdfs.conversion cache = lifespan_pdf_converter_cache() cache_key = config_cache_key(config.pdf_converter) pdf_converter_factory = async_enter_cm( @@ -215,6 +217,7 @@ async def preprocess_pdfs( self, batch: Path, project: str, + config: PDFPreprocessorConfig, *, force_reprocessing: bool, progress: Annotated[ @@ -222,12 +225,19 @@ async def preprocess_pdfs( ] = None, ) -> tuple[Path, Path]: worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) + worker_pdf_config = worker_config.preprocessing.pdfs.preprocessing workdir = worker_config.paths.workdir + logger.info("loading pdf preprocessor...") + # We don't cache the pdf preprocessor, it's lightweight + extras = worker_pdf_config.resolve_preprocessor_extras(config) + pdf_preprocessor = PDFPreprocessor.from_config(config, **extras) # We cache processing at the config level, we ignore package updates, # to discard the cache we just need to disable it in the args to overwrite - output_root = activity_workdir(workdir, project, caching_key="pdf-preprocessor") + result_cache_key = config_cache_key(config) + output_root = activity_workdir(workdir, project, caching_key=result_cache_key) output_root.mkdir(parents=True, exist_ok=True) successes, errors = await preprocess_pdfs_act( + pdf_preprocessor, batch, worker_config.paths, output_root=output_root, diff --git a/workers/passport-worker/passport_worker/config.py b/workers/passport-worker/passport_worker/config.py index 1d5e2f60..7ebd4989 100644 --- a/workers/passport-worker/passport_worker/config.py +++ b/workers/passport-worker/passport_worker/config.py @@ -1,7 +1,8 @@ +import os from concurrent.futures import ProcessPoolExecutor from datetime import timedelta from enum import StrEnum -from typing import ClassVar +from typing import Any, ClassVar import datashare_python from datashare_python.config import ( @@ -15,6 +16,8 @@ from icij_common.registrable import RegistrableConfig from pydantic import Field +from .objects import DefaultPDFPreprocessorConfig, PDFPreprocessorConfigBase + _ALL_LOGGERS = [datashare_python.__name__, __name__, "__main__"] _DEFAULT_LOGGERS = { @@ -68,13 +71,50 @@ class PDFConversionWorkerConfig(DatashareModel): ) +class LiteparsePDFPreprocessorWorkerConfig(BaseModel): + concurrency: int = Field(default_factory=os.cpu_count) + page_batch_size: int = 8 + + +class PDFPreprocessingWorkerConfig(BaseModel): + liteparse: LiteparsePDFPreprocessorWorkerConfig = Field( + default_factory=LiteparsePDFPreprocessorWorkerConfig + ) + + def resolve_preprocessor_extras( + self, preprocessor_config: PDFPreprocessorConfigBase + ) -> dict[str, Any]: + from .liteparse_ import LiteparsePDFPreprocessorConfig # noqa: PLC0415 + + match preprocessor_config: + case DefaultPDFPreprocessorConfig(): + return dict() + case LiteparsePDFPreprocessorConfig(): + return self.liteparse.model_dump() + raise TypeError(f"unexpected type: {preprocessor_config.__class__.__name__}") + + +class PDFProcessingWorkerConfig(BaseModel): + conversion: PDFConversionWorkerConfig = Field( + default_factory=PDFConversionWorkerConfig + ) + preprocessing: PDFPreprocessingWorkerConfig = Field( + default_factory=PDFPreprocessingWorkerConfig + ) + + def resolve_pdf_preprocessor_extras( + self, preprocessor_config: PDFPreprocessorConfigBase + ) -> dict[str, Any]: + return self.preprocessing.resolve_preprocessor_extras(preprocessor_config) + + class PreprocessingWorkerConfig(DatashareModel): target_n_pages_per_batch: int = 200 images: ImagePreprocessingWorkerConfig = Field( default_factory=ImagePreprocessingWorkerConfig ) - pdfs: PDFConversionWorkerConfig = Field(default_factory=PDFConversionWorkerConfig) + pdfs: PDFProcessingWorkerConfig = Field(default_factory=PDFProcessingWorkerConfig) def to_image_preprocessing_executor(self) -> ProcessPoolExecutor: return self.images.to_image_preprocessing_executor() diff --git a/workers/passport-worker/passport_worker/liteparse_.py b/workers/passport-worker/passport_worker/liteparse_.py new file mode 100644 index 00000000..845e0b33 --- /dev/null +++ b/workers/passport-worker/passport_worker/liteparse_.py @@ -0,0 +1,110 @@ +import asyncio +import logging +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Self + +from datashare_python.utils import run_with_concurrency +from liteparse.parser import LiteParse +from liteparse.types import PageComplexityStats + +from .objects import ( + LayoutReason, + LiteparsePDFPreprocessorConfig, + OCRReason, + PDFPreprocessorType, +) +from .preprocessing import DefaultPDFPreprocessor, PDFPreprocessor + +logger = logging.getLogger(__name__) + + +@PDFPreprocessor.register(PDFPreprocessorType.LITEPARSE) +class LiteparsePDFPreprocessor(PDFPreprocessor[LiteparsePDFPreprocessorConfig]): + def __init__( + self, + config: LiteparsePDFPreprocessorConfig, + inner: DefaultPDFPreprocessor, + page_batch_size: int = 8, + concurrency: int | None = None, + ): + super().__init__(config) + self._inner = inner + self._page_batch_size = page_batch_size + self._concurrency = concurrency or os.cpu_count() + + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + complexities = await self._compute_complexities(path, force_reprocessing) + ocr_reasons = self._config.page_filter.accept_ocr_reasons + layout_reasons = self._config.page_filter.accept_layout_reasons + skip = set() + for i, c in enumerate(complexities): + might_be_passport = may_have_passport( + c, ocr_reasons=ocr_reasons, layout_reasons=layout_reasons + ) + if might_be_passport: + skip.add(i) + pages = await self._inner.process_pdf( + path, skip, output_dir=output_dir, force_reprocessing=force_reprocessing + ) + return pages + + async def _compute_complexities( + self, path: Path, n_pages: int + ) -> list["PageComplexityStats"]: + batches = ( + pages for pages in _chunk_docs(n_pages, chunk_size=self._page_batch_size) + ) + aws = (_compute_batch_complexities(path, batch) for batch in batches) + complexities = [] + async for res in run_with_concurrency(aws, self._concurrency): + complexities.extend(res) + complexities = sorted(complexities, key=lambda c: c.page_number) + return complexities + + @classmethod + def _from_config(cls, config: LiteparsePDFPreprocessorConfig, **extras) -> Self: # noqa: ARG003 + inner = DefaultPDFPreprocessor(config.preprocessor) + return cls(config, inner) + + +def may_have_passport( + complexity: PageComplexityStats, + ocr_reasons: set[OCRReason], + layout_reasons: set[LayoutReason], +) -> bool: + if any(r in ocr_reasons for r in complexity.reasons): + return True + if complexity.layout is not None: + return any(r in layout_reasons for r in complexity.layout.reasons) + return False + + +Range = tuple[int, int] + + +def _chunk_docs(n_pages: int, *, chunk_size: int) -> Iterator[Range]: + for i in range(0, n_pages, chunk_size): + start = i + end = min(start + n_pages, n_pages) + rng = (start, end) + yield rng + + +async def _compute_batch_complexities( + path: Path, rng: Range +) -> list[PageComplexityStats]: + target_pages = _liteparse_range(rng) + parser = LiteParse(ocr_enabled=False, target_pages=target_pages) + with parser: + comps = await asyncio.to_thread(parser.is_complex, path) + return comps + + +def _liteparse_range(rng: Range) -> str: + # page range is a Python range, liteparse indexes from 1 and upper bound + # is inclusive + return f"{rng[0] + 1}-{rng[1]}" diff --git a/workers/passport-worker/passport_worker/objects.py b/workers/passport-worker/passport_worker/objects.py index 99630071..61544aec 100644 --- a/workers/passport-worker/passport_worker/objects.py +++ b/workers/passport-worker/passport_worker/objects.py @@ -21,7 +21,7 @@ WorkerFile, WorkerPaths, ) -from icij_common.pydantic_utils import safe_copy +from icij_common.pydantic_utils import make_enum_discriminator, safe_copy, tagged_union from icij_common.registrable import RegistrableConfig from passport_service.constants import ( DEFAULT_DETECTION_THRESHOLD, @@ -30,7 +30,7 @@ DEFAULT_NMS_THRESHOLD, ) from passport_service.objects import Passport -from pydantic import Field +from pydantic import Discriminator, Field DocumentSearchQuery = dict[str, Any] DocId = str @@ -71,12 +71,73 @@ class DefaultImagePreprocessorConfig(ImagePreprocessorConfigBase): ImagePreprocessorConfig = DefaultImagePreprocessorConfig +class PDFPreprocessorType(StrEnum): + DEFAULT = "default" + LITEPARSE = "liteparse" + + +class PDFPreprocessorConfigBase(DatashareModel, RegistrableConfig): + registry_key: ClassVar[str] = Field(frozen=True, default="type") + type: ClassVar[PDFPreprocessorType] + + +class DefaultPDFPreprocessorConfig(PDFPreprocessorConfigBase): + type: ClassVar[PDFPreprocessorType] = Field( + frozen=True, default=PDFPreprocessorType.DEFAULT + ) + + +class OCRReason(StrEnum): + SCANNED = "scanned" + NO_TEXT = "no-text" + SPARSE_TEXT = "sparse-text" + EMBEDDED_IMAGES = "embedded-images" + GARBLED = "garbled" + VECTOR_TEXT = "vector-text" + ANNOTATION_TEXT = "annotation-text" + + +class LayoutReason(StrEnum): + MULTI_COLUMN = "multi-column" + TABLE_LIKELY = "table-likely" + DENSE_GRAPHICS = "dense-graphics" + + +def _all_ocr_reasons() -> set[OCRReason]: + return set(OCRReason) + + +class LiteparsePageFilter(BaseModel): + accept_ocr_reasons: set[OCRReason] = Field(default_factory=_all_ocr_reasons) + accept_layout_reasons: set[LayoutReason] = Field(default_factory=set) + + +class LiteparsePDFPreprocessorConfig(PDFPreprocessorConfigBase): + type: ClassVar[PDFPreprocessorType] = Field( + frozen=True, default=PDFPreprocessorType.LITEPARSE + ) + preprocessor: DefaultPDFPreprocessorConfig = Field( + default_factory=DefaultPDFPreprocessorConfig + ) + page_filter: LiteparsePageFilter = Field(default_factory=LiteparsePageFilter) + + +PDFPreprocessorConfig = tagged_union( + PDFPreprocessorConfigBase.__subclasses__(), lambda t: t.type.default +) +pdf_preprocessor_discriminator = make_enum_discriminator("type", PDFPreprocessorType) + + class PreprocessingConfig(DatashareModel): use_caching: bool = True images: ImagePreprocessorConfig = Field( default_factory=DefaultImagePreprocessorConfig ) + pdfs: PDFPreprocessorConfig = Field( + discriminator=Discriminator(pdf_preprocessor_discriminator), + default_factory=DefaultPDFPreprocessorConfig, + ) class PassportDetectorType(StrEnum): diff --git a/workers/passport-worker/passport_worker/preprocessing.py b/workers/passport-worker/passport_worker/preprocessing.py index 67069d64..f4132a6f 100644 --- a/workers/passport-worker/passport_worker/preprocessing.py +++ b/workers/passport-worker/passport_worker/preprocessing.py @@ -1,11 +1,13 @@ import asyncio import logging from abc import abstractmethod +from collections.abc import AsyncGenerator from concurrent.futures import ProcessPoolExecutor +from contextlib import asynccontextmanager from functools import partial from pathlib import Path from types import TracebackType -from typing import Protocol, Self, TypeVar +from typing import Self, TypeVar from aiofile import async_open from aiohttp import ClientResponseError @@ -22,13 +24,14 @@ from icij_common.registrable import RegistrableFromConfig from passport_service import GotenbergClient from passport_service.constants import Colorspace -from passport_service.core import process_image, process_pdf +from passport_service.core import async_process_pdf, process_image from passport_service.exceptions import ( REPORTED_ERRORS, ProcessingTimeout, UnsupportedDocExtension, ) from passport_service.utils import run_with_concurrency +from pymupdf import Document, pymupdf from passport_worker.config import GotenbergPDFConverterConfig, PDFConverterType from passport_worker.constants import pil_supported_extensions @@ -36,10 +39,13 @@ DefaultImagePreprocessorConfig, ImagePreprocessorType, PassportProcessingError, + PDFPreprocessorType, ProcessedPage, WorkerPaths, ) +from .config import DefaultPDFPreprocessorConfig, PDFPreprocessorConfigBase + logger = logging.getLogger(__name__) @@ -113,17 +119,47 @@ def _from_config(cls, config: GotenbergPDFConverterConfig, **extras) -> Self: # return cls(config) -class PDFPreprocessor(Protocol): - def __call__( - self, - pdf_path: Path, - pdf_bytes: bytes, - output_dir: Path, - *, - force_reprocessing: bool, +class PDFPreprocessor[C: PDFPreprocessorConfigBase](RegistrableFromConfig): + def __init__(self, config: C): + self._config = config + + @abstractmethod + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool ) -> list[Path]: ... +@PDFPreprocessor.register(PDFPreprocessorType.DEFAULT) +class DefaultPDFPreprocessor(PDFPreprocessor): + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + pages = await self.process_pdf( + path, + output_dir=output_dir, + force_reprocessing=force_reprocessing, + skip=set(), + ) + return pages + + async def process_pdf( + self, path: Path, skip: set[int], *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + async with async_open_pdf(path) as pdf_doc: + return await async_process_pdf( + pdf_doc, + path, + output_dir=output_dir, + force_reprocessing=force_reprocessing, + colorspace=Colorspace.RGB, + skip=skip, + ) + + @classmethod + def _from_config(cls, config: DefaultPDFPreprocessorConfig, **extras) -> Self: # noqa: ARG003 + return cls(config) + + def preprocess_images_act( batch: Path, paths: WorkerPaths, @@ -235,16 +271,14 @@ def _preprocess_image_doc( async def preprocess_pdfs_act( + pdf_preprocessor: PDFPreprocessor, batch: Path, paths: WorkerPaths, - pdf_preprocessor: PDFPreprocessor | None = None, *, output_root: Path, force_reprocessing: bool, progress: AsyncProgressRateHandler | None = None, ) -> tuple[list[ProcessedPage], list[PassportProcessingError]]: - if pdf_preprocessor is None: - pdf_preprocessor = partial(process_pdf, colorspace=Colorspace.RGB) docs = [d async for d in async_read_jsonl_as(batch, PROCESSED_FILE_TA)] n_docs = len(docs) if progress is not None: @@ -303,14 +337,10 @@ async def _preprocess_pdf( output_root: Path, ) -> list[ProcessedPage]: pdf_path = doc.locate(paths) - async with async_open(pdf_path, "rb") as f: - pdf_bytes = await f.read() output_dir = output_root / safe_dir(doc.doc_id) / doc.doc_id output_dir.mkdir(parents=True, exist_ok=True) - pages = await asyncio.to_thread( - pdf_processor, + pages = await pdf_processor( pdf_path, - pdf_bytes, output_dir=output_dir, force_reprocessing=force_reprocessing, ) @@ -322,8 +352,6 @@ async def _preprocess_pdf( async def is_valid_pdf(path: Path) -> bool: - import pymupdf # noqa: PLC0415 - if not path.exists(): return False async with async_open(path, "rb") as f: @@ -338,3 +366,14 @@ async def is_valid_pdf(path: Path) -> bool: finally: if doc is not None: doc.close() + + +@asynccontextmanager +async def async_open_pdf(path: Path) -> AsyncGenerator[Document]: + doc = None + try: + doc = await asyncio.to_thread(pymupdf.open, path) + yield doc + finally: + if doc is not None: + doc.close() diff --git a/workers/passport-worker/passport_worker/workflows.py b/workers/passport-worker/passport_worker/workflows.py index ed2ebe1f..637034ac 100644 --- a/workers/passport-worker/passport_worker/workflows.py +++ b/workers/passport-worker/passport_worker/workflows.py @@ -166,7 +166,7 @@ async def preprocess( pdf_batches = preprocessing_batches.pdfs + pdf_paths preprocess_pdfs_tasks = _process_pdfs_tasks( pdf_batches, - args.project, + args, force_reprocessing=force_reprocessing, worker_config=worker_config, ) @@ -225,17 +225,19 @@ def _convert_to_pdfs_tasks( def _process_pdfs_tasks( batches: Batches, - project: str, + args: PassportDetectionArgs, *, force_reprocessing: bool, worker_config: PassportWorkerConfig, ) -> list[Coroutine]: all_tasks = [] + config = args.config.preprocessing.pdfs + project = args.project for b in batches: all_tasks.append( execute_activity( PassportDetectionActivities.preprocess_pdfs, - args=(b, project, force_reprocessing), + args=(b, project, config, force_reprocessing), task_queue=TaskQueue.IO, timeouts=worker_config.timeouts.pdf_preprocessing, ) diff --git a/workers/passport-worker/pyproject.toml b/workers/passport-worker/pyproject.toml index ca37fe21..8250aca5 100644 --- a/workers/passport-worker/pyproject.toml +++ b/workers/passport-worker/pyproject.toml @@ -11,19 +11,24 @@ requires-python = ">=3.12,<3.14" dependencies = [ "datashare-python~=0.12.12", "icij-common[elasticsearch-8]~=0.9.3", - "icij-passport-core==0.11.9", + "icij-passport-core==0.11.13", + # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + # The LiteparsePDFPreprocessor was benched for this specific version + # bumping it might result in passport false negative + "liteparse==2.19.0", + # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ ] [project.optional-dependencies] preprocessing = [ - "icij-passport-core[preprocessing]==0.11.9", + "icij-passport-core[preprocessing]==0.11.13", "pandas~=3.0", ] inference-gpu = [ - "icij-passport-core[inference,gpu]==0.11.9" + "icij-passport-core[inference,gpu]==0.11.13" ] inference-cpu = [ - "icij-passport-core[inference,cpu]==0.11.9" + "icij-passport-core[inference,cpu]==0.11.13" ] [tool.uv.sources] @@ -85,4 +90,4 @@ log_file_date_format = "%Y-%m-%d %H:%M:%S" [tool.ruff] target-version = "py312" -extend = "../../qa/ruff.toml" \ No newline at end of file +extend = "../../qa/ruff.toml" diff --git a/workers/passport-worker/tests/conftest.py b/workers/passport-worker/tests/conftest.py index a3ca091b..8f2d5143 100644 --- a/workers/passport-worker/tests/conftest.py +++ b/workers/passport-worker/tests/conftest.py @@ -1,6 +1,7 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path +from typing import Self import pytest from _pytest.tmpdir import TempPathFactory @@ -39,6 +40,7 @@ from datashare_python.utils import artifacts_dir, ext_to_mime_types from icij_common.es import ESClient from icij_common.pydantic_utils import safe_copy +from icij_common.registrable import RegistrableConfig from passport_worker.activities import Activity from passport_worker.config import ( @@ -46,6 +48,7 @@ PassportWorkerConfig, PreprocessingWorkerConfig, ) +from passport_worker.preprocessing import PDFPreprocessor from passport_worker.workflows import TaskQueue from . import DOCS_PATH, TEST_RESOURCE_PATH @@ -434,3 +437,24 @@ def test_model_path() -> Path: if not test_model_path.exists(): raise ValueError("place a model in here or implement automatic model DL") return test_model_path + + +class MockPDFPreprocessor(PDFPreprocessor): + def __init__(self, res: list[list[Path] | Exception]): + self._res = iter(res) + + async def __call__( + self, + path: Path, # noqa: ARG002 + *, + output_dir: Path, # noqa: ARG002 + force_reprocessing: bool, # noqa: ARG002 + ) -> list[Path]: + r = next(self._res) + if isinstance(r, Exception): + raise r + return r + + @classmethod + def _from_config(cls, config: RegistrableConfig, **extras) -> Self: # noqa: ARG003 + return cls() diff --git a/workers/passport-worker/tests/resources/liteparse_test.pdf b/workers/passport-worker/tests/resources/liteparse_test.pdf new file mode 100644 index 00000000..94012f04 Binary files /dev/null and b/workers/passport-worker/tests/resources/liteparse_test.pdf differ diff --git a/workers/passport-worker/tests/test_inference.py b/workers/passport-worker/tests/test_inference.py index 1f232eff..54bc1fee 100644 --- a/workers/passport-worker/tests/test_inference.py +++ b/workers/passport-worker/tests/test_inference.py @@ -41,7 +41,6 @@ ProcessedPage, YOLOPassportDetectorConfig, ) -from passport_worker.preprocessing import PDFPreprocessor from tests import DOCS_PATH from tests.conftest import DS_FILE_0 @@ -91,22 +90,6 @@ def _from_config(cls, config: RegistrableConfig, **extras) -> FromConfig: pass -class MockPDFPreprocessor(PDFPreprocessor): - def __init__(self, res: list[list[Path] | Exception]): - self._res = iter(res) - - def __call__( - self, - pdf_path: Path, # noqa: ARG002 - pdf_bytes: bytes, # noqa: ARG002 - output_dir: Path, # noqa: ARG002 - ) -> list[Path]: - r = next(self._res) - if isinstance(r, Exception): - raise r - return r - - @pytest.fixture def symlinked_doc_0() -> DatashareFile: # Let's test with a symlinked file diff --git a/workers/passport-worker/tests/test_liteparse.py b/workers/passport-worker/tests/test_liteparse.py new file mode 100644 index 00000000..37d1bc8f --- /dev/null +++ b/workers/passport-worker/tests/test_liteparse.py @@ -0,0 +1,99 @@ +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from liteparse.types import LayoutComplexityStats, PageComplexityStats + +from passport_worker.liteparse_ import ( + LayoutReason, + LiteparsePDFPreprocessor, + LiteparsePDFPreprocessorConfig, + OCRReason, + may_have_passport, +) +from passport_worker.preprocessing import PDFPreprocessor +from tests import TEST_RESOURCE_PATH + + +def test_liteparse_pdf_preprocessor_from_config() -> None: + # Given + config = LiteparsePDFPreprocessorConfig() + # When + processor = PDFPreprocessor.from_config(config) + # Then + assert isinstance(processor, LiteparsePDFPreprocessor) + + +async def test_should_skip_pages(tmp_path: Path) -> None: + # Given + path = TEST_RESOURCE_PATH / "liteparse_test.pdf" + output_dir = Path(tmp_path) + inner = AsyncMock() + pages = [Path("some_pdf_page_as.png")] + inner.process_pdf.return_value = pages + config = LiteparsePDFPreprocessorConfig() + preprocessor = LiteparsePDFPreprocessor(config, inner) + + # When + res = await preprocessor(path, output_dir=output_dir, force_reprocessing=True) + + # Then + assert res == pages + expected_skip = {0} + inner.process_pdf.assert_called_once_with( + path, expected_skip, output_dir=output_dir, force_reprocessing=True + ) + + +@pytest.fixture(scope="session") +def test_complexity() -> PageComplexityStats: + return PageComplexityStats( + page_number=0, + text_length=2, + text_coverage=0.5, + has_substantial_images=True, + image_block_count=1, + image_coverage=0.5, + largest_image_coverage=0.8, + full_page_image=True, + uncovered_vector_area=0.5, + is_garbled=True, + page_area=0.5, + needs_ocr=True, + reasons=["scanned"], + layout=LayoutComplexityStats( + column_count=1, + ruled_table_count=1, + ruled_table_coverage=0.5, + text_table_run_count=1, + figure_count=1, + figure_coverage=0.5, + is_complex=True, + reasons=["dense-graphics"], + ), + ) + + +@pytest.mark.parametrize( + ("ocr_reasons", "layout_reasons", "expected"), + [ + (set(), set(), False), + ({OCRReason.SCANNED}, set(), True), + ({OCRReason.EMBEDDED_IMAGES}, set(), False), + (set(), {LayoutReason.DENSE_GRAPHICS}, True), + (set(), {LayoutReason.TABLE_LIKELY}, False), + ], +) +async def test_may_have_passport( + test_complexity: PageComplexityStats, + ocr_reasons: set[OCRReason], + layout_reasons: set[LayoutReason], + *, + expected: bool, +) -> None: + # When + might_have_passport = may_have_passport( + test_complexity, ocr_reasons=ocr_reasons, layout_reasons=layout_reasons + ) + # Then + assert might_have_passport is expected diff --git a/workers/passport-worker/tests/test_preprocessing.py b/workers/passport-worker/tests/test_preprocessing.py index e56d6b0f..82c6378b 100644 --- a/workers/passport-worker/tests/test_preprocessing.py +++ b/workers/passport-worker/tests/test_preprocessing.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Self import pytest from datashare_python.conftest import TEST_PROJECT @@ -83,10 +84,9 @@ def __init__(self, res: list[list[Path] | Exception]): self._res = iter(res) self.processed = [] - def __call__( + async def __call__( self, pdf_path: Path, - pdf_bytes: bytes, # noqa: ARG002 output_dir: Path, # noqa: ARG002 *, force_reprocessing: bool, @@ -99,6 +99,9 @@ def __call__( self.processed.append(pdf_path) return r + @classmethod + def _from_config(cls, config: RegistrableConfig, **extras) -> Self: ... + @pytest.fixture def symlinked_doc_0() -> DatashareFile: @@ -289,14 +292,14 @@ async def test_convert_to_pdfs_act_caching( @pytest.fixture def doc_1_pages( - test_worker_config: PassportWorkerConfig, doc_0: Document + test_worker_config: PassportWorkerConfig, doc_1: Document ) -> list[Path]: worker_paths = test_worker_config.paths workdir = worker_paths.workdir output_root = workdir.joinpath(TEST_PROJECT, "workflow_id") output_root.mkdir(parents=True, exist_ok=True) - doc_0_page_dir = output_root / safe_dir(doc_0.id) / doc_0.id - doc_0_pages = [doc_0_page_dir / "page_0.png", doc_0_page_dir / "page_1.png"] + doc_1_page_dir = output_root / safe_dir(doc_1.id) / doc_1.id + doc_0_pages = [doc_1_page_dir / "page_0.png", doc_1_page_dir / "page_1.png"] return doc_0_pages @@ -318,9 +321,9 @@ async def test_preprocess_pdfs_act( batch_path.write_text("\n".join(d.model_dump_json() for d in batch)) # When successes, errors = await preprocess_pdfs_act( + preprocessor, batch_path, worker_paths, - preprocessor, output_root=output_root, force_reprocessing=True, ) @@ -362,9 +365,9 @@ async def test_preprocess_pdfs_act_caching( p.write_bytes((DOCS_PATH / "passport.png").read_bytes()) # When successes, errors = await preprocess_pdfs_act( + preprocessor, batch_path, worker_paths, - preprocessor, output_root=output_root, force_reprocessing=False, ) diff --git a/workers/passport-worker/uv.dist.lock b/workers/passport-worker/uv.dist.lock index d7f2cbff..6409a98c 100644 --- a/workers/passport-worker/uv.dist.lock +++ b/workers/passport-worker/uv.dist.lock @@ -297,6 +297,7 @@ dependencies = [ { name = "datashare-python" }, { name = "icij-common", extra = ["elasticsearch-8"] }, { name = "icij-passport-core" }, + { name = "liteparse" }, ] [package.optional-dependencies] @@ -324,10 +325,11 @@ dev = [ requires-dist = [ { name = "datashare-python", specifier = "~=0.12.12" }, { name = "icij-common", extras = ["elasticsearch-8"], specifier = "~=0.9.3" }, - { name = "icij-passport-core", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.9" }, + { name = "icij-passport-core", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.13" }, + { name = "liteparse", specifier = ">=2.19.0" }, { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -537,15 +539,15 @@ elasticsearch-8 = [ [[package]] name = "icij-passport-core" -version = "0.11.9" +version = "0.11.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiostream" }, { name = "icij-common" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/b3/034a1e061d3b9bf9df8791b476fb4859566180e574db36c8b7c90fb2c85f/icij_passport_core-0.11.9.tar.gz", hash = "sha256:56c519c8c2a4d2193f5a240a999c3e3f7ddbcfed3a82e388e2b4aba9f7acadcc", size = 14490527, upload-time = "2026-11-10T17:40:00.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2e/6cf9adb636492021cc1c58e0df00cb5941afa7bbc06921b2cb7133efe27d/icij_passport_core-0.11.13.tar.gz", hash = "sha256:6e6ba913ec7ed887645b20c16be705fa9343c420b38949c097ae89d5408fe367", size = 14490699, upload-time = "2026-11-18T16:11:21.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/45/4e6e4c63fdb37b8744077b4f3b54940f0659429506d08d73378c80f4c43e/icij_passport_core-0.11.9-py3-none-any.whl", hash = "sha256:4a859da648994c731771c3538100e64c48404616e2fd7c4be7e1cf45523203af", size = 10959056, upload-time = "2026-11-10T17:39:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ff/410d9f224bfe7f2c3db953d848e73559dfc4df8e61eb84df577a9555dbca/icij_passport_core-0.11.13-py3-none-any.whl", hash = "sha256:a77d693b8bf21067e5f7162c3a2188eb0a5920a6d4af04ec2ad1fd86f04c8efe", size = 10959255, upload-time = "2026-11-18T16:11:18.964Z" }, ] [package.optional-dependencies] @@ -715,6 +717,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/04/e8d3b4e89cba70babf4bf58fce963dc6f627ae8192d2a45b495f42422880/Levenshtein-0.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:5a61606bad3afb9fcec0a2a21871319c3f7da933658d2e0e6e55ab4a34814f48", size = 90919, upload-time = "2023-10-09T13:38:52.507Z" }, ] +[[package]] +name = "liteparse" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/c1/562179f34bc184cb874d7f4fefb23c6f471d007c9cb5e5a1df47d4a73c3e/liteparse-2.19.0.tar.gz", hash = "sha256:b90f6cb350611853f6142edafefb715b5973ca8a2998e36b8837781332a089de", size = 688596, upload-time = "2026-11-13T02:37:49.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/15/403d15c29c40c9a20352d632ff0c679b907f0e7c17294607a313b0d8099f/liteparse-2.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffbc26835590160313fec3736c762ae57088d806adc4395dc87090e4e5607993", size = 11752816, upload-time = "2026-11-13T02:37:35.3Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/96128effcacc673e35a0313e34ca461199d08cdc3cb3e04807e461bd1c02/liteparse-2.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:52742af76cbf7dfb4b451d417eb99d87d1dacfceb2bfb89849db5e6b052f2618", size = 11203396, upload-time = "2026-11-13T02:37:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d0/ded0c9c41d71d90fe6ac9889d952ef44d89099d3b85301d8952d316abfcc/liteparse-2.19.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7eea6b66ff5b718f4672c45032ba642e836aef005f782836c73b261f2991a57f", size = 12692857, upload-time = "2026-11-13T02:37:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/fe/43/492849721200c35822e5f6eb6517170bd306bc5cfe96ba1bbf910b67e837/liteparse-2.19.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8d1663f742fd4b8e504ebea757a219658b10755088a619243858e376f9f98531", size = 12956108, upload-time = "2026-11-13T02:37:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/d96ec3444733d03a22a8cad1b592bce0501feef8f26e5e2a0ed88bc50f71/liteparse-2.19.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1c09f7911d502e044c8f41807213534808ffbd40a64d8abce053ef0699c6d5a1", size = 16089685, upload-time = "2026-11-13T02:37:43.256Z" }, + { url = "https://files.pythonhosted.org/packages/78/24/ff81674f6d338199e27ba5dd545925202e3bca8ea371acaa42818a652c7c/liteparse-2.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:f58b5e1eda4ac2fbcc17346c9e40162ed9e99c7a8bad95dff50f336e4bbb2f6f", size = 11146829, upload-time = "2026-11-13T02:37:45.566Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4f/0a200526b7858704ed6841c46702cc417bb27aa56fff579a6f5ab3a037ff/liteparse-2.19.0-cp310-abi3-win_arm64.whl", hash = "sha256:6daddd5770a56761ab4f0afd6b96d5a8670c845c9df06882a67eafcc21c05242", size = 10604750, upload-time = "2026-11-13T02:37:48.314Z" }, +] + [[package]] name = "lru-dict" version = "1.4.1" diff --git a/workers/passport-worker/uv.lock b/workers/passport-worker/uv.lock index f867ca00..361935dc 100644 --- a/workers/passport-worker/uv.lock +++ b/workers/passport-worker/uv.lock @@ -297,6 +297,7 @@ dependencies = [ { name = "datashare-python" }, { name = "icij-common", extra = ["elasticsearch-8"] }, { name = "icij-passport-core" }, + { name = "liteparse" }, ] [package.optional-dependencies] @@ -324,10 +325,11 @@ dev = [ requires-dist = [ { name = "datashare-python", editable = "../../datashare-python" }, { name = "icij-common", extras = ["elasticsearch-8"], specifier = "~=0.9.3" }, - { name = "icij-passport-core", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.9" }, + { name = "icij-passport-core", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.13" }, + { name = "liteparse", specifier = ">=2.19.0" }, { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -561,15 +563,15 @@ elasticsearch-8 = [ [[package]] name = "icij-passport-core" -version = "0.11.9" +version = "0.11.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiostream" }, { name = "icij-common" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/b3/034a1e061d3b9bf9df8791b476fb4859566180e574db36c8b7c90fb2c85f/icij_passport_core-0.11.9.tar.gz", hash = "sha256:56c519c8c2a4d2193f5a240a999c3e3f7ddbcfed3a82e388e2b4aba9f7acadcc", size = 14490527, upload-time = "2026-11-10T17:40:00.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2e/6cf9adb636492021cc1c58e0df00cb5941afa7bbc06921b2cb7133efe27d/icij_passport_core-0.11.13.tar.gz", hash = "sha256:6e6ba913ec7ed887645b20c16be705fa9343c420b38949c097ae89d5408fe367", size = 14490699, upload-time = "2026-11-18T16:11:21.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/45/4e6e4c63fdb37b8744077b4f3b54940f0659429506d08d73378c80f4c43e/icij_passport_core-0.11.9-py3-none-any.whl", hash = "sha256:4a859da648994c731771c3538100e64c48404616e2fd7c4be7e1cf45523203af", size = 10959056, upload-time = "2026-11-10T17:39:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ff/410d9f224bfe7f2c3db953d848e73559dfc4df8e61eb84df577a9555dbca/icij_passport_core-0.11.13-py3-none-any.whl", hash = "sha256:a77d693b8bf21067e5f7162c3a2188eb0a5920a6d4af04ec2ad1fd86f04c8efe", size = 10959255, upload-time = "2026-11-18T16:11:18.964Z" }, ] [package.optional-dependencies] @@ -739,6 +741,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/04/e8d3b4e89cba70babf4bf58fce963dc6f627ae8192d2a45b495f42422880/Levenshtein-0.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:5a61606bad3afb9fcec0a2a21871319c3f7da933658d2e0e6e55ab4a34814f48", size = 90919, upload-time = "2023-10-09T13:38:52.507Z" }, ] +[[package]] +name = "liteparse" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/c1/562179f34bc184cb874d7f4fefb23c6f471d007c9cb5e5a1df47d4a73c3e/liteparse-2.19.0.tar.gz", hash = "sha256:b90f6cb350611853f6142edafefb715b5973ca8a2998e36b8837781332a089de", size = 688596, upload-time = "2026-11-13T02:37:49.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/15/403d15c29c40c9a20352d632ff0c679b907f0e7c17294607a313b0d8099f/liteparse-2.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffbc26835590160313fec3736c762ae57088d806adc4395dc87090e4e5607993", size = 11752816, upload-time = "2026-11-13T02:37:35.3Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/96128effcacc673e35a0313e34ca461199d08cdc3cb3e04807e461bd1c02/liteparse-2.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:52742af76cbf7dfb4b451d417eb99d87d1dacfceb2bfb89849db5e6b052f2618", size = 11203396, upload-time = "2026-11-13T02:37:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d0/ded0c9c41d71d90fe6ac9889d952ef44d89099d3b85301d8952d316abfcc/liteparse-2.19.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7eea6b66ff5b718f4672c45032ba642e836aef005f782836c73b261f2991a57f", size = 12692857, upload-time = "2026-11-13T02:37:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/fe/43/492849721200c35822e5f6eb6517170bd306bc5cfe96ba1bbf910b67e837/liteparse-2.19.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8d1663f742fd4b8e504ebea757a219658b10755088a619243858e376f9f98531", size = 12956108, upload-time = "2026-11-13T02:37:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/d96ec3444733d03a22a8cad1b592bce0501feef8f26e5e2a0ed88bc50f71/liteparse-2.19.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1c09f7911d502e044c8f41807213534808ffbd40a64d8abce053ef0699c6d5a1", size = 16089685, upload-time = "2026-11-13T02:37:43.256Z" }, + { url = "https://files.pythonhosted.org/packages/78/24/ff81674f6d338199e27ba5dd545925202e3bca8ea371acaa42818a652c7c/liteparse-2.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:f58b5e1eda4ac2fbcc17346c9e40162ed9e99c7a8bad95dff50f336e4bbb2f6f", size = 11146829, upload-time = "2026-11-13T02:37:45.566Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4f/0a200526b7858704ed6841c46702cc417bb27aa56fff579a6f5ab3a037ff/liteparse-2.19.0-cp310-abi3-win_arm64.whl", hash = "sha256:6daddd5770a56761ab4f0afd6b96d5a8670c845c9df06882a67eafcc21c05242", size = 10604750, upload-time = "2026-11-13T02:37:48.314Z" }, +] + [[package]] name = "lru-dict" version = "1.4.1"