From 9089d830b31205ffc3b460f5c6fc4d4d4a1be631 Mon Sep 17 00:00:00 2001 From: Stommer Date: Sat, 24 Oct 2026 20:40:47 +0200 Subject: [PATCH] =?UTF-8?q?Erweitere=20die=20Paperless-Suche=20um=20Suchar?= =?UTF-8?q?t,=20weitere=20Filter=20und=20eine=20Vorschau=20je=20Treffer,?= =?UTF-8?q?=20damit=20sich=20das=20richtige=20Dokument=20finden=20und=20er?= =?UTF-8?q?kennen=20l=C3=A4sst.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standard ist jetzt Titel und Inhalt ohne Index; Titel, Inhalt und Volltextindex sind wählbar. Neue Filter: Speicherpfad, Tag-Verknüpfung (alle, eines, keines), "ohne" bei Korrespondent, Typ und Pfad, Hinzugefügt-Datum und Archivnummer. Treffer zeigen Vorschaubild, Textauszug, Tags, Typ und Seitenzahl; die Vorschaubilder laufen über den Server. Co-Authored-By: Claude Sonnet 5.5 --- backend/app/api/sources.py | 142 ++++----- backend/app/sourceport.py | 18 ++ backend/app/sources.py | 221 ++++++++++++-- backend/tests/test_connector_basis.py | 2 +- backend/tests/test_connector_followup.py | 148 ++++++++- backend/tests/test_correction_k1_k7.py | 2 +- docs/architecture/API_REFERENZ.md | 5 +- frontend/src/pages/Sources.jsx | 352 +++++++++++++++++----- frontend/src/sources/connector-config.jsx | 6 + frontend/src/styles.css | 49 +++ 10 files changed, 760 insertions(+), 185 deletions(-) diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index aa0d4b0..fd18171 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -28,6 +28,7 @@ from app.sources import ( list_connectors, owned_connector, paperless_content, + paperless_thumbnail, parse_connector, probe, save_connector, @@ -264,40 +265,53 @@ def _mail_bytes(data: bytes, media_type: str, filename: str, inline: bool = Fals ) +class SearchFilters: + """The query parameters every search route shares. A source answers only those it declares.""" + + def __init__( + self, + mode: str = "", + title: str = "", + correspondent: str = "", + document_type: str = "", + storage_path: str = "", + tag: str = "", + tag_mode: str = "", + document_from: str = "", + document_to: str = "", + added_from: str = "", + added_to: str = "", + asn: str = "", + sort: str = "", + cursor: str = "", + ): + self.values = { + "mode": mode, + "title": title, + "correspondent": correspondent, + "document_type": document_type, + "storage_path": storage_path, + "tag": tag, + "tag_mode": tag_mode, + "document_from": document_from, + "document_to": document_to, + "added_from": added_from, + "added_to": added_to, + "asn": asn, + "sort": sort, + "cursor": cursor, + } + + @router.get("/api/v1/sources/paperless/documents") def search_paperless( q: str = "", connector_id: UUID | None = None, - title: str = "", - correspondent: str = "", - document_type: str = "", - tag: str = "", - document_from: str = "", - document_to: str = "", - sort: str = "", - cursor: str = "", + filters: SearchFilters = Depends(), principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn), ): - return _run_search( - "paperless", - conn, - principal.account_id, - q.strip(), - connector_id, - "", - None, - { - "title": title, - "correspondent": correspondent, - "document_type": document_type, - "tag": tag, - "document_from": document_from, - "document_to": document_to, - "sort": sort, - "cursor": cursor, - }, - ) + return _run_search("paperless", conn, principal.account_id, q.strip(), connector_id, "", None, filters.values) @router.get("/api/v1/sources/{kind}/facets") @@ -328,6 +342,17 @@ def preview_paperless_document( ) +@router.get("/api/v1/sources/paperless/documents/{document_id}/thumb") +def thumbnail_paperless_document( + document_id: str, + connector_id: UUID, + principal: Principal = Depends(get_principal), + conn: Connection = Depends(get_conn), +): + data, media = paperless_thumbnail(conn, principal.account_id, document_id, connector_id) + return Response(content=data, media_type=media, headers=_PRIVATE) + + @router.get("/api/v1/sources/imap/mailboxes") def list_mailboxes( connector_id: UUID, @@ -343,39 +368,15 @@ def search_mail( q: str = "", mailbox: str = "", before: int | None = None, - title: str = "", - correspondent: str = "", - document_type: str = "", - tag: str = "", - document_from: str = "", - document_to: str = "", - sort: str = "", - cursor: str = "", uidvalidity: int | None = None, + filters: SearchFilters = Depends(), principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn), ): if before is not None and before < 1: raise unprocessable("Die Seitenangabe ist ungültig.") return _run_search( - "imap", - conn, - principal.account_id, - q.strip(), - connector_id, - mailbox.strip(), - before, - { - "title": title, - "correspondent": correspondent, - "document_type": document_type, - "tag": tag, - "document_from": document_from, - "document_to": document_to, - "sort": sort, - "cursor": cursor, - }, - uidvalidity, + "imap", conn, principal.account_id, q.strip(), connector_id, mailbox.strip(), before, filters.values, uidvalidity ) @@ -432,12 +433,18 @@ def _connector_parts(body: ConnectorIn, current: dict | None) -> tuple[str, dict _SEARCH_FLAGS = { + "mode": "mode", "title": "title", "correspondent": "correspondent", "document_type": "document_type", + "storage_path": "storage_path", "tag": "tags", + "tag_mode": "tag_mode", "document_from": "document_range", "document_to": "document_range", + "added_from": "added_range", + "added_to": "added_range", + "asn": "asn", "sort": "sort", "cursor": "continuation", } @@ -474,38 +481,13 @@ def search_source( connector_id: UUID | None = None, mailbox: str = "", before: int | None = None, - title: str = "", - correspondent: str = "", - document_type: str = "", - tag: str = "", - document_from: str = "", - document_to: str = "", - sort: str = "", - cursor: str = "", + filters: SearchFilters = Depends(), principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn), ): if before is not None and before < 1: raise unprocessable("Die Seitenangabe ist ungültig.") - return _run_search( - kind, - conn, - principal.account_id, - q.strip(), - connector_id, - mailbox.strip(), - before, - { - "title": title, - "correspondent": correspondent, - "document_type": document_type, - "tag": tag, - "document_from": document_from, - "document_to": document_to, - "sort": sort, - "cursor": cursor, - }, - ) + return _run_search(kind, conn, principal.account_id, q.strip(), connector_id, mailbox.strip(), before, filters.values) @router.get("/api/v1/sources/{kind}/navigate") diff --git a/backend/app/sourceport.py b/backend/app/sourceport.py index 87d1428..2b72cf3 100644 --- a/backend/app/sourceport.py +++ b/backend/app/sourceport.py @@ -16,13 +16,19 @@ PAPERLESS_CAPABILITIES = { "edit_metadata": False, "search": { "query": True, + "mode": True, "title": True, "correspondent": True, "document_type": True, + "storage_path": True, "tags": True, + "tag_mode": True, "document_range": True, + "added_range": True, + "asn": True, "sort": True, "continuation": True, + "result_preview": True, }, } @@ -36,13 +42,19 @@ IMAP_CAPABILITIES = { "edit_metadata": False, "search": { "query": True, + "mode": False, "title": False, "correspondent": False, "document_type": False, + "storage_path": False, "tags": False, + "tag_mode": False, "document_range": False, + "added_range": False, + "asn": False, "sort": False, "continuation": True, + "result_preview": False, }, } @@ -57,12 +69,18 @@ EDITORS = { } UNSUPPORTED_SEARCH = ( + "mode", "title", "correspondent", "document_type", + "storage_path", "tag", + "tag_mode", "document_from", "document_to", + "added_from", + "added_to", + "asn", "sort", "cursor", ) diff --git a/backend/app/sources.py b/backend/app/sources.py index b302840..9a9e053 100644 --- a/backend/app/sources.py +++ b/backend/app/sources.py @@ -3,14 +3,18 @@ from __future__ import annotations import base64 +import hashlib import imaplib import json import re import ssl +import threading +import time from dataclasses import dataclass from email.header import decode_header from email.parser import BytesParser from email.utils import parsedate_to_datetime +from html import unescape from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urlparse @@ -183,10 +187,24 @@ def _record_probe(conn, connector_id: UUID, ok: bool, detail: str) -> None: ) -def _paperless_choice(value, label: str) -> str | None: +PAPERLESS_PAGE_SIZE = 20 +_PAPERLESS_MODES = { + # Paperless-ngx offers these searches in its own filter bar. The first needs no search index. + "title_content": "title_content", + "title": "title__icontains", + "content": "content__icontains", + "index": "query", +} +_PAPERLESS_TAG_MODES = {"all": "tags__id__all", "any": "tags__id__in", "none": "tags__id__none"} +_PAPERLESS_SORTS = {"created", "-created", "added", "-added", "title", "-title", "score", "-score"} + + +def _paperless_choice(value, label: str, allow_none: bool = False) -> str | None: text = str(value or "").strip() if not text: return None + if allow_none and text == "none": + return text if not text.isdigit() or len(text) > 12: raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.") return text @@ -201,45 +219,67 @@ def _paperless_day(value, label: str) -> str | None: return text +def _paperless_range(options: dict, prefix: str, field: str, label: str, params: dict) -> None: + start = _paperless_day(options.get(f"{prefix}_from"), f"{label}, von") + end = _paperless_day(options.get(f"{prefix}_to"), f"{label}, bis") + if start and end and start > end: + raise unprocessable(f"{label}: Der Beginn liegt nach dem Ende.") + if start: + params[f"{field}__date__gte"] = start + if end: + params[f"{field}__date__lte"] = end + + def _paperless_params(query: str, options: dict | None) -> dict: """Paperless-ngx document date is the created field, not the upload or modified time.""" options = options or {} - params = {"page_size": "20"} + params = {"page_size": str(PAPERLESS_PAGE_SIZE)} text = " ".join(str(query or "").split()) + mode = str(options.get("mode") or "").strip() or "title_content" + if mode not in _PAPERLESS_MODES: + raise unprocessable("Diese Suchart gibt es nicht.") if text: - params["query"] = text + params[_PAPERLESS_MODES[mode]] = text title = " ".join(str(options.get("title") or "").split()) if title: + if "title__icontains" in params and params["title__icontains"] != title: + raise unprocessable("Die Suche nur im Titel und der Titelfilter sind beide gesetzt. Nutze eines davon.") params["title__icontains"] = title - correspondent = _paperless_choice(options.get("correspondent"), "Der Korrespondent") - if correspondent: - params["correspondent__id"] = correspondent - document_type = _paperless_choice(options.get("document_type"), "Der Dokumenttyp") - if document_type: - params["document_type__id"] = document_type + for name, field, label in ( + ("correspondent", "correspondent", "Der Korrespondent"), + ("document_type", "document_type", "Der Dokumenttyp"), + ("storage_path", "storage_path", "Der Speicherpfad"), + ): + chosen = _paperless_choice(options.get(name), label, allow_none=True) + if chosen == "none": + params[f"{field}__isnull"] = "true" + elif chosen: + params[f"{field}__id"] = chosen raw_tags = options.get("tag") or options.get("tags") or "" if isinstance(raw_tags, (list, tuple)): parts = [str(item) for item in raw_tags] else: parts = str(raw_tags).split(",") tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen] + tag_mode = str(options.get("tag_mode") or "").strip() or "all" + if tag_mode not in _PAPERLESS_TAG_MODES: + raise unprocessable("Diese Tag-Verknüpfung gibt es nicht.") if tag_ids: - params["tags__id__all"] = ",".join(tag_ids) - start = _paperless_day(options.get("document_from"), "Das Dokumentdatum von") - end = _paperless_day(options.get("document_to"), "Das Dokumentdatum bis") - if start and end and start > end: - raise unprocessable("Das Dokumentdatum von liegt nach dem Dokumentdatum bis.") - if start: - params["created__date__gte"] = start - if end: - params["created__date__lte"] = end + params[_PAPERLESS_TAG_MODES[tag_mode]] = ",".join(tag_ids) + _paperless_range(options, "document", "created", "Dokumentdatum", params) + _paperless_range(options, "added", "added", "Hinzugefügt am", params) + asn = str(options.get("asn") or "").strip() + if asn: + if not asn.isdigit() or len(asn) > 10: + raise unprocessable("Die Archivnummer ist keine Zahl.") + params["archive_serial_number"] = asn sort = str(options.get("sort") or "").strip() if sort: - if sort not in {"created", "-created", "title", "-title", "score", "-score"}: + if sort not in _PAPERLESS_SORTS: raise unprocessable("Diese Sortierung gibt es nicht.") if sort in {"score", "-score"}: - if not text: - raise unprocessable("Die Relevanz gilt zusammen mit einem Suchtext.") + if not text or mode != "index": + raise unprocessable("Die Relevanz gilt nur für die Suche im Volltextindex.") # Paperless-ngx 2.20.6 has no ordering field named score. A query without # ordering is the relevance order of the full-text index. else: @@ -279,42 +319,159 @@ def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | N params = _paperless_params(query, options) release_for_external(conn) payload = _paperless_get(base, row["secret"], "/api/documents/", params) + names = _paperless_lookup(row["id"], base, row["secret"]) assert_still_owned(conn, row["id"], owner_id) + terms = _search_terms(query, (options or {}).get("title")) items = [] for doc in payload.get("results") or []: created = str(doc.get("created") or "")[:10] + added = str(doc.get("added") or "")[:10] title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip() + pages = doc.get("page_count") + asn = doc.get("archive_serial_number") items.append( { "id": str(doc.get("id")), "title": title[:200], "document_on": created if len(created) == 10 else None, - "correspondent": _correspondent(doc), + "added_on": added if len(added) == 10 else None, + "correspondent": _correspondent(doc, names), + "document_type": _named(doc.get("document_type"), names["document_types"]), + "storage_path": _named(doc.get("storage_path"), names["storage_paths"]), + "tags": [name for name in (_named(tag, names["tags"]) for tag in doc.get("tags") or []) if name][:12], + "pages": pages if isinstance(pages, int) and pages > 0 else None, + "asn": asn if isinstance(asn, int) else None, + "snippet": _snippet(doc, terms), "open_url": f"{base}/documents/{doc.get('id')}/", } ) page = int(params.get("page") or 1) count = payload.get("count") if isinstance(count, int): - next_page = str(page + 1) if page * 20 < count else None + next_page = str(page + 1) if page * PAPERLESS_PAGE_SIZE < count else None else: - next_page = str(page + 1) if len(items) == 20 else None - return {"items": items, "connector_id": str(row["id"]), "next_page": next_page, "incomplete": False} + next_page = str(page + 1) if len(items) == PAPERLESS_PAGE_SIZE else None + return { + "items": items, + "connector_id": str(row["id"]), + "next_page": next_page, + "previous_page": str(page - 1) if page > 1 else None, + "total": count if isinstance(count, int) else None, + "incomplete": False, + } def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict: row = _paperless_row(conn, owner_id, connector_id) base = row["settings"]["base_url"].rstrip("/") - token = row["secret"] release_for_external(conn) - body = { + names = _paperless_lookup(row["id"], base, row["secret"]) + assert_still_owned(conn, row["id"], owner_id) + return { + "correspondents": names["correspondents"], + "document_types": names["document_types"], + "storage_paths": names["storage_paths"], + "tags": names["tags"], + "tag_mode": "all", + "tag_modes": list(_PAPERLESS_TAG_MODES), + "modes": list(_PAPERLESS_MODES), + } + + +def paperless_thumbnail(conn, owner_id: UUID, document_id: str, connector_id: UUID) -> tuple[bytes, str]: + if not document_id.isdigit() or len(document_id) > 12: + raise unprocessable("Die Paperless-Nummer ist ungültig.") + row = _paperless_row(conn, owner_id, connector_id) + base = row["settings"]["base_url"].rstrip("/") + release_for_external(conn) + fetched = _paperless_file(base, row["secret"], f"/api/documents/{document_id}/thumb/", 2 * 1024 * 1024) + assert_still_owned(conn, row["id"], owner_id) + if fetched is None: + raise not_found() + data, media, _name = fetched + if not media.startswith("image/"): + raise unprocessable("Paperless hat kein Vorschaubild geliefert.") + return data, media + + +_LOOKUP_TTL = 60.0 +_LOOKUP_LIMIT = 32 +_lookup_cache: dict[tuple, tuple[float, dict]] = {} +_lookup_lock = threading.Lock() + + +def _paperless_lookup(connector_id, base: str, token: str) -> dict: + """Names behind the ids of a result list. Bound to the instance and key that asked, kept for a minute.""" + key = (str(connector_id), base, hashlib.sha256(token.encode("utf-8")).hexdigest()) + now = time.monotonic() + with _lookup_lock: + hit = _lookup_cache.get(key) + if hit and now - hit[0] < _LOOKUP_TTL: + return hit[1] + names = { "correspondents": _paperless_names(base, token, "/api/correspondents/"), "document_types": _paperless_names(base, token, "/api/document_types/"), "tags": _paperless_names(base, token, "/api/tags/"), - "tag_mode": "all", + # An instance without storage paths, or a key that may not read them, still searches. + "storage_paths": _paperless_names_or_none(base, token, "/api/storage_paths/"), } - assert_still_owned(conn, row["id"], owner_id) - return body + with _lookup_lock: + if len(_lookup_cache) >= _LOOKUP_LIMIT: + _lookup_cache.clear() + _lookup_cache[key] = (now, names) + return names + + +def _paperless_names_or_none(base: str, token: str, path: str) -> list[dict]: + try: + return _paperless_names(base, token, path) + except Problem: + return [] + + +def _named(value, entries: list[dict]) -> str | None: + if value is None or isinstance(value, bool): + return None + wanted = str(value) + for entry in entries: + if entry["id"] == wanted: + return entry["name"] + return None + + +_TAG_MARKUP = re.compile(r"<[^>]{0,200}>") +_SNIPPET_LENGTH = 260 + + +def _search_terms(query: str, title) -> list[str]: + words = re.findall(r"[\w-]{3,}", f"{query or ''} {title or ''}".casefold()) + return list(dict.fromkeys(words))[:8] + + +def _snippet(doc: dict, terms: list[str]) -> str | None: + """A short plain-text excerpt: the index hit when Paperless sends one, else the text around the first match.""" + hit = doc.get("__search_hit__") + if isinstance(hit, dict): + lines = hit.get("highlights") + if isinstance(lines, str) and lines.strip(): + text = " ".join(unescape(_TAG_MARKUP.sub("", lines)).split()) + if text: + return text[:_SNIPPET_LENGTH] + content = doc.get("content") + if not isinstance(content, str): + return None + text = " ".join(content.split()) + if not text: + return None + folded = text.casefold() + at = min((index for index in (folded.find(term) for term in terms) if index >= 0), default=-1) + if at < 0: + return text[:_SNIPPET_LENGTH] + ("…" if len(text) > _SNIPPET_LENGTH else "") + first = max(0, at - 60) + if first: + first = text.rfind(" ", 0, first) + 1 or first + last = first + _SNIPPET_LENGTH + return ("…" if first else "") + text[first:last] + ("…" if last < len(text) else "") def _paperless_names(base: str, token: str, path: str) -> list[dict]: @@ -679,10 +836,12 @@ def _paperless_get(base: str, token: str, path: str, params: dict) -> dict: raise unprocessable("Paperless ist nicht erreichbar.") from exc -def _correspondent(doc: dict) -> str | None: +def _correspondent(doc: dict, names: dict | None = None) -> str | None: value = doc.get("correspondent_name") or doc.get("correspondent") if isinstance(value, str) and value.strip(): return value.strip()[:200] + if names is not None: + return _named(value, names["correspondents"]) return None diff --git a/backend/tests/test_connector_basis.py b/backend/tests/test_connector_basis.py index be1b56f..40473cc 100644 --- a/backend/tests/test_connector_basis.py +++ b/backend/tests/test_connector_basis.py @@ -216,7 +216,7 @@ def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monke ) assert filtered["items"][0]["title"] == "Treffer" assert calls[-1][3]["correspondent__id"] == "4" - assert calls[-1][3]["query"] == "Treffer" + assert calls[-1][3]["title_content"] == "Treffer" def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch): diff --git a/backend/tests/test_connector_followup.py b/backend/tests/test_connector_followup.py index c536e15..e877ae9 100644 --- a/backend/tests/test_connector_followup.py +++ b/backend/tests/test_connector_followup.py @@ -355,22 +355,34 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch params={ "connector_id": first["id"], "q": "nurinhalt", + "mode": "index", "title": "Vertrag", "correspondent": "4", "document_type": "8", + "storage_path": "6", "tag": "2,5", + "tag_mode": "any", "document_from": "2024-01-01", "document_to": "2024-12-31", + "added_from": "2024-02-01", + "added_to": "2024-02-28", + "asn": "1042", "sort": "-created", "cursor": "3", }, ) sent = calls[-1]["params"] assert sent["query"] == "nurinhalt" + assert "title_content" not in sent assert sent["title__icontains"] == "Vertrag" assert sent["correspondent__id"] == "4" assert sent["document_type__id"] == "8" - assert sent["tags__id__all"] == "2,5" + assert sent["storage_path__id"] == "6" + assert sent["tags__id__in"] == "2,5" + assert "tags__id__all" not in sent + assert sent["added__date__gte"] == "2024-02-01" + assert sent["added__date__lte"] == "2024-02-28" + assert sent["archive_serial_number"] == "1042" assert sent["created__date__gte"] == "2024-01-01" assert sent["created__date__lte"] == "2024-12-31" assert sent["ordering"] == "-created" @@ -378,9 +390,15 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch ada.request( "GET", "/api/v1/sources/paperless/documents", - params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"}, + params={"connector_id": first["id"], "q": "nurinhalt", "mode": "index", "sort": "-score"}, ) assert "ordering" not in calls[-1]["params"] + ada.request( + "GET", + "/api/v1/sources/paperless/documents", + params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"}, + status=422, + ) assert found["items"][0]["id"] == "41" assert found["connector_id"] == first["id"] preview = ada.client.get( @@ -431,6 +449,132 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch ) +def test_paperless_default_search_reads_title_and_content_and_shows_a_preview(client, db, monkeypatch): + import app.sources as sources + + calls = [] + body = "Vorspann " * 30 + "Die Kündigung des Mietvertrags erfolgt zum Jahresende. " + "Nachspann " * 30 + + def fake_get(base, token, path, params): + calls.append({"path": path, "params": dict(params)}) + if path == "/api/documents/": + return { + "count": 1, + "results": [ + { + "id": 7, + "title": "Schreiben", + "created": "2024-03-01T00:00:00+01:00", + "added": "2024-03-05T08:00:00Z", + "correspondent": 4, + "document_type": 8, + "storage_path": None, + "tags": [2, 5, 99], + "archive_serial_number": 1042, + "page_count": 3, + "content": body, + } + ], + } + if path == "/api/storage_paths/": + raise sources.unprocessable("Paperless hat die Anfrage nicht beantwortet.") + if path == "/api/tags/": + return {"results": [{"id": 2, "name": "Miete"}, {"id": 5, "name": "Wohnung"}]} + if path == "/api/correspondents/": + return {"results": [{"id": 4, "name": "Hausverwaltung"}]} + return {"results": [{"id": 8, "name": "Brief"}]} + + monkeypatch.setattr(sources, "_paperless_get", fake_get) + ada = Actor(client, "ada-preview@example.test", "Ada Preview") + register(db, ada.email, ada.name) + ada.login() + first = ada.request( + "POST", + "/api/v1/connectors", + json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"}, + status=201, + ) + found = ada.request( + "GET", + "/api/v1/sources/paperless/documents", + params={"connector_id": first["id"], "q": "kündigung", "correspondent": "none", "tag": "2", "tag_mode": "none"}, + ) + sent = [call for call in calls if call["path"] == "/api/documents/"][-1]["params"] + assert sent["title_content"] == "kündigung" + assert "query" not in sent + assert sent["correspondent__isnull"] == "true" + assert sent["tags__id__none"] == "2" + item = found["items"][0] + assert item["correspondent"] == "Hausverwaltung" + assert item["document_type"] == "Brief" + assert item["tags"] == ["Miete", "Wohnung"] + assert item["pages"] == 3 + assert item["asn"] == 1042 + assert item["document_on"] == "2024-03-01" + assert item["added_on"] == "2024-03-05" + assert "Kündigung des Mietvertrags" in item["snippet"] + assert len(item["snippet"]) <= 262 + assert item["snippet"].startswith("…") + assert found["total"] == 1 + assert found["previous_page"] is None + facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]}) + assert facets["storage_paths"] == [] + assert facets["tag_modes"] == ["all", "any", "none"] + for params in ( + {"mode": "regex"}, + {"tag_mode": "x"}, + {"asn": "x1"}, + {"storage_path": "a"}, + {"q": "miete", "mode": "title", "title": "vertrag"}, + ): + ada.request( + "GET", + "/api/v1/sources/paperless/documents", + params={"connector_id": first["id"], **params}, + status=422, + ) + + +def test_paperless_thumbnail_is_bound_to_the_owners_instance(client, db, monkeypatch): + import app.sources as sources + + seen = [] + + def fake_file(base, token, path, max_bytes=0): + seen.append((token, path)) + if path.endswith("/9/thumb/"): + return b"RIFFxxxxWEBP", "image/webp", "" + if path.endswith("/8/thumb/"): + return b"

", "text/html", "" + return None + + monkeypatch.setattr(sources, "_paperless_file", fake_file) + ada = Actor(client, "ada-thumb@example.test", "Ada Thumb") + register(db, ada.email, ada.name) + ada.login() + first = ada.request( + "POST", + "/api/v1/connectors", + json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"}, + status=201, + ) + url = "/api/v1/sources/paperless/documents/{}/thumb" + ok = ada.client.get(url.format(9), params={"connector_id": first["id"]}, headers=_headers(ada)) + assert ok.status_code == 200 + assert ok.headers["content-type"] == "image/webp" + assert ok.headers["cache-control"] == "private, no-store" + assert seen[-1] == ("token-alpha", "/api/documents/9/thumb/") + assert ada.client.get(url.format(8), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 422 + assert ada.client.get(url.format(5), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 404 + assert ada.client.get(url.format("x1"), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 422 + ben = Actor(client, "ben-thumb@example.test", "Ben Thumb") + register(db, ben.email, ben.name) + ben.login() + stranger = ben.client.get(url.format(9), params={"connector_id": first["id"]}, headers=_headers(ben)) + assert stranger.status_code == 404 + assert len(seen) == 3 + + def test_revoked_grant_hides_bytes_fetched_during_the_call(client, db, monkeypatch): import app.sources as sources diff --git a/backend/tests/test_correction_k1_k7.py b/backend/tests/test_correction_k1_k7.py index 6601057..c5e9924 100644 --- a/backend/tests/test_correction_k1_k7.py +++ b/backend/tests/test_correction_k1_k7.py @@ -429,7 +429,7 @@ def test_paperless_filters_change_the_hits(client, db, monkeypatch): return {"count": 1, "results": [{"id": 4, "name": "Haus"}]} matched = [] for item in catalog: - query = params.get("query") or "" + query = params.get("title_content") or params.get("query") or "" title = params.get("title__icontains") or "" if query and query.casefold() not in f"{item['title']} {item['content']}".casefold(): continue diff --git a/docs/architecture/API_REFERENZ.md b/docs/architecture/API_REFERENZ.md index 4d67450..12f2e72 100644 --- a/docs/architecture/API_REFERENZ.md +++ b/docs/architecture/API_REFERENZ.md @@ -196,7 +196,10 @@ Legende Spalte „Auth“: **–** öffentlich, **S** gültige Session, **S+R** | GET | `/api/v1/sources/{kind}/search` | Suche der registrierten Quelle im erklärten Umfang; unbekannte Art und nicht erklärte Filter antworten `422` | | GET | `/api/v1/sources/{kind}/navigate?connector_id=` | Fachnavigation, wenn die Quelle sie erklärt | | GET | `/api/v1/sources/{kind}/preview` | Vorschau vor dem Übernehmen, wenn die Quelle sie erklärt | -| GET | `/api/v1/sources/paperless/documents?q=&connector_id=` | Kompatibilitätshülle der Paperless-Suche; delegiert an dieselbe Suche | +| GET | `/api/v1/sources/paperless/documents?q=&connector_id=` | Paperless-Suche, 20 Treffer je Seite, `cursor` ist die Seitennummer. `mode`: `title_content` (Standard, Titel und Inhalt ohne Index), `title`, `content`, `index` (Volltextindex von Paperless). Filter: `title`, `correspondent`, `document_type`, `storage_path` (jeweils Nummer oder `none` für „ohne“), `tag` (Nummern, durch Komma) mit `tag_mode` `all` (Standard), `any` oder `none`, `document_from`/`document_to` (Dokumentdatum), `added_from`/`added_to` (Hinzugefügt), `asn`, `sort` (`created`, `added`, `title`, jeweils mit `-`; `-score` nur bei `mode=index` mit Suchtext). Ein ungültiger Wert antwortet `422`. Je Treffer: `title`, `document_on`, `added_on`, `correspondent`, `document_type`, `storage_path`, `tags` (Namen), `pages`, `asn`, `snippet` (Textauszug, reiner Text) und `open_url`; dazu `total`, `next_page`, `previous_page` | +| GET | `/api/v1/sources/{kind}/facets?connector_id=` | Auswahllisten der Instanz (Paperless: Korrespondenten, Dokumenttypen, Speicherpfade, Tags). Namen und Listen werden je Instanz und Schlüssel eine Minute lang vorgehalten | +| GET | `/api/v1/sources/paperless/documents/{id}/preview?connector_id=` | Datei zum Ansehen vor dem Übernehmen | +| GET | `/api/v1/sources/paperless/documents/{id}/thumb?connector_id=` | Vorschaubild der Trefferliste. Läuft über den Server der eigenen Instanz, `private, no-store`; fremde Instanz `404`, Antwort ohne Bild `422` | | GET | `/api/v1/sources/imap/mailboxes?connector_id=` | IMAP-Fächer | | GET | `/api/v1/sources/imap/messages?connector_id=&mailbox=&q=&before=` | Nachrichtenliste, 30 je Seite, Fortsetzung über `before` | | GET | `/api/v1/sources/imap/message?connector_id=&mailbox=&uid=` | Vorschau vor dem Übernehmen | diff --git a/frontend/src/pages/Sources.jsx b/frontend/src/pages/Sources.jsx index 87f7aee..6e494d2 100644 --- a/frontend/src/pages/Sources.jsx +++ b/frontend/src/pages/Sources.jsx @@ -7,7 +7,7 @@ import { NewConnector, useSourceModules } from "../sources/connector-config.jsx" import { SourceFrame, registerSourceModule } from "../sources/registry.jsx"; const SEARCH_LIMITS = { - paperless: "Suchtext gilt für Titel und Dokumentinhalt. Zusätzlich gibt es Titel, Korrespondent, Dokumenttyp, Tags, Dokumentdatum und Sortierung.", + paperless: "Die Suche läuft in Paperless selbst, über Titel und Inhalt, den Titel allein, den Inhalt allein oder den Volltextindex. Filter nach Korrespondent, Dokumenttyp, Speicherpfad, Tags, Dokumentdatum, Hinzugefügt-Datum und Archivnummer grenzen sie ein.", imap: "Die Suche gilt für das ganze gewählte Fach, auch über die erste Seite hinaus. Betreff, Absender und Nachrichtentext werden auf dem Server geprüft.", }; import { formatDate } from "./content.jsx"; @@ -69,25 +69,106 @@ function paperlessLabel(item) { return dated !== "Ohne Datum" ? `${dated} · ${item.title}` : item.title; } +function paperlessFacts(item) { + return [ + formatDate(item.document_on) !== "Ohne Datum" ? formatDate(item.document_on) : "", + item.correspondent, + item.document_type, + item.pages ? `${item.pages} ${item.pages === 1 ? "Seite" : "Seiten"}` : "", + item.asn ? `ASN ${item.asn}` : "", + ].filter(Boolean); +} + +const PAPERLESS_MODES = [ + ["title_content", "Titel und Inhalt"], + ["title", "Nur Titel"], + ["content", "Nur Inhalt"], + ["index", "Volltextindex"], +]; + +const PAPERLESS_MODE_HINTS = { + title_content: "Sucht den Text im Titel und im erkannten Inhalt, wie „Titel & Inhalt“ in Paperless.", + title: "Sucht den Text nur im Titel.", + content: "Sucht den Text nur im erkannten Inhalt der Dokumente.", + index: "Der Volltextindex von Paperless findet ganze Wörter, auch mit AND, OR und Anführungszeichen, und ordnet nach Relevanz. Ein neues Dokument erscheint erst, wenn Paperless es indexiert hat.", +}; + +const NO_FACETS = { correspondents: [], document_types: [], storage_paths: [], tags: [] }; + const EMPTY_FILTERS = { query: "", + mode: "title_content", title: "", correspondent: "", documentType: "", + storagePath: "", tags: [], + tagMode: "all", documentFrom: "", documentTo: "", + addedFrom: "", + addedTo: "", + asn: "", sort: "-created", }; +function activeFilterCount(filters) { + return [ + filters.title.trim(), + filters.correspondent, + filters.documentType, + filters.storagePath, + filters.tags.length ? "tags" : "", + filters.documentFrom, + filters.documentTo, + filters.addedFrom, + filters.addedTo, + filters.asn.trim(), + ].filter(Boolean).length; +} + +// The thumbnail comes through the server like every other Paperless call, so the key stays there. +function PaperlessThumb({ connectorId, id }) { + const [url, setUrl] = useState(""); + const [failed, setFailed] = useState(false); + + useEffect(() => { + let alive = true; + let made = ""; + setUrl(""); + setFailed(false); + fetchBlob(`/api/v1/sources/paperless/documents/${id}/thumb?connector_id=${connectorId}`) + .then((blob) => { + if (!alive) return; + made = URL.createObjectURL(blob); + setUrl(made); + }) + .catch(() => { + if (alive) setFailed(true); + }); + return () => { + alive = false; + if (made) URL.revokeObjectURL(made); + }; + }, [connectorId, id]); + + if (url) return ; + return ; +} + export function PaperlessPicker({ context, placement, chapterId = "", chapterTitle = "", targetLabel = "", onDone, onInserted }) { const [filters, setFilters] = useState(EMPTY_FILTERS); const [items, setItems] = useState([]); const [connectors, setConnectors] = useState([]); const [connectorId, setConnectorId] = useState(""); - const [facets, setFacets] = useState({ correspondents: [], document_types: [], tags: [] }); + const [facets, setFacets] = useState(NO_FACETS); + const [page, setPage] = useState(1); const [nextPage, setNextPage] = useState(null); + const [previousPage, setPreviousPage] = useState(null); + const [total, setTotal] = useState(null); + const [searched, setSearched] = useState(false); const [picked, setPicked] = useState(() => new Map()); + const [previewKey, setPreviewKey] = useState(""); const [previewUrl, setPreviewUrl] = useState(""); const [pending, setPending] = useState(false); const [error, setError] = useState(""); @@ -107,7 +188,7 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit useEffect(() => { if (!connectorId) { - setFacets({ correspondents: [], document_types: [], tags: [] }); + setFacets(NO_FACETS); return undefined; } const token = facetRef.current + 1; @@ -115,7 +196,7 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit let alive = true; api(`/api/v1/sources/paperless/facets?connector_id=${connectorId}`) .then((data) => { - if (alive && token === facetRef.current) setFacets(data); + if (alive && token === facetRef.current) setFacets({ ...NO_FACETS, ...data }); }) .catch((err) => { if (alive && token === facetRef.current) setError(err.message); @@ -129,32 +210,49 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit if (previewUrl) URL.revokeObjectURL(previewUrl); }, [previewUrl]); - function paramsFor(page) { + function paramsFor(pageNumber) { const params = new URLSearchParams(); + const text = filters.query.trim(); if (connectorId) params.set("connector_id", connectorId); - if (filters.query.trim()) params.set("q", filters.query.trim()); - if (filters.title.trim()) params.set("title", filters.title.trim()); + if (text) params.set("q", text); + if (text && filters.mode !== "title_content") params.set("mode", filters.mode); + // The search text already is the title in this mode. + if (filters.title.trim() && !(text && filters.mode === "title")) params.set("title", filters.title.trim()); if (filters.correspondent) params.set("correspondent", filters.correspondent); if (filters.documentType) params.set("document_type", filters.documentType); - if (filters.tags.length) params.set("tag", filters.tags.join(",")); + if (filters.storagePath) params.set("storage_path", filters.storagePath); + if (filters.tags.length) { + params.set("tag", filters.tags.join(",")); + if (filters.tagMode !== "all") params.set("tag_mode", filters.tagMode); + } if (filters.documentFrom) params.set("document_from", filters.documentFrom); if (filters.documentTo) params.set("document_to", filters.documentTo); + if (filters.addedFrom) params.set("added_from", filters.addedFrom); + if (filters.addedTo) params.set("added_to", filters.addedTo); + if (filters.asn.trim()) params.set("asn", filters.asn.trim()); if (filters.sort === "-score") { - if (filters.query.trim()) params.set("sort", "-score"); + if (text && filters.mode === "index") params.set("sort", "-score"); } else if (filters.sort) { params.set("sort", filters.sort); } - if (page) params.set("cursor", String(page)); + if (pageNumber) params.set("cursor", String(pageNumber)); return params; } + function closePreview() { + previewRef.current += 1; + if (previewUrl) URL.revokeObjectURL(previewUrl); + setPreviewUrl(""); + setPreviewKey(""); + } + function invalidate() { searchRef.current += 1; previewRef.current += 1; facetRef.current += 1; } - async function runSearch(event, page = "") { + async function runSearch(event, wanted = "") { event?.preventDefault(); setError(""); if (connectors.length > 1 && !connectorId) { @@ -165,10 +263,15 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit searchRef.current = token; const asked = connectorId; try { - const data = await api(`/api/v1/sources/paperless/documents?${paramsFor(page)}`); + const data = await api(`/api/v1/sources/paperless/documents?${paramsFor(wanted)}`); if (token !== searchRef.current) return; + closePreview(); setItems(data.items || []); + setPage(Number(wanted) || 1); setNextPage(data.next_page || data.continuation || null); + setPreviousPage(data.previous_page || null); + setTotal(typeof data.total === "number" ? data.total : null); + setSearched(true); if (!asked && data.connector_id) setConnectorId(data.connector_id); } catch (err) { if (token !== searchRef.current) return; @@ -177,9 +280,15 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit } function resetFilters() { + invalidate(); + closePreview(); setFilters(EMPTY_FILTERS); setItems([]); setNextPage(null); + setPreviousPage(null); + setTotal(null); + setSearched(false); + setPage(1); } function toggle(item) { @@ -195,17 +304,23 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit async function preview(item) { setError(""); - if (previewUrl) URL.revokeObjectURL(previewUrl); - setPreviewUrl(""); + const bound = item.connector_id || connectorId; + const key = `${bound}:${item.id}`; + if (key === previewKey) { + closePreview(); + return; + } + closePreview(); const token = previewRef.current + 1; previewRef.current = token; - const bound = item.connector_id || connectorId; + setPreviewKey(key); try { const blob = await fetchBlob(`/api/v1/sources/paperless/documents/${item.id}/preview?connector_id=${bound}`); if (token !== previewRef.current) return; setPreviewUrl(URL.createObjectURL(blob)); } catch (err) { if (token !== previewRef.current) return; + setPreviewKey(""); setError(err.message); } } @@ -262,6 +377,10 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit setFilters((current) => ({ ...current, [name]: value })); } + function setMode(mode) { + setFilters((current) => ({ ...current, mode, sort: current.sort === "-score" && mode !== "index" ? "-created" : current.sort })); + } + function toggleTag(id) { setFilters((current) => { const tags = current.tags.includes(id) ? current.tags.filter((item) => item !== id) : [...current.tags, id]; @@ -270,6 +389,18 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit } const held = [...picked.values()].filter((item) => !items.some((entry) => `${connectorId}:${entry.id}` === `${item.connector_id}:${item.id}`)); + const active = activeFilterCount(filters); + const relevance = filters.mode === "index" && filters.query.trim() !== ""; + + function choiceSelect(name, value, entries, empty) { + return ( + + ); + } return ( @@ -278,13 +409,16 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit Paperless-Zugang setFilter("query", event.target.value)} /> - - - - -

- Tags, alle gewählten - {facets.tags.length ? facets.tags.map((item) => ( -
- - + + + +
+ Tags + +
+ {facets.tags.length ? facets.tags.map((item) => ( + + )) :

Keine Tags an diesem Zugang.

} +
+
+ + + + + + +
-
@@ -356,7 +529,13 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit {picked.size ? `${picked.size} Belege einfügen` : "Belege auswählen"} ) : null} - {previewUrl ? : null} + {searched ? ( +

+ {items.length + ? `${total !== null ? `${total} Treffer` : "Treffer"}, Seite ${page}` + : "Dazu gibt es in Paperless keinen Treffer."} +

+ ) : null}
{held.map((item) => (
@@ -366,17 +545,52 @@ export function PaperlessPicker({ context, placement, chapterId = "", chapterTit
))} - {items.map((item) => ( -
- - -
- ))} + {items.map((item) => { + const key = `${connectorId}:${item.id}`; + const open = previewKey === key; + const facts = paperlessFacts(item); + return ( +
+ toggle(item)} + aria-label={`${item.title} auswählen`} + /> + +
+ {item.title} + {facts.length ? {facts.join(" · ")} : null} + {item.tags?.length ? ( +
    + {item.tags.map((tag) =>
  • {tag}
  • )} +
+ ) : null} + {item.snippet ?

{item.snippet}

: null} +
+ +
+
+ {open ? ( +
+ {previewUrl ? :

Vorschau wird geladen …

} +
+ ) : null} +
+ ); + })}
- {nextPage ? : null} + {previousPage || nextPage ? ( +
+ {previousPage ? : null} + {nextPage ? : null} +
+ ) : null} ); } diff --git a/frontend/src/sources/connector-config.jsx b/frontend/src/sources/connector-config.jsx index 7abe58e..c3ea68f 100644 --- a/frontend/src/sources/connector-config.jsx +++ b/frontend/src/sources/connector-config.jsx @@ -35,13 +35,19 @@ const OPERATIONS = [ const SEARCH_FILTERS = [ ["query", "Suchtext"], + ["mode", "Suchart (Titel und Inhalt, Titel, Inhalt, Volltextindex)"], ["title", "Titel"], ["correspondent", "Korrespondent"], ["document_type", "Dokumenttyp"], + ["storage_path", "Speicherpfad"], ["tags", "Tags"], + ["tag_mode", "Tag-Verknüpfung (alle, mindestens eines, keines)"], ["document_range", "Dokumentdatum"], + ["added_range", "Hinzugefügt am"], + ["asn", "Archivnummer"], ["sort", "Sortierung"], ["continuation", "Weitere Seiten"], + ["result_preview", "Vorschaubild und Textauszug je Treffer"], ]; function Availability({ items, source }) { diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 68c7ba7..2fa8748 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -1718,6 +1718,55 @@ button.pick-tree__twist { .note-source__bar { display: flex; flex-wrap: wrap; gap: 6px; } .note-source textarea { width: 100%; min-height: 8rem; } .source-filters { display: grid; gap: 8px; } +.paperless-search { display: grid; grid-template-columns: minmax(9rem, 12rem) minmax(0, 1fr) auto; gap: 8px; align-items: end; } +.paperless-more > summary { cursor: pointer; font-weight: 600; min-height: 32px; } +.paperless-tags { display: grid; gap: 2px; max-height: 12rem; overflow: auto; } +.paperless-card { + display: grid; + grid-template-columns: auto 5rem minmax(0, 1fr); + gap: 4px 10px; + align-items: start; + padding: 10px 2px; + border-top: 1px solid var(--border); +} +.paperless-card > input { width: 20px; min-width: 20px; min-height: 20px; margin-top: 2px; } +.paperless-card.is-open { background: var(--accent-light); } +.paperless-card__body { display: grid; gap: 4px; min-width: 0; } +.paperless-card__body strong { overflow-wrap: anywhere; } +.paperless-card__facts { color: var(--text2); font-size: 13px; } +.paperless-card__snippet { + display: -webkit-box; + margin: 0; + overflow: hidden; + color: var(--text2); + font-size: 13px; + line-height: 1.4; + overflow-wrap: anywhere; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; +} +.paperless-card__tags { display: flex; flex-wrap: wrap; gap: 4px; margin: 0; padding: 0; list-style: none; } +.paperless-card__tags li { padding: 0 8px; border-radius: 999px; background: var(--surface2); color: var(--text2); font-size: 12px; } +.paperless-card__view { grid-column: 1 / -1; } +.paperless-thumb { + display: grid; + place-items: center; + width: 5rem; + min-width: 5rem; + min-height: 6.5rem; + padding: 0; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 4px; + background: var(--surface2); +} +.paperless-thumb__image { display: block; width: 100%; height: 100%; object-fit: cover; object-position: top; } +.paperless-thumb__empty { color: var(--text3); font-size: 11px; } +@media (max-width: 600px) { + .paperless-search { grid-template-columns: minmax(0, 1fr); } + .paperless-card { grid-template-columns: auto 4rem minmax(0, 1fr); } + .paperless-thumb { width: 4rem; min-width: 4rem; min-height: 5.5rem; } +} .markdown pre, .markdown blockquote { margin: 0.4em 0; } .markdown blockquote { border-left: 3px solid var(--border2); padding-left: 10px; } .markdown pre { overflow: auto; padding: 8px; background: var(--surface2); border-radius: 6px; }