Erweitere die Paperless-Suche um Suchart, weitere Filter und eine Vorschau je Treffer, damit sich das richtige Dokument finden und erkennen lässt.
Standard ist jetzt Titel und Inhalt ohne Index; Titel, Inhalt und Volltextindex sind wählbar. Neue Filter: Speicherpfad, Tag-Verknüpfung (alle, eines, keines), "ohne" bei Korrespondent, Typ und Pfad, Hinzugefügt-Datum und Archivnummer. Treffer zeigen Vorschaubild, Textauszug, Tags, Typ und Seitenzahl; die Vorschaubilder laufen über den Server. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4327f5a434
commit
9089d830b3
10 files changed
+760
-185
No files matched your search
+62
-80
@@ -28,6 +28,7 @@ from app.sources import (
|
||||
list_connectors,
|
||||
owned_connector,
|
||||
paperless_content,
|
||||
paperless_thumbnail,
|
||||
parse_connector,
|
||||
probe,
|
||||
save_connector,
|
||||
@@ -264,40 +265,53 @@ def _mail_bytes(data: bytes, media_type: str, filename: str, inline: bool = Fals
|
||||
)
|
||||
|
||||
|
||||
class SearchFilters:
|
||||
"""The query parameters every search route shares. A source answers only those it declares."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str = "",
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
storage_path: str = "",
|
||||
tag: str = "",
|
||||
tag_mode: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
added_from: str = "",
|
||||
added_to: str = "",
|
||||
asn: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
):
|
||||
self.values = {
|
||||
"mode": mode,
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"storage_path": storage_path,
|
||||
"tag": tag,
|
||||
"tag_mode": tag_mode,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"added_from": added_from,
|
||||
"added_to": added_to,
|
||||
"asn": asn,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/paperless/documents")
|
||||
def search_paperless(
|
||||
q: str = "",
|
||||
connector_id: UUID | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
filters: SearchFilters = Depends(),
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
return _run_search(
|
||||
"paperless",
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
"",
|
||||
None,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
)
|
||||
return _run_search("paperless", conn, principal.account_id, q.strip(), connector_id, "", None, filters.values)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/facets")
|
||||
@@ -328,6 +342,17 @@ def preview_paperless_document(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/paperless/documents/{document_id}/thumb")
|
||||
def thumbnail_paperless_document(
|
||||
document_id: str,
|
||||
connector_id: UUID,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
data, media = paperless_thumbnail(conn, principal.account_id, document_id, connector_id)
|
||||
return Response(content=data, media_type=media, headers=_PRIVATE)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/imap/mailboxes")
|
||||
def list_mailboxes(
|
||||
connector_id: UUID,
|
||||
@@ -343,39 +368,15 @@ def search_mail(
|
||||
q: str = "",
|
||||
mailbox: str = "",
|
||||
before: int | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
uidvalidity: int | None = None,
|
||||
filters: SearchFilters = Depends(),
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
if before is not None and before < 1:
|
||||
raise unprocessable("Die Seitenangabe ist ungültig.")
|
||||
return _run_search(
|
||||
"imap",
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
mailbox.strip(),
|
||||
before,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
uidvalidity,
|
||||
"imap", conn, principal.account_id, q.strip(), connector_id, mailbox.strip(), before, filters.values, uidvalidity
|
||||
)
|
||||
|
||||
|
||||
@@ -432,12 +433,18 @@ def _connector_parts(body: ConnectorIn, current: dict | None) -> tuple[str, dict
|
||||
|
||||
|
||||
_SEARCH_FLAGS = {
|
||||
"mode": "mode",
|
||||
"title": "title",
|
||||
"correspondent": "correspondent",
|
||||
"document_type": "document_type",
|
||||
"storage_path": "storage_path",
|
||||
"tag": "tags",
|
||||
"tag_mode": "tag_mode",
|
||||
"document_from": "document_range",
|
||||
"document_to": "document_range",
|
||||
"added_from": "added_range",
|
||||
"added_to": "added_range",
|
||||
"asn": "asn",
|
||||
"sort": "sort",
|
||||
"cursor": "continuation",
|
||||
}
|
||||
@@ -474,38 +481,13 @@ def search_source(
|
||||
connector_id: UUID | None = None,
|
||||
mailbox: str = "",
|
||||
before: int | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
filters: SearchFilters = Depends(),
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
if before is not None and before < 1:
|
||||
raise unprocessable("Die Seitenangabe ist ungültig.")
|
||||
return _run_search(
|
||||
kind,
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
mailbox.strip(),
|
||||
before,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
)
|
||||
return _run_search(kind, conn, principal.account_id, q.strip(), connector_id, mailbox.strip(), before, filters.values)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/navigate")
|
||||
|
||||
@@ -16,13 +16,19 @@ PAPERLESS_CAPABILITIES = {
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"mode": True,
|
||||
"title": True,
|
||||
"correspondent": True,
|
||||
"document_type": True,
|
||||
"storage_path": True,
|
||||
"tags": True,
|
||||
"tag_mode": True,
|
||||
"document_range": True,
|
||||
"added_range": True,
|
||||
"asn": True,
|
||||
"sort": True,
|
||||
"continuation": True,
|
||||
"result_preview": True,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -36,13 +42,19 @@ IMAP_CAPABILITIES = {
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"mode": False,
|
||||
"title": False,
|
||||
"correspondent": False,
|
||||
"document_type": False,
|
||||
"storage_path": False,
|
||||
"tags": False,
|
||||
"tag_mode": False,
|
||||
"document_range": False,
|
||||
"added_range": False,
|
||||
"asn": False,
|
||||
"sort": False,
|
||||
"continuation": True,
|
||||
"result_preview": False,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -57,12 +69,18 @@ EDITORS = {
|
||||
}
|
||||
|
||||
UNSUPPORTED_SEARCH = (
|
||||
"mode",
|
||||
"title",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"storage_path",
|
||||
"tag",
|
||||
"tag_mode",
|
||||
"document_from",
|
||||
"document_to",
|
||||
"added_from",
|
||||
"added_to",
|
||||
"asn",
|
||||
"sort",
|
||||
"cursor",
|
||||
)
|
||||
|
||||
+190
-31
@@ -3,14 +3,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import imaplib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from email.header import decode_header
|
||||
from email.parser import BytesParser
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode, urlparse
|
||||
@@ -183,10 +187,24 @@ def _record_probe(conn, connector_id: UUID, ok: bool, detail: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _paperless_choice(value, label: str) -> str | None:
|
||||
PAPERLESS_PAGE_SIZE = 20
|
||||
_PAPERLESS_MODES = {
|
||||
# Paperless-ngx offers these searches in its own filter bar. The first needs no search index.
|
||||
"title_content": "title_content",
|
||||
"title": "title__icontains",
|
||||
"content": "content__icontains",
|
||||
"index": "query",
|
||||
}
|
||||
_PAPERLESS_TAG_MODES = {"all": "tags__id__all", "any": "tags__id__in", "none": "tags__id__none"}
|
||||
_PAPERLESS_SORTS = {"created", "-created", "added", "-added", "title", "-title", "score", "-score"}
|
||||
|
||||
|
||||
def _paperless_choice(value, label: str, allow_none: bool = False) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if allow_none and text == "none":
|
||||
return text
|
||||
if not text.isdigit() or len(text) > 12:
|
||||
raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.")
|
||||
return text
|
||||
@@ -201,45 +219,67 @@ def _paperless_day(value, label: str) -> str | None:
|
||||
return text
|
||||
|
||||
|
||||
def _paperless_range(options: dict, prefix: str, field: str, label: str, params: dict) -> None:
|
||||
start = _paperless_day(options.get(f"{prefix}_from"), f"{label}, von")
|
||||
end = _paperless_day(options.get(f"{prefix}_to"), f"{label}, bis")
|
||||
if start and end and start > end:
|
||||
raise unprocessable(f"{label}: Der Beginn liegt nach dem Ende.")
|
||||
if start:
|
||||
params[f"{field}__date__gte"] = start
|
||||
if end:
|
||||
params[f"{field}__date__lte"] = end
|
||||
|
||||
|
||||
def _paperless_params(query: str, options: dict | None) -> dict:
|
||||
"""Paperless-ngx document date is the created field, not the upload or modified time."""
|
||||
options = options or {}
|
||||
params = {"page_size": "20"}
|
||||
params = {"page_size": str(PAPERLESS_PAGE_SIZE)}
|
||||
text = " ".join(str(query or "").split())
|
||||
mode = str(options.get("mode") or "").strip() or "title_content"
|
||||
if mode not in _PAPERLESS_MODES:
|
||||
raise unprocessable("Diese Suchart gibt es nicht.")
|
||||
if text:
|
||||
params["query"] = text
|
||||
params[_PAPERLESS_MODES[mode]] = text
|
||||
title = " ".join(str(options.get("title") or "").split())
|
||||
if title:
|
||||
if "title__icontains" in params and params["title__icontains"] != title:
|
||||
raise unprocessable("Die Suche nur im Titel und der Titelfilter sind beide gesetzt. Nutze eines davon.")
|
||||
params["title__icontains"] = title
|
||||
correspondent = _paperless_choice(options.get("correspondent"), "Der Korrespondent")
|
||||
if correspondent:
|
||||
params["correspondent__id"] = correspondent
|
||||
document_type = _paperless_choice(options.get("document_type"), "Der Dokumenttyp")
|
||||
if document_type:
|
||||
params["document_type__id"] = document_type
|
||||
for name, field, label in (
|
||||
("correspondent", "correspondent", "Der Korrespondent"),
|
||||
("document_type", "document_type", "Der Dokumenttyp"),
|
||||
("storage_path", "storage_path", "Der Speicherpfad"),
|
||||
):
|
||||
chosen = _paperless_choice(options.get(name), label, allow_none=True)
|
||||
if chosen == "none":
|
||||
params[f"{field}__isnull"] = "true"
|
||||
elif chosen:
|
||||
params[f"{field}__id"] = chosen
|
||||
raw_tags = options.get("tag") or options.get("tags") or ""
|
||||
if isinstance(raw_tags, (list, tuple)):
|
||||
parts = [str(item) for item in raw_tags]
|
||||
else:
|
||||
parts = str(raw_tags).split(",")
|
||||
tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen]
|
||||
tag_mode = str(options.get("tag_mode") or "").strip() or "all"
|
||||
if tag_mode not in _PAPERLESS_TAG_MODES:
|
||||
raise unprocessable("Diese Tag-Verknüpfung gibt es nicht.")
|
||||
if tag_ids:
|
||||
params["tags__id__all"] = ",".join(tag_ids)
|
||||
start = _paperless_day(options.get("document_from"), "Das Dokumentdatum von")
|
||||
end = _paperless_day(options.get("document_to"), "Das Dokumentdatum bis")
|
||||
if start and end and start > end:
|
||||
raise unprocessable("Das Dokumentdatum von liegt nach dem Dokumentdatum bis.")
|
||||
if start:
|
||||
params["created__date__gte"] = start
|
||||
if end:
|
||||
params["created__date__lte"] = end
|
||||
params[_PAPERLESS_TAG_MODES[tag_mode]] = ",".join(tag_ids)
|
||||
_paperless_range(options, "document", "created", "Dokumentdatum", params)
|
||||
_paperless_range(options, "added", "added", "Hinzugefügt am", params)
|
||||
asn = str(options.get("asn") or "").strip()
|
||||
if asn:
|
||||
if not asn.isdigit() or len(asn) > 10:
|
||||
raise unprocessable("Die Archivnummer ist keine Zahl.")
|
||||
params["archive_serial_number"] = asn
|
||||
sort = str(options.get("sort") or "").strip()
|
||||
if sort:
|
||||
if sort not in {"created", "-created", "title", "-title", "score", "-score"}:
|
||||
if sort not in _PAPERLESS_SORTS:
|
||||
raise unprocessable("Diese Sortierung gibt es nicht.")
|
||||
if sort in {"score", "-score"}:
|
||||
if not text:
|
||||
raise unprocessable("Die Relevanz gilt zusammen mit einem Suchtext.")
|
||||
if not text or mode != "index":
|
||||
raise unprocessable("Die Relevanz gilt nur für die Suche im Volltextindex.")
|
||||
# Paperless-ngx 2.20.6 has no ordering field named score. A query without
|
||||
# ordering is the relevance order of the full-text index.
|
||||
else:
|
||||
@@ -279,42 +319,159 @@ def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | N
|
||||
params = _paperless_params(query, options)
|
||||
release_for_external(conn)
|
||||
payload = _paperless_get(base, row["secret"], "/api/documents/", params)
|
||||
names = _paperless_lookup(row["id"], base, row["secret"])
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
terms = _search_terms(query, (options or {}).get("title"))
|
||||
items = []
|
||||
for doc in payload.get("results") or []:
|
||||
created = str(doc.get("created") or "")[:10]
|
||||
added = str(doc.get("added") or "")[:10]
|
||||
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
|
||||
pages = doc.get("page_count")
|
||||
asn = doc.get("archive_serial_number")
|
||||
items.append(
|
||||
{
|
||||
"id": str(doc.get("id")),
|
||||
"title": title[:200],
|
||||
"document_on": created if len(created) == 10 else None,
|
||||
"correspondent": _correspondent(doc),
|
||||
"added_on": added if len(added) == 10 else None,
|
||||
"correspondent": _correspondent(doc, names),
|
||||
"document_type": _named(doc.get("document_type"), names["document_types"]),
|
||||
"storage_path": _named(doc.get("storage_path"), names["storage_paths"]),
|
||||
"tags": [name for name in (_named(tag, names["tags"]) for tag in doc.get("tags") or []) if name][:12],
|
||||
"pages": pages if isinstance(pages, int) and pages > 0 else None,
|
||||
"asn": asn if isinstance(asn, int) else None,
|
||||
"snippet": _snippet(doc, terms),
|
||||
"open_url": f"{base}/documents/{doc.get('id')}/",
|
||||
}
|
||||
)
|
||||
page = int(params.get("page") or 1)
|
||||
count = payload.get("count")
|
||||
if isinstance(count, int):
|
||||
next_page = str(page + 1) if page * 20 < count else None
|
||||
next_page = str(page + 1) if page * PAPERLESS_PAGE_SIZE < count else None
|
||||
else:
|
||||
next_page = str(page + 1) if len(items) == 20 else None
|
||||
return {"items": items, "connector_id": str(row["id"]), "next_page": next_page, "incomplete": False}
|
||||
next_page = str(page + 1) if len(items) == PAPERLESS_PAGE_SIZE else None
|
||||
return {
|
||||
"items": items,
|
||||
"connector_id": str(row["id"]),
|
||||
"next_page": next_page,
|
||||
"previous_page": str(page - 1) if page > 1 else None,
|
||||
"total": count if isinstance(count, int) else None,
|
||||
"incomplete": False,
|
||||
}
|
||||
|
||||
|
||||
def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
row = _paperless_row(conn, owner_id, connector_id)
|
||||
base = row["settings"]["base_url"].rstrip("/")
|
||||
token = row["secret"]
|
||||
release_for_external(conn)
|
||||
body = {
|
||||
names = _paperless_lookup(row["id"], base, row["secret"])
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return {
|
||||
"correspondents": names["correspondents"],
|
||||
"document_types": names["document_types"],
|
||||
"storage_paths": names["storage_paths"],
|
||||
"tags": names["tags"],
|
||||
"tag_mode": "all",
|
||||
"tag_modes": list(_PAPERLESS_TAG_MODES),
|
||||
"modes": list(_PAPERLESS_MODES),
|
||||
}
|
||||
|
||||
|
||||
def paperless_thumbnail(conn, owner_id: UUID, document_id: str, connector_id: UUID) -> tuple[bytes, str]:
|
||||
if not document_id.isdigit() or len(document_id) > 12:
|
||||
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
||||
row = _paperless_row(conn, owner_id, connector_id)
|
||||
base = row["settings"]["base_url"].rstrip("/")
|
||||
release_for_external(conn)
|
||||
fetched = _paperless_file(base, row["secret"], f"/api/documents/{document_id}/thumb/", 2 * 1024 * 1024)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
if fetched is None:
|
||||
raise not_found()
|
||||
data, media, _name = fetched
|
||||
if not media.startswith("image/"):
|
||||
raise unprocessable("Paperless hat kein Vorschaubild geliefert.")
|
||||
return data, media
|
||||
|
||||
|
||||
_LOOKUP_TTL = 60.0
|
||||
_LOOKUP_LIMIT = 32
|
||||
_lookup_cache: dict[tuple, tuple[float, dict]] = {}
|
||||
_lookup_lock = threading.Lock()
|
||||
|
||||
|
||||
def _paperless_lookup(connector_id, base: str, token: str) -> dict:
|
||||
"""Names behind the ids of a result list. Bound to the instance and key that asked, kept for a minute."""
|
||||
key = (str(connector_id), base, hashlib.sha256(token.encode("utf-8")).hexdigest())
|
||||
now = time.monotonic()
|
||||
with _lookup_lock:
|
||||
hit = _lookup_cache.get(key)
|
||||
if hit and now - hit[0] < _LOOKUP_TTL:
|
||||
return hit[1]
|
||||
names = {
|
||||
"correspondents": _paperless_names(base, token, "/api/correspondents/"),
|
||||
"document_types": _paperless_names(base, token, "/api/document_types/"),
|
||||
"tags": _paperless_names(base, token, "/api/tags/"),
|
||||
"tag_mode": "all",
|
||||
# An instance without storage paths, or a key that may not read them, still searches.
|
||||
"storage_paths": _paperless_names_or_none(base, token, "/api/storage_paths/"),
|
||||
}
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return body
|
||||
with _lookup_lock:
|
||||
if len(_lookup_cache) >= _LOOKUP_LIMIT:
|
||||
_lookup_cache.clear()
|
||||
_lookup_cache[key] = (now, names)
|
||||
return names
|
||||
|
||||
|
||||
def _paperless_names_or_none(base: str, token: str, path: str) -> list[dict]:
|
||||
try:
|
||||
return _paperless_names(base, token, path)
|
||||
except Problem:
|
||||
return []
|
||||
|
||||
|
||||
def _named(value, entries: list[dict]) -> str | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
wanted = str(value)
|
||||
for entry in entries:
|
||||
if entry["id"] == wanted:
|
||||
return entry["name"]
|
||||
return None
|
||||
|
||||
|
||||
_TAG_MARKUP = re.compile(r"<[^>]{0,200}>")
|
||||
_SNIPPET_LENGTH = 260
|
||||
|
||||
|
||||
def _search_terms(query: str, title) -> list[str]:
|
||||
words = re.findall(r"[\w-]{3,}", f"{query or ''} {title or ''}".casefold())
|
||||
return list(dict.fromkeys(words))[:8]
|
||||
|
||||
|
||||
def _snippet(doc: dict, terms: list[str]) -> str | None:
|
||||
"""A short plain-text excerpt: the index hit when Paperless sends one, else the text around the first match."""
|
||||
hit = doc.get("__search_hit__")
|
||||
if isinstance(hit, dict):
|
||||
lines = hit.get("highlights")
|
||||
if isinstance(lines, str) and lines.strip():
|
||||
text = " ".join(unescape(_TAG_MARKUP.sub("", lines)).split())
|
||||
if text:
|
||||
return text[:_SNIPPET_LENGTH]
|
||||
content = doc.get("content")
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
text = " ".join(content.split())
|
||||
if not text:
|
||||
return None
|
||||
folded = text.casefold()
|
||||
at = min((index for index in (folded.find(term) for term in terms) if index >= 0), default=-1)
|
||||
if at < 0:
|
||||
return text[:_SNIPPET_LENGTH] + ("…" if len(text) > _SNIPPET_LENGTH else "")
|
||||
first = max(0, at - 60)
|
||||
if first:
|
||||
first = text.rfind(" ", 0, first) + 1 or first
|
||||
last = first + _SNIPPET_LENGTH
|
||||
return ("…" if first else "") + text[first:last] + ("…" if last < len(text) else "")
|
||||
|
||||
|
||||
def _paperless_names(base: str, token: str, path: str) -> list[dict]:
|
||||
@@ -679,10 +836,12 @@ def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
|
||||
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
||||
|
||||
|
||||
def _correspondent(doc: dict) -> str | None:
|
||||
def _correspondent(doc: dict, names: dict | None = None) -> str | None:
|
||||
value = doc.get("correspondent_name") or doc.get("correspondent")
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()[:200]
|
||||
if names is not None:
|
||||
return _named(value, names["correspondents"])
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -216,7 +216,7 @@ def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monke
|
||||
)
|
||||
assert filtered["items"][0]["title"] == "Treffer"
|
||||
assert calls[-1][3]["correspondent__id"] == "4"
|
||||
assert calls[-1][3]["query"] == "Treffer"
|
||||
assert calls[-1][3]["title_content"] == "Treffer"
|
||||
|
||||
|
||||
def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch):
|
||||
|
||||
@@ -355,22 +355,34 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch
|
||||
params={
|
||||
"connector_id": first["id"],
|
||||
"q": "nurinhalt",
|
||||
"mode": "index",
|
||||
"title": "Vertrag",
|
||||
"correspondent": "4",
|
||||
"document_type": "8",
|
||||
"storage_path": "6",
|
||||
"tag": "2,5",
|
||||
"tag_mode": "any",
|
||||
"document_from": "2024-01-01",
|
||||
"document_to": "2024-12-31",
|
||||
"added_from": "2024-02-01",
|
||||
"added_to": "2024-02-28",
|
||||
"asn": "1042",
|
||||
"sort": "-created",
|
||||
"cursor": "3",
|
||||
},
|
||||
)
|
||||
sent = calls[-1]["params"]
|
||||
assert sent["query"] == "nurinhalt"
|
||||
assert "title_content" not in sent
|
||||
assert sent["title__icontains"] == "Vertrag"
|
||||
assert sent["correspondent__id"] == "4"
|
||||
assert sent["document_type__id"] == "8"
|
||||
assert sent["tags__id__all"] == "2,5"
|
||||
assert sent["storage_path__id"] == "6"
|
||||
assert sent["tags__id__in"] == "2,5"
|
||||
assert "tags__id__all" not in sent
|
||||
assert sent["added__date__gte"] == "2024-02-01"
|
||||
assert sent["added__date__lte"] == "2024-02-28"
|
||||
assert sent["archive_serial_number"] == "1042"
|
||||
assert sent["created__date__gte"] == "2024-01-01"
|
||||
assert sent["created__date__lte"] == "2024-12-31"
|
||||
assert sent["ordering"] == "-created"
|
||||
@@ -378,9 +390,15 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch
|
||||
ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"},
|
||||
params={"connector_id": first["id"], "q": "nurinhalt", "mode": "index", "sort": "-score"},
|
||||
)
|
||||
assert "ordering" not in calls[-1]["params"]
|
||||
ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"},
|
||||
status=422,
|
||||
)
|
||||
assert found["items"][0]["id"] == "41"
|
||||
assert found["connector_id"] == first["id"]
|
||||
preview = ada.client.get(
|
||||
@@ -431,6 +449,132 @@ def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch
|
||||
)
|
||||
|
||||
|
||||
def test_paperless_default_search_reads_title_and_content_and_shows_a_preview(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
calls = []
|
||||
body = "Vorspann " * 30 + "Die Kündigung des Mietvertrags erfolgt zum Jahresende. " + "Nachspann " * 30
|
||||
|
||||
def fake_get(base, token, path, params):
|
||||
calls.append({"path": path, "params": dict(params)})
|
||||
if path == "/api/documents/":
|
||||
return {
|
||||
"count": 1,
|
||||
"results": [
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Schreiben",
|
||||
"created": "2024-03-01T00:00:00+01:00",
|
||||
"added": "2024-03-05T08:00:00Z",
|
||||
"correspondent": 4,
|
||||
"document_type": 8,
|
||||
"storage_path": None,
|
||||
"tags": [2, 5, 99],
|
||||
"archive_serial_number": 1042,
|
||||
"page_count": 3,
|
||||
"content": body,
|
||||
}
|
||||
],
|
||||
}
|
||||
if path == "/api/storage_paths/":
|
||||
raise sources.unprocessable("Paperless hat die Anfrage nicht beantwortet.")
|
||||
if path == "/api/tags/":
|
||||
return {"results": [{"id": 2, "name": "Miete"}, {"id": 5, "name": "Wohnung"}]}
|
||||
if path == "/api/correspondents/":
|
||||
return {"results": [{"id": 4, "name": "Hausverwaltung"}]}
|
||||
return {"results": [{"id": 8, "name": "Brief"}]}
|
||||
|
||||
monkeypatch.setattr(sources, "_paperless_get", fake_get)
|
||||
ada = Actor(client, "ada-preview@example.test", "Ada Preview")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
first = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||||
status=201,
|
||||
)
|
||||
found = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "q": "kündigung", "correspondent": "none", "tag": "2", "tag_mode": "none"},
|
||||
)
|
||||
sent = [call for call in calls if call["path"] == "/api/documents/"][-1]["params"]
|
||||
assert sent["title_content"] == "kündigung"
|
||||
assert "query" not in sent
|
||||
assert sent["correspondent__isnull"] == "true"
|
||||
assert sent["tags__id__none"] == "2"
|
||||
item = found["items"][0]
|
||||
assert item["correspondent"] == "Hausverwaltung"
|
||||
assert item["document_type"] == "Brief"
|
||||
assert item["tags"] == ["Miete", "Wohnung"]
|
||||
assert item["pages"] == 3
|
||||
assert item["asn"] == 1042
|
||||
assert item["document_on"] == "2024-03-01"
|
||||
assert item["added_on"] == "2024-03-05"
|
||||
assert "Kündigung des Mietvertrags" in item["snippet"]
|
||||
assert len(item["snippet"]) <= 262
|
||||
assert item["snippet"].startswith("…")
|
||||
assert found["total"] == 1
|
||||
assert found["previous_page"] is None
|
||||
facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]})
|
||||
assert facets["storage_paths"] == []
|
||||
assert facets["tag_modes"] == ["all", "any", "none"]
|
||||
for params in (
|
||||
{"mode": "regex"},
|
||||
{"tag_mode": "x"},
|
||||
{"asn": "x1"},
|
||||
{"storage_path": "a"},
|
||||
{"q": "miete", "mode": "title", "title": "vertrag"},
|
||||
):
|
||||
ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], **params},
|
||||
status=422,
|
||||
)
|
||||
|
||||
|
||||
def test_paperless_thumbnail_is_bound_to_the_owners_instance(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
seen = []
|
||||
|
||||
def fake_file(base, token, path, max_bytes=0):
|
||||
seen.append((token, path))
|
||||
if path.endswith("/9/thumb/"):
|
||||
return b"RIFFxxxxWEBP", "image/webp", ""
|
||||
if path.endswith("/8/thumb/"):
|
||||
return b"<p>", "text/html", ""
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(sources, "_paperless_file", fake_file)
|
||||
ada = Actor(client, "ada-thumb@example.test", "Ada Thumb")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
first = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||||
status=201,
|
||||
)
|
||||
url = "/api/v1/sources/paperless/documents/{}/thumb"
|
||||
ok = ada.client.get(url.format(9), params={"connector_id": first["id"]}, headers=_headers(ada))
|
||||
assert ok.status_code == 200
|
||||
assert ok.headers["content-type"] == "image/webp"
|
||||
assert ok.headers["cache-control"] == "private, no-store"
|
||||
assert seen[-1] == ("token-alpha", "/api/documents/9/thumb/")
|
||||
assert ada.client.get(url.format(8), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 422
|
||||
assert ada.client.get(url.format(5), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 404
|
||||
assert ada.client.get(url.format("x1"), params={"connector_id": first["id"]}, headers=_headers(ada)).status_code == 422
|
||||
ben = Actor(client, "ben-thumb@example.test", "Ben Thumb")
|
||||
register(db, ben.email, ben.name)
|
||||
ben.login()
|
||||
stranger = ben.client.get(url.format(9), params={"connector_id": first["id"]}, headers=_headers(ben))
|
||||
assert stranger.status_code == 404
|
||||
assert len(seen) == 3
|
||||
|
||||
|
||||
def test_revoked_grant_hides_bytes_fetched_during_the_call(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
|
||||
@@ -429,7 +429,7 @@ def test_paperless_filters_change_the_hits(client, db, monkeypatch):
|
||||
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
|
||||
matched = []
|
||||
for item in catalog:
|
||||
query = params.get("query") or ""
|
||||
query = params.get("title_content") or params.get("query") or ""
|
||||
title = params.get("title__icontains") or ""
|
||||
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user