Mache große IMAP-Fächer, die Paperless-Suche und die interne Markdown-Notiz über die gemeinsamen Schnittstellen benutzbar.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
c3a07cee64
commit
7e39fbb2bd
29 files changed
+6053
-320
No files matched your search
+10
-2
@@ -4,6 +4,14 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def discard_stored(item) -> None:
|
||||
discard = getattr(item, "discard", None)
|
||||
if callable(discard):
|
||||
discard()
|
||||
return
|
||||
Path(item).unlink(missing_ok=True)
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Depends, Request
|
||||
@@ -35,11 +43,11 @@ def get_conn(request: Request):
|
||||
yield conn
|
||||
conn.commit()
|
||||
for path in getattr(request.state, "delete_after_commit", []):
|
||||
Path(path).unlink(missing_ok=True)
|
||||
discard_stored(path)
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
for path in getattr(request.state, "cleanup_files", []):
|
||||
Path(path).unlink(missing_ok=True)
|
||||
discard_stored(path)
|
||||
raise
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from app.core.errors import not_found, unprocessable, unsupported
|
||||
from app.mailpack import parse_mail
|
||||
from app.media.storage import content_disposition
|
||||
from app.ops import create_external
|
||||
from app.retrieval import RetrievalFailure
|
||||
from app.sourcebind import assign_reference
|
||||
from app.sourceport import capability_payload, with_capabilities
|
||||
from app.sourceregistry import get_source, require_source
|
||||
@@ -23,8 +24,8 @@ from app.sources import (
|
||||
delete_connector,
|
||||
imap_preview,
|
||||
list_connectors,
|
||||
paperless_content,
|
||||
probe,
|
||||
read_linked_mail,
|
||||
save_connector,
|
||||
update_connector,
|
||||
)
|
||||
@@ -158,7 +159,13 @@ def _opened_ref(conn, principal: Principal, element_id: UUID, context_kind: str
|
||||
|
||||
|
||||
def _complete_mail(conn, principal: Principal, element, context, ref):
|
||||
raw = read_linked_mail(conn, principal.account_id, element, context, ref)
|
||||
module = get_source(ref["source"]) if ref is not None else None
|
||||
if module is None or module.representation != "email":
|
||||
raise unprocessable("Das ist keine E-Mail.")
|
||||
opened = module.open_original(conn, principal.account_id, element, context, ref)
|
||||
if isinstance(opened, RetrievalFailure):
|
||||
raise unprocessable(opened.message)
|
||||
raw = opened[0] if isinstance(opened, tuple) else opened
|
||||
parsed = parse_mail(raw)
|
||||
if parsed.missing:
|
||||
names = ", ".join(parsed.missing[:5])
|
||||
@@ -246,6 +253,34 @@ def search_paperless(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/facets")
|
||||
def source_facets(
|
||||
kind: str,
|
||||
connector_id: UUID,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
module = require_source(kind)
|
||||
if module.facets is None:
|
||||
raise unsupported("Diese Quelle hat keine Auswahllisten.")
|
||||
return module.facets(conn, principal.account_id, connector_id)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/paperless/documents/{document_id}/preview")
|
||||
def preview_paperless_document(
|
||||
document_id: str,
|
||||
connector_id: UUID,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
data, media, name = paperless_content(conn, principal.account_id, document_id, connector_id)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=media,
|
||||
headers={**_PRIVATE, "Content-Disposition": content_disposition(name).replace("attachment;", "inline;", 1)},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/imap/mailboxes")
|
||||
def list_mailboxes(
|
||||
connector_id: UUID,
|
||||
@@ -269,6 +304,7 @@ def search_mail(
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
uidvalidity: int | None = None,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
@@ -292,6 +328,7 @@ def search_mail(
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
uidvalidity,
|
||||
)
|
||||
|
||||
|
||||
@@ -357,7 +394,7 @@ _SEARCH_FLAGS = {
|
||||
}
|
||||
|
||||
|
||||
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict) -> dict:
|
||||
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict, uidvalidity: int | None = None) -> dict:
|
||||
module = require_source(kind)
|
||||
search = module.capabilities["search"]
|
||||
refused = [name for name, value in filters.items() if str(value or "").strip() and not search.get(_SEARCH_FLAGS.get(name, name))]
|
||||
@@ -365,7 +402,11 @@ def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, m
|
||||
refused.append("before")
|
||||
if refused:
|
||||
raise unsupported("Diese Suche unterstützt den Filter nicht: " + ", ".join(refused) + ".")
|
||||
found = module.find(conn, owner, connector_id, query, {"mailbox": mailbox, "before": before})
|
||||
options = {"mailbox": mailbox, "before": before}
|
||||
if uidvalidity is not None:
|
||||
options["uidvalidity"] = uidvalidity
|
||||
options.update({key: value for key, value in filters.items() if str(value or "").strip()})
|
||||
found = module.find(conn, owner, connector_id, query, options)
|
||||
bound = str(found.get("connector_id") or connector_id or "")
|
||||
return with_capabilities(kind, bound, found)
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ from app.access.policy import (
|
||||
)
|
||||
from app.core.errors import Problem, conflict, not_found, unprocessable
|
||||
from app.core.timeutil import dump_date, dump_dt
|
||||
from app.media.port import LEGACY_PROFILE, LocalAdapter, instance_for_write, publish, read_clear_bytes, stored_path
|
||||
from app.media.port import LEGACY_PROFILE, StoredHandle, adapter_for, instance_for_write, publish, read_clear_bytes, removal_for, stored_path
|
||||
from app.media.storage import display_filename, sha256
|
||||
from app.retrieval import FilePayload, MailPayload, RetrievalFailure, load_external, load_file
|
||||
|
||||
@@ -936,9 +936,9 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
|
||||
instance = instance_for_write(conn, media_root)
|
||||
object_key = f"dispatch/{snapshot_id}"
|
||||
content_revision = str(uuid4())
|
||||
adapter = LocalAdapter(instance["root_path"])
|
||||
adapter = adapter_for(instance)
|
||||
publish(adapter, object_key, payload.data)
|
||||
cleanup.append(stored_path(conn, instance["id"], instance["generation"], object_key, media_root))
|
||||
cleanup.append(StoredHandle(adapter, object_key))
|
||||
digest = sha256(payload.data)
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -1005,8 +1005,8 @@ def _snapshot_disk(conn, snapshot_id: UUID, media_root: str) -> Path:
|
||||
(snapshot_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return stored_path(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
|
||||
return stored_path(
|
||||
return removal_for(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
|
||||
return removal_for(
|
||||
conn,
|
||||
row["storage_instance_id"],
|
||||
row["storage_generation"],
|
||||
|
||||
@@ -496,8 +496,8 @@ def _note_text(conn, element_id: UUID) -> str:
|
||||
parts = []
|
||||
for block in document.get("blocks") or []:
|
||||
if isinstance(block, dict):
|
||||
text = str(block.get("text") or "").strip()
|
||||
if text:
|
||||
text = str(block.get("text") or "")
|
||||
if text.strip():
|
||||
parts.append(text)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
@@ -208,6 +208,47 @@ def instance_for_write(conn, media_root: str) -> dict:
|
||||
).fetchone()
|
||||
|
||||
|
||||
class StoredHandle:
|
||||
"""Rollback and delete go through the adapter that owns the bytes."""
|
||||
|
||||
def __init__(self, adapter: StoragePort, object_key: str):
|
||||
self._adapter = adapter
|
||||
self._key = object_key
|
||||
|
||||
def discard(self) -> None:
|
||||
self._adapter.delete(self._key)
|
||||
|
||||
|
||||
_FACTORIES: dict[str, callable] = {}
|
||||
|
||||
|
||||
def _local_factory(row: dict) -> StoragePort:
|
||||
return LocalAdapter(row["root_path"])
|
||||
|
||||
|
||||
_FACTORIES["local"] = _local_factory
|
||||
|
||||
|
||||
def adapter_for(row: dict) -> StoragePort:
|
||||
factory = _FACTORIES.get(row["kind"])
|
||||
if factory is None or not row.get("root_path") and row["kind"] == "local":
|
||||
raise storage_unavailable()
|
||||
return factory(row)
|
||||
|
||||
|
||||
def removal_for(conn, instance_id, generation: str | None, object_key: str, media_root: str) -> StoredHandle:
|
||||
row = None
|
||||
if instance_id is not None:
|
||||
row = conn.execute(
|
||||
"SELECT id, kind, generation, root_path FROM storage_instances WHERE id = %s",
|
||||
(instance_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
root = resolve_root(conn, instance_id, generation, media_root)
|
||||
return StoredHandle(LocalAdapter(root), object_key)
|
||||
return StoredHandle(adapter_for(row), object_key)
|
||||
|
||||
|
||||
def resolve_root(conn, instance_id, generation: str | None, media_root: str) -> str:
|
||||
if instance_id is not None:
|
||||
row = conn.execute(
|
||||
@@ -236,7 +277,14 @@ def read_clear_bytes(
|
||||
expected_sha256: str | None,
|
||||
) -> bytes:
|
||||
require_cleartext(protection_profile)
|
||||
data = LocalAdapter(resolve_root(conn, instance_id, generation, media_root)).open(object_key)
|
||||
row = None
|
||||
if instance_id is not None:
|
||||
row = conn.execute(
|
||||
"SELECT id, kind, generation, root_path FROM storage_instances WHERE id = %s",
|
||||
(instance_id,),
|
||||
).fetchone()
|
||||
adapter = adapter_for(row) if row is not None else LocalAdapter(resolve_root(conn, instance_id, generation, media_root))
|
||||
data = adapter.open(object_key)
|
||||
if expected_sha256 and sha256(data) != expected_sha256:
|
||||
raise Problem(
|
||||
422,
|
||||
|
||||
+4
-6
@@ -35,8 +35,7 @@ from app.access.policy import (
|
||||
from app.content.documents import clean_document
|
||||
from app.core.errors import forbidden, not_found, unprocessable, unsupported
|
||||
from app.core.timeutil import utcnow
|
||||
from app.media.port import LEGACY_PROFILE, instance_for_write, publish, stored_path
|
||||
from app.media.port import LocalAdapter
|
||||
from app.media.port import LEGACY_PROFILE, StoredHandle, adapter_for, instance_for_write, publish, removal_for
|
||||
from app.media.storage import display_filename, sha256, sniff
|
||||
|
||||
MAX_CHAPTER_DEPTH = 32
|
||||
@@ -1034,10 +1033,9 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
|
||||
content_revision = str(uuid4())
|
||||
instance = instance_for_write(conn, media_root)
|
||||
object_key = str(asset_id)
|
||||
adapter = LocalAdapter(instance["root_path"])
|
||||
adapter = adapter_for(instance)
|
||||
publish(adapter, object_key, data)
|
||||
path = stored_path(conn, instance["id"], instance["generation"], object_key, media_root)
|
||||
rollback_files.append(path)
|
||||
rollback_files.append(StoredHandle(adapter, object_key))
|
||||
digest = sha256(data)
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -1097,7 +1095,7 @@ def delete_element(conn, actor: UUID, element_id: UUID, expected: int, media_roo
|
||||
if asset is not None and asset["status"] == "active":
|
||||
conn.execute("UPDATE media_assets SET status = 'purging' WHERE id = %s", (asset["id"],))
|
||||
delete_after.append(
|
||||
stored_path(
|
||||
removal_for(
|
||||
conn,
|
||||
asset["storage_instance_id"],
|
||||
asset["storage_generation"],
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Rights stay in one place. A source module only fetches its own bytes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.access.policy import Ctx, audit, mediated_source_owner
|
||||
from app.core.errors import Problem, not_found, unprocessable
|
||||
from app.sourcebind import resolve_for_read
|
||||
|
||||
ORIGINAL_UNAVAILABLE = "Das Original ist über die Freigabe nicht abrufbar."
|
||||
|
||||
|
||||
@dataclass
|
||||
class BorrowedOriginal:
|
||||
row: dict
|
||||
external_id: str
|
||||
mediated: bool
|
||||
actor_id: UUID
|
||||
owner_id: UUID
|
||||
element: dict
|
||||
context: Ctx | None
|
||||
kind: str
|
||||
|
||||
|
||||
def release_for_external(conn) -> None:
|
||||
"""External calls must not sit inside an open domain transaction."""
|
||||
if conn.info.transaction_status != psycopg.pq.TransactionStatus.IDLE:
|
||||
conn.commit()
|
||||
|
||||
|
||||
def connector_owner(conn, connector_id: UUID | None) -> UUID | None:
|
||||
if connector_id is None:
|
||||
return None
|
||||
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (connector_id,)).fetchone()
|
||||
return row["owner_id"] if row else None
|
||||
|
||||
|
||||
def assert_still_owned(conn, connector_id, owner_id: UUID) -> None:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM connectors WHERE id = %s AND owner_id = %s",
|
||||
(connector_id, owner_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
|
||||
|
||||
def audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
|
||||
audit(
|
||||
conn,
|
||||
actor_id,
|
||||
element["home_scope_id"],
|
||||
"source_read_mediated",
|
||||
"element",
|
||||
element["id"],
|
||||
context,
|
||||
result="ok" if outcome == "ok" else "unavailable",
|
||||
detail={
|
||||
"source": source,
|
||||
"connector_id": str(connector_id) if connector_id else None,
|
||||
"outcome": outcome,
|
||||
},
|
||||
)
|
||||
if outcome != "ok":
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _uuid(value) -> UUID | None:
|
||||
if isinstance(value, UUID):
|
||||
return value
|
||||
try:
|
||||
return UUID(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def borrow_original(conn, actor_id: UUID, element: dict, context: Ctx | None, ref, kind: str) -> BorrowedOriginal:
|
||||
"""Resolve the owner's connector. A grantee never supplies another account's connector."""
|
||||
detail = (ref or {}).get("detail") or {}
|
||||
external_id = str((ref or {}).get("external_id") or "")
|
||||
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
|
||||
stored_owner = connector_owner(conn, stored)
|
||||
mediated = not (element["owner_subject_id"] == actor_id or stored_owner == actor_id)
|
||||
if mediated:
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
else:
|
||||
owner_id = actor_id
|
||||
result = resolve_for_read(conn, owner_id, kind, ref, None)
|
||||
if result.row is None:
|
||||
if mediated:
|
||||
audit_mediated(conn, actor_id, element, context, kind, stored, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
raise unprocessable(result.message)
|
||||
return BorrowedOriginal(
|
||||
row=result.row,
|
||||
external_id=external_id,
|
||||
mediated=mediated,
|
||||
actor_id=actor_id,
|
||||
owner_id=owner_id,
|
||||
element=element,
|
||||
context=context,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
|
||||
def _recheck(conn, borrowed: BorrowedOriginal) -> None:
|
||||
if borrowed.mediated:
|
||||
owner_id = mediated_source_owner(conn, borrowed.actor_id, borrowed.element, borrowed.context)
|
||||
if owner_id != borrowed.owner_id:
|
||||
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (borrowed.row["id"],)).fetchone()
|
||||
if row is None or row["owner_id"] != borrowed.owner_id:
|
||||
if borrowed.mediated:
|
||||
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
raise not_found()
|
||||
|
||||
|
||||
def complete_borrow(conn, borrowed: BorrowedOriginal, fetch):
|
||||
"""Run the adapter fetch outside the open transaction, then check the grant again."""
|
||||
release_for_external(conn)
|
||||
try:
|
||||
value = fetch(borrowed.row)
|
||||
except Problem:
|
||||
if borrowed.mediated:
|
||||
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
raise
|
||||
_recheck(conn, borrowed)
|
||||
if borrowed.mediated:
|
||||
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "ok")
|
||||
return value
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Declared source and editor capabilities.
|
||||
|
||||
A common contract does not invent abilities the adapter does not have.
|
||||
Search filters that are false stay unavailable until a later delivery.
|
||||
A filter is offered only when that module implements it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,13 +16,13 @@ PAPERLESS_CAPABILITIES = {
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"title": False,
|
||||
"correspondent": False,
|
||||
"document_type": False,
|
||||
"tags": False,
|
||||
"document_range": False,
|
||||
"sort": False,
|
||||
"continuation": False,
|
||||
"title": True,
|
||||
"correspondent": True,
|
||||
"document_type": True,
|
||||
"tags": True,
|
||||
"document_range": True,
|
||||
"sort": True,
|
||||
"continuation": True,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -94,5 +94,9 @@ def with_capabilities(kind: str, connector_id: str, payload: dict) -> dict:
|
||||
body["connector_id"] = connector_id
|
||||
body["capabilities"] = spec
|
||||
body["unsupported"] = [name for name, enabled in spec["search"].items() if not enabled]
|
||||
body["continuation"] = body.get("next_before") if spec["search"].get("continuation") else None
|
||||
if spec["search"].get("continuation"):
|
||||
body["continuation"] = body.get("next_before") if body.get("next_before") is not None else body.get("next_page")
|
||||
else:
|
||||
body["continuation"] = None
|
||||
body["incomplete"] = bool(body.get("incomplete"))
|
||||
return body
|
||||
@@ -21,6 +21,7 @@ class SourceModule:
|
||||
export_original: Callable
|
||||
navigate: Callable | None = None
|
||||
preview: Callable | None = None
|
||||
facets: Callable | None = None
|
||||
|
||||
|
||||
_MODULES: dict[str, SourceModule] = {}
|
||||
|
||||
+330
-141
@@ -18,12 +18,21 @@ from uuid import UUID
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.access.policy import Ctx, audit, mediated_source_owner
|
||||
from app.access.policy import Ctx
|
||||
from app.core.errors import Problem, not_found, unprocessable
|
||||
from app.core.settings import get_settings
|
||||
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
|
||||
from app.media.storage import sniff
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceaccess import (
|
||||
ORIGINAL_UNAVAILABLE,
|
||||
assert_still_owned,
|
||||
audit_mediated as _audit_mediated,
|
||||
borrow_original,
|
||||
complete_borrow,
|
||||
connector_owner as _connector_owner,
|
||||
release_for_external,
|
||||
)
|
||||
from app.sourceport import CAPABILITIES
|
||||
from app.sourceregistry import SourceModule, get_source, register_source
|
||||
|
||||
@@ -81,30 +90,109 @@ def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
module = get_source(row["kind"])
|
||||
if module is None:
|
||||
raise unprocessable("Diese Quelle ist nicht registriert.")
|
||||
return module.probe(row)
|
||||
release_for_external(conn)
|
||||
result = module.probe(row)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return result
|
||||
|
||||
|
||||
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None) -> tuple[list[dict], str]:
|
||||
def _paperless_choice(value, label: str) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if not text.isdigit() or len(text) > 12:
|
||||
raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.")
|
||||
return text
|
||||
|
||||
|
||||
def _paperless_day(value, label: str) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", text):
|
||||
raise unprocessable(f"{label} ist kein Datum.")
|
||||
return text
|
||||
|
||||
|
||||
def _paperless_params(query: str, options: dict | None) -> dict:
|
||||
"""Paperless-ngx document date is the created field, not the upload or modified time."""
|
||||
options = options or {}
|
||||
params = {"page_size": "20"}
|
||||
text = " ".join(str(query or "").split())
|
||||
if text:
|
||||
params["query"] = text
|
||||
title = " ".join(str(options.get("title") or "").split())
|
||||
if title:
|
||||
params["title__icontains"] = title
|
||||
correspondent = _paperless_choice(options.get("correspondent"), "Der Korrespondent")
|
||||
if correspondent:
|
||||
params["correspondent__id"] = correspondent
|
||||
document_type = _paperless_choice(options.get("document_type"), "Der Dokumenttyp")
|
||||
if document_type:
|
||||
params["document_type__id"] = document_type
|
||||
raw_tags = options.get("tag") or options.get("tags") or ""
|
||||
if isinstance(raw_tags, (list, tuple)):
|
||||
parts = [str(item) for item in raw_tags]
|
||||
else:
|
||||
parts = str(raw_tags).split(",")
|
||||
tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen]
|
||||
if tag_ids:
|
||||
params["tags__id__all"] = ",".join(tag_ids)
|
||||
start = _paperless_day(options.get("document_from"), "Das Dokumentdatum von")
|
||||
end = _paperless_day(options.get("document_to"), "Das Dokumentdatum bis")
|
||||
if start and end and start > end:
|
||||
raise unprocessable("Das Dokumentdatum von liegt nach dem Dokumentdatum bis.")
|
||||
if start:
|
||||
params["created__date__gte"] = start
|
||||
if end:
|
||||
params["created__date__lte"] = end
|
||||
sort = str(options.get("sort") or "").strip()
|
||||
if sort:
|
||||
if sort not in {"created", "-created", "title", "-title", "score", "-score"}:
|
||||
raise unprocessable("Diese Sortierung gibt es nicht.")
|
||||
if sort in {"score", "-score"}:
|
||||
if not text:
|
||||
raise unprocessable("Die Relevanz gilt zusammen mit einem Suchtext.")
|
||||
# Paperless-ngx 2.20.6 has no ordering field named score. A query without
|
||||
# ordering is the relevance order of the full-text index.
|
||||
else:
|
||||
params["ordering"] = sort
|
||||
cursor = str(options.get("cursor") or "").strip()
|
||||
if cursor:
|
||||
if not cursor.isdigit() or not 1 <= int(cursor) <= 10000:
|
||||
raise unprocessable("Die Seite ist ungültig.")
|
||||
params["page"] = cursor
|
||||
return params
|
||||
|
||||
|
||||
def _paperless_row(conn, owner_id: UUID, connector_id: UUID | None):
|
||||
if connector_id is not None:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "paperless":
|
||||
raise unprocessable("Das ist kein Paperless-Zugang.")
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
|
||||
row = rows[0]
|
||||
return row
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
|
||||
return rows[0]
|
||||
|
||||
|
||||
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None, options: dict | None = None) -> dict:
|
||||
row = _paperless_row(conn, owner_id, connector_id)
|
||||
base = row["settings"]["base_url"].rstrip("/")
|
||||
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
|
||||
params = _paperless_params(query, options)
|
||||
release_for_external(conn)
|
||||
payload = _paperless_get(base, row["secret"], "/api/documents/", params)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
items = []
|
||||
for doc in payload.get("results") or []:
|
||||
created = str(doc.get("created") or "")[:10]
|
||||
@@ -118,18 +206,56 @@ def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | N
|
||||
"open_url": f"{base}/documents/{doc.get('id')}/",
|
||||
}
|
||||
)
|
||||
return items, str(row["id"])
|
||||
page = int(params.get("page") or 1)
|
||||
count = payload.get("count")
|
||||
if isinstance(count, int):
|
||||
next_page = str(page + 1) if page * 20 < count else None
|
||||
else:
|
||||
next_page = str(page + 1) if len(items) == 20 else None
|
||||
return {"items": items, "connector_id": str(row["id"]), "next_page": next_page, "incomplete": False}
|
||||
|
||||
|
||||
def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
row = _paperless_row(conn, owner_id, connector_id)
|
||||
base = row["settings"]["base_url"].rstrip("/")
|
||||
token = row["secret"]
|
||||
release_for_external(conn)
|
||||
body = {
|
||||
"correspondents": _paperless_names(base, token, "/api/correspondents/"),
|
||||
"document_types": _paperless_names(base, token, "/api/document_types/"),
|
||||
"tags": _paperless_names(base, token, "/api/tags/"),
|
||||
"tag_mode": "all",
|
||||
}
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return body
|
||||
|
||||
|
||||
def _paperless_names(base: str, token: str, path: str) -> list[dict]:
|
||||
found = []
|
||||
for page in range(1, 21):
|
||||
payload = _paperless_get(base, token, path, {"page": str(page), "page_size": "100"})
|
||||
rows = payload.get("results") or []
|
||||
for item in rows:
|
||||
if item.get("id") is None:
|
||||
continue
|
||||
found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200]})
|
||||
if len(rows) < 100:
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "imap":
|
||||
raise unprocessable("Das ist kein E-Mail-Konto.")
|
||||
release_for_external(conn)
|
||||
client = _imap_login(row["settings"], row["secret"])
|
||||
try:
|
||||
return _imap_mailboxes(client)
|
||||
items = _imap_mailboxes(client)
|
||||
finally:
|
||||
_imap_logout(client)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return items
|
||||
|
||||
|
||||
def imap_messages(
|
||||
@@ -139,6 +265,7 @@ def imap_messages(
|
||||
query: str,
|
||||
mailbox: str,
|
||||
before: int | None,
|
||||
uidvalidity: int | None = None,
|
||||
) -> dict:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "imap":
|
||||
@@ -147,11 +274,14 @@ def imap_messages(
|
||||
raise unprocessable("Der Fachname ist ungültig.")
|
||||
if len(query) > 200:
|
||||
raise unprocessable("Die Suche ist zu lang.")
|
||||
release_for_external(conn)
|
||||
client = _imap_login(row["settings"], row["secret"])
|
||||
try:
|
||||
return _imap_page(client, mailbox, query, before)
|
||||
page = _imap_page(client, mailbox, query, before, uidvalidity)
|
||||
finally:
|
||||
_imap_logout(client)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return page
|
||||
|
||||
|
||||
def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict:
|
||||
@@ -208,23 +338,11 @@ def _paperless_count(base: str, token: str) -> int:
|
||||
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
|
||||
if not document_id.isdigit() or len(document_id) > 12:
|
||||
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
||||
if connector_id is not None:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "paperless":
|
||||
raise unprocessable("Das ist kein Paperless-Zugang.")
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für Paperless ist kein eindeutiger Zugang zugeordnet.")
|
||||
row = rows[0]
|
||||
return _paperless_bytes(row, document_id)
|
||||
row = _paperless_row(conn, owner_id, connector_id)
|
||||
release_for_external(conn)
|
||||
fetched = _paperless_bytes(row, document_id)
|
||||
assert_still_owned(conn, row["id"], owner_id)
|
||||
return fetched
|
||||
|
||||
|
||||
def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
|
||||
@@ -249,35 +367,18 @@ def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
|
||||
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
|
||||
|
||||
|
||||
ORIGINAL_UNAVAILABLE = "Das Original ist über die Freigabe nicht abrufbar."
|
||||
|
||||
|
||||
def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
||||
"""The stored document. A grantee never uses a connector of their own account."""
|
||||
detail = (ref or {}).get("detail") or {}
|
||||
external_id = str((ref or {}).get("external_id") or "")
|
||||
detail = (ref or {}).get("detail") or {}
|
||||
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
|
||||
stored_owner = _connector_owner(conn, stored)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
if not external_id.isdigit() or len(external_id) > 12:
|
||||
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, stored) == actor_id
|
||||
if not external_id.isdigit() or len(external_id) > 12:
|
||||
if owner_path:
|
||||
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
||||
result = resolve_for_read(conn, actor_id, "paperless", ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
return _paperless_bytes(result.row, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None or not external_id.isdigit() or len(external_id) > 12:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
result = resolve_for_read(conn, owner_id, "paperless", ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", stored, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
data, media, name = _paperless_bytes(result.row, external_id)
|
||||
except Problem:
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "ok")
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, "paperless")
|
||||
data, media, name = complete_borrow(conn, borrowed, lambda row: _paperless_bytes(row, external_id))
|
||||
if external_id and external_id in (name or ""):
|
||||
name = "dokument.pdf"
|
||||
return data, media, name or "dokument.pdf"
|
||||
@@ -292,28 +393,18 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
|
||||
mailbox = str(detail.get("mailbox") or "").strip()
|
||||
uid = _mail_uid(detail.get("uid"))
|
||||
external_id = str(ref["external_id"])
|
||||
stored_owner = _connector_owner(conn, connector_id)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
result = resolve_for_read(conn, actor_id, "imap", ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, connector_id) == actor_id
|
||||
if owner_path:
|
||||
if not mailbox or uid is None:
|
||||
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
return load_imap_original(conn, actor_id, result.row["id"], mailbox, uid, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None or connector_id is None or not mailbox or uid is None:
|
||||
elif connector_id is None or not mailbox or uid is None:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
result = resolve_for_read(conn, owner_id, "imap", ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
raw = load_imap_original(conn, owner_id, result.row["id"], mailbox, uid, external_id)
|
||||
except Problem:
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "ok")
|
||||
return raw
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
|
||||
|
||||
def fetch(row):
|
||||
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
|
||||
|
||||
return complete_borrow(conn, borrowed, fetch)
|
||||
|
||||
|
||||
def _uuid(value) -> UUID | None:
|
||||
@@ -335,13 +426,6 @@ def _mail_uid(value) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def _connector_owner(conn, connector_id: UUID | None) -> UUID | None:
|
||||
if connector_id is None:
|
||||
return None
|
||||
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (connector_id,)).fetchone()
|
||||
return row["owner_id"] if row else None
|
||||
|
||||
|
||||
def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
|
||||
return conn.execute(
|
||||
"""
|
||||
@@ -353,26 +437,6 @@ def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
|
||||
).fetchone()
|
||||
|
||||
|
||||
def _audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
|
||||
audit(
|
||||
conn,
|
||||
actor_id,
|
||||
element["home_scope_id"],
|
||||
"source_read_mediated",
|
||||
"element",
|
||||
element["id"],
|
||||
context,
|
||||
result="ok" if outcome == "ok" else "unavailable",
|
||||
detail={
|
||||
"source": source,
|
||||
"connector_id": str(connector_id) if connector_id else None,
|
||||
"outcome": outcome,
|
||||
},
|
||||
)
|
||||
if outcome != "ok":
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _paperless_file(
|
||||
base: str,
|
||||
token: str,
|
||||
@@ -534,6 +598,8 @@ def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
|
||||
|
||||
|
||||
PAGE_SIZE = 30
|
||||
UID_WINDOW = 400
|
||||
UID_ROUNDS = 8
|
||||
_LIST_LINE = re.compile(
|
||||
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
|
||||
)
|
||||
@@ -565,7 +631,96 @@ def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
|
||||
return items
|
||||
|
||||
|
||||
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None) -> dict:
|
||||
def _untagged_int(client: imaplib.IMAP4, name: str) -> int | None:
|
||||
values = (getattr(client, "untagged_responses", None) or {}).get(name) or []
|
||||
if not values:
|
||||
return None
|
||||
raw = values[-1]
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("ascii", "replace")
|
||||
try:
|
||||
return int(str(raw).strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _uids_from(data, low: int | None, high: int | None) -> list[int]:
|
||||
if data is None or isinstance(data, (bytes, str)):
|
||||
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
||||
chunks = []
|
||||
for part in data:
|
||||
if part is None:
|
||||
continue
|
||||
if isinstance(part, bytes):
|
||||
chunks.append(part)
|
||||
elif isinstance(part, str):
|
||||
chunks.append(part.encode("utf-8"))
|
||||
else:
|
||||
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
||||
found = []
|
||||
for piece in b" ".join(chunks).split():
|
||||
if not piece.isdigit():
|
||||
continue
|
||||
uid = int(piece)
|
||||
if low is not None and uid < low:
|
||||
continue
|
||||
if high is not None and uid > high:
|
||||
continue
|
||||
found.append(uid)
|
||||
return found
|
||||
|
||||
|
||||
def _uid_call(client: imaplib.IMAP4, *args):
|
||||
try:
|
||||
return client.uid(*args)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
|
||||
|
||||
|
||||
def _search_status(status) -> None:
|
||||
if status in {"NO", "BAD"}:
|
||||
raise unprocessable("Das Postfach hat die Suche abgelehnt.")
|
||||
if status != "OK":
|
||||
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
||||
|
||||
|
||||
def _uid_window(client: imaplib.IMAP4, query: str, low: int, high: int) -> list[int]:
|
||||
needle = " ".join(query.split())
|
||||
uid_range = f"{low}:{high}"
|
||||
if not needle:
|
||||
status, data = _uid_call(client, "SEARCH", None, "UID", uid_range)
|
||||
else:
|
||||
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
if needle.isascii():
|
||||
status, data = _uid_call(
|
||||
client, "SEARCH", None, "UID", uid_range,
|
||||
"OR", "OR", "SUBJECT", quoted, "FROM", quoted, "TEXT", quoted,
|
||||
)
|
||||
else:
|
||||
encoded = quoted.encode("utf-8")
|
||||
status, data = _uid_call(
|
||||
client, "SEARCH", "CHARSET", "UTF-8", "UID", uid_range,
|
||||
"OR", "OR", "SUBJECT", encoded, "FROM", encoded, "TEXT", encoded,
|
||||
)
|
||||
_search_status(status)
|
||||
return _uids_from(data, low, high)
|
||||
|
||||
|
||||
def _highest_uid(client: imaplib.IMAP4) -> int:
|
||||
uidnext = _untagged_int(client, "UIDNEXT")
|
||||
if uidnext is not None:
|
||||
return max(0, uidnext - 1)
|
||||
status, data = _uid_call(client, "SEARCH", None, "UID", "*")
|
||||
_search_status(status)
|
||||
found = _uids_from(data, None, None)
|
||||
return max(found) if found else 0
|
||||
|
||||
|
||||
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None, claimed_validity: int | None = None) -> dict:
|
||||
folders = _imap_mailboxes(client)
|
||||
chosen = mailbox or "INBOX"
|
||||
match = next((item for item in folders if item["name"] == chosen), None)
|
||||
@@ -573,38 +728,60 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
||||
try:
|
||||
status, _ = client.select(chosen, readonly=True)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
if status != "OK":
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
||||
status, data = _uid_search(client, query)
|
||||
if status != "OK" or not data or not data[0]:
|
||||
return {"mailbox": chosen, "items": [], "next_before": None}
|
||||
uids = sorted(int(part) for part in data[0].split() if part.isdigit())
|
||||
if before is not None:
|
||||
uids = [uid for uid in uids if uid < before]
|
||||
page = uids[-PAGE_SIZE:]
|
||||
next_before = page[0] if len(uids) > PAGE_SIZE else None
|
||||
items = []
|
||||
for uid in reversed(page):
|
||||
item = _fetch_header(client, uid, chosen)
|
||||
if item is not None:
|
||||
items.append(item)
|
||||
return {"mailbox": chosen, "items": items, "next_before": next_before}
|
||||
|
||||
|
||||
def _uid_search(client: imaplib.IMAP4, query: str):
|
||||
needle = " ".join(query.split())
|
||||
if not needle:
|
||||
return client.uid("SEARCH", None, "ALL")
|
||||
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
try:
|
||||
if needle.isascii():
|
||||
return client.uid("SEARCH", None, "OR", "SUBJECT", quoted, "FROM", quoted)
|
||||
encoded = quoted.encode("utf-8")
|
||||
return client.uid("SEARCH", "CHARSET", "UTF-8", "OR", "SUBJECT", encoded, "FROM", encoded)
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
|
||||
validity = _untagged_int(client, "UIDVALIDITY")
|
||||
restarted = False
|
||||
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
|
||||
before = None
|
||||
restarted = True
|
||||
high = before - 1 if before is not None else _highest_uid(client)
|
||||
collected = []
|
||||
attempts = 0
|
||||
failures = 0
|
||||
low = 1
|
||||
rounds = 0
|
||||
while high >= 1 and len(collected) < PAGE_SIZE and rounds < UID_ROUNDS:
|
||||
low = max(1, high - UID_WINDOW + 1)
|
||||
for uid in sorted(_uid_window(client, query, low, high), reverse=True):
|
||||
if len(collected) >= PAGE_SIZE:
|
||||
break
|
||||
attempts += 1
|
||||
item = _fetch_header(client, uid, chosen)
|
||||
if item is None:
|
||||
failures += 1
|
||||
continue
|
||||
collected.append(item)
|
||||
rounds += 1
|
||||
if len(collected) >= PAGE_SIZE or low == 1:
|
||||
break
|
||||
high = low - 1
|
||||
if attempts and failures == attempts:
|
||||
raise unprocessable("Die Nachrichtenköpfe dieses Fachs konnten nicht gelesen werden.")
|
||||
incomplete = bool(collected) is False and low > 1 and high >= 1 and rounds >= UID_ROUNDS
|
||||
if len(collected) < PAGE_SIZE and low > 1 and rounds >= UID_ROUNDS:
|
||||
incomplete = True
|
||||
oldest = collected[-1]["uid"] if collected else None
|
||||
if len(collected) >= PAGE_SIZE and oldest and oldest > 1:
|
||||
next_before = oldest
|
||||
elif incomplete:
|
||||
next_before = low
|
||||
else:
|
||||
next_before = None
|
||||
return {
|
||||
"mailbox": chosen,
|
||||
"items": collected,
|
||||
"next_before": next_before,
|
||||
"incomplete": incomplete,
|
||||
"restarted": restarted,
|
||||
"uidvalidity": validity,
|
||||
}
|
||||
|
||||
|
||||
def _check_mailbox(mailbox: str) -> None:
|
||||
@@ -1052,16 +1229,23 @@ def _imap_config(payload: dict) -> tuple[str, dict, str]:
|
||||
|
||||
|
||||
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
del options
|
||||
items, bound = paperless_documents(conn, owner_id, query, connector_id)
|
||||
return {"items": items, "connector_id": bound}
|
||||
return paperless_documents(conn, owner_id, query, connector_id, options)
|
||||
|
||||
|
||||
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
if connector_id is None:
|
||||
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
|
||||
before = options.get("before")
|
||||
return imap_messages(conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before)
|
||||
validity = options.get("uidvalidity")
|
||||
return imap_messages(
|
||||
conn,
|
||||
owner_id,
|
||||
connector_id,
|
||||
query,
|
||||
str(options.get("mailbox") or ""),
|
||||
before,
|
||||
int(validity) if validity else None,
|
||||
)
|
||||
|
||||
|
||||
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
||||
@@ -1133,11 +1317,15 @@ class _PaperlessAdapter:
|
||||
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
||||
|
||||
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
|
||||
return paperless_documents(conn, owner_id, query, connector_id)
|
||||
found = paperless_documents(conn, owner_id, query, connector_id)
|
||||
return found["items"], found["connector_id"]
|
||||
|
||||
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
return _paperless_find(conn, owner_id, connector_id, query, options)
|
||||
|
||||
def facets(self, conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
return paperless_facets(conn, owner_id, connector_id)
|
||||
|
||||
|
||||
class _ImapAdapter:
|
||||
kind = "imap"
|
||||
@@ -1174,6 +1362,7 @@ def _install_sources() -> None:
|
||||
export_original=_export_paperless,
|
||||
navigate=None,
|
||||
preview=None,
|
||||
facets=paperless.facets,
|
||||
))
|
||||
register_source(SourceModule(
|
||||
kind="imap",
|
||||
|
||||
@@ -7,12 +7,11 @@ from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from app.access.policy import mediated_source_owner
|
||||
from app.core.errors import not_found, unprocessable
|
||||
from app.retrieval import FilePayload, RetrievalFailure
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceregistry import SourceModule, register_source, unregister_source
|
||||
from app.sources import ORIGINAL_UNAVAILABLE, _audit_mediated, _connector_owner
|
||||
from app.sourceaccess import borrow_original, complete_borrow
|
||||
|
||||
KIND = "sample"
|
||||
|
||||
@@ -88,28 +87,8 @@ def preview(conn, owner_id: UUID, connector_id: UUID, options: dict) -> dict:
|
||||
|
||||
|
||||
def open_original(conn, actor_id: UUID, element: dict, context, ref) -> tuple[bytes, str, str]:
|
||||
external_id = str((ref or {}).get("external_id") or "")
|
||||
stored = ref.get("connector_id") if ref else None
|
||||
stored_owner = _connector_owner(conn, stored)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
result = resolve_for_read(conn, actor_id, KIND, ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
return _bytes(result.row, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
result = resolve_for_read(conn, owner_id, KIND, ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, stored, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
data, media, name = _bytes(result.row, external_id)
|
||||
except Exception:
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "ok")
|
||||
return data, media, name
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, KIND)
|
||||
return complete_borrow(conn, borrowed, lambda row: _bytes(row, borrowed.external_id))
|
||||
|
||||
|
||||
def export_original(conn, actor_id: UUID, external_id: str, detail: dict, assigned, ref):
|
||||
|
||||
@@ -207,16 +207,16 @@ def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monke
|
||||
)
|
||||
assert found["items"][0]["title"] == "Treffer"
|
||||
assert found["connector_id"] == older
|
||||
assert found["capabilities"]["search"]["correspondent"] is False
|
||||
assert "correspondent" in found["unsupported"]
|
||||
filtered = ada.client.get(
|
||||
assert found["capabilities"]["search"]["correspondent"] is True
|
||||
assert "correspondent" not in found["unsupported"]
|
||||
filtered = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"q": "Treffer", "connector_id": older, "correspondent": "Haus"},
|
||||
headers=headers,
|
||||
params={"q": "Treffer", "connector_id": older, "correspondent": "4"},
|
||||
)
|
||||
assert filtered.status_code == 422
|
||||
assert filtered.json()["code"] == "unsupported"
|
||||
assert "Haus" not in json.dumps(filtered.json())
|
||||
assert filtered["items"][0]["title"] == "Treffer"
|
||||
assert calls[-1][3]["correspondent__id"] == "4"
|
||||
assert calls[-1][3]["query"] == "Treffer"
|
||||
|
||||
|
||||
def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch):
|
||||
|
||||
@@ -0,0 +1,770 @@
|
||||
"""Follow-up checks for paging, filters, storage resolution and notes.
|
||||
|
||||
IMAP and Paperless doubles stand in for the external servers. They do not
|
||||
prove a live mailbox or a live Paperless instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from email.parser import BytesParser
|
||||
|
||||
from app.media.port import MemoryAdapter
|
||||
from app.sourceaccess import borrow_original, complete_borrow
|
||||
from app.sourceregistry import SourceModule, unregister_source, register_source
|
||||
from tests.support import ORIGIN, Actor, document, register
|
||||
|
||||
PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\nM\n"
|
||||
LETTER = (
|
||||
b"From: Ada <ada@example.test>\r\n"
|
||||
b"To: Ben <ben@example.test>\r\n"
|
||||
b"Subject: Brief\r\n"
|
||||
b"Message-ID: <brief@example.test>\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b'Content-Type: multipart/mixed; boundary="grenze"\r\n'
|
||||
b"\r\n"
|
||||
b"--grenze\r\n"
|
||||
b"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
b"\r\n"
|
||||
b"Hallo aus der zweiten Quelle.\r\n"
|
||||
b"--grenze\r\n"
|
||||
b"Content-Type: application/pdf\r\n"
|
||||
b'Content-Disposition: attachment; filename="plan.pdf"\r\n'
|
||||
b"\r\n"
|
||||
b"%PDF-plan\r\n"
|
||||
b"--grenze--\r\n"
|
||||
)
|
||||
|
||||
|
||||
def _headers(actor: Actor) -> dict:
|
||||
return {
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": actor.csrf,
|
||||
"X-Scope-Kind": actor.scope["kind"],
|
||||
"X-Scope-Id": actor.scope["id"],
|
||||
}
|
||||
|
||||
|
||||
def _install_letterbox() -> None:
|
||||
def parse_config(payload: dict):
|
||||
name = str(payload.get("name") or "Briefkasten").strip() or "Briefkasten"
|
||||
return name, {"catalog": "letters"}, "letter-secret"
|
||||
|
||||
def probe(row):
|
||||
return {"ok": True, "detail": "Der Briefkasten antwortet."}
|
||||
|
||||
def find(conn, owner_id, connector_id, query, options):
|
||||
del conn, owner_id, query, options
|
||||
return {"items": [{"id": "<brief@example.test>", "title": "Brief"}], "connector_id": str(connector_id)}
|
||||
|
||||
def open_original(conn, actor_id, element, context, ref):
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, "letterbox")
|
||||
return complete_borrow(conn, borrowed, lambda row: (LETTER, "message/rfc822", "nachricht.eml"))
|
||||
|
||||
def export_original(conn, actor_id, external_id, detail, assigned, ref):
|
||||
del conn, actor_id, external_id, detail, assigned, ref
|
||||
from app.retrieval import FilePayload
|
||||
|
||||
return FilePayload("nachricht.eml", "message/rfc822", LETTER, "")
|
||||
|
||||
register_source(SourceModule(
|
||||
kind="letterbox",
|
||||
label="Briefkasten",
|
||||
representation="email",
|
||||
capabilities={
|
||||
"kind": "letterbox",
|
||||
"navigate": False,
|
||||
"read_original": True,
|
||||
"read_preview": False,
|
||||
"read_attachment": True,
|
||||
"edit_content": False,
|
||||
"edit_metadata": False,
|
||||
"search": {"query": True, "continuation": False},
|
||||
},
|
||||
parse_config=parse_config,
|
||||
probe=probe,
|
||||
find=find,
|
||||
open_original=open_original,
|
||||
export_original=export_original,
|
||||
))
|
||||
|
||||
|
||||
class RangeIMAP:
|
||||
searches: list = []
|
||||
validity = b"3"
|
||||
uidnext = b"50001"
|
||||
fail_headers = False
|
||||
search_status = "OK"
|
||||
parts = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.untagged_responses = {}
|
||||
|
||||
def login(self, username, secret):
|
||||
assert secret == "Postfach-Geheim"
|
||||
return "OK", [b"logged"]
|
||||
|
||||
def list(self):
|
||||
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
|
||||
|
||||
def select(self, mailbox, readonly=False):
|
||||
assert readonly is True
|
||||
self.untagged_responses = {"UIDVALIDITY": [type(self).validity], "UIDNEXT": [type(self).uidnext]}
|
||||
return "OK", [b"1"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
if command == "SEARCH":
|
||||
type(self).searches.append(args)
|
||||
if type(self).search_status != "OK":
|
||||
return type(self).search_status, [None]
|
||||
if "*" in args and not any(isinstance(arg, str) and ":" in arg for arg in args):
|
||||
return "OK", [b"50000"]
|
||||
low, high = _range_of(args)
|
||||
if type(self).parts and low <= 5 <= high:
|
||||
return "OK", [b"4 5", b"3"]
|
||||
needle = _needle(args)
|
||||
if needle:
|
||||
hit = 80 if low <= 80 <= high else None
|
||||
if needle.startswith("=?") or "nderung" in needle or "Änderung" in needle:
|
||||
hit = 80 if low <= 80 <= high else None
|
||||
return ("OK", [str(hit).encode()]) if hit else ("OK", [b""])
|
||||
found = [str(uid).encode() for uid in range(low, high + 1)]
|
||||
return "OK", [b" ".join(found)]
|
||||
uid = int(args[0])
|
||||
if type(self).fail_headers:
|
||||
return "NO", [None]
|
||||
subject = "Seltener Treffer" if uid == 80 else f"Nachricht {uid}"
|
||||
header = (
|
||||
f"From: Ada <ada@example.test>\r\nSubject: {subject}\r\n"
|
||||
f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||||
).encode()
|
||||
return "OK", [(b"body", header), b")"]
|
||||
|
||||
def logout(self):
|
||||
return "BYE", [b""]
|
||||
|
||||
|
||||
def _range_of(args) -> tuple[int, int]:
|
||||
for arg in args:
|
||||
text = arg.decode() if isinstance(arg, bytes) else str(arg)
|
||||
if ":" in text and text.replace(":", "").isdigit():
|
||||
low, high = text.split(":", 1)
|
||||
return int(low), int(high)
|
||||
raise AssertionError(args)
|
||||
|
||||
|
||||
def _needle(args) -> str:
|
||||
for arg in args:
|
||||
text = arg.decode() if isinstance(arg, bytes) else str(arg)
|
||||
if text.startswith('"') or "nderung" in text or "Änderung" in text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _account(monkeypatch, actor: Actor, fake) -> str:
|
||||
import app.sources as sources
|
||||
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", fake)
|
||||
created = actor.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={
|
||||
"kind": "imap",
|
||||
"name": "Kanzlei",
|
||||
"host": "imap.example.test",
|
||||
"port": 993,
|
||||
"username": "ada@example.test",
|
||||
"password": "Postfach-Geheim",
|
||||
"security": "ssl",
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
return created["id"]
|
||||
|
||||
|
||||
def _spans(searches) -> list[int]:
|
||||
spans = []
|
||||
for args in searches:
|
||||
if any(arg == "*" for arg in args):
|
||||
continue
|
||||
low, high = _range_of(args)
|
||||
spans.append(high - low + 1)
|
||||
return spans
|
||||
|
||||
|
||||
def test_large_folder_is_bounded_and_a_late_hit_is_not_empty(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
ada = Actor(client, "ada-range@example.test", "Ada Bereich")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
RangeIMAP.searches = []
|
||||
RangeIMAP.validity = b"3"
|
||||
RangeIMAP.uidnext = b"50001"
|
||||
RangeIMAP.fail_headers = False
|
||||
RangeIMAP.search_status = "OK"
|
||||
RangeIMAP.parts = False
|
||||
account = _account(monkeypatch, ada, RangeIMAP)
|
||||
first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
|
||||
assert first["items"][0]["title"] == "Nachricht 50000"
|
||||
assert len(first["items"]) == sources.PAGE_SIZE
|
||||
assert first["incomplete"] is False
|
||||
assert first["next_before"] is not None
|
||||
assert _spans(RangeIMAP.searches)
|
||||
assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW
|
||||
assert all("ALL" not in args for args in RangeIMAP.searches)
|
||||
RangeIMAP.searches = []
|
||||
page = first
|
||||
seen = {item["uid"] for item in page["items"]}
|
||||
hops = 0
|
||||
while 80 not in seen:
|
||||
hops += 1
|
||||
assert hops < 40
|
||||
assert page["next_before"] or page["incomplete"]
|
||||
page = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/messages",
|
||||
params={"connector_id": account, "mailbox": "INBOX", "q": "Seltener", "before": page["next_before"]},
|
||||
)
|
||||
if not page["items"] and page["incomplete"]:
|
||||
assert page["next_before"]
|
||||
seen.update(item["uid"] for item in page["items"])
|
||||
assert any(item["title"] == "Seltener Treffer" for item in page["items"])
|
||||
assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW
|
||||
|
||||
|
||||
def test_imap_errors_are_not_reported_as_an_empty_folder(client, db, monkeypatch):
|
||||
ada = Actor(client, "ada-imap-state@example.test", "Ada Zustand")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
RangeIMAP.searches = []
|
||||
RangeIMAP.uidnext = b"1"
|
||||
RangeIMAP.fail_headers = False
|
||||
RangeIMAP.search_status = "OK"
|
||||
RangeIMAP.parts = False
|
||||
account = _account(monkeypatch, ada, RangeIMAP)
|
||||
empty = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
|
||||
assert empty["items"] == []
|
||||
assert empty["incomplete"] is False
|
||||
RangeIMAP.uidnext = b"31"
|
||||
RangeIMAP.search_status = "NO"
|
||||
refused = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/messages",
|
||||
params={"connector_id": account, "mailbox": "INBOX"},
|
||||
status=422,
|
||||
)
|
||||
assert "abgelehnt" in refused["detail"]
|
||||
assert "leer" not in refused["detail"].casefold()
|
||||
RangeIMAP.search_status = "OK"
|
||||
RangeIMAP.fail_headers = True
|
||||
failed = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/messages",
|
||||
params={"connector_id": account, "mailbox": "INBOX"},
|
||||
status=422,
|
||||
)
|
||||
assert "Nachrichtenköpfe" in failed["detail"]
|
||||
assert "leer" not in failed["detail"].casefold()
|
||||
|
||||
|
||||
def test_uidvalidity_change_restarts_and_split_responses_join(client, db, monkeypatch):
|
||||
ada = Actor(client, "ada-validity@example.test", "Ada Geltung")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
RangeIMAP.searches = []
|
||||
RangeIMAP.validity = b"4"
|
||||
RangeIMAP.uidnext = b"11"
|
||||
RangeIMAP.fail_headers = False
|
||||
RangeIMAP.search_status = "OK"
|
||||
RangeIMAP.parts = False
|
||||
account = _account(monkeypatch, ada, RangeIMAP)
|
||||
first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
|
||||
RangeIMAP.validity = b"9"
|
||||
restarted = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/messages",
|
||||
params={
|
||||
"connector_id": account,
|
||||
"mailbox": "INBOX",
|
||||
**({"before": first["next_before"]} if first["next_before"] else {}),
|
||||
"uidvalidity": first["uidvalidity"],
|
||||
},
|
||||
)
|
||||
assert restarted["restarted"] is True
|
||||
assert restarted["items"][0]["title"] == "Nachricht 10"
|
||||
RangeIMAP.validity = b"4"
|
||||
RangeIMAP.parts = True
|
||||
RangeIMAP.uidnext = b"6"
|
||||
joined = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
|
||||
assert [item["uid"] for item in joined["items"]] == [5, 4, 3]
|
||||
|
||||
|
||||
def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_get(base, token, path, params):
|
||||
calls.append({"path": path, "params": dict(params), "token": token})
|
||||
if path != "/api/documents/":
|
||||
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
|
||||
page = int(params.get("page") or 1)
|
||||
start = (page - 1) * 20
|
||||
return {
|
||||
"count": 45,
|
||||
"results": [
|
||||
{"id": number + 1, "title": f"Titel {number + 1}", "created": "2024-03-01"}
|
||||
for number in range(start, min(start + 20, 45))
|
||||
],
|
||||
}
|
||||
|
||||
def fake_bytes(row, document_id):
|
||||
return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf"
|
||||
|
||||
monkeypatch.setattr(sources, "_paperless_get", fake_get)
|
||||
monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes)
|
||||
ada = Actor(client, "ada-filter@example.test", "Ada Filter")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
first = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||||
status=201,
|
||||
)
|
||||
second = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Zweitarchiv", "base_url": "https://paper.example.test", "token": "token-bravo"},
|
||||
status=201,
|
||||
)
|
||||
facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]})
|
||||
assert facets["tag_mode"] == "all"
|
||||
assert facets["correspondents"] == [{"id": "4", "name": "Haus"}]
|
||||
ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "document_from": "2024-05-02", "document_to": "2024-01-01"},
|
||||
status=422,
|
||||
)
|
||||
found = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={
|
||||
"connector_id": first["id"],
|
||||
"q": "nurinhalt",
|
||||
"title": "Vertrag",
|
||||
"correspondent": "4",
|
||||
"document_type": "8",
|
||||
"tag": "2,5",
|
||||
"document_from": "2024-01-01",
|
||||
"document_to": "2024-12-31",
|
||||
"sort": "-created",
|
||||
"cursor": "3",
|
||||
},
|
||||
)
|
||||
sent = calls[-1]["params"]
|
||||
assert sent["query"] == "nurinhalt"
|
||||
assert sent["title__icontains"] == "Vertrag"
|
||||
assert sent["correspondent__id"] == "4"
|
||||
assert sent["document_type__id"] == "8"
|
||||
assert sent["tags__id__all"] == "2,5"
|
||||
assert sent["created__date__gte"] == "2024-01-01"
|
||||
assert sent["created__date__lte"] == "2024-12-31"
|
||||
assert sent["ordering"] == "-created"
|
||||
assert sent["page"] == "3"
|
||||
ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"},
|
||||
)
|
||||
assert "ordering" not in calls[-1]["params"]
|
||||
assert found["items"][0]["id"] == "41"
|
||||
assert found["connector_id"] == first["id"]
|
||||
preview = ada.client.get(
|
||||
"/api/v1/sources/paperless/documents/41/preview",
|
||||
params={"connector_id": first["id"]},
|
||||
headers=_headers(ada),
|
||||
)
|
||||
assert preview.status_code == 200
|
||||
assert preview.content.startswith(b"%PDF-token-alpha-41")
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Suche"}, status=201)
|
||||
linked = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Seite drei",
|
||||
"source": "paperless",
|
||||
"external_id": "41",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": first["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
other = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Dieselbe Nummer",
|
||||
"source": "paperless",
|
||||
"external_id": "41",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": second["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
opened = ada.client.get(f"/api/v1/external-refs/{linked['id']}/content", headers=_headers(ada))
|
||||
twin = ada.client.get(f"/api/v1/external-refs/{other['id']}/content", headers=_headers(ada))
|
||||
assert opened.content.startswith(b"%PDF-token-alpha-41")
|
||||
assert twin.content.startswith(b"%PDF-token-bravo-41")
|
||||
ben = Actor(client, "ben-filter@example.test", "Ben Filter")
|
||||
register(db, ben.email, ben.name)
|
||||
ben.login()
|
||||
ben.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]}, status=404)
|
||||
ben.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": first["id"], "q": "nurinhalt"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
|
||||
def test_revoked_grant_hides_bytes_fetched_during_the_call(client, db, monkeypatch):
|
||||
import app.sources as sources
|
||||
|
||||
def fake_bytes(row, document_id):
|
||||
db.execute("UPDATE access_grants SET can_read = false")
|
||||
db.commit()
|
||||
return b"%PDF-1.4\nsecret\n", "application/pdf", "geheim.pdf"
|
||||
|
||||
monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes)
|
||||
ada = Actor(client, "ada-revoke@example.test", "Ada Widerruf")
|
||||
ben = Actor(client, "ben-revoke@example.test", "Ben Widerruf")
|
||||
register(db, ada.email, ada.name)
|
||||
register(db, ben.email, ben.name)
|
||||
ada.login()
|
||||
paper = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||||
status=201,
|
||||
)
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Freigabe"}, status=201)
|
||||
linked = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Vertrag",
|
||||
"source": "paperless",
|
||||
"external_id": "15",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": paper["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{matter['id']}/grants",
|
||||
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True},
|
||||
status=201,
|
||||
)
|
||||
ben.login()
|
||||
hidden = ben.client.get(
|
||||
f"/api/v1/external-refs/{linked['id']}/content",
|
||||
params={"context_kind": "matter", "context_id": matter["id"]},
|
||||
headers=_headers(ben),
|
||||
)
|
||||
assert hidden.status_code == 422
|
||||
assert b"secret" not in hidden.content
|
||||
|
||||
|
||||
def test_email_representation_uses_the_mail_reader_without_imap(client, db):
|
||||
ada = Actor(client, "ada-letter@example.test", "Ada Brief")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
_install_letterbox()
|
||||
try:
|
||||
box = ada.request("POST", "/api/v1/connectors", json={"kind": "letterbox", "name": "Briefe"}, status=201)
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
|
||||
linked = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Brief",
|
||||
"source": "letterbox",
|
||||
"external_id": "<brief@example.test>",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": box["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
mail = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
|
||||
assert mail["attachments"][0]["filename"] == "plan.pdf"
|
||||
attachment = ada.client.get(
|
||||
f"/api/v1/external-refs/{linked['id']}/mail/attachments/0",
|
||||
headers=_headers(ada),
|
||||
)
|
||||
assert attachment.status_code == 200
|
||||
assert attachment.content.startswith(b"%PDF-plan")
|
||||
raw = ada.client.get(f"/api/v1/external-refs/{linked['id']}/mail/original", headers=_headers(ada))
|
||||
parsed = BytesParser().parsebytes(raw.content)
|
||||
assert parsed.get("subject") == "Brief"
|
||||
finally:
|
||||
unregister_source("letterbox")
|
||||
|
||||
|
||||
def test_domain_storage_uses_the_resolved_adapter(client, db, monkeypatch):
|
||||
import app.media.port as port
|
||||
|
||||
memory = MemoryAdapter()
|
||||
monkeypatch.setitem(port._FACTORIES, "local", lambda row: memory)
|
||||
ada = Actor(client, "ada-store@example.test", "Ada Speicher")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Ablage"}, status=201)
|
||||
ctx = {"kind": "matter", "id": matter["id"]}
|
||||
note = ada.request(
|
||||
"POST",
|
||||
"/api/v1/notes",
|
||||
json={"title": "Schreiben", "document": document("Text"), "context": ctx},
|
||||
status=201,
|
||||
)
|
||||
uploaded = ada.request(
|
||||
"POST",
|
||||
"/api/v1/files",
|
||||
data={"title": "Plan", "context_kind": "matter", "context_id": matter["id"], "document_on": "2024-01-02"},
|
||||
files={"upload": ("plan.pdf", PDF, "application/pdf")},
|
||||
status=201,
|
||||
)
|
||||
detail = ada.request("GET", f"/api/v1/matters/{matter['id']}")
|
||||
host = next(item for item in detail["bindings"] if item["element"]["id"] == note["id"])
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/elements/{note['id']}/attachments",
|
||||
json={
|
||||
"expected_version": host["element"]["version"],
|
||||
"context": ctx,
|
||||
"attachment_element_ids": [uploaded["id"]],
|
||||
"fold": "attachment",
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
folded = ada.request("GET", f"/api/v1/matters/{matter['id']}")
|
||||
host = next(item for item in folded["bindings"] if item["element"]["id"] == note["id"])
|
||||
dispatch = ada.request(
|
||||
"POST",
|
||||
f"/api/v1/elements/{note['id']}/dispatches",
|
||||
json={
|
||||
"expected_version": host["element"]["version"],
|
||||
"context": ctx,
|
||||
"channel": "post",
|
||||
"sent_on": "2024-02-02",
|
||||
"attachment_element_ids": [uploaded["id"]],
|
||||
"capture_editions": True,
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
content = ada.client.get(f"/api/v1/files/{uploaded['id']}/content", headers=_headers(ada))
|
||||
assert content.content == PDF
|
||||
item = next(entry for entry in dispatch["items"] if entry["element_id"] == uploaded["id"])
|
||||
snapshot = ada.client.get(
|
||||
f"/api/v1/dispatches/{dispatch['id']}/items/{item['id']}/content",
|
||||
params={"context_kind": "matter", "context_id": matter["id"]},
|
||||
headers=_headers(ada),
|
||||
)
|
||||
assert snapshot.content == PDF
|
||||
asset = db.execute(
|
||||
"SELECT object_key, storage_generation, sha256 FROM media_assets WHERE element_id = %s",
|
||||
(uploaded["id"],),
|
||||
).fetchone()
|
||||
assert memory.open(asset["object_key"]) == PDF
|
||||
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
|
||||
selected = [
|
||||
node["id"]
|
||||
for node in _walk(tree)
|
||||
if node.get("exportable") and node.get("kind") in {"binding", "attachment", "dispatch_item"}
|
||||
]
|
||||
package = ada.client.post(
|
||||
f"/api/v1/matters/{matter['id']}/export",
|
||||
json={"context": ctx, "selected": selected},
|
||||
headers={**_headers(ada), "Content-Type": "application/json"},
|
||||
)
|
||||
assert package.status_code == 200, package.text
|
||||
archive = zipfile.ZipFile(io.BytesIO(package.content))
|
||||
assert any(archive.read(name) == PDF for name in archive.namelist())
|
||||
version = next(
|
||||
item["element"]["version"]
|
||||
for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["bindings"]
|
||||
if item["element"]["id"] == uploaded["id"]
|
||||
)
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/elements/{uploaded['id']}/delete",
|
||||
json={"expected_version": version},
|
||||
status=204,
|
||||
)
|
||||
try:
|
||||
memory.open(asset["object_key"])
|
||||
except Exception:
|
||||
removed = True
|
||||
else:
|
||||
removed = False
|
||||
assert removed is True
|
||||
assert asset["storage_generation"]
|
||||
|
||||
|
||||
def test_note_markdown_is_shared_exported_and_kept_on_conflict(client, db):
|
||||
ada = Actor(client, "ada-note-md@example.test", "Ada Notiz")
|
||||
ben = Actor(client, "ben-note-md@example.test", "Ben Notiz")
|
||||
register(db, ada.email, ada.name)
|
||||
register(db, ben.email, ben.name)
|
||||
ada.login()
|
||||
left = ada.request("POST", "/api/v1/matters", json={"title": "Links"}, status=201)
|
||||
right = ada.request("POST", "/api/v1/matters", json={"title": "Rechts"}, status=201)
|
||||
body = "\n".join([
|
||||
"# Überschrift",
|
||||
"",
|
||||
"Ein **fetter** und *kursiver* Satz.",
|
||||
"",
|
||||
"- erster",
|
||||
"- zweiter",
|
||||
"",
|
||||
"> Zitat",
|
||||
"",
|
||||
"Siehe [Quelle](https://example.test/hinweis).",
|
||||
"",
|
||||
"`code`",
|
||||
"",
|
||||
"```",
|
||||
"block",
|
||||
"```",
|
||||
"",
|
||||
"<script>alert(1)</script>",
|
||||
"",
|
||||
"",
|
||||
])
|
||||
created = ada.request(
|
||||
"POST",
|
||||
"/api/v1/notes",
|
||||
json={
|
||||
"title": "Merkzettel",
|
||||
"document": {"version": 1, "blocks": [{"type": "markdown", "text": body}]},
|
||||
"context": {"kind": "matter", "id": left["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
ada.request(
|
||||
"POST",
|
||||
"/api/v1/bindings",
|
||||
json={
|
||||
"element_id": created["id"],
|
||||
"source_context": {"kind": "matter", "id": left["id"]},
|
||||
"target": {"kind": "matter", "id": right["id"]},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
detail = ada.request("GET", f"/api/v1/matters/{left['id']}")
|
||||
element = next(item["element"] for item in detail["bindings"] if item["element"]["id"] == created["id"])
|
||||
changed = "Geändert in beiden Vorgängen.\n\n" + body
|
||||
ada.request(
|
||||
"PATCH",
|
||||
f"/api/v1/notes/{created['id']}",
|
||||
json={
|
||||
"expected_version": element["version"],
|
||||
"document": {"version": 1, "blocks": [{"type": "markdown", "text": changed}]},
|
||||
"context": {"kind": "matter", "id": right["id"]},
|
||||
},
|
||||
)
|
||||
again = ada.request("GET", f"/api/v1/matters/{left['id']}")
|
||||
shared = next(item["element"] for item in again["bindings"] if item["element"]["id"] == created["id"])
|
||||
assert changed in _text(shared)
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/bindings/{next(item['id'] for item in again['bindings'] if item['element']['id'] == created['id'])}/annotations",
|
||||
json={"body": "Nur links"},
|
||||
params={"context_kind": "matter", "context_id": left["id"]},
|
||||
status=201,
|
||||
)
|
||||
left_detail = ada.request("GET", f"/api/v1/matters/{left['id']}")
|
||||
right_detail = ada.request("GET", f"/api/v1/matters/{right['id']}")
|
||||
left_binding = next(item for item in left_detail["bindings"] if item["element"]["id"] == created["id"])
|
||||
right_binding = next(item for item in right_detail["bindings"] if item["element"]["id"] == created["id"])
|
||||
assert any(note["body"] == "Nur links" for note in left_binding["annotations"])
|
||||
assert all(note["body"] != "Nur links" for note in right_binding["annotations"])
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{left['id']}/grants",
|
||||
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False},
|
||||
status=201,
|
||||
)
|
||||
ben.login()
|
||||
ben.request(
|
||||
"PATCH",
|
||||
f"/api/v1/notes/{created['id']}",
|
||||
json={
|
||||
"expected_version": shared["version"],
|
||||
"document": {"version": 1, "blocks": [{"type": "markdown", "text": "fremd"}]},
|
||||
"context": {"kind": "matter", "id": left["id"]},
|
||||
},
|
||||
status=403,
|
||||
)
|
||||
ada.login()
|
||||
current = next(
|
||||
item["element"]["version"]
|
||||
for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"]
|
||||
if item["element"]["id"] == created["id"]
|
||||
)
|
||||
conflict = ada.request(
|
||||
"PATCH",
|
||||
f"/api/v1/notes/{created['id']}",
|
||||
json={
|
||||
"expected_version": current - 1,
|
||||
"document": {"version": 1, "blocks": [{"type": "markdown", "text": "darf nicht überschreiben"}]},
|
||||
"context": {"kind": "matter", "id": left["id"]},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
assert conflict["code"] == "conflict"
|
||||
stored = next(
|
||||
item["element"]
|
||||
for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"]
|
||||
if item["element"]["id"] == created["id"]
|
||||
)
|
||||
assert "darf nicht überschreiben" not in _text(stored)
|
||||
assert "<script>alert(1)</script>" in _text(stored)
|
||||
tree = ada.request("GET", f"/api/v1/matters/{left['id']}/export-tree")["tree"]
|
||||
selected = [node["id"] for node in _walk(tree) if node.get("kind") == "binding" and node.get("exportable")]
|
||||
package = ada.client.post(
|
||||
f"/api/v1/matters/{left['id']}/export",
|
||||
json={"context": {"kind": "matter", "id": left["id"]}, "selected": selected},
|
||||
headers={**_headers(ada), "Content-Type": "application/json"},
|
||||
)
|
||||
assert package.status_code == 200, package.text
|
||||
archive = zipfile.ZipFile(io.BytesIO(package.content))
|
||||
exported = "\n".join(
|
||||
archive.read(name).decode("utf-8", errors="replace")
|
||||
for name in archive.namelist()
|
||||
if name.endswith((".md", ".txt"))
|
||||
)
|
||||
assert "```" in exported
|
||||
assert "<script>alert(1)</script>" in exported
|
||||
assert "https://example.test/bild.png" in exported
|
||||
|
||||
|
||||
def _text(element: dict) -> str:
|
||||
return "\n".join(block.get("text") or "" for block in (element.get("document") or {}).get("blocks") or [])
|
||||
|
||||
|
||||
def _walk(node):
|
||||
if isinstance(node, dict) and "kind" in node:
|
||||
yield node
|
||||
for child in node.get("children") or []:
|
||||
yield from _walk(child)
|
||||
elif isinstance(node, dict):
|
||||
for value in node.values():
|
||||
yield from _walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
yield from _walk(item)
|
||||
Reference in new issue
Block a user