Mache große IMAP-Fächer, die Paperless-Suche und die interne Markdown-Notiz über die gemeinsamen Schnittstellen benutzbar.
Deploy Development / deploy (push) Successful in 41s
Test Suite / pytest-backend (push) Failing after 2m55s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 23s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-20 07:34:14 +02:00
1 parent c3a07cee64
commit 7e39fbb2bd
29 files changed
+6053 -320

No files matched your search

+10 -2
View File
@@ -4,6 +4,14 @@ from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
def discard_stored(item) -> None:
discard = getattr(item, "discard", None)
if callable(discard):
discard()
return
Path(item).unlink(missing_ok=True)
from uuid import UUID
from fastapi import Depends, Request
@@ -35,11 +43,11 @@ def get_conn(request: Request):
yield conn
conn.commit()
for path in getattr(request.state, "delete_after_commit", []):
Path(path).unlink(missing_ok=True)
discard_stored(path)
except Exception:
conn.rollback()
for path in getattr(request.state, "cleanup_files", []):
Path(path).unlink(missing_ok=True)
discard_stored(path)
raise
+45 -4
View File
@@ -16,6 +16,7 @@ from app.core.errors import not_found, unprocessable, unsupported
from app.mailpack import parse_mail
from app.media.storage import content_disposition
from app.ops import create_external
from app.retrieval import RetrievalFailure
from app.sourcebind import assign_reference
from app.sourceport import capability_payload, with_capabilities
from app.sourceregistry import get_source, require_source
@@ -23,8 +24,8 @@ from app.sources import (
delete_connector,
imap_preview,
list_connectors,
paperless_content,
probe,
read_linked_mail,
save_connector,
update_connector,
)
@@ -158,7 +159,13 @@ def _opened_ref(conn, principal: Principal, element_id: UUID, context_kind: str
def _complete_mail(conn, principal: Principal, element, context, ref):
raw = read_linked_mail(conn, principal.account_id, element, context, ref)
module = get_source(ref["source"]) if ref is not None else None
if module is None or module.representation != "email":
raise unprocessable("Das ist keine E-Mail.")
opened = module.open_original(conn, principal.account_id, element, context, ref)
if isinstance(opened, RetrievalFailure):
raise unprocessable(opened.message)
raw = opened[0] if isinstance(opened, tuple) else opened
parsed = parse_mail(raw)
if parsed.missing:
names = ", ".join(parsed.missing[:5])
@@ -246,6 +253,34 @@ def search_paperless(
)
@router.get("/api/v1/sources/{kind}/facets")
def source_facets(
kind: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
module = require_source(kind)
if module.facets is None:
raise unsupported("Diese Quelle hat keine Auswahllisten.")
return module.facets(conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/paperless/documents/{document_id}/preview")
def preview_paperless_document(
document_id: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
data, media, name = paperless_content(conn, principal.account_id, document_id, connector_id)
return Response(
content=data,
media_type=media,
headers={**_PRIVATE, "Content-Disposition": content_disposition(name).replace("attachment;", "inline;", 1)},
)
@router.get("/api/v1/sources/imap/mailboxes")
def list_mailboxes(
connector_id: UUID,
@@ -269,6 +304,7 @@ def search_mail(
document_to: str = "",
sort: str = "",
cursor: str = "",
uidvalidity: int | None = None,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
@@ -292,6 +328,7 @@ def search_mail(
"sort": sort,
"cursor": cursor,
},
uidvalidity,
)
@@ -357,7 +394,7 @@ _SEARCH_FLAGS = {
}
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict) -> dict:
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict, uidvalidity: int | None = None) -> dict:
module = require_source(kind)
search = module.capabilities["search"]
refused = [name for name, value in filters.items() if str(value or "").strip() and not search.get(_SEARCH_FLAGS.get(name, name))]
@@ -365,7 +402,11 @@ def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, m
refused.append("before")
if refused:
raise unsupported("Diese Suche unterstützt den Filter nicht: " + ", ".join(refused) + ".")
found = module.find(conn, owner, connector_id, query, {"mailbox": mailbox, "before": before})
options = {"mailbox": mailbox, "before": before}
if uidvalidity is not None:
options["uidvalidity"] = uidvalidity
options.update({key: value for key, value in filters.items() if str(value or "").strip()})
found = module.find(conn, owner, connector_id, query, options)
bound = str(found.get("connector_id") or connector_id or "")
return with_capabilities(kind, bound, found)
+5 -5
View File
@@ -28,7 +28,7 @@ from app.access.policy import (
)
from app.core.errors import Problem, conflict, not_found, unprocessable
from app.core.timeutil import dump_date, dump_dt
from app.media.port import LEGACY_PROFILE, LocalAdapter, instance_for_write, publish, read_clear_bytes, stored_path
from app.media.port import LEGACY_PROFILE, StoredHandle, adapter_for, instance_for_write, publish, read_clear_bytes, removal_for, stored_path
from app.media.storage import display_filename, sha256
from app.retrieval import FilePayload, MailPayload, RetrievalFailure, load_external, load_file
@@ -936,9 +936,9 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
instance = instance_for_write(conn, media_root)
object_key = f"dispatch/{snapshot_id}"
content_revision = str(uuid4())
adapter = LocalAdapter(instance["root_path"])
adapter = adapter_for(instance)
publish(adapter, object_key, payload.data)
cleanup.append(stored_path(conn, instance["id"], instance["generation"], object_key, media_root))
cleanup.append(StoredHandle(adapter, object_key))
digest = sha256(payload.data)
conn.execute(
"""
@@ -1005,8 +1005,8 @@ def _snapshot_disk(conn, snapshot_id: UUID, media_root: str) -> Path:
(snapshot_id,),
).fetchone()
if row is None:
return stored_path(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
return stored_path(
return removal_for(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
return removal_for(
conn,
row["storage_instance_id"],
row["storage_generation"],
+2 -2
View File
@@ -496,8 +496,8 @@ def _note_text(conn, element_id: UUID) -> str:
parts = []
for block in document.get("blocks") or []:
if isinstance(block, dict):
text = str(block.get("text") or "").strip()
if text:
text = str(block.get("text") or "")
if text.strip():
parts.append(text)
return "\n\n".join(parts)
+49 -1
View File
@@ -208,6 +208,47 @@ def instance_for_write(conn, media_root: str) -> dict:
).fetchone()
class StoredHandle:
"""Rollback and delete go through the adapter that owns the bytes."""
def __init__(self, adapter: StoragePort, object_key: str):
self._adapter = adapter
self._key = object_key
def discard(self) -> None:
self._adapter.delete(self._key)
_FACTORIES: dict[str, callable] = {}
def _local_factory(row: dict) -> StoragePort:
return LocalAdapter(row["root_path"])
_FACTORIES["local"] = _local_factory
def adapter_for(row: dict) -> StoragePort:
factory = _FACTORIES.get(row["kind"])
if factory is None or not row.get("root_path") and row["kind"] == "local":
raise storage_unavailable()
return factory(row)
def removal_for(conn, instance_id, generation: str | None, object_key: str, media_root: str) -> StoredHandle:
row = None
if instance_id is not None:
row = conn.execute(
"SELECT id, kind, generation, root_path FROM storage_instances WHERE id = %s",
(instance_id,),
).fetchone()
if row is None:
root = resolve_root(conn, instance_id, generation, media_root)
return StoredHandle(LocalAdapter(root), object_key)
return StoredHandle(adapter_for(row), object_key)
def resolve_root(conn, instance_id, generation: str | None, media_root: str) -> str:
if instance_id is not None:
row = conn.execute(
@@ -236,7 +277,14 @@ def read_clear_bytes(
expected_sha256: str | None,
) -> bytes:
require_cleartext(protection_profile)
data = LocalAdapter(resolve_root(conn, instance_id, generation, media_root)).open(object_key)
row = None
if instance_id is not None:
row = conn.execute(
"SELECT id, kind, generation, root_path FROM storage_instances WHERE id = %s",
(instance_id,),
).fetchone()
adapter = adapter_for(row) if row is not None else LocalAdapter(resolve_root(conn, instance_id, generation, media_root))
data = adapter.open(object_key)
if expected_sha256 and sha256(data) != expected_sha256:
raise Problem(
422,
+4 -6
View File
@@ -35,8 +35,7 @@ from app.access.policy import (
from app.content.documents import clean_document
from app.core.errors import forbidden, not_found, unprocessable, unsupported
from app.core.timeutil import utcnow
from app.media.port import LEGACY_PROFILE, instance_for_write, publish, stored_path
from app.media.port import LocalAdapter
from app.media.port import LEGACY_PROFILE, StoredHandle, adapter_for, instance_for_write, publish, removal_for
from app.media.storage import display_filename, sha256, sniff
MAX_CHAPTER_DEPTH = 32
@@ -1034,10 +1033,9 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
content_revision = str(uuid4())
instance = instance_for_write(conn, media_root)
object_key = str(asset_id)
adapter = LocalAdapter(instance["root_path"])
adapter = adapter_for(instance)
publish(adapter, object_key, data)
path = stored_path(conn, instance["id"], instance["generation"], object_key, media_root)
rollback_files.append(path)
rollback_files.append(StoredHandle(adapter, object_key))
digest = sha256(data)
conn.execute(
"""
@@ -1097,7 +1095,7 @@ def delete_element(conn, actor: UUID, element_id: UUID, expected: int, media_roo
if asset is not None and asset["status"] == "active":
conn.execute("UPDATE media_assets SET status = 'purging' WHERE id = %s", (asset["id"],))
delete_after.append(
stored_path(
removal_for(
conn,
asset["storage_instance_id"],
asset["storage_generation"],
+138
View File
@@ -0,0 +1,138 @@
"""Rights stay in one place. A source module only fetches its own bytes."""
from __future__ import annotations
from dataclasses import dataclass
from uuid import UUID
import psycopg
from app.access.policy import Ctx, audit, mediated_source_owner
from app.core.errors import Problem, not_found, unprocessable
from app.sourcebind import resolve_for_read
ORIGINAL_UNAVAILABLE = "Das Original ist über die Freigabe nicht abrufbar."
@dataclass
class BorrowedOriginal:
row: dict
external_id: str
mediated: bool
actor_id: UUID
owner_id: UUID
element: dict
context: Ctx | None
kind: str
def release_for_external(conn) -> None:
"""External calls must not sit inside an open domain transaction."""
if conn.info.transaction_status != psycopg.pq.TransactionStatus.IDLE:
conn.commit()
def connector_owner(conn, connector_id: UUID | None) -> UUID | None:
if connector_id is None:
return None
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (connector_id,)).fetchone()
return row["owner_id"] if row else None
def assert_still_owned(conn, connector_id, owner_id: UUID) -> None:
row = conn.execute(
"SELECT id FROM connectors WHERE id = %s AND owner_id = %s",
(connector_id, owner_id),
).fetchone()
if row is None:
raise not_found()
def audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
audit(
conn,
actor_id,
element["home_scope_id"],
"source_read_mediated",
"element",
element["id"],
context,
result="ok" if outcome == "ok" else "unavailable",
detail={
"source": source,
"connector_id": str(connector_id) if connector_id else None,
"outcome": outcome,
},
)
if outcome != "ok":
conn.commit()
def _uuid(value) -> UUID | None:
if isinstance(value, UUID):
return value
try:
return UUID(str(value))
except (TypeError, ValueError):
return None
def borrow_original(conn, actor_id: UUID, element: dict, context: Ctx | None, ref, kind: str) -> BorrowedOriginal:
"""Resolve the owner's connector. A grantee never supplies another account's connector."""
detail = (ref or {}).get("detail") or {}
external_id = str((ref or {}).get("external_id") or "")
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
stored_owner = connector_owner(conn, stored)
mediated = not (element["owner_subject_id"] == actor_id or stored_owner == actor_id)
if mediated:
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None:
raise unprocessable(ORIGINAL_UNAVAILABLE)
else:
owner_id = actor_id
result = resolve_for_read(conn, owner_id, kind, ref, None)
if result.row is None:
if mediated:
audit_mediated(conn, actor_id, element, context, kind, stored, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
raise unprocessable(result.message)
return BorrowedOriginal(
row=result.row,
external_id=external_id,
mediated=mediated,
actor_id=actor_id,
owner_id=owner_id,
element=element,
context=context,
kind=kind,
)
def _recheck(conn, borrowed: BorrowedOriginal) -> None:
if borrowed.mediated:
owner_id = mediated_source_owner(conn, borrowed.actor_id, borrowed.element, borrowed.context)
if owner_id != borrowed.owner_id:
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (borrowed.row["id"],)).fetchone()
if row is None or row["owner_id"] != borrowed.owner_id:
if borrowed.mediated:
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
raise not_found()
def complete_borrow(conn, borrowed: BorrowedOriginal, fetch):
"""Run the adapter fetch outside the open transaction, then check the grant again."""
release_for_external(conn)
try:
value = fetch(borrowed.row)
except Problem:
if borrowed.mediated:
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
raise
_recheck(conn, borrowed)
if borrowed.mediated:
audit_mediated(conn, borrowed.actor_id, borrowed.element, borrowed.context, borrowed.kind, borrowed.row["id"], "ok")
return value
+13 -9
View File
@@ -1,7 +1,7 @@
"""Declared source and editor capabilities.
A common contract does not invent abilities the adapter does not have.
Search filters that are false stay unavailable until a later delivery.
A filter is offered only when that module implements it.
"""
from __future__ import annotations
@@ -16,13 +16,13 @@ PAPERLESS_CAPABILITIES = {
"edit_metadata": False,
"search": {
"query": True,
"title": False,
"correspondent": False,
"document_type": False,
"tags": False,
"document_range": False,
"sort": False,
"continuation": False,
"title": True,
"correspondent": True,
"document_type": True,
"tags": True,
"document_range": True,
"sort": True,
"continuation": True,
},
}
@@ -94,5 +94,9 @@ def with_capabilities(kind: str, connector_id: str, payload: dict) -> dict:
body["connector_id"] = connector_id
body["capabilities"] = spec
body["unsupported"] = [name for name, enabled in spec["search"].items() if not enabled]
body["continuation"] = body.get("next_before") if spec["search"].get("continuation") else None
if spec["search"].get("continuation"):
body["continuation"] = body.get("next_before") if body.get("next_before") is not None else body.get("next_page")
else:
body["continuation"] = None
body["incomplete"] = bool(body.get("incomplete"))
return body
+1
View File
@@ -21,6 +21,7 @@ class SourceModule:
export_original: Callable
navigate: Callable | None = None
preview: Callable | None = None
facets: Callable | None = None
_MODULES: dict[str, SourceModule] = {}
+330 -141
View File
@@ -18,12 +18,21 @@ from uuid import UUID
from psycopg.types.json import Json
from app.access.policy import Ctx, audit, mediated_source_owner
from app.access.policy import Ctx
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceaccess import (
ORIGINAL_UNAVAILABLE,
assert_still_owned,
audit_mediated as _audit_mediated,
borrow_original,
complete_borrow,
connector_owner as _connector_owner,
release_for_external,
)
from app.sourceport import CAPABILITIES
from app.sourceregistry import SourceModule, get_source, register_source
@@ -81,30 +90,109 @@ def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
module = get_source(row["kind"])
if module is None:
raise unprocessable("Diese Quelle ist nicht registriert.")
return module.probe(row)
release_for_external(conn)
result = module.probe(row)
assert_still_owned(conn, row["id"], owner_id)
return result
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None) -> tuple[list[dict], str]:
def _paperless_choice(value, label: str) -> str | None:
text = str(value or "").strip()
if not text:
return None
if not text.isdigit() or len(text) > 12:
raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.")
return text
def _paperless_day(value, label: str) -> str | None:
text = str(value or "").strip()
if not text:
return None
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", text):
raise unprocessable(f"{label} ist kein Datum.")
return text
def _paperless_params(query: str, options: dict | None) -> dict:
"""Paperless-ngx document date is the created field, not the upload or modified time."""
options = options or {}
params = {"page_size": "20"}
text = " ".join(str(query or "").split())
if text:
params["query"] = text
title = " ".join(str(options.get("title") or "").split())
if title:
params["title__icontains"] = title
correspondent = _paperless_choice(options.get("correspondent"), "Der Korrespondent")
if correspondent:
params["correspondent__id"] = correspondent
document_type = _paperless_choice(options.get("document_type"), "Der Dokumenttyp")
if document_type:
params["document_type__id"] = document_type
raw_tags = options.get("tag") or options.get("tags") or ""
if isinstance(raw_tags, (list, tuple)):
parts = [str(item) for item in raw_tags]
else:
parts = str(raw_tags).split(",")
tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen]
if tag_ids:
params["tags__id__all"] = ",".join(tag_ids)
start = _paperless_day(options.get("document_from"), "Das Dokumentdatum von")
end = _paperless_day(options.get("document_to"), "Das Dokumentdatum bis")
if start and end and start > end:
raise unprocessable("Das Dokumentdatum von liegt nach dem Dokumentdatum bis.")
if start:
params["created__date__gte"] = start
if end:
params["created__date__lte"] = end
sort = str(options.get("sort") or "").strip()
if sort:
if sort not in {"created", "-created", "title", "-title", "score", "-score"}:
raise unprocessable("Diese Sortierung gibt es nicht.")
if sort in {"score", "-score"}:
if not text:
raise unprocessable("Die Relevanz gilt zusammen mit einem Suchtext.")
# Paperless-ngx 2.20.6 has no ordering field named score. A query without
# ordering is the relevance order of the full-text index.
else:
params["ordering"] = sort
cursor = str(options.get("cursor") or "").strip()
if cursor:
if not cursor.isdigit() or not 1 <= int(cursor) <= 10000:
raise unprocessable("Die Seite ist ungültig.")
params["page"] = cursor
return params
def _paperless_row(conn, owner_id: UUID, connector_id: UUID | None):
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if not rows:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
if len(rows) != 1:
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
row = rows[0]
return row
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if not rows:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
if len(rows) != 1:
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
return rows[0]
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None, options: dict | None = None) -> dict:
row = _paperless_row(conn, owner_id, connector_id)
base = row["settings"]["base_url"].rstrip("/")
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
params = _paperless_params(query, options)
release_for_external(conn)
payload = _paperless_get(base, row["secret"], "/api/documents/", params)
assert_still_owned(conn, row["id"], owner_id)
items = []
for doc in payload.get("results") or []:
created = str(doc.get("created") or "")[:10]
@@ -118,18 +206,56 @@ def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | N
"open_url": f"{base}/documents/{doc.get('id')}/",
}
)
return items, str(row["id"])
page = int(params.get("page") or 1)
count = payload.get("count")
if isinstance(count, int):
next_page = str(page + 1) if page * 20 < count else None
else:
next_page = str(page + 1) if len(items) == 20 else None
return {"items": items, "connector_id": str(row["id"]), "next_page": next_page, "incomplete": False}
def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = _paperless_row(conn, owner_id, connector_id)
base = row["settings"]["base_url"].rstrip("/")
token = row["secret"]
release_for_external(conn)
body = {
"correspondents": _paperless_names(base, token, "/api/correspondents/"),
"document_types": _paperless_names(base, token, "/api/document_types/"),
"tags": _paperless_names(base, token, "/api/tags/"),
"tag_mode": "all",
}
assert_still_owned(conn, row["id"], owner_id)
return body
def _paperless_names(base: str, token: str, path: str) -> list[dict]:
found = []
for page in range(1, 21):
payload = _paperless_get(base, token, path, {"page": str(page), "page_size": "100"})
rows = payload.get("results") or []
for item in rows:
if item.get("id") is None:
continue
found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200]})
if len(rows) < 100:
break
return found
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
release_for_external(conn)
client = _imap_login(row["settings"], row["secret"])
try:
return _imap_mailboxes(client)
items = _imap_mailboxes(client)
finally:
_imap_logout(client)
assert_still_owned(conn, row["id"], owner_id)
return items
def imap_messages(
@@ -139,6 +265,7 @@ def imap_messages(
query: str,
mailbox: str,
before: int | None,
uidvalidity: int | None = None,
) -> dict:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
@@ -147,11 +274,14 @@ def imap_messages(
raise unprocessable("Der Fachname ist ungültig.")
if len(query) > 200:
raise unprocessable("Die Suche ist zu lang.")
release_for_external(conn)
client = _imap_login(row["settings"], row["secret"])
try:
return _imap_page(client, mailbox, query, before)
page = _imap_page(client, mailbox, query, before, uidvalidity)
finally:
_imap_logout(client)
assert_still_owned(conn, row["id"], owner_id)
return page
def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict:
@@ -208,23 +338,11 @@ def _paperless_count(base: str, token: str) -> int:
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
if not document_id.isdigit() or len(document_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if len(rows) != 1:
raise unprocessable("Für Paperless ist kein eindeutiger Zugang zugeordnet.")
row = rows[0]
return _paperless_bytes(row, document_id)
row = _paperless_row(conn, owner_id, connector_id)
release_for_external(conn)
fetched = _paperless_bytes(row, document_id)
assert_still_owned(conn, row["id"], owner_id)
return fetched
def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
@@ -249,35 +367,18 @@ def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
ORIGINAL_UNAVAILABLE = "Das Original ist über die Freigabe nicht abrufbar."
def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
"""The stored document. A grantee never uses a connector of their own account."""
detail = (ref or {}).get("detail") or {}
external_id = str((ref or {}).get("external_id") or "")
detail = (ref or {}).get("detail") or {}
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
stored_owner = _connector_owner(conn, stored)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
if not external_id.isdigit() or len(external_id) > 12:
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, stored) == actor_id
if not external_id.isdigit() or len(external_id) > 12:
if owner_path:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
result = resolve_for_read(conn, actor_id, "paperless", ref, None)
if result.row is None:
raise unprocessable(result.message)
return _paperless_bytes(result.row, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or not external_id.isdigit() or len(external_id) > 12:
raise unprocessable(ORIGINAL_UNAVAILABLE)
result = resolve_for_read(conn, owner_id, "paperless", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "paperless", stored, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
data, media, name = _paperless_bytes(result.row, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "ok")
borrowed = borrow_original(conn, actor_id, element, context, ref, "paperless")
data, media, name = complete_borrow(conn, borrowed, lambda row: _paperless_bytes(row, external_id))
if external_id and external_id in (name or ""):
name = "dokument.pdf"
return data, media, name or "dokument.pdf"
@@ -292,28 +393,18 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
mailbox = str(detail.get("mailbox") or "").strip()
uid = _mail_uid(detail.get("uid"))
external_id = str(ref["external_id"])
stored_owner = _connector_owner(conn, connector_id)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
result = resolve_for_read(conn, actor_id, "imap", ref, None)
if result.row is None:
raise unprocessable(result.message)
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, connector_id) == actor_id
if owner_path:
if not mailbox or uid is None:
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
return load_imap_original(conn, actor_id, result.row["id"], mailbox, uid, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or connector_id is None or not mailbox or uid is None:
elif connector_id is None or not mailbox or uid is None:
raise unprocessable(ORIGINAL_UNAVAILABLE)
result = resolve_for_read(conn, owner_id, "imap", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
raw = load_imap_original(conn, owner_id, result.row["id"], mailbox, uid, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "ok")
return raw
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
def fetch(row):
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
return complete_borrow(conn, borrowed, fetch)
def _uuid(value) -> UUID | None:
@@ -335,13 +426,6 @@ def _mail_uid(value) -> int | None:
return None
def _connector_owner(conn, connector_id: UUID | None) -> UUID | None:
if connector_id is None:
return None
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (connector_id,)).fetchone()
return row["owner_id"] if row else None
def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
return conn.execute(
"""
@@ -353,26 +437,6 @@ def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
).fetchone()
def _audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
audit(
conn,
actor_id,
element["home_scope_id"],
"source_read_mediated",
"element",
element["id"],
context,
result="ok" if outcome == "ok" else "unavailable",
detail={
"source": source,
"connector_id": str(connector_id) if connector_id else None,
"outcome": outcome,
},
)
if outcome != "ok":
conn.commit()
def _paperless_file(
base: str,
token: str,
@@ -534,6 +598,8 @@ def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
PAGE_SIZE = 30
UID_WINDOW = 400
UID_ROUNDS = 8
_LIST_LINE = re.compile(
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
)
@@ -565,7 +631,96 @@ def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
return items
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None) -> dict:
def _untagged_int(client: imaplib.IMAP4, name: str) -> int | None:
values = (getattr(client, "untagged_responses", None) or {}).get(name) or []
if not values:
return None
raw = values[-1]
if isinstance(raw, bytes):
raw = raw.decode("ascii", "replace")
try:
return int(str(raw).strip())
except ValueError:
return None
def _uids_from(data, low: int | None, high: int | None) -> list[int]:
if data is None or isinstance(data, (bytes, str)):
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
chunks = []
for part in data:
if part is None:
continue
if isinstance(part, bytes):
chunks.append(part)
elif isinstance(part, str):
chunks.append(part.encode("utf-8"))
else:
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
found = []
for piece in b" ".join(chunks).split():
if not piece.isdigit():
continue
uid = int(piece)
if low is not None and uid < low:
continue
if high is not None and uid > high:
continue
found.append(uid)
return found
def _uid_call(client: imaplib.IMAP4, *args):
try:
return client.uid(*args)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
def _search_status(status) -> None:
if status in {"NO", "BAD"}:
raise unprocessable("Das Postfach hat die Suche abgelehnt.")
if status != "OK":
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
def _uid_window(client: imaplib.IMAP4, query: str, low: int, high: int) -> list[int]:
needle = " ".join(query.split())
uid_range = f"{low}:{high}"
if not needle:
status, data = _uid_call(client, "SEARCH", None, "UID", uid_range)
else:
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
if needle.isascii():
status, data = _uid_call(
client, "SEARCH", None, "UID", uid_range,
"OR", "OR", "SUBJECT", quoted, "FROM", quoted, "TEXT", quoted,
)
else:
encoded = quoted.encode("utf-8")
status, data = _uid_call(
client, "SEARCH", "CHARSET", "UTF-8", "UID", uid_range,
"OR", "OR", "SUBJECT", encoded, "FROM", encoded, "TEXT", encoded,
)
_search_status(status)
return _uids_from(data, low, high)
def _highest_uid(client: imaplib.IMAP4) -> int:
uidnext = _untagged_int(client, "UIDNEXT")
if uidnext is not None:
return max(0, uidnext - 1)
status, data = _uid_call(client, "SEARCH", None, "UID", "*")
_search_status(status)
found = _uids_from(data, None, None)
return max(found) if found else 0
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None, claimed_validity: int | None = None) -> dict:
folders = _imap_mailboxes(client)
chosen = mailbox or "INBOX"
match = next((item for item in folders if item["name"] == chosen), None)
@@ -573,38 +728,60 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(chosen, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
status, data = _uid_search(client, query)
if status != "OK" or not data or not data[0]:
return {"mailbox": chosen, "items": [], "next_before": None}
uids = sorted(int(part) for part in data[0].split() if part.isdigit())
if before is not None:
uids = [uid for uid in uids if uid < before]
page = uids[-PAGE_SIZE:]
next_before = page[0] if len(uids) > PAGE_SIZE else None
items = []
for uid in reversed(page):
item = _fetch_header(client, uid, chosen)
if item is not None:
items.append(item)
return {"mailbox": chosen, "items": items, "next_before": next_before}
def _uid_search(client: imaplib.IMAP4, query: str):
needle = " ".join(query.split())
if not needle:
return client.uid("SEARCH", None, "ALL")
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
try:
if needle.isascii():
return client.uid("SEARCH", None, "OR", "SUBJECT", quoted, "FROM", quoted)
encoded = quoted.encode("utf-8")
return client.uid("SEARCH", "CHARSET", "UTF-8", "OR", "SUBJECT", encoded, "FROM", encoded)
except imaplib.IMAP4.error as exc:
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
validity = _untagged_int(client, "UIDVALIDITY")
restarted = False
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
before = None
restarted = True
high = before - 1 if before is not None else _highest_uid(client)
collected = []
attempts = 0
failures = 0
low = 1
rounds = 0
while high >= 1 and len(collected) < PAGE_SIZE and rounds < UID_ROUNDS:
low = max(1, high - UID_WINDOW + 1)
for uid in sorted(_uid_window(client, query, low, high), reverse=True):
if len(collected) >= PAGE_SIZE:
break
attempts += 1
item = _fetch_header(client, uid, chosen)
if item is None:
failures += 1
continue
collected.append(item)
rounds += 1
if len(collected) >= PAGE_SIZE or low == 1:
break
high = low - 1
if attempts and failures == attempts:
raise unprocessable("Die Nachrichtenköpfe dieses Fachs konnten nicht gelesen werden.")
incomplete = bool(collected) is False and low > 1 and high >= 1 and rounds >= UID_ROUNDS
if len(collected) < PAGE_SIZE and low > 1 and rounds >= UID_ROUNDS:
incomplete = True
oldest = collected[-1]["uid"] if collected else None
if len(collected) >= PAGE_SIZE and oldest and oldest > 1:
next_before = oldest
elif incomplete:
next_before = low
else:
next_before = None
return {
"mailbox": chosen,
"items": collected,
"next_before": next_before,
"incomplete": incomplete,
"restarted": restarted,
"uidvalidity": validity,
}
def _check_mailbox(mailbox: str) -> None:
@@ -1052,16 +1229,23 @@ def _imap_config(payload: dict) -> tuple[str, dict, str]:
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
del options
items, bound = paperless_documents(conn, owner_id, query, connector_id)
return {"items": items, "connector_id": bound}
return paperless_documents(conn, owner_id, query, connector_id, options)
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
if connector_id is None:
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
before = options.get("before")
return imap_messages(conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before)
validity = options.get("uidvalidity")
return imap_messages(
conn,
owner_id,
connector_id,
query,
str(options.get("mailbox") or ""),
before,
int(validity) if validity else None,
)
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
@@ -1133,11 +1317,15 @@ class _PaperlessAdapter:
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
return paperless_documents(conn, owner_id, query, connector_id)
found = paperless_documents(conn, owner_id, query, connector_id)
return found["items"], found["connector_id"]
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
return _paperless_find(conn, owner_id, connector_id, query, options)
def facets(self, conn, owner_id: UUID, connector_id: UUID) -> dict:
return paperless_facets(conn, owner_id, connector_id)
class _ImapAdapter:
kind = "imap"
@@ -1174,6 +1362,7 @@ def _install_sources() -> None:
export_original=_export_paperless,
navigate=None,
preview=None,
facets=paperless.facets,
))
register_source(SourceModule(
kind="imap",