Schließe die Connector-Kapselung ab, damit Paperless, IMAP und weitere Quellen über dieselbe Anbindung laufen.
Deploy Development / deploy (push) Successful in 28s
Test Suite / pytest-backend (push) Failing after 2m18s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 10s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-19 15:00:50 +02:00
1 parent 75a3949f7c
commit d6b70d8f92
75 files changed
+9477 -414

No files matched your search

+16 -4
View File
@@ -93,8 +93,8 @@ from app.auth.security import (
)
from app.core.errors import Problem, forbidden, not_found, too_many, unprocessable
from app.core.timeutil import dump_date, dump_dt, utcnow
from app.media.storage import content_disposition, read_bytes
from app.media.storage import asset_path
from app.media.port import read_clear_bytes
from app.media.storage import content_disposition
from app.ops import (
add_annotation,
assign_task,
@@ -1119,12 +1119,24 @@ def download_file(
if not rights.read:
raise not_found()
asset = conn.execute(
"SELECT id, original_filename, content_type, status FROM media_assets WHERE element_id = %s",
"""
SELECT id, original_filename, content_type, sha256, status,
storage_instance_id, storage_generation, object_key, protection_profile
FROM media_assets WHERE element_id = %s
""",
(element_id,),
).fetchone()
if asset is None or asset["status"] != "active":
raise not_found()
payload = read_bytes(asset_path(request.app.state.settings.media_root, asset["id"]))
payload = read_clear_bytes(
conn,
instance_id=asset["storage_instance_id"],
generation=asset["storage_generation"],
object_key=asset["object_key"] or str(asset["id"]),
media_root=request.app.state.settings.media_root,
protection_profile=asset["protection_profile"],
expected_sha256=asset["sha256"],
)
return Response(
content=payload,
media_type=asset["content_type"],
+13 -2
View File
@@ -278,7 +278,7 @@ class ExternalDetail(APIModel):
class ExternalIn(APIModel):
title: str = Field(min_length=1, max_length=200)
source: Literal["paperless", "url", "imap"]
source: str = Field(min_length=1, max_length=40)
external_id: str = Field(min_length=1, max_length=200)
context: ContextIn
placement: ContextIn | None = None
@@ -288,7 +288,13 @@ class ExternalIn(APIModel):
class ConnectorIn(APIModel):
kind: Literal["paperless", "imap"]
"""Kind is checked against the source registry. Source fields stay optional.
Paperless and IMAP still send their existing fields. A further module
uses ``config`` and validates that bag itself.
"""
kind: str = Field(min_length=1, max_length=40)
name: str = Field(min_length=1, max_length=120)
base_url: str | None = None
token: str | None = None
@@ -298,6 +304,11 @@ class ConnectorIn(APIModel):
password: str | None = None
mailbox: str | None = Field(default=None, max_length=200)
security: Literal["ssl", "starttls"] | None = None
config: dict[str, str] | None = None
class SourceAssignmentIn(APIModel):
connector_id: UUID
class DeleteElementIn(APIModel):
+212 -35
View File
@@ -8,26 +8,25 @@ from fastapi import APIRouter, Depends
from fastapi.responses import JSONResponse, Response
from psycopg import Connection
from app.access.policy import element_rights, get_element, parse_query_context
from app.access.policy import Ctx, element_rights, get_element, parse_query_context
from app.api.deps import Principal, get_conn, get_principal
from app.api.routes import _created, _ctx_model
from app.api.schemas import ConnectorIn, ExternalIn
from app.core.errors import not_found, unprocessable
from app.api.schemas import ConnectorIn, ExternalIn, SourceAssignmentIn
from app.core.errors import not_found, unprocessable, unsupported
from app.mailpack import parse_mail
from app.media.storage import content_disposition
from app.ops import create_external
from app.sourcebind import assign_reference
from app.sourceport import capability_payload, with_capabilities
from app.sourceregistry import get_source, require_source
from app.sources import (
clean_base_url,
delete_connector,
imap_mailboxes,
imap_messages,
imap_preview,
list_connectors,
paperless_documents,
probe,
read_linked_mail,
read_linked_paperless,
save_connector,
update_connector,
)
_PRIVATE = {
@@ -44,6 +43,12 @@ def get_connectors(principal: Principal = Depends(get_principal), conn: Connecti
return {"items": list_connectors(conn, principal.account_id)}
@router.get("/api/v1/sources/capabilities")
def source_capabilities(principal: Principal = Depends(get_principal)):
del principal
return capability_payload()
@router.post("/api/v1/connectors", status_code=201)
def post_connector(body: ConnectorIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
name, settings, secret = _connector_parts(body)
@@ -51,6 +56,22 @@ def post_connector(body: ConnectorIn, principal: Principal = Depends(get_princip
return row
@router.patch("/api/v1/connectors/{connector_id}")
def patch_connector(
connector_id: UUID,
body: ConnectorIn,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
name, settings, secret = _connector_parts(body)
current = next((item for item in list_connectors(conn, principal.account_id) if item["id"] == str(connector_id)), None)
if current is None:
raise not_found()
if current["kind"] != body.kind:
raise unprocessable("Der Zugang bleibt bei seiner Quellenart.")
return update_connector(conn, principal.account_id, connector_id, name, settings, secret)
@router.delete("/api/v1/connectors/{connector_id}", status_code=204)
def remove_connector(connector_id: UUID, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
delete_connector(conn, principal.account_id, connector_id)
@@ -62,7 +83,7 @@ def probe_connector(connector_id: UUID, principal: Principal = Depends(get_princ
@router.get("/api/v1/external-refs/{element_id}/content")
def view_paperless(
def view_external_content(
element_id: UUID,
context_kind: str | None = None,
context_id: str | None = None,
@@ -70,9 +91,10 @@ def view_paperless(
conn: Connection = Depends(get_conn),
):
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
if ref is None or ref["source"] != "paperless":
raise unprocessable("Nur ein Paperless-Dokument lässt sich hier als Datei ansehen.")
data, media_type, filename = read_linked_paperless(conn, principal.account_id, element, context, ref)
module = get_source(ref["source"]) if ref is not None else None
if module is None or module.open_original is None:
raise unprocessable("Für diese Quelle ist kein Dateiabruf vorgesehen.")
data, media_type, filename = module.open_original(conn, principal.account_id, element, context, ref)
safe_name = (filename or "dokument.pdf").replace('"', "")
return Response(
content=data,
@@ -126,7 +148,10 @@ def _opened_ref(conn, principal: Principal, element_id: UUID, context_kind: str
if not rights.read:
raise not_found()
ref = conn.execute(
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
"""
SELECT source, external_id, detail, connector_id, resolution
FROM external_references WHERE element_id = %s
""",
(element_id,),
).fetchone()
return element, context, ref
@@ -186,8 +211,39 @@ def _mail_bytes(data: bytes, media_type: str, filename: str, inline: bool = Fals
@router.get("/api/v1/sources/paperless/documents")
def search_paperless(q: str = "", principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
return {"items": paperless_documents(conn, principal.account_id, q.strip())}
def search_paperless(
q: str = "",
connector_id: UUID | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return _run_search(
"paperless",
conn,
principal.account_id,
q.strip(),
connector_id,
"",
None,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
)
@router.get("/api/v1/sources/imap/mailboxes")
@@ -196,7 +252,7 @@ def list_mailboxes(
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return {"items": imap_mailboxes(conn, principal.account_id, connector_id)}
return _run_navigate("imap", conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/imap/messages")
@@ -205,12 +261,38 @@ def search_mail(
q: str = "",
mailbox: str = "",
before: int | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
if before is not None and before < 1:
raise unprocessable("Die Seitenangabe ist ungültig.")
return imap_messages(conn, principal.account_id, connector_id, q.strip(), mailbox.strip(), before)
return _run_search(
"imap",
conn,
principal.account_id,
q.strip(),
connector_id,
mailbox.strip(),
before,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
)
@router.get("/api/v1/sources/imap/message")
@@ -228,8 +310,8 @@ def read_mail(
@router.post("/api/v1/sources/link", status_code=201)
def link_source(body: ExternalIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
if body.source not in {"paperless", "imap"}:
raise unprocessable("Verknüpft werden Paperless-Dokumente und E-Mails.")
if get_source(body.source) is None:
raise unprocessable("Verknüpft werden registrierte Quellen.")
element = create_external(
conn,
principal.account_id,
@@ -248,20 +330,115 @@ def link_source(body: ExternalIn, principal: Principal = Depends(get_principal),
return _created(f"/api/v1/elements/{element['id']}", {"id": str(element["id"]), "title": element["title"]})
@router.post("/api/v1/external-refs/{element_id}/assignment")
def assign_source(
element_id: UUID,
body: SourceAssignmentIn,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
assign_reference(conn, principal.account_id, element_id, body.connector_id)
return {"id": str(element_id), "connector_id": str(body.connector_id), "resolution": "bound"}
def _connector_parts(body: ConnectorIn) -> tuple[str, dict, str]:
if body.kind == "paperless":
if not body.base_url or not body.token or len(body.token.strip()) < 8:
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
return body.name.strip(), {"base_url": clean_base_url(body.base_url)}, body.token.strip()
if not body.host or not body.username or not body.password:
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
host = body.host.strip()
if " " in host or len(host) > 200:
raise unprocessable("Der Mailserver ist ungültig.")
return body.name.strip(), {
"host": host,
"port": body.port or 993,
"username": body.username.strip(),
"mailbox": (body.mailbox or "INBOX").strip() or "INBOX",
"security": body.security or "ssl",
}, body.password
return require_source(body.kind).parse_config(body.model_dump())
_SEARCH_FLAGS = {
"title": "title",
"correspondent": "correspondent",
"document_type": "document_type",
"tag": "tags",
"document_from": "document_range",
"document_to": "document_range",
"sort": "sort",
"cursor": "continuation",
}
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict) -> dict:
module = require_source(kind)
search = module.capabilities["search"]
refused = [name for name, value in filters.items() if str(value or "").strip() and not search.get(_SEARCH_FLAGS.get(name, name))]
if before is not None and not search.get("continuation"):
refused.append("before")
if refused:
raise unsupported("Diese Suche unterstützt den Filter nicht: " + ", ".join(refused) + ".")
found = module.find(conn, owner, connector_id, query, {"mailbox": mailbox, "before": before})
bound = str(found.get("connector_id") or connector_id or "")
return with_capabilities(kind, bound, found)
def _run_navigate(kind: str, conn, owner, connector_id: UUID) -> dict:
module = require_source(kind)
if not module.capabilities.get("navigate") or module.navigate is None:
raise unsupported("Diese Quelle hat keine Fachnavigation.")
return {"items": module.navigate(conn, owner, connector_id)}
@router.get("/api/v1/sources/{kind}/search")
def search_source(
kind: str,
q: str = "",
connector_id: UUID | None = None,
mailbox: str = "",
before: int | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
if before is not None and before < 1:
raise unprocessable("Die Seitenangabe ist ungültig.")
return _run_search(
kind,
conn,
principal.account_id,
q.strip(),
connector_id,
mailbox.strip(),
before,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
)
@router.get("/api/v1/sources/{kind}/navigate")
def navigate_source(
kind: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return _run_navigate(kind, conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/{kind}/preview")
def preview_source(
kind: str,
connector_id: UUID,
mailbox: str = "",
uid: int = 0,
item: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
module = require_source(kind)
if not module.capabilities.get("read_preview") or module.preview is None:
raise unsupported("Diese Quelle hat keine Vorschau vor dem Übernehmen.")
return module.preview(conn, principal.account_id, connector_id, {"mailbox": mailbox, "uid": uid, "item": item})
+4
View File
@@ -50,6 +50,10 @@ def unprocessable(detail: str) -> Problem:
return Problem(422, "Eingabe ungültig", detail, "unprocessable")
def unsupported(detail: str) -> Problem:
return Problem(422, "Nicht unterstützt", detail, "unsupported")
def too_many() -> Problem:
return Problem(429, "Zu viele Versuche", "Bitte kurz warten und es erneut versuchen.", "too_many_requests")
+66 -23
View File
@@ -26,9 +26,10 @@ from app.access.policy import (
require_case,
require_matter,
)
from app.core.errors import conflict, not_found, unprocessable
from app.core.errors import Problem, conflict, not_found, unprocessable
from app.core.timeutil import dump_date, dump_dt
from app.media.storage import asset_path, display_filename, read_bytes, sha256, write_bytes
from app.media.port import LEGACY_PROFILE, LocalAdapter, instance_for_write, publish, read_clear_bytes, stored_path
from app.media.storage import display_filename, sha256
from app.retrieval import FilePayload, MailPayload, RetrievalFailure, load_external, load_file
CHANNELS = {"email": "per E-Mail", "post": "per Post", "other": "sonstiger Weg"}
@@ -415,9 +416,25 @@ def load_snapshot_file(conn, snapshot_id: UUID, media_root: str) -> FilePayload
row = conn.execute("SELECT * FROM dispatch_snapshots WHERE id = %s", (snapshot_id,)).fetchone()
if row is None:
return RetrievalFailure("missing", "Die gesicherte Fassung ist nicht mehr vorhanden.")
profile = row["protection_profile"] or "legacy-plaintext"
if profile != "legacy-plaintext":
return RetrievalFailure(
"unsupported",
"Dieser Inhalt ist geschützt und kann hier nicht im Klartext gelesen werden.",
)
try:
data = read_bytes(_snapshot_path(media_root, row["id"]))
except Exception:
data = read_clear_bytes(
conn,
instance_id=row["storage_instance_id"],
generation=row["storage_generation"],
object_key=row["object_key"] or f"dispatch/{row['id']}",
media_root=media_root,
protection_profile=profile,
expected_sha256=row["sha256"],
)
except Problem as exc:
if exc.code == "processing":
return RetrievalFailure("missing", "Die gesicherte Fassung lässt sich nicht unverändert lesen.")
return RetrievalFailure("missing", "Die gesicherte Fassung ist nicht mehr vorhanden.")
if sha256(data) != row["sha256"]:
return RetrievalFailure("missing", "Die gesicherte Fassung lässt sich nicht unverändert lesen.")
@@ -573,7 +590,7 @@ def purge_element_dispatch_files(conn, element_id: UUID, media_root: str) -> lis
(item_id,),
).fetchone()
if snap is not None:
paths.append(_snapshot_path(media_root, snap["id"]))
paths.append(_snapshot_disk(conn, snap["id"], media_root))
conn.execute("DELETE FROM dispatch_snapshots WHERE id = %s", (snap["id"],))
conn.execute(
"""
@@ -596,7 +613,7 @@ def purge_element_dispatch_files(conn, element_id: UUID, media_root: str) -> lis
(evidence_id,),
).fetchone()
if snap is not None:
paths.append(_snapshot_path(media_root, snap["id"]))
paths.append(_snapshot_disk(conn, snap["id"], media_root))
conn.execute("DELETE FROM dispatch_snapshots WHERE id = %s", (snap["id"],))
conn.execute(
"""
@@ -916,17 +933,22 @@ def _working_bytes(conn, actor, element, media_root: str):
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
snapshot_id = uuid4()
path = _snapshot_path(media_root, snapshot_id)
write_bytes(path, payload.data)
cleanup.append(path)
instance = instance_for_write(conn, media_root)
object_key = f"dispatch/{snapshot_id}"
content_revision = str(uuid4())
adapter = LocalAdapter(instance["root_path"])
publish(adapter, object_key, payload.data)
cleanup.append(stored_path(conn, instance["id"], instance["generation"], object_key, media_root))
digest = sha256(payload.data)
conn.execute(
"""
INSERT INTO dispatch_snapshots (
id, dispatch_item_id, evidence_id, home_scope_id, owner_subject_id,
original_filename, content_type, byte_size, sha256
original_filename, content_type, byte_size, sha256,
storage_instance_id, storage_generation, object_key, content_revision,
protection_profile, key_responsibility
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'none')
""",
(
snapshot_id,
@@ -938,6 +960,11 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
payload.media_type or "application/octet-stream",
len(payload.data),
digest,
instance["id"],
instance["generation"],
object_key,
content_revision,
LEGACY_PROFILE,
),
)
return snapshot_id
@@ -956,20 +983,36 @@ def _snapshot_file(conn, media_root: str, dispatch_item_id: UUID | None = None,
).fetchone()
if row is None:
raise not_found()
data = read_bytes(_snapshot_path(media_root, row["id"]))
if sha256(data) != row["sha256"]:
raise not_found()
profile = row["protection_profile"] or "legacy-plaintext"
data = read_clear_bytes(
conn,
instance_id=row["storage_instance_id"],
generation=row["storage_generation"],
object_key=row["object_key"] or f"dispatch/{row['id']}",
media_root=media_root,
protection_profile=profile,
expected_sha256=row["sha256"],
)
return FilePayload(row["original_filename"], row["content_type"], data, row["sha256"])
def _snapshot_path(media_root: str, snapshot_id: UUID) -> Path:
root = Path(media_root).resolve()
folder = (root / "dispatch").resolve()
folder.mkdir(parents=True, exist_ok=True)
target = (folder / str(snapshot_id)).resolve()
if target.parent != folder:
raise unprocessable("Ungültiger Speicherpfad.")
return target
def _snapshot_disk(conn, snapshot_id: UUID, media_root: str) -> Path:
row = conn.execute(
"""
SELECT id, storage_instance_id, storage_generation, object_key
FROM dispatch_snapshots WHERE id = %s
""",
(snapshot_id,),
).fetchone()
if row is None:
return stored_path(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
return stored_path(
conn,
row["storage_instance_id"],
row["storage_generation"],
row["object_key"] or f"dispatch/{row['id']}",
media_root,
)
def _snapshot_id(conn, dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> str | None:
@@ -1416,5 +1459,5 @@ def _delete_dispatch_files(conn, dispatch_id: UUID, media_root: str) -> list[Pat
(dispatch_id, dispatch_id),
).fetchall()
for row in rows:
paths.append(_snapshot_path(media_root, row["id"]))
paths.append(_snapshot_disk(conn, row["id"], media_root))
return paths
+13
View File
@@ -49,9 +49,22 @@ def migrate(settings: Settings | None = None) -> list[str]:
)
applied.append(version)
_ensure_runtime_role(conn, settings)
_prepare_source_storage(settings)
return applied
def _prepare_source_storage(settings: Settings) -> None:
from psycopg.rows import dict_row
from app.media.port import ensure_legacy_instance
from app.sourcebind import backfill_source_bindings
with psycopg.connect(settings.dsn(), row_factory=dict_row) as conn:
backfill_source_bindings(conn)
ensure_legacy_instance(conn, settings.media_root)
conn.commit()
def _ensure_runtime_role(conn: psycopg.Connection, settings: Settings) -> None:
role = sql.Identifier(settings.db_runtime_user)
database = sql.Identifier(settings.db_name)
+10 -6
View File
@@ -39,6 +39,7 @@ _KIND_LABEL = {
"file": "Datei",
"email": "E-Mail",
"paperless": "Paperless",
"source": "Quelle",
"url": "Webadresse",
"annotation": "Anmerkung",
"chapter": "Kapitel",
@@ -233,7 +234,7 @@ def _plan(conn, actor, matter_id, context, selected, assignments, media_root) ->
continue
if node.content_kind == "file":
result = load_file(conn, UUID(node.element_id), media_root)
elif node.content_kind in {"email", "paperless", "url"}:
elif node.content_kind in {"email", "paperless", "url", "source"}:
result = load_external(conn, actor, UUID(node.element_id), assignments.get(node.id))
else:
result = RetrievalFailure("capability", "Für diesen Inhalt ist kein Originalabruf vorgesehen.")
@@ -461,13 +462,16 @@ def _content_kind(element: dict) -> str:
if element["kind"] == "file":
return "file"
source = (element.get("external") or {}).get("source")
if source == "imap":
return "email"
if source == "paperless":
return "paperless"
if source == "url":
return "url"
return element["kind"]
from app.sourceregistry import get_source
module = get_source(source) if source else None
if module is None:
return "source"
if module.representation == "email":
return "email"
return "source"
def _chosen(root: ExportNode, selected: list[str]) -> set[str]:
+247
View File
@@ -0,0 +1,247 @@
"""Opaque storage port. The adapter stores bytes and does not decide who may read them.
Cleartext for a reader goes through require_cleartext. Locked profiles stay
bytes on disk and are not decrypted here. No new cryptography is implemented.
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from uuid import UUID
from app.core.errors import Problem, storage_unavailable, unprocessable
from app.media.storage import read_bytes, sha256, write_bytes
LEGACY_GENERATION = "legacy-local-1"
LEGACY_PROFILE = "legacy-plaintext"
LOCKED_PROFILES = frozenset({"managed-encrypted", "client-encrypted"})
PROFILES = frozenset({LEGACY_PROFILE, *LOCKED_PROFILES})
def require_cleartext(profile: str | None) -> None:
name = profile or LEGACY_PROFILE
if name == LEGACY_PROFILE:
return
if name in LOCKED_PROFILES:
raise Problem(
422,
"Geschützter Inhalt",
"Dieser Inhalt ist geschützt. Ein Klartextabruf ist in dieser Lieferung nicht möglich.",
"unsupported",
)
raise Problem(
422,
"Nicht unterstützt",
"Dieses Schutzprofil wird nicht unterstützt.",
"unsupported",
)
def check_object_key(object_key: str) -> str:
key = (object_key or "").replace("\\", "/").strip()
parts = key.split("/")
if not key or key.startswith("/") or any(part in {"", ".", ".."} for part in parts):
raise unprocessable("Ungültiger Speicherpfad.")
try:
if len(parts) == 1:
UUID(parts[0])
elif len(parts) == 2 and parts[0] == "dispatch":
UUID(parts[1])
else:
raise ValueError
except ValueError:
raise unprocessable("Ungültiger Speicherpfad.") from None
return key
def object_path(root: Path, object_key: str) -> Path:
key = check_object_key(object_key)
base = Path(root).resolve()
base.mkdir(parents=True, exist_ok=True)
target = (base / key).resolve()
if base != target and base not in target.parents:
raise unprocessable("Ungültiger Speicherpfad.")
return target
class LocalAdapter:
"""Files stay under one recorded root. A client path is not an address."""
def __init__(self, root: str | Path) -> None:
self.root = Path(root)
def put(self, object_key: str, data: bytes) -> None:
path = object_path(self.root, object_key)
path.parent.mkdir(parents=True, exist_ok=True)
write_bytes(path, data)
def open(self, object_key: str) -> bytes:
return read_bytes(object_path(self.root, object_key))
def delete(self, object_key: str) -> None:
path = object_path(self.root, object_key)
try:
path.unlink(missing_ok=True)
except OSError as exc:
raise storage_unavailable() from exc
def describe(self, object_key: str) -> dict:
path = object_path(self.root, object_key)
try:
if not path.is_file():
raise storage_unavailable()
return {"byte_size": path.stat().st_size, "object_key": object_key}
except OSError as exc:
raise storage_unavailable() from exc
class MemoryAdapter:
"""In-memory stand-in for the same port. It is not a second product store."""
def __init__(self) -> None:
self._items: dict[str, bytes] = {}
self.fail_put = False
def put(self, object_key: str, data: bytes) -> None:
if self.fail_put:
raise storage_unavailable()
key = check_object_key(object_key)
self._items[key] = bytes(data)
def open(self, object_key: str) -> bytes:
key = check_object_key(object_key)
try:
return self._items[key]
except KeyError as exc:
raise storage_unavailable() from exc
def delete(self, object_key: str) -> None:
key = check_object_key(object_key)
self._items.pop(key, None)
def describe(self, object_key: str) -> dict:
data = self.open(object_key)
return {"byte_size": len(data), "object_key": object_key}
def publish(adapter, object_key: str, data: bytes) -> None:
"""Make bytes readable only after a checked roundtrip. A failed put leaves nothing published."""
adapter.put(object_key, data)
try:
opened = adapter.open(object_key)
except Problem:
adapter.delete(object_key)
raise
if opened != data:
adapter.delete(object_key)
raise storage_unavailable()
def ensure_legacy_instance(conn, media_root: str) -> dict:
"""Bind the legacy generation to the current root once. Later roots do not retarget it."""
root = str(Path(media_root).resolve())
conn.execute(
"""
INSERT INTO storage_instances (kind, generation, root_path)
VALUES ('local', %s, %s)
ON CONFLICT (generation) DO NOTHING
""",
(LEGACY_GENERATION, root),
)
conn.execute(
"""
UPDATE storage_instances
SET root_path = %s
WHERE generation = %s AND root_path IS NULL
""",
(root, LEGACY_GENERATION),
)
row = conn.execute(
"""
SELECT id, kind, generation, root_path
FROM storage_instances
WHERE generation = %s
""",
(LEGACY_GENERATION,),
).fetchone()
conn.execute(
"""
UPDATE media_assets
SET storage_instance_id = %s
WHERE storage_generation = %s AND storage_instance_id IS NULL
""",
(row["id"], LEGACY_GENERATION),
)
conn.execute(
"""
UPDATE dispatch_snapshots
SET storage_instance_id = %s
WHERE storage_generation = %s AND storage_instance_id IS NULL
""",
(row["id"], LEGACY_GENERATION),
)
return row
def instance_for_write(conn, media_root: str) -> dict:
legacy = ensure_legacy_instance(conn, media_root)
root = str(Path(media_root).resolve())
if legacy["root_path"] == root:
return legacy
generation = "local-" + hashlib.sha256(root.encode("utf-8")).hexdigest()[:16]
conn.execute(
"""
INSERT INTO storage_instances (kind, generation, root_path)
VALUES ('local', %s, %s)
ON CONFLICT (generation) DO NOTHING
""",
(generation, root),
)
return conn.execute(
"""
SELECT id, kind, generation, root_path
FROM storage_instances
WHERE generation = %s
""",
(generation,),
).fetchone()
def resolve_root(conn, instance_id, generation: str | None, media_root: str) -> str:
if instance_id is not None:
row = conn.execute(
"SELECT root_path FROM storage_instances WHERE id = %s",
(instance_id,),
).fetchone()
if row is not None and row["root_path"]:
return row["root_path"]
if generation in {None, LEGACY_GENERATION}:
return ensure_legacy_instance(conn, media_root)["root_path"]
return str(Path(media_root).resolve())
def stored_path(conn, instance_id, generation: str | None, object_key: str, media_root: str) -> Path:
return object_path(Path(resolve_root(conn, instance_id, generation, media_root)), object_key)
def read_clear_bytes(
conn,
*,
instance_id,
generation: str | None,
object_key: str,
media_root: str,
protection_profile: str | None,
expected_sha256: str | None,
) -> bytes:
require_cleartext(protection_profile)
data = LocalAdapter(resolve_root(conn, instance_id, generation, media_root)).open(object_key)
if expected_sha256 and sha256(data) != expected_sha256:
raise Problem(
422,
"Inhalt nicht geprüft",
"Die gespeicherte Datei stimmt nicht mit dem erfassten Stand überein.",
"processing",
)
return data
+39 -10
View File
@@ -33,9 +33,11 @@ from app.access.policy import (
task_rights,
)
from app.content.documents import clean_document
from app.core.errors import forbidden, not_found, unprocessable
from app.core.errors import forbidden, not_found, unprocessable, unsupported
from app.core.timeutil import utcnow
from app.media.storage import asset_path, display_filename, sha256, sniff, write_bytes
from app.media.port import LEGACY_PROFILE, instance_for_write, publish, stored_path
from app.media.port import LocalAdapter
from app.media.storage import display_filename, sha256, sniff
MAX_CHAPTER_DEPTH = 32
@@ -929,7 +931,7 @@ def update_note(conn, actor: UUID, element_id: UUID, expected: int, document: di
document = clean_document(document)
element, rights = require_element(conn, actor, element_id, context, "edit")
if element["kind"] != "note":
raise not_found()
raise unsupported("Diese Fassung lässt sich nicht als Notiz bearbeiten.")
if not rights.edit:
raise forbidden()
version = bump_version(conn, "elements", element_id, expected)
@@ -998,6 +1000,10 @@ def add_annotation(conn, actor: UUID, binding_id: UUID, body: str, context: Ctx
def create_external(conn, actor: UUID, scope_id: UUID, title: str, source: str, external_id: str, context: Ctx, placement: Ctx | None, chapter_id: UUID | None, document_on, sent_on, received_on, detail: dict | None = None, communication_direction: str | None = None) -> dict:
from app.sourceregistry import accepts_reference
if not accepts_reference(source):
raise unprocessable("Diese Quelle ist nicht registriert.")
target, rights_context = _placement(context, placement)
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "external_ref")
@@ -1005,8 +1011,11 @@ def create_external(conn, actor: UUID, scope_id: UUID, title: str, source: str,
"INSERT INTO external_references (element_id, source, external_id, detail) VALUES (%s, %s, %s, %s)",
(element["id"], source, external_id.strip()[:200], Json(detail or {})),
)
from app.sourcebind import attach_new_reference
attach_new_reference(conn, element["id"])
binding = insert_binding(conn, element["id"], target.kind, target.id, chapter_id, actor)
adopted = (detail or {}).get("direction") if source == "imap" else None
adopted = (detail or {}).get("direction")
chosen = chosen_direction(communication_direction) or (adopted if adopted in ("in", "out") else None)
store_direction(conn, binding["id"], chosen, chosen is not None)
record_dates(conn, actor, scope_id, element["id"], rights_context or target, document_on, sent_on, received_on)
@@ -1022,16 +1031,23 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "file")
asset_id = uuid4()
path = asset_path(media_root, asset_id)
write_bytes(path, data)
content_revision = str(uuid4())
instance = instance_for_write(conn, media_root)
object_key = str(asset_id)
adapter = LocalAdapter(instance["root_path"])
publish(adapter, object_key, data)
path = stored_path(conn, instance["id"], instance["generation"], object_key, media_root)
rollback_files.append(path)
digest = sha256(data)
conn.execute(
"""
INSERT INTO media_assets (
id, element_id, owner_subject_id, home_scope_id, original_filename,
content_type, byte_size, sha256, status
content_type, byte_size, sha256, status, storage_instance_id,
storage_generation, object_key, content_revision, protection_profile,
key_responsibility
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'active')
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'active', %s, %s, %s, %s, %s, 'none')
""",
(
asset_id,
@@ -1041,7 +1057,12 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
display_filename(filename),
content_type,
len(data),
sha256(data),
digest,
instance["id"],
instance["generation"],
object_key,
content_revision,
LEGACY_PROFILE,
),
)
conn.execute(
@@ -1075,7 +1096,15 @@ def delete_element(conn, actor: UUID, element_id: UUID, expected: int, media_roo
)
if asset is not None and asset["status"] == "active":
conn.execute("UPDATE media_assets SET status = 'purging' WHERE id = %s", (asset["id"],))
delete_after.append(asset_path(media_root, asset["id"]))
delete_after.append(
stored_path(
conn,
asset["storage_instance_id"],
asset["storage_generation"],
asset["object_key"] or str(asset["id"]),
media_root,
)
)
conn.execute("UPDATE media_assets SET status = 'purged' WHERE id = %s", (asset["id"],))
audit(conn, actor, element["home_scope_id"], "element_deleted", "element", element_id)
+31 -11
View File
@@ -117,10 +117,16 @@ def _preview_meta(kind: str, content_type: str | None, source: str | None) -> di
return {"available": True, "media": "image", "source": "file"}
if kind == "file" and media == "application/pdf":
return {"available": True, "media": "pdf", "source": "file"}
if kind == "external_ref" and source == "paperless":
return {"available": True, "media": "paperless", "source": "paperless"}
if kind == "external_ref" and source == "imap":
return {"available": True, "media": "email", "source": "imap"}
if kind == "external_ref" and source:
from app.sourceregistry import get_source
module = get_source(source)
if module is not None:
return {
"available": bool(module.capabilities.get("read_preview")),
"media": module.representation,
"source": source,
}
if kind == "file":
return {"available": False, "media": "file", "source": "file"}
return {"available": False, "media": "none", "source": source}
@@ -175,35 +181,49 @@ def _element_body(conn, actor: UUID, element: dict) -> dict:
elif element["kind"] == "file":
asset = conn.execute(
"""
SELECT original_filename, content_type, byte_size, status
SELECT original_filename, content_type, byte_size, status, protection_profile
FROM media_assets WHERE element_id = %s
""",
(element["id"],),
).fetchone()
if asset is not None:
profile = asset["protection_profile"] or "legacy-plaintext"
payload["file"] = {
"filename": asset["original_filename"],
"content_type": asset["content_type"],
"byte_size": asset["byte_size"],
"status": asset["status"],
"protection_profile": profile,
}
payload["preview"] = _preview_meta("file", asset["content_type"], "file")
if profile != "legacy-plaintext":
payload["preview"] = {"available": False, "media": "locked", "source": "file"}
else:
payload["preview"] = _preview_meta("file", asset["content_type"], "file")
elif element["kind"] == "external_ref":
ref = conn.execute(
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
"""
SELECT source, external_id, detail, connector_id, resolution
FROM external_references WHERE element_id = %s
""",
(element["id"],),
).fetchone()
if ref is not None:
detail = dict(ref["detail"] or {})
connector_owner = _connector_owner(conn, detail.get("connector_id"))
connector_owner = _connector_owner(conn, ref["connector_id"] or detail.get("connector_id"))
revealed = actor == element["owner_subject_id"] or (connector_owner is not None and actor == connector_owner)
if not revealed:
detail = {key: value for key, value in detail.items() if key not in _HIDDEN_SOURCE_FIELDS}
external = {"source": ref["source"], "detail": detail}
resolution = ref["resolution"] or "unresolved"
external = {"source": ref["source"], "detail": detail, "resolution": resolution}
if revealed:
external["external_id"] = ref["external_id"]
payload["external"] = external
payload["preview"] = _preview_meta("external_ref", None, ref["source"])
from app.sourceregistry import get_source
if resolution in {"ambiguous", "unresolved"} and get_source(ref["source"]) is not None:
payload["preview"] = {"available": False, "media": "unassigned", "source": ref["source"]}
else:
payload["preview"] = _preview_meta("external_ref", None, ref["source"])
return payload
@@ -237,7 +257,7 @@ def communication_from_row(conn, binding: dict, element: dict) -> dict:
"SELECT source, detail FROM external_references WHERE element_id = %s",
(element["id"],),
).fetchone()
detail = (ref["detail"] or {}) if ref is not None and ref["source"] == "imap" else {}
detail = (ref["detail"] or {}) if ref is not None else {}
if detail.get("direction") in ("in", "out"):
source = detail["direction"]
return {"direction": source, "explicit": False}
+33 -111
View File
@@ -10,17 +10,12 @@ from dataclasses import dataclass, field
from uuid import UUID
from app.core.errors import Problem
from app.mailpack import ParsedMail, identities_match, parse_mail
from app.media.storage import asset_path, read_bytes
from app.sources import COMPLETE_MAIL_LIMIT, pull_paperless_original, pull_rfc822
from app.mailpack import ParsedMail
from app.media.port import read_clear_bytes
from app.sources import COMPLETE_MAIL_LIMIT
EXPORT_BYTE_LIMIT = COMPLETE_MAIL_LIMIT
_TOO_LARGE = (
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen."
)
@dataclass(frozen=True)
class RetrievalFailure:
@@ -45,132 +40,59 @@ class MailPayload:
def load_external(conn, actor_id: UUID, element_id: UUID, assigned: UUID | None) -> FilePayload | MailPayload | RetrievalFailure:
from app.sourceregistry import get_source
ref = conn.execute(
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
"""
SELECT source, external_id, detail, connector_id, resolution
FROM external_references WHERE element_id = %s
""",
(element_id,),
).fetchone()
if ref is None:
return RetrievalFailure("missing", "Die Quellenreferenz fehlt.")
source = ref["source"]
detail = ref["detail"] or {}
if source == "paperless":
return _load_paperless(conn, actor_id, ref["external_id"], detail, assigned)
if source == "imap":
return _load_imap(conn, actor_id, ref["external_id"], detail, assigned)
if source == "url":
return RetrievalFailure("capability", "Für eine Webadresse ist kein Originalabruf vorgesehen.")
return RetrievalFailure("capability", "Für diese Quelle ist kein Originalabruf vorgesehen.")
module = get_source(ref["source"])
if module is None or module.export_original is None:
return RetrievalFailure("capability", "Für diese Quelle ist kein Originalabruf vorgesehen.")
return module.export_original(conn, actor_id, ref["external_id"], ref["detail"] or {}, assigned, ref)
def load_file(conn, element_id: UUID, media_root: str) -> FilePayload | RetrievalFailure:
asset = conn.execute(
"""
SELECT id, original_filename, content_type, sha256, status
SELECT id, original_filename, content_type, sha256, status,
storage_instance_id, storage_generation, object_key, protection_profile
FROM media_assets WHERE element_id = %s
""",
(element_id,),
).fetchone()
if asset is None or asset["status"] != "active":
return RetrievalFailure("missing", "Die Originaldatei ist nicht mehr vorhanden.")
if (asset["protection_profile"] or "legacy-plaintext") != "legacy-plaintext":
return RetrievalFailure(
"unsupported",
"Dieser Inhalt ist geschützt und kann hier nicht im Klartext gelesen werden.",
)
try:
data = read_bytes(asset_path(media_root, asset["id"]))
except Problem:
data = read_clear_bytes(
conn,
instance_id=asset["storage_instance_id"],
generation=asset["storage_generation"],
object_key=asset["object_key"] or str(asset["id"]),
media_root=media_root,
protection_profile=asset["protection_profile"],
expected_sha256=asset["sha256"],
)
except Problem as exc:
if exc.code == "unsupported":
return RetrievalFailure("unsupported", exc.detail)
return RetrievalFailure("missing", "Die Originaldatei ist nicht mehr vorhanden.")
if len(data) > EXPORT_BYTE_LIMIT:
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
return FilePayload(asset["original_filename"], asset["content_type"], data, asset["sha256"])
def _load_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None):
stored = _uuid(detail.get("connector_id"))
resolved = _resolve_connector(conn, actor_id, "paperless", stored, assigned)
if isinstance(resolved, RetrievalFailure):
return resolved
try:
data, media, name = pull_paperless_original(
resolved["settings"]["base_url"].rstrip("/"),
resolved["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
except Problem as exc:
return _from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
return FilePayload(name or f"paperless-{external_id}", media, data, "")
def _load_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None):
stored = _uuid(detail.get("connector_id"))
resolved = _resolve_connector(conn, actor_id, "imap", stored, assigned)
if isinstance(resolved, RetrievalFailure):
return resolved
mailbox = str(detail.get("mailbox") or "").strip()
uid = detail.get("uid")
if isinstance(uid, str) and uid.isdigit():
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
try:
raw = pull_rfc822(resolved["settings"], resolved["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return _from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT:
return RetrievalFailure("too_large", _TOO_LARGE)
parsed = parse_mail(raw)
if not identities_match(parsed.message_id, str(external_id)):
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
if parsed.missing:
names = ", ".join(parsed.missing[:5])
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
return MailPayload(raw, parsed)
def _resolve_connector(conn, actor_id: UUID, kind: str, stored: UUID | None, assigned: UUID | None):
rows = conn.execute(
"""
SELECT id, kind, name, settings, secret
FROM connectors
WHERE owner_id = %s AND kind = %s
ORDER BY name, id
""",
(actor_id, kind),
).fetchall()
if stored is not None and assigned is None:
match = next((row for row in rows if row["id"] == stored), None)
if match is None:
return RetrievalFailure("missing", "Der gespeicherte Zugang für diese Quelle ist nicht mehr vorhanden.")
return match
chosen = assigned or stored
if chosen is not None:
match = next((row for row in rows if row["id"] == chosen), None)
if match is None:
return RetrievalFailure(
"assignment",
"Der gewählte Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.",
)
return match
if len(rows) == 1:
return rows[0]
if not rows:
return RetrievalFailure("missing", "Für diese Quelle ist kein Zugang vorhanden.")
return RetrievalFailure(
"ambiguous",
"Für diese Quelle ist kein eindeutiger Zugang zugeordnet. Die Zuordnung muss bewusst gewählt werden.",
tuple({"id": str(row["id"]), "name": row["name"]} for row in rows),
)
def _uuid(value) -> UUID | None:
if not value:
return None
try:
return UUID(str(value))
except ValueError:
return None
def _from_problem(exc: Problem) -> RetrievalFailure:
def failure_from_problem(exc: Problem) -> RetrievalFailure:
text = exc.detail
lowered = text.casefold()
if "exportgrenze" in lowered or "zu groß" in lowered:
+211
View File
@@ -0,0 +1,211 @@
"""Bind a reference to one connector instance.
The newest connection is never a fallback. A stored id that no longer
exists stays missing. Ambiguous stock waits for an explicit assignment.
"""
from __future__ import annotations
from dataclasses import dataclass
from uuid import UUID
from psycopg.types.json import Json
from app.access.policy import get_element
from app.core.errors import not_found, unprocessable
from app.sourceregistry import get_source
@dataclass(frozen=True)
class ResolveResult:
row: dict | None
code: str
message: str
choices: tuple[dict, ...] = ()
def _uuid(value) -> UUID | None:
if isinstance(value, UUID):
return value
if not value:
return None
try:
return UUID(str(value))
except (TypeError, ValueError):
return None
def stored_id(ref) -> UUID | None:
if not ref:
return None
detail = ref["detail"] if "detail" in ref.keys() else None
detail = detail or {}
return _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
def _connectors(conn, owner_id: UUID, kind: str) -> list[dict]:
return conn.execute(
"""
SELECT id, owner_id, kind, name, settings, secret
FROM connectors
WHERE owner_id = %s AND kind = %s
ORDER BY name, id
""",
(owner_id, kind),
).fetchall()
def resolve_for_read(conn, owner_id: UUID, kind: str, ref, assigned: UUID | None) -> ResolveResult:
"""Resolve one access. Assignment applies only when the reference has no stored id."""
stored = stored_id(ref)
resolution = (ref["resolution"] if ref and "resolution" in ref.keys() and ref["resolution"] else None) or "unresolved"
rows = _connectors(conn, owner_id, kind)
if stored is not None:
match = next((row for row in rows if row["id"] == stored), None)
if match is None:
return ResolveResult(None, "missing", "Der gespeicherte Zugang für diese Quelle ist nicht mehr vorhanden.")
return ResolveResult(match, "ok", "")
if assigned is not None:
match = next((row for row in rows if row["id"] == assigned), None)
if match is None:
return ResolveResult(
None,
"assignment",
"Der gewählte Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.",
)
return ResolveResult(match, "ok", "")
if resolution == "ambiguous" or len(rows) != 1:
if not rows:
return ResolveResult(None, "missing", "Für diese Quelle ist kein Zugang vorhanden.")
return ResolveResult(
None,
"ambiguous",
"Für diese Quelle ist kein eindeutiger Zugang zugeordnet. Die Zuordnung muss bewusst gewählt werden.",
tuple({"id": str(row["id"]), "name": row["name"]} for row in rows),
)
return ResolveResult(rows[0], "ok", "")
def _set_binding(conn, element_id: UUID, connector_id: UUID | None, resolution: str, detail: dict | None = None) -> None:
if detail is None:
conn.execute(
"""
UPDATE external_references
SET connector_id = %s, resolution = %s
WHERE element_id = %s
""",
(connector_id, resolution, element_id),
)
return
conn.execute(
"""
UPDATE external_references
SET connector_id = %s, resolution = %s, detail = %s
WHERE element_id = %s
""",
(connector_id, resolution, Json(detail), element_id),
)
def attach_new_reference(conn, element_id: UUID) -> None:
row = conn.execute(
"""
SELECT er.source, er.detail, e.owner_subject_id
FROM external_references er
JOIN elements e ON e.id = er.element_id
WHERE er.element_id = %s
""",
(element_id,),
).fetchone()
if row is None:
return
if get_source(row["source"]) is None:
_set_binding(conn, element_id, None, "none")
return
detail = dict(row["detail"] or {})
stored = _uuid(detail.get("connector_id"))
if stored is not None:
match = conn.execute(
"""
SELECT id FROM connectors
WHERE id = %s AND owner_id = %s AND kind = %s
""",
(stored, row["owner_subject_id"], row["source"]),
).fetchone()
_set_binding(conn, element_id, stored, "bound" if match else "unresolved")
return
rows = _connectors(conn, row["owner_subject_id"], row["source"])
if len(rows) == 1:
detail["connector_id"] = str(rows[0]["id"])
_set_binding(conn, element_id, rows[0]["id"], "bound", detail)
return
if len(rows) > 1:
_set_binding(conn, element_id, None, "ambiguous")
return
_set_binding(conn, element_id, None, "unresolved")
def backfill_source_bindings(conn) -> None:
"""Idempotent. Bound and ambiguous rows stay. Unique unresolved stock is bound once."""
conn.execute(
"""
UPDATE external_references
SET resolution = 'none'
WHERE source = 'url' AND resolution = 'unresolved'
"""
)
rows = conn.execute(
"""
SELECT er.element_id, er.source, er.detail, e.owner_subject_id
FROM external_references er
JOIN elements e ON e.id = er.element_id
WHERE er.source <> 'url'
AND er.resolution = 'unresolved'
AND er.connector_id IS NULL
"""
).fetchall()
for row in rows:
if get_source(row["source"]) is None:
continue
detail = dict(row["detail"] or {})
stored = _uuid(detail.get("connector_id"))
if stored is not None:
match = conn.execute(
"""
SELECT id FROM connectors
WHERE id = %s AND owner_id = %s AND kind = %s
""",
(stored, row["owner_subject_id"], row["source"]),
).fetchone()
_set_binding(conn, row["element_id"], stored, "bound" if match else "unresolved")
continue
found = _connectors(conn, row["owner_subject_id"], row["source"])
if len(found) == 1:
detail["connector_id"] = str(found[0]["id"])
_set_binding(conn, row["element_id"], found[0]["id"], "bound", detail)
elif len(found) > 1:
_set_binding(conn, row["element_id"], None, "ambiguous")
def assign_reference(conn, actor_id: UUID, element_id: UUID, connector_id: UUID) -> None:
element = get_element(conn, element_id)
if element is None or element["kind"] != "external_ref" or element["owner_subject_id"] != actor_id:
raise not_found()
ref = conn.execute(
"SELECT source, detail FROM external_references WHERE element_id = %s",
(element_id,),
).fetchone()
if ref is None or get_source(ref["source"]) is None:
raise unprocessable("Diese Referenz hat keinen Quellzugang.")
connector = conn.execute(
"""
SELECT id, kind FROM connectors
WHERE id = %s AND owner_id = %s
""",
(connector_id, actor_id),
).fetchone()
if connector is None or connector["kind"] != ref["source"]:
raise unprocessable("Der Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.")
detail = dict(ref["detail"] or {})
detail["connector_id"] = str(connector_id)
_set_binding(conn, element_id, connector_id, "bound", detail)
+98
View File
@@ -0,0 +1,98 @@
"""Declared source and editor capabilities.
A common contract does not invent abilities the adapter does not have.
Search filters that are false stay unavailable until a later delivery.
"""
from __future__ import annotations
PAPERLESS_CAPABILITIES = {
"kind": "paperless",
"navigate": False,
"read_original": True,
"read_preview": True,
"read_attachment": False,
"edit_content": False,
"edit_metadata": False,
"search": {
"query": True,
"title": False,
"correspondent": False,
"document_type": False,
"tags": False,
"document_range": False,
"sort": False,
"continuation": False,
},
}
IMAP_CAPABILITIES = {
"kind": "imap",
"navigate": True,
"read_original": True,
"read_preview": True,
"read_attachment": True,
"edit_content": False,
"edit_metadata": False,
"search": {
"query": True,
"title": False,
"correspondent": False,
"document_type": False,
"tags": False,
"document_range": False,
"sort": False,
"continuation": True,
},
}
CAPABILITIES = {
"paperless": PAPERLESS_CAPABILITIES,
"imap": IMAP_CAPABILITIES,
}
EDITORS = {
"note": True,
"markdown_file": False,
}
UNSUPPORTED_SEARCH = (
"title",
"correspondent",
"document_type",
"tag",
"document_from",
"document_to",
"sort",
"cursor",
)
def capability_payload() -> dict:
from app.sourceregistry import registered_sources
return {
"sources": [module.capabilities for module in registered_sources()],
"editors": EDITORS,
"protection_profiles": {
"legacy-plaintext": "cleartext",
"managed-encrypted": "locked",
"client-encrypted": "locked",
},
}
def unsupported_filters(values: dict[str, str]) -> list[str]:
return [name for name in UNSUPPORTED_SEARCH if str(values.get(name) or "").strip()]
def with_capabilities(kind: str, connector_id: str, payload: dict) -> dict:
from app.sourceregistry import require_source
body = dict(payload)
spec = require_source(kind).capabilities
body["connector_id"] = connector_id
body["capabilities"] = spec
body["unsupported"] = [name for name, enabled in spec["search"].items() if not enabled]
body["continuation"] = body.get("next_before") if spec["search"].get("continuation") else None
return body
+53
View File
@@ -0,0 +1,53 @@
"""Registered source modules. Unknown kinds are rejected, not guessed."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable
from app.core.errors import unprocessable
@dataclass
class SourceModule:
kind: str
label: str
representation: str
capabilities: dict
parse_config: Callable[[dict], tuple[str, dict, str]]
probe: Callable
find: Callable
open_original: Callable
export_original: Callable
navigate: Callable | None = None
preview: Callable | None = None
_MODULES: dict[str, SourceModule] = {}
def register_source(module: SourceModule) -> None:
_MODULES[module.kind] = module
def unregister_source(kind: str) -> None:
_MODULES.pop(kind, None)
def registered_sources() -> list[SourceModule]:
return list(_MODULES.values())
def get_source(kind: str) -> SourceModule | None:
return _MODULES.get(kind)
def require_source(kind: str) -> SourceModule:
module = _MODULES.get(kind)
if module is None:
raise unprocessable("Diese Quelle ist nicht registriert.")
return module
def accepts_reference(source: str) -> bool:
return source == "url" or source in _MODULES
+255 -76
View File
@@ -23,25 +23,12 @@ from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceport import CAPABILITIES
from app.sourceregistry import SourceModule, get_source, register_source
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
if kind == "paperless":
existing = conn.execute(
"SELECT id FROM connectors WHERE owner_id = %s AND kind = 'paperless'",
(owner_id,),
).fetchone()
if existing:
row = conn.execute(
"""
UPDATE connectors
SET name = %s, settings = %s, secret = %s
WHERE id = %s
RETURNING id, kind, name, settings, created_at
""",
(name, Json(settings), secret, existing["id"]),
).fetchone()
return _public(row)
row = conn.execute(
"""
INSERT INTO connectors (owner_id, kind, name, settings, secret)
@@ -53,6 +40,21 @@ def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, s
return _public(row)
def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict:
"""Credential or name change on the same instance. The id does not move."""
current = _owned(conn, owner_id, connector_id)
row = conn.execute(
"""
UPDATE connectors
SET name = %s, settings = %s, secret = %s
WHERE id = %s AND owner_id = %s
RETURNING id, kind, name, settings, created_at
""",
(name, Json(settings), secret, current["id"], owner_id),
).fetchone()
return _public(row)
def list_connectors(conn, owner_id: UUID) -> list[dict]:
rows = conn.execute(
"""
@@ -76,24 +78,31 @@ def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = _owned(conn, owner_id, connector_id)
if row["kind"] == "paperless":
count = _paperless_count(row["settings"]["base_url"], row["secret"])
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
_imap_login(row["settings"], row["secret"])
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
module = get_source(row["kind"])
if module is None:
raise unprocessable("Diese Quelle ist nicht registriert.")
return module.probe(row)
def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
row = conn.execute(
"""
SELECT settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY created_at DESC LIMIT 1
""",
(owner_id,),
).fetchone()
if row is None:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None) -> tuple[list[dict], str]:
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if not rows:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
if len(rows) != 1:
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
row = rows[0]
base = row["settings"]["base_url"].rstrip("/")
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
items = []
@@ -109,7 +118,7 @@ def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
"open_url": f"{base}/documents/{doc.get('id')}/",
}
)
return items
return items, str(row["id"])
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
@@ -177,6 +186,7 @@ def _public(row) -> dict:
"name": row["name"],
"settings": row["settings"],
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
"capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None),
}
@@ -195,19 +205,25 @@ def _paperless_count(base: str, token: str) -> int:
return int(payload.get("count") or 0)
def paperless_content(conn, owner_id: UUID, document_id: str) -> tuple[bytes, str, str]:
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
if not document_id.isdigit() or len(document_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
row = conn.execute(
"""
SELECT settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY created_at DESC LIMIT 1
""",
(owner_id,),
).fetchone()
if row is None:
raise unprocessable("Paperless ist für dieses Konto nicht verbunden.")
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if len(rows) != 1:
raise unprocessable("Für Paperless ist kein eindeutiger Zugang zugeordnet.")
row = rows[0]
return _paperless_bytes(row, document_id)
@@ -240,23 +256,28 @@ def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | No
"""The stored document. A grantee never uses a connector of their own account."""
detail = (ref or {}).get("detail") or {}
external_id = str((ref or {}).get("external_id") or "")
stored = _uuid(detail.get("connector_id"))
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
stored_owner = _connector_owner(conn, stored)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
return paperless_content(conn, actor_id, external_id)
if not external_id.isdigit() or len(external_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
result = resolve_for_read(conn, actor_id, "paperless", ref, None)
if result.row is None:
raise unprocessable(result.message)
return _paperless_bytes(result.row, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or not external_id.isdigit() or len(external_id) > 12:
raise unprocessable(ORIGINAL_UNAVAILABLE)
row = _mediated_paperless(conn, owner_id, stored, element)
if row is None:
result = resolve_for_read(conn, owner_id, "paperless", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "paperless", stored, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
data, media, name = _paperless_bytes(row, external_id)
data, media, name = _paperless_bytes(result.row, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "paperless", row["id"], "unavailable")
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "paperless", row["id"], "ok")
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "ok")
if external_id and external_id in (name or ""):
name = "dokument.pdf"
return data, media, name or "dokument.pdf"
@@ -267,28 +288,31 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
detail = (ref or {}).get("detail") or {}
if ref is None or ref["source"] != "imap":
raise unprocessable("Das ist keine E-Mail.")
connector_id = _uuid(detail.get("connector_id"))
connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
mailbox = str(detail.get("mailbox") or "").strip()
uid = _mail_uid(detail.get("uid"))
external_id = str(ref["external_id"])
stored_owner = _connector_owner(conn, connector_id)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
if connector_id is None or not mailbox or uid is None:
result = resolve_for_read(conn, actor_id, "imap", ref, None)
if result.row is None:
raise unprocessable(result.message)
if not mailbox or uid is None:
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
return load_imap_original(conn, actor_id, connector_id, mailbox, uid, external_id)
return load_imap_original(conn, actor_id, result.row["id"], mailbox, uid, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or connector_id is None or not mailbox or uid is None:
raise unprocessable(ORIGINAL_UNAVAILABLE)
row = _connector_for(conn, connector_id, owner_id, "imap")
if row is None:
result = resolve_for_read(conn, owner_id, "imap", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
raw = load_imap_original(conn, owner_id, connector_id, mailbox, uid, external_id)
raw = load_imap_original(conn, owner_id, result.row["id"], mailbox, uid, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "ok")
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "ok")
return raw
@@ -329,23 +353,6 @@ def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
).fetchone()
def _mediated_paperless(conn, owner_id: UUID, stored: UUID | None, element: dict):
if stored is not None:
return _connector_for(conn, stored, owner_id, "paperless")
if element["owner_subject_id"] != owner_id:
return None
return conn.execute(
"""
SELECT id, owner_id, kind, settings, secret
FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY created_at DESC
LIMIT 1
""",
(owner_id,),
).fetchone()
def _audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
audit(
conn,
@@ -1014,3 +1021,175 @@ def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> l
if len(found) > 5:
return found
return found
def _paperless_config(payload: dict) -> tuple[str, dict, str]:
base_url = str(payload.get("base_url") or "").strip()
token = str(payload.get("token") or "").strip()
name = str(payload.get("name") or "").strip()
if not base_url or len(token) < 8:
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
return name, {"base_url": clean_base_url(base_url)}, token
def _imap_config(payload: dict) -> tuple[str, dict, str]:
host = str(payload.get("host") or "").strip()
username = str(payload.get("username") or "").strip()
password = str(payload.get("password") or "")
name = str(payload.get("name") or "").strip()
if not host or not username or not password:
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
if " " in host or len(host) > 200:
raise unprocessable("Der Mailserver ist ungültig.")
port = payload.get("port") or 993
return name, {
"host": host,
"port": int(port),
"username": username,
"mailbox": str(payload.get("mailbox") or "INBOX").strip() or "INBOX",
"security": payload.get("security") or "ssl",
}, password
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
del options
items, bound = paperless_documents(conn, owner_id, query, connector_id)
return {"items": items, "connector_id": bound}
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
if connector_id is None:
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
before = options.get("before")
return imap_messages(conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before)
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem
del detail
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
try:
data, media, name = pull_paperless_original(
resolved.row["settings"]["base_url"].rstrip("/"),
resolved.row["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
except Problem as exc:
return failure_from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
return FilePayload(name or f"paperless-{external_id}", media, data, "")
def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
from app.mailpack import identities_match, parse_mail
from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem
resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
mailbox = str(detail.get("mailbox") or "").strip()
uid = detail.get("uid")
if isinstance(uid, str) and uid.isdigit():
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
try:
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT:
return RetrievalFailure(
"too_large",
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen.",
)
parsed = parse_mail(raw)
if not identities_match(parsed.message_id, str(external_id)):
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
if parsed.missing:
names = ", ".join(parsed.missing[:5])
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
return MailPayload(raw, parsed)
def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
raw = read_linked_mail(conn, actor_id, element, context, ref)
return raw, "message/rfc822", "nachricht.eml"
class _PaperlessAdapter:
kind = "paperless"
def capabilities(self) -> dict:
return CAPABILITIES[self.kind]
def probe(self, row) -> dict:
count = _paperless_count(row["settings"]["base_url"], row["secret"])
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
return paperless_documents(conn, owner_id, query, connector_id)
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
return _paperless_find(conn, owner_id, connector_id, query, options)
class _ImapAdapter:
kind = "imap"
def capabilities(self) -> dict:
return CAPABILITIES[self.kind]
def probe(self, row) -> dict:
_imap_login(row["settings"], row["secret"])
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
return imap_mailboxes(conn, owner_id, connector_id)
def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict:
return imap_messages(conn, owner_id, connector_id, query, mailbox, before)
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
return _imap_find(conn, owner_id, connector_id, query, options)
def _install_sources() -> None:
paperless = _PaperlessAdapter()
imap = _ImapAdapter()
register_source(SourceModule(
kind="paperless",
label="Paperless",
representation="pdf",
capabilities=CAPABILITIES["paperless"],
parse_config=_paperless_config,
probe=paperless.probe,
find=paperless.find,
open_original=read_linked_paperless,
export_original=_export_paperless,
navigate=None,
preview=None,
))
register_source(SourceModule(
kind="imap",
label="E-Mail",
representation="email",
capabilities=CAPABILITIES["imap"],
parse_config=_imap_config,
probe=imap.probe,
find=imap.find,
open_original=_imap_bytes,
export_original=_export_imap,
navigate=imap.navigate,
preview=lambda conn, owner, connector_id, options: imap_preview(
conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0)
),
))
_install_sources()
@@ -0,0 +1,66 @@
-- Source instance binding and opaque storage generation.
-- Connector ids are not a foreign key: deleting an access must not
-- retarget or remove an existing reference.
-- The legacy root path is bound in the migrator, not in this file.
CREATE TABLE storage_instances (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
kind text NOT NULL,
generation text NOT NULL,
root_path text,
created_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT storage_instances_kind CHECK (kind IN ('local')),
CONSTRAINT storage_instances_generation UNIQUE (generation),
CONSTRAINT storage_instances_generation_len CHECK (char_length(generation) BETWEEN 1 AND 80)
);
ALTER TABLE media_assets
ADD COLUMN storage_instance_id uuid REFERENCES storage_instances (id),
ADD COLUMN storage_generation text,
ADD COLUMN object_key text,
ADD COLUMN content_revision text,
ADD COLUMN protection_profile text NOT NULL DEFAULT 'legacy-plaintext',
ADD COLUMN key_responsibility text NOT NULL DEFAULT 'none',
ADD CONSTRAINT media_assets_protection CHECK (
protection_profile IN ('legacy-plaintext', 'managed-encrypted', 'client-encrypted')
),
ADD CONSTRAINT media_assets_key_responsibility CHECK (
key_responsibility IN ('none', 'operator', 'client')
);
ALTER TABLE dispatch_snapshots
ADD COLUMN storage_instance_id uuid REFERENCES storage_instances (id),
ADD COLUMN storage_generation text,
ADD COLUMN object_key text,
ADD COLUMN content_revision text,
ADD COLUMN protection_profile text NOT NULL DEFAULT 'legacy-plaintext',
ADD COLUMN key_responsibility text NOT NULL DEFAULT 'none',
ADD CONSTRAINT dispatch_snapshots_protection CHECK (
protection_profile IN ('legacy-plaintext', 'managed-encrypted', 'client-encrypted')
),
ADD CONSTRAINT dispatch_snapshots_key_responsibility CHECK (
key_responsibility IN ('none', 'operator', 'client')
);
UPDATE media_assets
SET storage_generation = 'legacy-local-1',
object_key = id::text,
content_revision = id::text,
protection_profile = 'legacy-plaintext',
key_responsibility = 'none'
WHERE object_key IS NULL;
UPDATE dispatch_snapshots
SET storage_generation = 'legacy-local-1',
object_key = 'dispatch/' || id::text,
content_revision = id::text,
protection_profile = 'legacy-plaintext',
key_responsibility = 'none'
WHERE object_key IS NULL;
ALTER TABLE external_references
ADD COLUMN connector_id uuid,
ADD COLUMN resolution text NOT NULL DEFAULT 'unresolved',
ADD CONSTRAINT external_references_resolution CHECK (
resolution IN ('bound', 'ambiguous', 'unresolved', 'none')
);
@@ -0,0 +1,5 @@
-- Connector kinds are validated by the registered source modules.
-- A database checklist must not grow for every additional source.
ALTER TABLE connectors DROP CONSTRAINT IF EXISTS connectors_kind;
ALTER TABLE external_references DROP CONSTRAINT IF EXISTS external_references_source;
+182
View File
@@ -0,0 +1,182 @@
"""Third source used only to prove that registration is enough.
This module is not a product connector and is not imported by the application.
"""
from __future__ import annotations
from uuid import UUID
from app.access.policy import mediated_source_owner
from app.core.errors import not_found, unprocessable
from app.retrieval import FilePayload, RetrievalFailure
from app.sourcebind import resolve_for_read
from app.sourceregistry import SourceModule, register_source, unregister_source
from app.sources import ORIGINAL_UNAVAILABLE, _audit_mediated, _connector_owner
KIND = "sample"
CATALOG = {
"alpha": {
"brief": {
"id": "brief",
"title": "Probenbrief",
"text": "Der Probenbrief bleibt lesbar.",
},
"notiz": {
"id": "notiz",
"title": "Randnotiz",
"text": "Die Randnotiz bleibt in der Quelle.",
},
}
}
CAPABILITIES = {
"kind": KIND,
"navigate": False,
"read_original": True,
"read_preview": True,
"read_attachment": False,
"edit_content": False,
"edit_metadata": False,
"search": {
"query": True,
"title": False,
"correspondent": False,
"document_type": False,
"tags": False,
"document_range": False,
"sort": False,
"continuation": False,
},
}
def parse_config(payload: dict) -> tuple[str, dict, str]:
config = payload.get("config") or {}
catalog = str(config.get("catalog") or "").strip()
name = str(payload.get("name") or "").strip() or catalog
if catalog not in CATALOG:
raise unprocessable("Der Katalog dieser Beispielquelle ist unbekannt.")
return name, {"catalog": catalog}, ""
def probe(row) -> dict:
items = _items(row)
return {"ok": True, "detail": f"Beispielquelle antwortet. {len(items)} Einträge sichtbar."}
def find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
del options
row = _connector(conn, owner_id, connector_id)
needle = query.casefold()
items = list(_items(row).values())
if needle:
items = [
item for item in items
if needle in item["title"].casefold() or needle in item["text"].casefold()
]
return {"items": items, "connector_id": str(row["id"])}
def preview(conn, owner_id: UUID, connector_id: UUID, options: dict) -> dict:
row = _connector(conn, owner_id, connector_id)
item = _items(row).get(str(options.get("item") or ""))
if item is None:
raise unprocessable("Dieser Eintrag ist in der Beispielquelle nicht vorhanden.")
return {"id": item["id"], "title": item["title"], "text": item["text"], "media": "text"}
def open_original(conn, actor_id: UUID, element: dict, context, ref) -> tuple[bytes, str, str]:
external_id = str((ref or {}).get("external_id") or "")
stored = ref.get("connector_id") if ref else None
stored_owner = _connector_owner(conn, stored)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
result = resolve_for_read(conn, actor_id, KIND, ref, None)
if result.row is None:
raise unprocessable(result.message)
return _bytes(result.row, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None:
raise unprocessable(ORIGINAL_UNAVAILABLE)
result = resolve_for_read(conn, owner_id, KIND, ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, KIND, stored, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
data, media, name = _bytes(result.row, external_id)
except Exception:
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "ok")
return data, media, name
def export_original(conn, actor_id: UUID, external_id: str, detail: dict, assigned, ref):
del detail
result = resolve_for_read(conn, actor_id, KIND, ref, assigned)
if result.row is None:
return RetrievalFailure(result.code, result.message, result.choices)
try:
data, media, name = _bytes(result.row, str(external_id))
except Exception as exc:
return RetrievalFailure("missing", str(getattr(exc, "detail", exc)))
return FilePayload(name, media, data, "")
def install_sample_source() -> None:
register_source(SourceModule(
kind=KIND,
label="Beispielquelle",
representation="text",
capabilities=CAPABILITIES,
parse_config=parse_config,
probe=probe,
find=find,
open_original=open_original,
export_original=export_original,
navigate=None,
preview=preview,
))
def remove_sample_source() -> None:
unregister_source(KIND)
def _connector(conn, owner_id: UUID, connector_id: UUID | None):
if connector_id is not None:
row = conn.execute(
"""
SELECT id, owner_id, kind, name, settings, secret
FROM connectors WHERE id = %s AND owner_id = %s
""",
(connector_id, owner_id),
).fetchone()
if row is None or row["kind"] != KIND:
raise not_found()
return row
rows = conn.execute(
"""
SELECT id, owner_id, kind, name, settings, secret
FROM connectors
WHERE owner_id = %s AND kind = %s
ORDER BY name, id
""",
(owner_id, KIND),
).fetchall()
if len(rows) != 1:
raise unprocessable("Für diese Quelle ist kein eindeutiger Zugang zugeordnet.")
return rows[0]
def _items(row) -> dict:
catalog = (row["settings"] or {}).get("catalog")
return CATALOG.get(catalog) or {}
def _bytes(row, external_id: str) -> tuple[bytes, str, str]:
item = _items(row).get(external_id)
if item is None:
raise unprocessable("Dieser Eintrag ist in der Beispielquelle nicht vorhanden.")
return item["text"].encode("utf-8"), "text/plain", f"{item['id']}.txt"
+333
View File
@@ -0,0 +1,333 @@
"""Section A: source binding and the storage port, with today's behaviour kept."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.errors import Problem
from app.media.port import (
LEGACY_GENERATION,
LocalAdapter,
MemoryAdapter,
check_object_key,
instance_for_write,
publish,
read_clear_bytes,
)
from app.sourcebind import backfill_source_bindings
from tests.support import ORIGIN, Actor, register
PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n"
def _raw(actor: Actor, path: str):
return actor.client.get(
path,
headers={
"Origin": ORIGIN,
"X-CSRF-Token": actor.csrf,
"X-Scope-Kind": actor.scope["kind"],
"X-Scope-Id": actor.scope["id"],
},
)
def _case(actor: Actor) -> dict:
return actor.request("POST", "/api/v1/cases", json={"title": "Bestand"}, status=201)
def _paperless(actor: Actor, name: str, token: str) -> str:
row = actor.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": name, "base_url": "https://paper.example.test", "token": token},
status=201,
)
return row["id"]
def _link(actor: Actor, case_id: str, connector_id: str | None, external_id: str, title: str) -> str:
body = {
"title": title,
"source": "paperless",
"external_id": external_id,
"context": {"kind": "case", "id": case_id},
}
if connector_id:
body["detail"] = {"connector_id": connector_id}
created = actor.request("POST", "/api/v1/external-refs", json=body, status=201)
return created["id"]
def test_storage_adapters_roundtrip_and_reject_escape(tmp_path):
payload = b"opaque-bytes"
key = str(uuid4())
revision = "rev-1"
for adapter in (LocalAdapter(tmp_path), MemoryAdapter()):
publish(adapter, key, payload)
assert adapter.open(key) == payload
assert adapter.describe(key)["byte_size"] == len(payload)
assert adapter.open(key) == payload
assert revision == "rev-1"
memory = MemoryAdapter()
memory.fail_put = True
with pytest.raises(Problem) as failed:
publish(memory, str(uuid4()), payload)
assert failed.value.code == "storage_unavailable"
assert memory._items == {}
with pytest.raises(Problem):
check_object_key("../secret")
with pytest.raises(Problem):
check_object_key("/etc/passwd")
with pytest.raises(Problem):
LocalAdapter(tmp_path).open("dispatch/../../" + str(uuid4()))
def test_changed_root_does_not_retarget_legacy(db, database):
legacy = db.execute(
"SELECT id, root_path FROM storage_instances WHERE generation = %s",
(LEGACY_GENERATION,),
).fetchone()
key = str(uuid4())
original = b"%PDF-legacy-root"
publish(LocalAdapter(legacy["root_path"]), key, original)
other = Path(database.media_root).parent / f"other-root-{uuid4().hex}"
other.mkdir()
moved = instance_for_write(db, str(other))
assert moved["id"] != legacy["id"]
assert moved["generation"] != LEGACY_GENERATION
publish(LocalAdapter(moved["root_path"]), key, b"%PDF-other-root")
seen = read_clear_bytes(
db,
instance_id=legacy["id"],
generation=LEGACY_GENERATION,
object_key=key,
media_root=str(other),
protection_profile="legacy-plaintext",
expected_sha256=None,
)
assert seen == original
db.rollback()
def test_existing_file_keeps_id_bytes_and_revision(client, db):
register(db, "ada-file@example.test", "Ada File")
ada = Actor(client, "ada-file@example.test", "Ada File")
ada.login()
case = _case(ada)
upload = ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", PDF, "application/pdf")},
status=201,
)
stored = db.execute(
"""
SELECT a.id, a.object_key, a.content_revision, a.storage_generation, a.protection_profile,
s.root_path
FROM media_assets a
JOIN storage_instances s ON s.id = a.storage_instance_id
WHERE a.element_id = %s
""",
(upload["id"],),
).fetchone()
assert stored["protection_profile"] == "legacy-plaintext"
assert stored["object_key"] == str(stored["id"])
assert Path(stored["root_path"]) == Path(client.app.state.settings.media_root).resolve()
assert stored["storage_generation"]
first = stored["content_revision"]
opened = _raw(ada, f"/api/v1/files/{upload['id']}/content")
assert opened.status_code == 200
assert opened.content == PDF
again = db.execute(
"SELECT content_revision FROM media_assets WHERE element_id = %s",
(upload["id"],),
).fetchone()
assert again["content_revision"] == first
refused = ada.client.patch(
f"/api/v1/notes/{upload['id']}",
json={"expected_version": 1, "document": {"version": 1, "blocks": [{"type": "paragraph", "text": "nein"}]}},
headers={
"Origin": ORIGIN,
"X-CSRF-Token": ada.csrf,
"X-Scope-Kind": ada.scope["kind"],
"X-Scope-Id": ada.scope["id"],
},
)
assert refused.status_code == 422
assert refused.json()["code"] == "unsupported"
def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monkeypatch):
register(db, "ada-two@example.test", "Ada Two")
ada = Actor(client, "ada-two@example.test", "Ada Two")
ada.login()
case = _case(ada)
older = _paperless(ada, "Zweitens", "token-older")
newer = _paperless(ada, "Erstens", "token-newer")
assert older != newer
def fake_bytes(row, document_id):
return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf"
monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes)
left = _link(ada, case["id"], older, "15", "Altbestand")
right = _link(ada, case["id"], newer, "15", "Zweiter Bestand")
db.execute("UPDATE connectors SET created_at = now() - interval '2 days' WHERE id = %s", (newer,))
db.execute("UPDATE connectors SET created_at = now() WHERE id = %s", (older,))
db.commit()
assert _raw(ada, f"/api/v1/external-refs/{left}/content").content.startswith(b"%PDF-token-older-15")
assert _raw(ada, f"/api/v1/external-refs/{right}/content").content.startswith(b"%PDF-token-newer-15")
calls = []
def fake_get(*args, **kwargs):
calls.append(args)
return {"results": [{"id": 15, "title": "Treffer", "created": "2024-01-02"}]}
monkeypatch.setattr("app.sources._paperless_get", fake_get)
headers = {
"Origin": ORIGIN,
"X-CSRF-Token": ada.csrf,
"X-Scope-Kind": ada.scope["kind"],
"X-Scope-Id": ada.scope["id"],
}
refused = ada.client.get("/api/v1/sources/paperless/documents", params={"q": "Treffer"}, headers=headers)
assert refused.status_code == 422
assert calls == []
found = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"q": "Treffer", "connector_id": older},
)
assert found["items"][0]["title"] == "Treffer"
assert found["connector_id"] == older
assert found["capabilities"]["search"]["correspondent"] is False
assert "correspondent" in found["unsupported"]
filtered = ada.client.get(
"/api/v1/sources/paperless/documents",
params={"q": "Treffer", "connector_id": older, "correspondent": "Haus"},
headers=headers,
)
assert filtered.status_code == 422
assert filtered.json()["code"] == "unsupported"
assert "Haus" not in json.dumps(filtered.json())
def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch):
register(db, "ada-old@example.test", "Ada Old")
ada = Actor(client, "ada-old@example.test", "Ada Old")
ada.login()
case = _case(ada)
loose = _link(ada, case["id"], None, "15", "Ohne Instanz")
_paperless(ada, "Alpha", "token-alpha")
second = _paperless(ada, "Beta", "token-beta")
db.execute(
"""
UPDATE external_references
SET connector_id = NULL, resolution = 'unresolved', detail = '{}'::jsonb
WHERE element_id = %s
""",
(loose,),
)
backfill_source_bindings(db)
state = db.execute(
"SELECT connector_id, resolution FROM external_references WHERE element_id = %s",
(loose,),
).fetchone()
db.commit()
assert state["resolution"] == "ambiguous"
assert state["connector_id"] is None
blocked = _raw(ada, f"/api/v1/external-refs/{loose}/content")
assert blocked.status_code == 422
assert "eindeutiger Zugang" in blocked.json()["detail"]
def fake_bytes(row, document_id):
del document_id
return f"%PDF-{row['secret']}\n".encode(), "application/pdf", "dokument.pdf"
monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes)
ada.request(
"POST",
f"/api/v1/external-refs/{loose}/assignment",
json={"connector_id": second},
)
_paperless(ada, "Gamma", "token-gamma")
assert _raw(ada, f"/api/v1/external-refs/{loose}/content").content.startswith(b"%PDF-token-beta")
def test_locked_profile_serves_metadata_not_cleartext(client, db):
register(db, "ada-lock@example.test", "Ada Lock")
ada = Actor(client, "ada-lock@example.test", "Ada Lock")
ada.login()
case = _case(ada)
upload = ada.request(
"POST",
"/api/v1/files",
data={"title": "Geschuetzt", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("geheim.pdf", PDF, "application/pdf")},
status=201,
)
dummy = b"cipher-dummy-not-clear"
asset = db.execute(
"SELECT id, object_key, storage_instance_id FROM media_assets WHERE element_id = %s",
(upload["id"],),
).fetchone()
root = db.execute(
"SELECT root_path FROM storage_instances WHERE id = %s",
(asset["storage_instance_id"],),
).fetchone()["root_path"]
path = Path(root) / asset["object_key"]
path.write_bytes(dummy)
db.execute(
"""
UPDATE media_assets
SET protection_profile = 'client-encrypted',
key_responsibility = 'client',
sha256 = %s
WHERE element_id = %s
""",
(hashlib.sha256(dummy).hexdigest(), upload["id"]),
)
db.commit()
detail = ada.request("GET", f"/api/v1/cases/{case['id']}")
rendered = json.dumps(detail)
assert "client-encrypted" in rendered
assert "Geschuetzt" in rendered
assert dummy.decode() not in rendered
opened = _raw(ada, f"/api/v1/files/{upload['id']}/content")
assert opened.status_code == 422
assert opened.json()["code"] == "unsupported"
assert path.read_bytes() == dummy
def test_credential_update_keeps_the_instance(client, db):
register(db, "ada-key@example.test", "Ada Key")
ada = Actor(client, "ada-key@example.test", "Ada Key")
ada.login()
first = _paperless(ada, "Archiv", "token-first-1")
second = _paperless(ada, "Anderes Archiv", "token-second-2")
renewed = ada.request(
"PATCH",
f"/api/v1/connectors/{first}",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-renewed"},
)
assert renewed["id"] == first
secrets = {
str(row["id"]): row["secret"]
for row in db.execute("SELECT id, secret FROM connectors").fetchall()
}
assert secrets[first] == "token-renewed"
assert secrets[second] == "token-second-2"
caps = ada.request("GET", "/api/v1/sources/capabilities")
assert caps["editors"]["note"] is True
assert caps["editors"]["markdown_file"] is False
assert caps["protection_profiles"]["client-encrypted"] == "locked"
paperless = next(item for item in caps["sources"] if item["kind"] == "paperless")
assert paperless["search"]["query"] is True
assert paperless["edit_content"] is False
+1 -1
View File
@@ -230,7 +230,7 @@ def test_dispatch_keeps_the_secured_stand_and_distinct_mail_parts(client, db, mo
"Fachlicher Zusatz: die Frist bleibt der 1. Juni.",
[("rechnung.pdf", PDF_A), ("lage.pdf", PDF_B), ("konto.pdf", PDF_C), ("lage.pdf", PDF_D)],
)
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
ada = Actor(client, "ada-versand@example.test", "Ada Versand")
register(db, ada.email, ada.name)
ada.login()
+7 -7
View File
@@ -141,9 +141,9 @@ def test_handover_package_follows_the_confirmed_selection(client, db, monkeypatc
"Der vollständige Brieftext bleibt erhalten.",
[("lage.pdf", PDF_A), ("lage.pdf", PDF_B)],
)
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
monkeypatch.setattr(
"app.retrieval.pull_paperless_original",
"app.sources.pull_paperless_original",
lambda *args, **kwargs: (PDF_B, "application/pdf", "archiv.pdf"),
)
ada = Actor(client, "ada-export@example.test", "Ada Export")
@@ -491,7 +491,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
calls.append(args)
return _eml("<stored@example.test>", "darf nicht ersatzweise kommen", [])
monkeypatch.setattr("app.retrieval.pull_rfc822", pull)
monkeypatch.setattr("app.sources.pull_rfc822", pull)
context = {"kind": "case", "id": case["id"]}
tree = ada.request(
"GET",
@@ -547,7 +547,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
assert {item["name"] for item in pending["issues"][0]["choices"]} == {"Neu", "Weiter"}
monkeypatch.setattr("app.retrieval.EXPORT_BYTE_LIMIT", 40)
huge = _eml("<offen@example.test>", "GEKÜRZT-VERBOTEN " * 20, [])
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: huge)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: huge)
assigned = ada.request(
"POST",
f"/api/v1/matters/{matter['id']}/export/review",
@@ -557,7 +557,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
assert "GEKÜRZT-VERBOTEN" not in json.dumps(assigned)
wrong = _eml("<andere@example.test>", "FALSCHE-NACHRICHT", [])
monkeypatch.setattr("app.retrieval.EXPORT_BYTE_LIMIT", 80 * 1024 * 1024)
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: wrong)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: wrong)
identity = ada.request(
"POST",
f"/api/v1/matters/{matter['id']}/export/review",
@@ -609,7 +609,7 @@ def test_nested_chapter_export_keeps_path_and_complete_mail(client, db, monkeypa
"Die Antwort bleibt vollständig.",
[("lage.pdf", PDF_A), ("konto.pdf", PDF_B)],
)
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
ada = Actor(client, "ada-nested-export@example.test", "Ada Verschachtelt")
register(db, ada.email, ada.name)
ada.login()
@@ -844,7 +844,7 @@ def test_chapter_entries_share_one_numbering_sequence(client, db, monkeypatch):
files={"upload": ("drittes.pdf", PDF_C, "application/pdf")},
status=201,
)
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
exportable = {"binding", "annotation", "attachment", "dispatch_item", "evidence"}
selected = [node["id"] for node in _walk(tree) if node["exportable"] and node["kind"] in exportable]
+225
View File
@@ -0,0 +1,225 @@
"""A third source appears only by registering its module."""
from __future__ import annotations
import io
import subprocess
import zipfile
from pathlib import Path
import pytest
from tests.sample_source import install_sample_source, remove_sample_source
from tests.support import ORIGIN, Actor, document, register
ROOT = Path(__file__).resolve().parents[2]
TEXT = "Der Probenbrief bleibt lesbar."
@pytest.fixture
def sample_source():
install_sample_source()
try:
yield
finally:
remove_sample_source()
def test_registered_sample_source_links_reads_and_exports(client, db, sample_source):
del sample_source
ada = Actor(client, "ada-sample@example.test", "Ada Probe")
ben = Actor(client, "ben-sample@example.test", "Ben Probe")
register(db, ada.email, ada.name)
register(db, ben.email, ben.name)
ada.login()
refused = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "missing", "name": "Unbekannt"},
status=422,
)
assert refused["code"] == "unprocessable"
ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "sample", "name": "Ohne Katalog", "config": {"catalog": "kein-katalog"}},
status=422,
)
connector_id = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "sample", "name": "Katalog Alpha", "config": {"catalog": "alpha"}},
status=201,
)["id"]
probed = ada.request("POST", f"/api/v1/connectors/{connector_id}/probe")
assert probed["ok"] is True
assert "2 Einträge" in probed["detail"]
blocked = ada.request(
"GET",
"/api/v1/sources/sample/search",
params={"q": "Probe", "correspondent": "geheim-wert"},
status=422,
)
assert blocked["code"] == "unsupported"
assert "geheim-wert" not in blocked["detail"]
ada.request("GET", "/api/v1/sources/sample/navigate", params={"connector_id": connector_id}, status=422)
found = ada.request("GET", "/api/v1/sources/sample/search", params={"q": "Proben", "connector_id": connector_id})
assert [item["id"] for item in found["items"]] == ["brief"]
preview = ada.request(
"GET",
"/api/v1/sources/sample/preview",
params={"connector_id": connector_id, "item": "brief"},
)
assert preview["text"] == TEXT
matter = ada.request("POST", "/api/v1/matters", json={"title": "Probenvorgang"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Probenbrief",
"source": "sample",
"external_id": "brief",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": connector_id},
},
status=201,
)
opened = _raw(ada, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
assert opened.status_code == 200
assert opened.content == TEXT.encode("utf-8")
assert opened.headers["content-type"].startswith("text/plain")
detail = ada.request("GET", f"/api/v1/matters/{matter['id']}")
element = next(item["element"] for item in detail["bindings"] if item["element"]["id"] == linked["id"])
assert element["preview"]["media"] == "text"
written = ada.request(
"PATCH",
f"/api/v1/notes/{linked['id']}",
json={
"expected_version": element["version"],
"document": document("Das darf nicht geschrieben werden."),
"context": {"kind": "matter", "id": matter["id"]},
},
status=422,
)
assert written["code"] == "unsupported"
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
selected = [node["id"] for node in _walk(tree) if node["kind"] == "binding" and node.get("exportable")]
body = {"context": {"kind": "matter", "id": matter["id"]}, "selected": selected}
review = ada.request("POST", f"/api/v1/matters/{matter['id']}/export/review", json=body)
assert review["ready"] is True, review
package = _post_zip(ada, matter["id"], body)
assert package.status_code == 200, package.text
archive = zipfile.ZipFile(io.BytesIO(package.content))
assert any(archive.read(name) == TEXT.encode("utf-8") for name in archive.namelist())
ben.login()
ben.request(
"GET",
"/api/v1/sources/sample/search",
params={"connector_id": connector_id, "q": "Proben"},
status=404,
)
hidden = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
assert hidden.status_code == 404
ada.login()
ada.request(
"POST",
f"/api/v1/matters/{matter['id']}/grants",
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": False},
status=201,
)
ben.login()
denied = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
assert denied.status_code == 422
assert denied.json()["detail"] == "Das Original ist über die Freigabe nicht abrufbar."
ada.login()
grant_id = next(item["id"] for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["grants"])
ada.request("DELETE", f"/api/v1/grants/{grant_id}", status=204)
ada.request(
"POST",
f"/api/v1/matters/{matter['id']}/grants",
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True},
status=201,
)
ben.login()
shared = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
assert shared.status_code == 200
assert shared.content == TEXT.encode("utf-8")
shown = ben.request("GET", f"/api/v1/matters/{matter['id']}")
visible = next(item["element"] for item in shown["bindings"] if item["element"]["id"] == linked["id"])
assert "connector_id" not in (visible["external"].get("detail") or {})
ada.login()
grant_id = next(item["id"] for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["grants"])
ada.request("DELETE", f"/api/v1/grants/{grant_id}", status=204)
ben.login()
revoked = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
assert revoked.status_code == 404
def test_sample_registration_does_not_edit_generic_modules():
forbidden = [
"backend/app/api/schemas.py",
"backend/app/exporting.py",
"backend/app/present.py",
"frontend/src/pages/Cases.jsx",
"frontend/src/pages/Matters.jsx",
"frontend/src/pages/ReadingPane.jsx",
"frontend/src/pages/ExportPackage.jsx",
"frontend/src/App.jsx",
"frontend/src/pages/Sources.jsx",
]
for relative in forbidden:
text = (ROOT / relative).read_text(encoding="utf-8")
assert '"sample"' not in text and "'sample'" not in text, relative
completed = subprocess.run(
[
"node",
"--input-type=module",
"-e",
(
"import { installSampleSource, removeSampleSource } from './src/sources/sample-source.js';"
"import { sourceModules } from './src/sources/registry-core.js';"
"installSampleSource();"
"if (!sourceModules().some((item) => item.kind === 'sample' && item.representation === 'text')) process.exit(2);"
"removeSampleSource();"
"if (sourceModules().some((item) => item.kind === 'sample')) process.exit(3);"
),
],
cwd=ROOT / "frontend",
check=False,
capture_output=True,
text=True,
)
assert completed.returncode == 0, completed.stderr
def _raw(actor: Actor, path: str):
return actor.client.get(
path,
headers={
"Origin": ORIGIN,
"X-CSRF-Token": actor.csrf,
"X-Scope-Kind": actor.scope["kind"],
"X-Scope-Id": actor.scope["id"],
},
)
def _post_zip(actor: Actor, matter_id: str, body: dict):
return actor.client.post(
f"/api/v1/matters/{matter_id}/export",
json=body,
headers={
"Origin": ORIGIN,
"X-CSRF-Token": actor.csrf,
"Content-Type": "application/json",
"X-Scope-Kind": actor.scope["kind"],
"X-Scope-Id": actor.scope["id"],
},
)
def _walk(node):
yield node
for child in node.get("children") or []:
yield from _walk(child)