Schließe die Connector-Kapselung ab, damit Paperless, IMAP und weitere Quellen über dieselbe Anbindung laufen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
75a3949f7c
commit
d6b70d8f92
75 files changed
+9477
-414
No files matched your search
@@ -93,8 +93,8 @@ from app.auth.security import (
|
||||
)
|
||||
from app.core.errors import Problem, forbidden, not_found, too_many, unprocessable
|
||||
from app.core.timeutil import dump_date, dump_dt, utcnow
|
||||
from app.media.storage import content_disposition, read_bytes
|
||||
from app.media.storage import asset_path
|
||||
from app.media.port import read_clear_bytes
|
||||
from app.media.storage import content_disposition
|
||||
from app.ops import (
|
||||
add_annotation,
|
||||
assign_task,
|
||||
@@ -1119,12 +1119,24 @@ def download_file(
|
||||
if not rights.read:
|
||||
raise not_found()
|
||||
asset = conn.execute(
|
||||
"SELECT id, original_filename, content_type, status FROM media_assets WHERE element_id = %s",
|
||||
"""
|
||||
SELECT id, original_filename, content_type, sha256, status,
|
||||
storage_instance_id, storage_generation, object_key, protection_profile
|
||||
FROM media_assets WHERE element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
if asset is None or asset["status"] != "active":
|
||||
raise not_found()
|
||||
payload = read_bytes(asset_path(request.app.state.settings.media_root, asset["id"]))
|
||||
payload = read_clear_bytes(
|
||||
conn,
|
||||
instance_id=asset["storage_instance_id"],
|
||||
generation=asset["storage_generation"],
|
||||
object_key=asset["object_key"] or str(asset["id"]),
|
||||
media_root=request.app.state.settings.media_root,
|
||||
protection_profile=asset["protection_profile"],
|
||||
expected_sha256=asset["sha256"],
|
||||
)
|
||||
return Response(
|
||||
content=payload,
|
||||
media_type=asset["content_type"],
|
||||
|
||||
@@ -278,7 +278,7 @@ class ExternalDetail(APIModel):
|
||||
|
||||
class ExternalIn(APIModel):
|
||||
title: str = Field(min_length=1, max_length=200)
|
||||
source: Literal["paperless", "url", "imap"]
|
||||
source: str = Field(min_length=1, max_length=40)
|
||||
external_id: str = Field(min_length=1, max_length=200)
|
||||
context: ContextIn
|
||||
placement: ContextIn | None = None
|
||||
@@ -288,7 +288,13 @@ class ExternalIn(APIModel):
|
||||
|
||||
|
||||
class ConnectorIn(APIModel):
|
||||
kind: Literal["paperless", "imap"]
|
||||
"""Kind is checked against the source registry. Source fields stay optional.
|
||||
|
||||
Paperless and IMAP still send their existing fields. A further module
|
||||
uses ``config`` and validates that bag itself.
|
||||
"""
|
||||
|
||||
kind: str = Field(min_length=1, max_length=40)
|
||||
name: str = Field(min_length=1, max_length=120)
|
||||
base_url: str | None = None
|
||||
token: str | None = None
|
||||
@@ -298,6 +304,11 @@ class ConnectorIn(APIModel):
|
||||
password: str | None = None
|
||||
mailbox: str | None = Field(default=None, max_length=200)
|
||||
security: Literal["ssl", "starttls"] | None = None
|
||||
config: dict[str, str] | None = None
|
||||
|
||||
|
||||
class SourceAssignmentIn(APIModel):
|
||||
connector_id: UUID
|
||||
|
||||
|
||||
class DeleteElementIn(APIModel):
|
||||
|
||||
+212
-35
@@ -8,26 +8,25 @@ from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from psycopg import Connection
|
||||
|
||||
from app.access.policy import element_rights, get_element, parse_query_context
|
||||
from app.access.policy import Ctx, element_rights, get_element, parse_query_context
|
||||
from app.api.deps import Principal, get_conn, get_principal
|
||||
from app.api.routes import _created, _ctx_model
|
||||
from app.api.schemas import ConnectorIn, ExternalIn
|
||||
from app.core.errors import not_found, unprocessable
|
||||
from app.api.schemas import ConnectorIn, ExternalIn, SourceAssignmentIn
|
||||
from app.core.errors import not_found, unprocessable, unsupported
|
||||
from app.mailpack import parse_mail
|
||||
from app.media.storage import content_disposition
|
||||
from app.ops import create_external
|
||||
from app.sourcebind import assign_reference
|
||||
from app.sourceport import capability_payload, with_capabilities
|
||||
from app.sourceregistry import get_source, require_source
|
||||
from app.sources import (
|
||||
clean_base_url,
|
||||
delete_connector,
|
||||
imap_mailboxes,
|
||||
imap_messages,
|
||||
imap_preview,
|
||||
list_connectors,
|
||||
paperless_documents,
|
||||
probe,
|
||||
read_linked_mail,
|
||||
read_linked_paperless,
|
||||
save_connector,
|
||||
update_connector,
|
||||
)
|
||||
|
||||
_PRIVATE = {
|
||||
@@ -44,6 +43,12 @@ def get_connectors(principal: Principal = Depends(get_principal), conn: Connecti
|
||||
return {"items": list_connectors(conn, principal.account_id)}
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/capabilities")
|
||||
def source_capabilities(principal: Principal = Depends(get_principal)):
|
||||
del principal
|
||||
return capability_payload()
|
||||
|
||||
|
||||
@router.post("/api/v1/connectors", status_code=201)
|
||||
def post_connector(body: ConnectorIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
|
||||
name, settings, secret = _connector_parts(body)
|
||||
@@ -51,6 +56,22 @@ def post_connector(body: ConnectorIn, principal: Principal = Depends(get_princip
|
||||
return row
|
||||
|
||||
|
||||
@router.patch("/api/v1/connectors/{connector_id}")
|
||||
def patch_connector(
|
||||
connector_id: UUID,
|
||||
body: ConnectorIn,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
name, settings, secret = _connector_parts(body)
|
||||
current = next((item for item in list_connectors(conn, principal.account_id) if item["id"] == str(connector_id)), None)
|
||||
if current is None:
|
||||
raise not_found()
|
||||
if current["kind"] != body.kind:
|
||||
raise unprocessable("Der Zugang bleibt bei seiner Quellenart.")
|
||||
return update_connector(conn, principal.account_id, connector_id, name, settings, secret)
|
||||
|
||||
|
||||
@router.delete("/api/v1/connectors/{connector_id}", status_code=204)
|
||||
def remove_connector(connector_id: UUID, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
|
||||
delete_connector(conn, principal.account_id, connector_id)
|
||||
@@ -62,7 +83,7 @@ def probe_connector(connector_id: UUID, principal: Principal = Depends(get_princ
|
||||
|
||||
|
||||
@router.get("/api/v1/external-refs/{element_id}/content")
|
||||
def view_paperless(
|
||||
def view_external_content(
|
||||
element_id: UUID,
|
||||
context_kind: str | None = None,
|
||||
context_id: str | None = None,
|
||||
@@ -70,9 +91,10 @@ def view_paperless(
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
|
||||
if ref is None or ref["source"] != "paperless":
|
||||
raise unprocessable("Nur ein Paperless-Dokument lässt sich hier als Datei ansehen.")
|
||||
data, media_type, filename = read_linked_paperless(conn, principal.account_id, element, context, ref)
|
||||
module = get_source(ref["source"]) if ref is not None else None
|
||||
if module is None or module.open_original is None:
|
||||
raise unprocessable("Für diese Quelle ist kein Dateiabruf vorgesehen.")
|
||||
data, media_type, filename = module.open_original(conn, principal.account_id, element, context, ref)
|
||||
safe_name = (filename or "dokument.pdf").replace('"', "")
|
||||
return Response(
|
||||
content=data,
|
||||
@@ -126,7 +148,10 @@ def _opened_ref(conn, principal: Principal, element_id: UUID, context_kind: str
|
||||
if not rights.read:
|
||||
raise not_found()
|
||||
ref = conn.execute(
|
||||
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
|
||||
"""
|
||||
SELECT source, external_id, detail, connector_id, resolution
|
||||
FROM external_references WHERE element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
return element, context, ref
|
||||
@@ -186,8 +211,39 @@ def _mail_bytes(data: bytes, media_type: str, filename: str, inline: bool = Fals
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/paperless/documents")
|
||||
def search_paperless(q: str = "", principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
|
||||
return {"items": paperless_documents(conn, principal.account_id, q.strip())}
|
||||
def search_paperless(
|
||||
q: str = "",
|
||||
connector_id: UUID | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
return _run_search(
|
||||
"paperless",
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
"",
|
||||
None,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/imap/mailboxes")
|
||||
@@ -196,7 +252,7 @@ def list_mailboxes(
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
return {"items": imap_mailboxes(conn, principal.account_id, connector_id)}
|
||||
return _run_navigate("imap", conn, principal.account_id, connector_id)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/imap/messages")
|
||||
@@ -205,12 +261,38 @@ def search_mail(
|
||||
q: str = "",
|
||||
mailbox: str = "",
|
||||
before: int | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
if before is not None and before < 1:
|
||||
raise unprocessable("Die Seitenangabe ist ungültig.")
|
||||
return imap_messages(conn, principal.account_id, connector_id, q.strip(), mailbox.strip(), before)
|
||||
return _run_search(
|
||||
"imap",
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
mailbox.strip(),
|
||||
before,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/imap/message")
|
||||
@@ -228,8 +310,8 @@ def read_mail(
|
||||
|
||||
@router.post("/api/v1/sources/link", status_code=201)
|
||||
def link_source(body: ExternalIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
|
||||
if body.source not in {"paperless", "imap"}:
|
||||
raise unprocessable("Verknüpft werden Paperless-Dokumente und E-Mails.")
|
||||
if get_source(body.source) is None:
|
||||
raise unprocessable("Verknüpft werden registrierte Quellen.")
|
||||
element = create_external(
|
||||
conn,
|
||||
principal.account_id,
|
||||
@@ -248,20 +330,115 @@ def link_source(body: ExternalIn, principal: Principal = Depends(get_principal),
|
||||
return _created(f"/api/v1/elements/{element['id']}", {"id": str(element["id"]), "title": element["title"]})
|
||||
|
||||
|
||||
@router.post("/api/v1/external-refs/{element_id}/assignment")
|
||||
def assign_source(
|
||||
element_id: UUID,
|
||||
body: SourceAssignmentIn,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
assign_reference(conn, principal.account_id, element_id, body.connector_id)
|
||||
return {"id": str(element_id), "connector_id": str(body.connector_id), "resolution": "bound"}
|
||||
|
||||
|
||||
def _connector_parts(body: ConnectorIn) -> tuple[str, dict, str]:
|
||||
if body.kind == "paperless":
|
||||
if not body.base_url or not body.token or len(body.token.strip()) < 8:
|
||||
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
|
||||
return body.name.strip(), {"base_url": clean_base_url(body.base_url)}, body.token.strip()
|
||||
if not body.host or not body.username or not body.password:
|
||||
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
|
||||
host = body.host.strip()
|
||||
if " " in host or len(host) > 200:
|
||||
raise unprocessable("Der Mailserver ist ungültig.")
|
||||
return body.name.strip(), {
|
||||
"host": host,
|
||||
"port": body.port or 993,
|
||||
"username": body.username.strip(),
|
||||
"mailbox": (body.mailbox or "INBOX").strip() or "INBOX",
|
||||
"security": body.security or "ssl",
|
||||
}, body.password
|
||||
return require_source(body.kind).parse_config(body.model_dump())
|
||||
|
||||
|
||||
_SEARCH_FLAGS = {
|
||||
"title": "title",
|
||||
"correspondent": "correspondent",
|
||||
"document_type": "document_type",
|
||||
"tag": "tags",
|
||||
"document_from": "document_range",
|
||||
"document_to": "document_range",
|
||||
"sort": "sort",
|
||||
"cursor": "continuation",
|
||||
}
|
||||
|
||||
|
||||
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict) -> dict:
|
||||
module = require_source(kind)
|
||||
search = module.capabilities["search"]
|
||||
refused = [name for name, value in filters.items() if str(value or "").strip() and not search.get(_SEARCH_FLAGS.get(name, name))]
|
||||
if before is not None and not search.get("continuation"):
|
||||
refused.append("before")
|
||||
if refused:
|
||||
raise unsupported("Diese Suche unterstützt den Filter nicht: " + ", ".join(refused) + ".")
|
||||
found = module.find(conn, owner, connector_id, query, {"mailbox": mailbox, "before": before})
|
||||
bound = str(found.get("connector_id") or connector_id or "")
|
||||
return with_capabilities(kind, bound, found)
|
||||
|
||||
|
||||
def _run_navigate(kind: str, conn, owner, connector_id: UUID) -> dict:
|
||||
module = require_source(kind)
|
||||
if not module.capabilities.get("navigate") or module.navigate is None:
|
||||
raise unsupported("Diese Quelle hat keine Fachnavigation.")
|
||||
return {"items": module.navigate(conn, owner, connector_id)}
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/search")
|
||||
def search_source(
|
||||
kind: str,
|
||||
q: str = "",
|
||||
connector_id: UUID | None = None,
|
||||
mailbox: str = "",
|
||||
before: int | None = None,
|
||||
title: str = "",
|
||||
correspondent: str = "",
|
||||
document_type: str = "",
|
||||
tag: str = "",
|
||||
document_from: str = "",
|
||||
document_to: str = "",
|
||||
sort: str = "",
|
||||
cursor: str = "",
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
if before is not None and before < 1:
|
||||
raise unprocessable("Die Seitenangabe ist ungültig.")
|
||||
return _run_search(
|
||||
kind,
|
||||
conn,
|
||||
principal.account_id,
|
||||
q.strip(),
|
||||
connector_id,
|
||||
mailbox.strip(),
|
||||
before,
|
||||
{
|
||||
"title": title,
|
||||
"correspondent": correspondent,
|
||||
"document_type": document_type,
|
||||
"tag": tag,
|
||||
"document_from": document_from,
|
||||
"document_to": document_to,
|
||||
"sort": sort,
|
||||
"cursor": cursor,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/navigate")
|
||||
def navigate_source(
|
||||
kind: str,
|
||||
connector_id: UUID,
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
return _run_navigate(kind, conn, principal.account_id, connector_id)
|
||||
|
||||
|
||||
@router.get("/api/v1/sources/{kind}/preview")
|
||||
def preview_source(
|
||||
kind: str,
|
||||
connector_id: UUID,
|
||||
mailbox: str = "",
|
||||
uid: int = 0,
|
||||
item: str = "",
|
||||
principal: Principal = Depends(get_principal),
|
||||
conn: Connection = Depends(get_conn),
|
||||
):
|
||||
module = require_source(kind)
|
||||
if not module.capabilities.get("read_preview") or module.preview is None:
|
||||
raise unsupported("Diese Quelle hat keine Vorschau vor dem Übernehmen.")
|
||||
return module.preview(conn, principal.account_id, connector_id, {"mailbox": mailbox, "uid": uid, "item": item})
|
||||
@@ -50,6 +50,10 @@ def unprocessable(detail: str) -> Problem:
|
||||
return Problem(422, "Eingabe ungültig", detail, "unprocessable")
|
||||
|
||||
|
||||
def unsupported(detail: str) -> Problem:
|
||||
return Problem(422, "Nicht unterstützt", detail, "unsupported")
|
||||
|
||||
|
||||
def too_many() -> Problem:
|
||||
return Problem(429, "Zu viele Versuche", "Bitte kurz warten und es erneut versuchen.", "too_many_requests")
|
||||
|
||||
|
||||
@@ -26,9 +26,10 @@ from app.access.policy import (
|
||||
require_case,
|
||||
require_matter,
|
||||
)
|
||||
from app.core.errors import conflict, not_found, unprocessable
|
||||
from app.core.errors import Problem, conflict, not_found, unprocessable
|
||||
from app.core.timeutil import dump_date, dump_dt
|
||||
from app.media.storage import asset_path, display_filename, read_bytes, sha256, write_bytes
|
||||
from app.media.port import LEGACY_PROFILE, LocalAdapter, instance_for_write, publish, read_clear_bytes, stored_path
|
||||
from app.media.storage import display_filename, sha256
|
||||
from app.retrieval import FilePayload, MailPayload, RetrievalFailure, load_external, load_file
|
||||
|
||||
CHANNELS = {"email": "per E-Mail", "post": "per Post", "other": "sonstiger Weg"}
|
||||
@@ -415,9 +416,25 @@ def load_snapshot_file(conn, snapshot_id: UUID, media_root: str) -> FilePayload
|
||||
row = conn.execute("SELECT * FROM dispatch_snapshots WHERE id = %s", (snapshot_id,)).fetchone()
|
||||
if row is None:
|
||||
return RetrievalFailure("missing", "Die gesicherte Fassung ist nicht mehr vorhanden.")
|
||||
profile = row["protection_profile"] or "legacy-plaintext"
|
||||
if profile != "legacy-plaintext":
|
||||
return RetrievalFailure(
|
||||
"unsupported",
|
||||
"Dieser Inhalt ist geschützt und kann hier nicht im Klartext gelesen werden.",
|
||||
)
|
||||
try:
|
||||
data = read_bytes(_snapshot_path(media_root, row["id"]))
|
||||
except Exception:
|
||||
data = read_clear_bytes(
|
||||
conn,
|
||||
instance_id=row["storage_instance_id"],
|
||||
generation=row["storage_generation"],
|
||||
object_key=row["object_key"] or f"dispatch/{row['id']}",
|
||||
media_root=media_root,
|
||||
protection_profile=profile,
|
||||
expected_sha256=row["sha256"],
|
||||
)
|
||||
except Problem as exc:
|
||||
if exc.code == "processing":
|
||||
return RetrievalFailure("missing", "Die gesicherte Fassung lässt sich nicht unverändert lesen.")
|
||||
return RetrievalFailure("missing", "Die gesicherte Fassung ist nicht mehr vorhanden.")
|
||||
if sha256(data) != row["sha256"]:
|
||||
return RetrievalFailure("missing", "Die gesicherte Fassung lässt sich nicht unverändert lesen.")
|
||||
@@ -573,7 +590,7 @@ def purge_element_dispatch_files(conn, element_id: UUID, media_root: str) -> lis
|
||||
(item_id,),
|
||||
).fetchone()
|
||||
if snap is not None:
|
||||
paths.append(_snapshot_path(media_root, snap["id"]))
|
||||
paths.append(_snapshot_disk(conn, snap["id"], media_root))
|
||||
conn.execute("DELETE FROM dispatch_snapshots WHERE id = %s", (snap["id"],))
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -596,7 +613,7 @@ def purge_element_dispatch_files(conn, element_id: UUID, media_root: str) -> lis
|
||||
(evidence_id,),
|
||||
).fetchone()
|
||||
if snap is not None:
|
||||
paths.append(_snapshot_path(media_root, snap["id"]))
|
||||
paths.append(_snapshot_disk(conn, snap["id"], media_root))
|
||||
conn.execute("DELETE FROM dispatch_snapshots WHERE id = %s", (snap["id"],))
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -916,17 +933,22 @@ def _working_bytes(conn, actor, element, media_root: str):
|
||||
|
||||
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
|
||||
snapshot_id = uuid4()
|
||||
path = _snapshot_path(media_root, snapshot_id)
|
||||
write_bytes(path, payload.data)
|
||||
cleanup.append(path)
|
||||
instance = instance_for_write(conn, media_root)
|
||||
object_key = f"dispatch/{snapshot_id}"
|
||||
content_revision = str(uuid4())
|
||||
adapter = LocalAdapter(instance["root_path"])
|
||||
publish(adapter, object_key, payload.data)
|
||||
cleanup.append(stored_path(conn, instance["id"], instance["generation"], object_key, media_root))
|
||||
digest = sha256(payload.data)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO dispatch_snapshots (
|
||||
id, dispatch_item_id, evidence_id, home_scope_id, owner_subject_id,
|
||||
original_filename, content_type, byte_size, sha256
|
||||
original_filename, content_type, byte_size, sha256,
|
||||
storage_instance_id, storage_generation, object_key, content_revision,
|
||||
protection_profile, key_responsibility
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'none')
|
||||
""",
|
||||
(
|
||||
snapshot_id,
|
||||
@@ -938,6 +960,11 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
|
||||
payload.media_type or "application/octet-stream",
|
||||
len(payload.data),
|
||||
digest,
|
||||
instance["id"],
|
||||
instance["generation"],
|
||||
object_key,
|
||||
content_revision,
|
||||
LEGACY_PROFILE,
|
||||
),
|
||||
)
|
||||
return snapshot_id
|
||||
@@ -956,20 +983,36 @@ def _snapshot_file(conn, media_root: str, dispatch_item_id: UUID | None = None,
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
data = read_bytes(_snapshot_path(media_root, row["id"]))
|
||||
if sha256(data) != row["sha256"]:
|
||||
raise not_found()
|
||||
profile = row["protection_profile"] or "legacy-plaintext"
|
||||
data = read_clear_bytes(
|
||||
conn,
|
||||
instance_id=row["storage_instance_id"],
|
||||
generation=row["storage_generation"],
|
||||
object_key=row["object_key"] or f"dispatch/{row['id']}",
|
||||
media_root=media_root,
|
||||
protection_profile=profile,
|
||||
expected_sha256=row["sha256"],
|
||||
)
|
||||
return FilePayload(row["original_filename"], row["content_type"], data, row["sha256"])
|
||||
|
||||
|
||||
def _snapshot_path(media_root: str, snapshot_id: UUID) -> Path:
|
||||
root = Path(media_root).resolve()
|
||||
folder = (root / "dispatch").resolve()
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
target = (folder / str(snapshot_id)).resolve()
|
||||
if target.parent != folder:
|
||||
raise unprocessable("Ungültiger Speicherpfad.")
|
||||
return target
|
||||
def _snapshot_disk(conn, snapshot_id: UUID, media_root: str) -> Path:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, storage_instance_id, storage_generation, object_key
|
||||
FROM dispatch_snapshots WHERE id = %s
|
||||
""",
|
||||
(snapshot_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return stored_path(conn, None, "legacy-local-1", f"dispatch/{snapshot_id}", media_root)
|
||||
return stored_path(
|
||||
conn,
|
||||
row["storage_instance_id"],
|
||||
row["storage_generation"],
|
||||
row["object_key"] or f"dispatch/{row['id']}",
|
||||
media_root,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_id(conn, dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> str | None:
|
||||
@@ -1416,5 +1459,5 @@ def _delete_dispatch_files(conn, dispatch_id: UUID, media_root: str) -> list[Pat
|
||||
(dispatch_id, dispatch_id),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
paths.append(_snapshot_path(media_root, row["id"]))
|
||||
paths.append(_snapshot_disk(conn, row["id"], media_root))
|
||||
return paths
|
||||
@@ -49,9 +49,22 @@ def migrate(settings: Settings | None = None) -> list[str]:
|
||||
)
|
||||
applied.append(version)
|
||||
_ensure_runtime_role(conn, settings)
|
||||
_prepare_source_storage(settings)
|
||||
return applied
|
||||
|
||||
|
||||
def _prepare_source_storage(settings: Settings) -> None:
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.media.port import ensure_legacy_instance
|
||||
from app.sourcebind import backfill_source_bindings
|
||||
|
||||
with psycopg.connect(settings.dsn(), row_factory=dict_row) as conn:
|
||||
backfill_source_bindings(conn)
|
||||
ensure_legacy_instance(conn, settings.media_root)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _ensure_runtime_role(conn: psycopg.Connection, settings: Settings) -> None:
|
||||
role = sql.Identifier(settings.db_runtime_user)
|
||||
database = sql.Identifier(settings.db_name)
|
||||
|
||||
@@ -39,6 +39,7 @@ _KIND_LABEL = {
|
||||
"file": "Datei",
|
||||
"email": "E-Mail",
|
||||
"paperless": "Paperless",
|
||||
"source": "Quelle",
|
||||
"url": "Webadresse",
|
||||
"annotation": "Anmerkung",
|
||||
"chapter": "Kapitel",
|
||||
@@ -233,7 +234,7 @@ def _plan(conn, actor, matter_id, context, selected, assignments, media_root) ->
|
||||
continue
|
||||
if node.content_kind == "file":
|
||||
result = load_file(conn, UUID(node.element_id), media_root)
|
||||
elif node.content_kind in {"email", "paperless", "url"}:
|
||||
elif node.content_kind in {"email", "paperless", "url", "source"}:
|
||||
result = load_external(conn, actor, UUID(node.element_id), assignments.get(node.id))
|
||||
else:
|
||||
result = RetrievalFailure("capability", "Für diesen Inhalt ist kein Originalabruf vorgesehen.")
|
||||
@@ -461,13 +462,16 @@ def _content_kind(element: dict) -> str:
|
||||
if element["kind"] == "file":
|
||||
return "file"
|
||||
source = (element.get("external") or {}).get("source")
|
||||
if source == "imap":
|
||||
return "email"
|
||||
if source == "paperless":
|
||||
return "paperless"
|
||||
if source == "url":
|
||||
return "url"
|
||||
return element["kind"]
|
||||
from app.sourceregistry import get_source
|
||||
|
||||
module = get_source(source) if source else None
|
||||
if module is None:
|
||||
return "source"
|
||||
if module.representation == "email":
|
||||
return "email"
|
||||
return "source"
|
||||
|
||||
|
||||
def _chosen(root: ExportNode, selected: list[str]) -> set[str]:
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Opaque storage port. The adapter stores bytes and does not decide who may read them.
|
||||
|
||||
Cleartext for a reader goes through require_cleartext. Locked profiles stay
|
||||
bytes on disk and are not decrypted here. No new cryptography is implemented.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.errors import Problem, storage_unavailable, unprocessable
|
||||
from app.media.storage import read_bytes, sha256, write_bytes
|
||||
|
||||
LEGACY_GENERATION = "legacy-local-1"
|
||||
LEGACY_PROFILE = "legacy-plaintext"
|
||||
LOCKED_PROFILES = frozenset({"managed-encrypted", "client-encrypted"})
|
||||
PROFILES = frozenset({LEGACY_PROFILE, *LOCKED_PROFILES})
|
||||
|
||||
|
||||
def require_cleartext(profile: str | None) -> None:
|
||||
name = profile or LEGACY_PROFILE
|
||||
if name == LEGACY_PROFILE:
|
||||
return
|
||||
if name in LOCKED_PROFILES:
|
||||
raise Problem(
|
||||
422,
|
||||
"Geschützter Inhalt",
|
||||
"Dieser Inhalt ist geschützt. Ein Klartextabruf ist in dieser Lieferung nicht möglich.",
|
||||
"unsupported",
|
||||
)
|
||||
raise Problem(
|
||||
422,
|
||||
"Nicht unterstützt",
|
||||
"Dieses Schutzprofil wird nicht unterstützt.",
|
||||
"unsupported",
|
||||
)
|
||||
|
||||
|
||||
def check_object_key(object_key: str) -> str:
|
||||
key = (object_key or "").replace("\\", "/").strip()
|
||||
parts = key.split("/")
|
||||
if not key or key.startswith("/") or any(part in {"", ".", ".."} for part in parts):
|
||||
raise unprocessable("Ungültiger Speicherpfad.")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
UUID(parts[0])
|
||||
elif len(parts) == 2 and parts[0] == "dispatch":
|
||||
UUID(parts[1])
|
||||
else:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise unprocessable("Ungültiger Speicherpfad.") from None
|
||||
return key
|
||||
|
||||
|
||||
def object_path(root: Path, object_key: str) -> Path:
|
||||
key = check_object_key(object_key)
|
||||
base = Path(root).resolve()
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
target = (base / key).resolve()
|
||||
if base != target and base not in target.parents:
|
||||
raise unprocessable("Ungültiger Speicherpfad.")
|
||||
return target
|
||||
|
||||
|
||||
class LocalAdapter:
|
||||
"""Files stay under one recorded root. A client path is not an address."""
|
||||
|
||||
def __init__(self, root: str | Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def put(self, object_key: str, data: bytes) -> None:
|
||||
path = object_path(self.root, object_key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_bytes(path, data)
|
||||
|
||||
def open(self, object_key: str) -> bytes:
|
||||
return read_bytes(object_path(self.root, object_key))
|
||||
|
||||
def delete(self, object_key: str) -> None:
|
||||
path = object_path(self.root, object_key)
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
raise storage_unavailable() from exc
|
||||
|
||||
def describe(self, object_key: str) -> dict:
|
||||
path = object_path(self.root, object_key)
|
||||
try:
|
||||
if not path.is_file():
|
||||
raise storage_unavailable()
|
||||
return {"byte_size": path.stat().st_size, "object_key": object_key}
|
||||
except OSError as exc:
|
||||
raise storage_unavailable() from exc
|
||||
|
||||
|
||||
class MemoryAdapter:
|
||||
"""In-memory stand-in for the same port. It is not a second product store."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._items: dict[str, bytes] = {}
|
||||
self.fail_put = False
|
||||
|
||||
def put(self, object_key: str, data: bytes) -> None:
|
||||
if self.fail_put:
|
||||
raise storage_unavailable()
|
||||
key = check_object_key(object_key)
|
||||
self._items[key] = bytes(data)
|
||||
|
||||
def open(self, object_key: str) -> bytes:
|
||||
key = check_object_key(object_key)
|
||||
try:
|
||||
return self._items[key]
|
||||
except KeyError as exc:
|
||||
raise storage_unavailable() from exc
|
||||
|
||||
def delete(self, object_key: str) -> None:
|
||||
key = check_object_key(object_key)
|
||||
self._items.pop(key, None)
|
||||
|
||||
def describe(self, object_key: str) -> dict:
|
||||
data = self.open(object_key)
|
||||
return {"byte_size": len(data), "object_key": object_key}
|
||||
|
||||
|
||||
def publish(adapter, object_key: str, data: bytes) -> None:
|
||||
"""Make bytes readable only after a checked roundtrip. A failed put leaves nothing published."""
|
||||
adapter.put(object_key, data)
|
||||
try:
|
||||
opened = adapter.open(object_key)
|
||||
except Problem:
|
||||
adapter.delete(object_key)
|
||||
raise
|
||||
if opened != data:
|
||||
adapter.delete(object_key)
|
||||
raise storage_unavailable()
|
||||
|
||||
|
||||
def ensure_legacy_instance(conn, media_root: str) -> dict:
|
||||
"""Bind the legacy generation to the current root once. Later roots do not retarget it."""
|
||||
root = str(Path(media_root).resolve())
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO storage_instances (kind, generation, root_path)
|
||||
VALUES ('local', %s, %s)
|
||||
ON CONFLICT (generation) DO NOTHING
|
||||
""",
|
||||
(LEGACY_GENERATION, root),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE storage_instances
|
||||
SET root_path = %s
|
||||
WHERE generation = %s AND root_path IS NULL
|
||||
""",
|
||||
(root, LEGACY_GENERATION),
|
||||
)
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, kind, generation, root_path
|
||||
FROM storage_instances
|
||||
WHERE generation = %s
|
||||
""",
|
||||
(LEGACY_GENERATION,),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE media_assets
|
||||
SET storage_instance_id = %s
|
||||
WHERE storage_generation = %s AND storage_instance_id IS NULL
|
||||
""",
|
||||
(row["id"], LEGACY_GENERATION),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE dispatch_snapshots
|
||||
SET storage_instance_id = %s
|
||||
WHERE storage_generation = %s AND storage_instance_id IS NULL
|
||||
""",
|
||||
(row["id"], LEGACY_GENERATION),
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def instance_for_write(conn, media_root: str) -> dict:
|
||||
legacy = ensure_legacy_instance(conn, media_root)
|
||||
root = str(Path(media_root).resolve())
|
||||
if legacy["root_path"] == root:
|
||||
return legacy
|
||||
generation = "local-" + hashlib.sha256(root.encode("utf-8")).hexdigest()[:16]
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO storage_instances (kind, generation, root_path)
|
||||
VALUES ('local', %s, %s)
|
||||
ON CONFLICT (generation) DO NOTHING
|
||||
""",
|
||||
(generation, root),
|
||||
)
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT id, kind, generation, root_path
|
||||
FROM storage_instances
|
||||
WHERE generation = %s
|
||||
""",
|
||||
(generation,),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def resolve_root(conn, instance_id, generation: str | None, media_root: str) -> str:
|
||||
if instance_id is not None:
|
||||
row = conn.execute(
|
||||
"SELECT root_path FROM storage_instances WHERE id = %s",
|
||||
(instance_id,),
|
||||
).fetchone()
|
||||
if row is not None and row["root_path"]:
|
||||
return row["root_path"]
|
||||
if generation in {None, LEGACY_GENERATION}:
|
||||
return ensure_legacy_instance(conn, media_root)["root_path"]
|
||||
return str(Path(media_root).resolve())
|
||||
|
||||
|
||||
def stored_path(conn, instance_id, generation: str | None, object_key: str, media_root: str) -> Path:
|
||||
return object_path(Path(resolve_root(conn, instance_id, generation, media_root)), object_key)
|
||||
|
||||
|
||||
def read_clear_bytes(
|
||||
conn,
|
||||
*,
|
||||
instance_id,
|
||||
generation: str | None,
|
||||
object_key: str,
|
||||
media_root: str,
|
||||
protection_profile: str | None,
|
||||
expected_sha256: str | None,
|
||||
) -> bytes:
|
||||
require_cleartext(protection_profile)
|
||||
data = LocalAdapter(resolve_root(conn, instance_id, generation, media_root)).open(object_key)
|
||||
if expected_sha256 and sha256(data) != expected_sha256:
|
||||
raise Problem(
|
||||
422,
|
||||
"Inhalt nicht geprüft",
|
||||
"Die gespeicherte Datei stimmt nicht mit dem erfassten Stand überein.",
|
||||
"processing",
|
||||
)
|
||||
return data
|
||||
+39
-10
@@ -33,9 +33,11 @@ from app.access.policy import (
|
||||
task_rights,
|
||||
)
|
||||
from app.content.documents import clean_document
|
||||
from app.core.errors import forbidden, not_found, unprocessable
|
||||
from app.core.errors import forbidden, not_found, unprocessable, unsupported
|
||||
from app.core.timeutil import utcnow
|
||||
from app.media.storage import asset_path, display_filename, sha256, sniff, write_bytes
|
||||
from app.media.port import LEGACY_PROFILE, instance_for_write, publish, stored_path
|
||||
from app.media.port import LocalAdapter
|
||||
from app.media.storage import display_filename, sha256, sniff
|
||||
|
||||
MAX_CHAPTER_DEPTH = 32
|
||||
|
||||
@@ -929,7 +931,7 @@ def update_note(conn, actor: UUID, element_id: UUID, expected: int, document: di
|
||||
document = clean_document(document)
|
||||
element, rights = require_element(conn, actor, element_id, context, "edit")
|
||||
if element["kind"] != "note":
|
||||
raise not_found()
|
||||
raise unsupported("Diese Fassung lässt sich nicht als Notiz bearbeiten.")
|
||||
if not rights.edit:
|
||||
raise forbidden()
|
||||
version = bump_version(conn, "elements", element_id, expected)
|
||||
@@ -998,6 +1000,10 @@ def add_annotation(conn, actor: UUID, binding_id: UUID, body: str, context: Ctx
|
||||
|
||||
|
||||
def create_external(conn, actor: UUID, scope_id: UUID, title: str, source: str, external_id: str, context: Ctx, placement: Ctx | None, chapter_id: UUID | None, document_on, sent_on, received_on, detail: dict | None = None, communication_direction: str | None = None) -> dict:
|
||||
from app.sourceregistry import accepts_reference
|
||||
|
||||
if not accepts_reference(source):
|
||||
raise unprocessable("Diese Quelle ist nicht registriert.")
|
||||
target, rights_context = _placement(context, placement)
|
||||
require_container_edit(conn, actor, target, rights_context)
|
||||
element = _new_element(conn, actor, scope_id, title, "external_ref")
|
||||
@@ -1005,8 +1011,11 @@ def create_external(conn, actor: UUID, scope_id: UUID, title: str, source: str,
|
||||
"INSERT INTO external_references (element_id, source, external_id, detail) VALUES (%s, %s, %s, %s)",
|
||||
(element["id"], source, external_id.strip()[:200], Json(detail or {})),
|
||||
)
|
||||
from app.sourcebind import attach_new_reference
|
||||
|
||||
attach_new_reference(conn, element["id"])
|
||||
binding = insert_binding(conn, element["id"], target.kind, target.id, chapter_id, actor)
|
||||
adopted = (detail or {}).get("direction") if source == "imap" else None
|
||||
adopted = (detail or {}).get("direction")
|
||||
chosen = chosen_direction(communication_direction) or (adopted if adopted in ("in", "out") else None)
|
||||
store_direction(conn, binding["id"], chosen, chosen is not None)
|
||||
record_dates(conn, actor, scope_id, element["id"], rights_context or target, document_on, sent_on, received_on)
|
||||
@@ -1022,16 +1031,23 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
|
||||
require_container_edit(conn, actor, target, rights_context)
|
||||
element = _new_element(conn, actor, scope_id, title, "file")
|
||||
asset_id = uuid4()
|
||||
path = asset_path(media_root, asset_id)
|
||||
write_bytes(path, data)
|
||||
content_revision = str(uuid4())
|
||||
instance = instance_for_write(conn, media_root)
|
||||
object_key = str(asset_id)
|
||||
adapter = LocalAdapter(instance["root_path"])
|
||||
publish(adapter, object_key, data)
|
||||
path = stored_path(conn, instance["id"], instance["generation"], object_key, media_root)
|
||||
rollback_files.append(path)
|
||||
digest = sha256(data)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO media_assets (
|
||||
id, element_id, owner_subject_id, home_scope_id, original_filename,
|
||||
content_type, byte_size, sha256, status
|
||||
content_type, byte_size, sha256, status, storage_instance_id,
|
||||
storage_generation, object_key, content_revision, protection_profile,
|
||||
key_responsibility
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'active')
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'active', %s, %s, %s, %s, %s, 'none')
|
||||
""",
|
||||
(
|
||||
asset_id,
|
||||
@@ -1041,7 +1057,12 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
|
||||
display_filename(filename),
|
||||
content_type,
|
||||
len(data),
|
||||
sha256(data),
|
||||
digest,
|
||||
instance["id"],
|
||||
instance["generation"],
|
||||
object_key,
|
||||
content_revision,
|
||||
LEGACY_PROFILE,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
@@ -1075,7 +1096,15 @@ def delete_element(conn, actor: UUID, element_id: UUID, expected: int, media_roo
|
||||
)
|
||||
if asset is not None and asset["status"] == "active":
|
||||
conn.execute("UPDATE media_assets SET status = 'purging' WHERE id = %s", (asset["id"],))
|
||||
delete_after.append(asset_path(media_root, asset["id"]))
|
||||
delete_after.append(
|
||||
stored_path(
|
||||
conn,
|
||||
asset["storage_instance_id"],
|
||||
asset["storage_generation"],
|
||||
asset["object_key"] or str(asset["id"]),
|
||||
media_root,
|
||||
)
|
||||
)
|
||||
conn.execute("UPDATE media_assets SET status = 'purged' WHERE id = %s", (asset["id"],))
|
||||
audit(conn, actor, element["home_scope_id"], "element_deleted", "element", element_id)
|
||||
|
||||
|
||||
+31
-11
@@ -117,10 +117,16 @@ def _preview_meta(kind: str, content_type: str | None, source: str | None) -> di
|
||||
return {"available": True, "media": "image", "source": "file"}
|
||||
if kind == "file" and media == "application/pdf":
|
||||
return {"available": True, "media": "pdf", "source": "file"}
|
||||
if kind == "external_ref" and source == "paperless":
|
||||
return {"available": True, "media": "paperless", "source": "paperless"}
|
||||
if kind == "external_ref" and source == "imap":
|
||||
return {"available": True, "media": "email", "source": "imap"}
|
||||
if kind == "external_ref" and source:
|
||||
from app.sourceregistry import get_source
|
||||
|
||||
module = get_source(source)
|
||||
if module is not None:
|
||||
return {
|
||||
"available": bool(module.capabilities.get("read_preview")),
|
||||
"media": module.representation,
|
||||
"source": source,
|
||||
}
|
||||
if kind == "file":
|
||||
return {"available": False, "media": "file", "source": "file"}
|
||||
return {"available": False, "media": "none", "source": source}
|
||||
@@ -175,35 +181,49 @@ def _element_body(conn, actor: UUID, element: dict) -> dict:
|
||||
elif element["kind"] == "file":
|
||||
asset = conn.execute(
|
||||
"""
|
||||
SELECT original_filename, content_type, byte_size, status
|
||||
SELECT original_filename, content_type, byte_size, status, protection_profile
|
||||
FROM media_assets WHERE element_id = %s
|
||||
""",
|
||||
(element["id"],),
|
||||
).fetchone()
|
||||
if asset is not None:
|
||||
profile = asset["protection_profile"] or "legacy-plaintext"
|
||||
payload["file"] = {
|
||||
"filename": asset["original_filename"],
|
||||
"content_type": asset["content_type"],
|
||||
"byte_size": asset["byte_size"],
|
||||
"status": asset["status"],
|
||||
"protection_profile": profile,
|
||||
}
|
||||
payload["preview"] = _preview_meta("file", asset["content_type"], "file")
|
||||
if profile != "legacy-plaintext":
|
||||
payload["preview"] = {"available": False, "media": "locked", "source": "file"}
|
||||
else:
|
||||
payload["preview"] = _preview_meta("file", asset["content_type"], "file")
|
||||
elif element["kind"] == "external_ref":
|
||||
ref = conn.execute(
|
||||
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
|
||||
"""
|
||||
SELECT source, external_id, detail, connector_id, resolution
|
||||
FROM external_references WHERE element_id = %s
|
||||
""",
|
||||
(element["id"],),
|
||||
).fetchone()
|
||||
if ref is not None:
|
||||
detail = dict(ref["detail"] or {})
|
||||
connector_owner = _connector_owner(conn, detail.get("connector_id"))
|
||||
connector_owner = _connector_owner(conn, ref["connector_id"] or detail.get("connector_id"))
|
||||
revealed = actor == element["owner_subject_id"] or (connector_owner is not None and actor == connector_owner)
|
||||
if not revealed:
|
||||
detail = {key: value for key, value in detail.items() if key not in _HIDDEN_SOURCE_FIELDS}
|
||||
external = {"source": ref["source"], "detail": detail}
|
||||
resolution = ref["resolution"] or "unresolved"
|
||||
external = {"source": ref["source"], "detail": detail, "resolution": resolution}
|
||||
if revealed:
|
||||
external["external_id"] = ref["external_id"]
|
||||
payload["external"] = external
|
||||
payload["preview"] = _preview_meta("external_ref", None, ref["source"])
|
||||
from app.sourceregistry import get_source
|
||||
|
||||
if resolution in {"ambiguous", "unresolved"} and get_source(ref["source"]) is not None:
|
||||
payload["preview"] = {"available": False, "media": "unassigned", "source": ref["source"]}
|
||||
else:
|
||||
payload["preview"] = _preview_meta("external_ref", None, ref["source"])
|
||||
return payload
|
||||
|
||||
|
||||
@@ -237,7 +257,7 @@ def communication_from_row(conn, binding: dict, element: dict) -> dict:
|
||||
"SELECT source, detail FROM external_references WHERE element_id = %s",
|
||||
(element["id"],),
|
||||
).fetchone()
|
||||
detail = (ref["detail"] or {}) if ref is not None and ref["source"] == "imap" else {}
|
||||
detail = (ref["detail"] or {}) if ref is not None else {}
|
||||
if detail.get("direction") in ("in", "out"):
|
||||
source = detail["direction"]
|
||||
return {"direction": source, "explicit": False}
|
||||
|
||||
+33
-111
@@ -10,17 +10,12 @@ from dataclasses import dataclass, field
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.errors import Problem
|
||||
from app.mailpack import ParsedMail, identities_match, parse_mail
|
||||
from app.media.storage import asset_path, read_bytes
|
||||
from app.sources import COMPLETE_MAIL_LIMIT, pull_paperless_original, pull_rfc822
|
||||
from app.mailpack import ParsedMail
|
||||
from app.media.port import read_clear_bytes
|
||||
from app.sources import COMPLETE_MAIL_LIMIT
|
||||
|
||||
EXPORT_BYTE_LIMIT = COMPLETE_MAIL_LIMIT
|
||||
|
||||
_TOO_LARGE = (
|
||||
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
||||
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetrievalFailure:
|
||||
@@ -45,132 +40,59 @@ class MailPayload:
|
||||
|
||||
|
||||
def load_external(conn, actor_id: UUID, element_id: UUID, assigned: UUID | None) -> FilePayload | MailPayload | RetrievalFailure:
|
||||
from app.sourceregistry import get_source
|
||||
|
||||
ref = conn.execute(
|
||||
"SELECT source, external_id, detail FROM external_references WHERE element_id = %s",
|
||||
"""
|
||||
SELECT source, external_id, detail, connector_id, resolution
|
||||
FROM external_references WHERE element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
if ref is None:
|
||||
return RetrievalFailure("missing", "Die Quellenreferenz fehlt.")
|
||||
source = ref["source"]
|
||||
detail = ref["detail"] or {}
|
||||
if source == "paperless":
|
||||
return _load_paperless(conn, actor_id, ref["external_id"], detail, assigned)
|
||||
if source == "imap":
|
||||
return _load_imap(conn, actor_id, ref["external_id"], detail, assigned)
|
||||
if source == "url":
|
||||
return RetrievalFailure("capability", "Für eine Webadresse ist kein Originalabruf vorgesehen.")
|
||||
return RetrievalFailure("capability", "Für diese Quelle ist kein Originalabruf vorgesehen.")
|
||||
module = get_source(ref["source"])
|
||||
if module is None or module.export_original is None:
|
||||
return RetrievalFailure("capability", "Für diese Quelle ist kein Originalabruf vorgesehen.")
|
||||
return module.export_original(conn, actor_id, ref["external_id"], ref["detail"] or {}, assigned, ref)
|
||||
|
||||
|
||||
def load_file(conn, element_id: UUID, media_root: str) -> FilePayload | RetrievalFailure:
|
||||
asset = conn.execute(
|
||||
"""
|
||||
SELECT id, original_filename, content_type, sha256, status
|
||||
SELECT id, original_filename, content_type, sha256, status,
|
||||
storage_instance_id, storage_generation, object_key, protection_profile
|
||||
FROM media_assets WHERE element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
if asset is None or asset["status"] != "active":
|
||||
return RetrievalFailure("missing", "Die Originaldatei ist nicht mehr vorhanden.")
|
||||
if (asset["protection_profile"] or "legacy-plaintext") != "legacy-plaintext":
|
||||
return RetrievalFailure(
|
||||
"unsupported",
|
||||
"Dieser Inhalt ist geschützt und kann hier nicht im Klartext gelesen werden.",
|
||||
)
|
||||
try:
|
||||
data = read_bytes(asset_path(media_root, asset["id"]))
|
||||
except Problem:
|
||||
data = read_clear_bytes(
|
||||
conn,
|
||||
instance_id=asset["storage_instance_id"],
|
||||
generation=asset["storage_generation"],
|
||||
object_key=asset["object_key"] or str(asset["id"]),
|
||||
media_root=media_root,
|
||||
protection_profile=asset["protection_profile"],
|
||||
expected_sha256=asset["sha256"],
|
||||
)
|
||||
except Problem as exc:
|
||||
if exc.code == "unsupported":
|
||||
return RetrievalFailure("unsupported", exc.detail)
|
||||
return RetrievalFailure("missing", "Die Originaldatei ist nicht mehr vorhanden.")
|
||||
if len(data) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
|
||||
return FilePayload(asset["original_filename"], asset["content_type"], data, asset["sha256"])
|
||||
|
||||
|
||||
def _load_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None):
|
||||
stored = _uuid(detail.get("connector_id"))
|
||||
resolved = _resolve_connector(conn, actor_id, "paperless", stored, assigned)
|
||||
if isinstance(resolved, RetrievalFailure):
|
||||
return resolved
|
||||
try:
|
||||
data, media, name = pull_paperless_original(
|
||||
resolved["settings"]["base_url"].rstrip("/"),
|
||||
resolved["secret"],
|
||||
str(external_id),
|
||||
EXPORT_BYTE_LIMIT,
|
||||
)
|
||||
except Problem as exc:
|
||||
return _from_problem(exc)
|
||||
if len(data) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
|
||||
return FilePayload(name or f"paperless-{external_id}", media, data, "")
|
||||
|
||||
|
||||
def _load_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None):
|
||||
stored = _uuid(detail.get("connector_id"))
|
||||
resolved = _resolve_connector(conn, actor_id, "imap", stored, assigned)
|
||||
if isinstance(resolved, RetrievalFailure):
|
||||
return resolved
|
||||
mailbox = str(detail.get("mailbox") or "").strip()
|
||||
uid = detail.get("uid")
|
||||
if isinstance(uid, str) and uid.isdigit():
|
||||
uid = int(uid)
|
||||
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
||||
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
try:
|
||||
raw = pull_rfc822(resolved["settings"], resolved["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return _from_problem(exc)
|
||||
if len(raw) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure("too_large", _TOO_LARGE)
|
||||
parsed = parse_mail(raw)
|
||||
if not identities_match(parsed.message_id, str(external_id)):
|
||||
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
|
||||
if parsed.missing:
|
||||
names = ", ".join(parsed.missing[:5])
|
||||
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
|
||||
return MailPayload(raw, parsed)
|
||||
|
||||
|
||||
def _resolve_connector(conn, actor_id: UUID, kind: str, stored: UUID | None, assigned: UUID | None):
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, kind, name, settings, secret
|
||||
FROM connectors
|
||||
WHERE owner_id = %s AND kind = %s
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(actor_id, kind),
|
||||
).fetchall()
|
||||
if stored is not None and assigned is None:
|
||||
match = next((row for row in rows if row["id"] == stored), None)
|
||||
if match is None:
|
||||
return RetrievalFailure("missing", "Der gespeicherte Zugang für diese Quelle ist nicht mehr vorhanden.")
|
||||
return match
|
||||
chosen = assigned or stored
|
||||
if chosen is not None:
|
||||
match = next((row for row in rows if row["id"] == chosen), None)
|
||||
if match is None:
|
||||
return RetrievalFailure(
|
||||
"assignment",
|
||||
"Der gewählte Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.",
|
||||
)
|
||||
return match
|
||||
if len(rows) == 1:
|
||||
return rows[0]
|
||||
if not rows:
|
||||
return RetrievalFailure("missing", "Für diese Quelle ist kein Zugang vorhanden.")
|
||||
return RetrievalFailure(
|
||||
"ambiguous",
|
||||
"Für diese Quelle ist kein eindeutiger Zugang zugeordnet. Die Zuordnung muss bewusst gewählt werden.",
|
||||
tuple({"id": str(row["id"]), "name": row["name"]} for row in rows),
|
||||
)
|
||||
|
||||
|
||||
def _uuid(value) -> UUID | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _from_problem(exc: Problem) -> RetrievalFailure:
|
||||
def failure_from_problem(exc: Problem) -> RetrievalFailure:
|
||||
text = exc.detail
|
||||
lowered = text.casefold()
|
||||
if "exportgrenze" in lowered or "zu groß" in lowered:
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Bind a reference to one connector instance.
|
||||
|
||||
The newest connection is never a fallback. A stored id that no longer
|
||||
exists stays missing. Ambiguous stock waits for an explicit assignment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.access.policy import get_element
|
||||
from app.core.errors import not_found, unprocessable
|
||||
from app.sourceregistry import get_source
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolveResult:
|
||||
row: dict | None
|
||||
code: str
|
||||
message: str
|
||||
choices: tuple[dict, ...] = ()
|
||||
|
||||
|
||||
def _uuid(value) -> UUID | None:
|
||||
if isinstance(value, UUID):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def stored_id(ref) -> UUID | None:
|
||||
if not ref:
|
||||
return None
|
||||
detail = ref["detail"] if "detail" in ref.keys() else None
|
||||
detail = detail or {}
|
||||
return _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
|
||||
|
||||
|
||||
def _connectors(conn, owner_id: UUID, kind: str) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT id, owner_id, kind, name, settings, secret
|
||||
FROM connectors
|
||||
WHERE owner_id = %s AND kind = %s
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id, kind),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def resolve_for_read(conn, owner_id: UUID, kind: str, ref, assigned: UUID | None) -> ResolveResult:
|
||||
"""Resolve one access. Assignment applies only when the reference has no stored id."""
|
||||
stored = stored_id(ref)
|
||||
resolution = (ref["resolution"] if ref and "resolution" in ref.keys() and ref["resolution"] else None) or "unresolved"
|
||||
rows = _connectors(conn, owner_id, kind)
|
||||
if stored is not None:
|
||||
match = next((row for row in rows if row["id"] == stored), None)
|
||||
if match is None:
|
||||
return ResolveResult(None, "missing", "Der gespeicherte Zugang für diese Quelle ist nicht mehr vorhanden.")
|
||||
return ResolveResult(match, "ok", "")
|
||||
if assigned is not None:
|
||||
match = next((row for row in rows if row["id"] == assigned), None)
|
||||
if match is None:
|
||||
return ResolveResult(
|
||||
None,
|
||||
"assignment",
|
||||
"Der gewählte Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.",
|
||||
)
|
||||
return ResolveResult(match, "ok", "")
|
||||
if resolution == "ambiguous" or len(rows) != 1:
|
||||
if not rows:
|
||||
return ResolveResult(None, "missing", "Für diese Quelle ist kein Zugang vorhanden.")
|
||||
return ResolveResult(
|
||||
None,
|
||||
"ambiguous",
|
||||
"Für diese Quelle ist kein eindeutiger Zugang zugeordnet. Die Zuordnung muss bewusst gewählt werden.",
|
||||
tuple({"id": str(row["id"]), "name": row["name"]} for row in rows),
|
||||
)
|
||||
return ResolveResult(rows[0], "ok", "")
|
||||
|
||||
|
||||
def _set_binding(conn, element_id: UUID, connector_id: UUID | None, resolution: str, detail: dict | None = None) -> None:
|
||||
if detail is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET connector_id = %s, resolution = %s
|
||||
WHERE element_id = %s
|
||||
""",
|
||||
(connector_id, resolution, element_id),
|
||||
)
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET connector_id = %s, resolution = %s, detail = %s
|
||||
WHERE element_id = %s
|
||||
""",
|
||||
(connector_id, resolution, Json(detail), element_id),
|
||||
)
|
||||
|
||||
|
||||
def attach_new_reference(conn, element_id: UUID) -> None:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT er.source, er.detail, e.owner_subject_id
|
||||
FROM external_references er
|
||||
JOIN elements e ON e.id = er.element_id
|
||||
WHERE er.element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
if get_source(row["source"]) is None:
|
||||
_set_binding(conn, element_id, None, "none")
|
||||
return
|
||||
detail = dict(row["detail"] or {})
|
||||
stored = _uuid(detail.get("connector_id"))
|
||||
if stored is not None:
|
||||
match = conn.execute(
|
||||
"""
|
||||
SELECT id FROM connectors
|
||||
WHERE id = %s AND owner_id = %s AND kind = %s
|
||||
""",
|
||||
(stored, row["owner_subject_id"], row["source"]),
|
||||
).fetchone()
|
||||
_set_binding(conn, element_id, stored, "bound" if match else "unresolved")
|
||||
return
|
||||
rows = _connectors(conn, row["owner_subject_id"], row["source"])
|
||||
if len(rows) == 1:
|
||||
detail["connector_id"] = str(rows[0]["id"])
|
||||
_set_binding(conn, element_id, rows[0]["id"], "bound", detail)
|
||||
return
|
||||
if len(rows) > 1:
|
||||
_set_binding(conn, element_id, None, "ambiguous")
|
||||
return
|
||||
_set_binding(conn, element_id, None, "unresolved")
|
||||
|
||||
|
||||
def backfill_source_bindings(conn) -> None:
|
||||
"""Idempotent. Bound and ambiguous rows stay. Unique unresolved stock is bound once."""
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET resolution = 'none'
|
||||
WHERE source = 'url' AND resolution = 'unresolved'
|
||||
"""
|
||||
)
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT er.element_id, er.source, er.detail, e.owner_subject_id
|
||||
FROM external_references er
|
||||
JOIN elements e ON e.id = er.element_id
|
||||
WHERE er.source <> 'url'
|
||||
AND er.resolution = 'unresolved'
|
||||
AND er.connector_id IS NULL
|
||||
"""
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
if get_source(row["source"]) is None:
|
||||
continue
|
||||
detail = dict(row["detail"] or {})
|
||||
stored = _uuid(detail.get("connector_id"))
|
||||
if stored is not None:
|
||||
match = conn.execute(
|
||||
"""
|
||||
SELECT id FROM connectors
|
||||
WHERE id = %s AND owner_id = %s AND kind = %s
|
||||
""",
|
||||
(stored, row["owner_subject_id"], row["source"]),
|
||||
).fetchone()
|
||||
_set_binding(conn, row["element_id"], stored, "bound" if match else "unresolved")
|
||||
continue
|
||||
found = _connectors(conn, row["owner_subject_id"], row["source"])
|
||||
if len(found) == 1:
|
||||
detail["connector_id"] = str(found[0]["id"])
|
||||
_set_binding(conn, row["element_id"], found[0]["id"], "bound", detail)
|
||||
elif len(found) > 1:
|
||||
_set_binding(conn, row["element_id"], None, "ambiguous")
|
||||
|
||||
|
||||
def assign_reference(conn, actor_id: UUID, element_id: UUID, connector_id: UUID) -> None:
|
||||
element = get_element(conn, element_id)
|
||||
if element is None or element["kind"] != "external_ref" or element["owner_subject_id"] != actor_id:
|
||||
raise not_found()
|
||||
ref = conn.execute(
|
||||
"SELECT source, detail FROM external_references WHERE element_id = %s",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
if ref is None or get_source(ref["source"]) is None:
|
||||
raise unprocessable("Diese Referenz hat keinen Quellzugang.")
|
||||
connector = conn.execute(
|
||||
"""
|
||||
SELECT id, kind FROM connectors
|
||||
WHERE id = %s AND owner_id = %s
|
||||
""",
|
||||
(connector_id, actor_id),
|
||||
).fetchone()
|
||||
if connector is None or connector["kind"] != ref["source"]:
|
||||
raise unprocessable("Der Zugang gehört nicht zu diesem Konto oder passt nicht zur Quelle.")
|
||||
detail = dict(ref["detail"] or {})
|
||||
detail["connector_id"] = str(connector_id)
|
||||
_set_binding(conn, element_id, connector_id, "bound", detail)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Declared source and editor capabilities.
|
||||
|
||||
A common contract does not invent abilities the adapter does not have.
|
||||
Search filters that are false stay unavailable until a later delivery.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
PAPERLESS_CAPABILITIES = {
|
||||
"kind": "paperless",
|
||||
"navigate": False,
|
||||
"read_original": True,
|
||||
"read_preview": True,
|
||||
"read_attachment": False,
|
||||
"edit_content": False,
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"title": False,
|
||||
"correspondent": False,
|
||||
"document_type": False,
|
||||
"tags": False,
|
||||
"document_range": False,
|
||||
"sort": False,
|
||||
"continuation": False,
|
||||
},
|
||||
}
|
||||
|
||||
IMAP_CAPABILITIES = {
|
||||
"kind": "imap",
|
||||
"navigate": True,
|
||||
"read_original": True,
|
||||
"read_preview": True,
|
||||
"read_attachment": True,
|
||||
"edit_content": False,
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"title": False,
|
||||
"correspondent": False,
|
||||
"document_type": False,
|
||||
"tags": False,
|
||||
"document_range": False,
|
||||
"sort": False,
|
||||
"continuation": True,
|
||||
},
|
||||
}
|
||||
|
||||
CAPABILITIES = {
|
||||
"paperless": PAPERLESS_CAPABILITIES,
|
||||
"imap": IMAP_CAPABILITIES,
|
||||
}
|
||||
|
||||
EDITORS = {
|
||||
"note": True,
|
||||
"markdown_file": False,
|
||||
}
|
||||
|
||||
UNSUPPORTED_SEARCH = (
|
||||
"title",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"tag",
|
||||
"document_from",
|
||||
"document_to",
|
||||
"sort",
|
||||
"cursor",
|
||||
)
|
||||
|
||||
|
||||
def capability_payload() -> dict:
|
||||
from app.sourceregistry import registered_sources
|
||||
|
||||
return {
|
||||
"sources": [module.capabilities for module in registered_sources()],
|
||||
"editors": EDITORS,
|
||||
"protection_profiles": {
|
||||
"legacy-plaintext": "cleartext",
|
||||
"managed-encrypted": "locked",
|
||||
"client-encrypted": "locked",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def unsupported_filters(values: dict[str, str]) -> list[str]:
|
||||
return [name for name in UNSUPPORTED_SEARCH if str(values.get(name) or "").strip()]
|
||||
|
||||
|
||||
def with_capabilities(kind: str, connector_id: str, payload: dict) -> dict:
|
||||
from app.sourceregistry import require_source
|
||||
|
||||
body = dict(payload)
|
||||
spec = require_source(kind).capabilities
|
||||
body["connector_id"] = connector_id
|
||||
body["capabilities"] = spec
|
||||
body["unsupported"] = [name for name, enabled in spec["search"].items() if not enabled]
|
||||
body["continuation"] = body.get("next_before") if spec["search"].get("continuation") else None
|
||||
return body
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Registered source modules. Unknown kinds are rejected, not guessed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from app.core.errors import unprocessable
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceModule:
|
||||
kind: str
|
||||
label: str
|
||||
representation: str
|
||||
capabilities: dict
|
||||
parse_config: Callable[[dict], tuple[str, dict, str]]
|
||||
probe: Callable
|
||||
find: Callable
|
||||
open_original: Callable
|
||||
export_original: Callable
|
||||
navigate: Callable | None = None
|
||||
preview: Callable | None = None
|
||||
|
||||
|
||||
_MODULES: dict[str, SourceModule] = {}
|
||||
|
||||
|
||||
def register_source(module: SourceModule) -> None:
|
||||
_MODULES[module.kind] = module
|
||||
|
||||
|
||||
def unregister_source(kind: str) -> None:
|
||||
_MODULES.pop(kind, None)
|
||||
|
||||
|
||||
def registered_sources() -> list[SourceModule]:
|
||||
return list(_MODULES.values())
|
||||
|
||||
|
||||
def get_source(kind: str) -> SourceModule | None:
|
||||
return _MODULES.get(kind)
|
||||
|
||||
|
||||
def require_source(kind: str) -> SourceModule:
|
||||
module = _MODULES.get(kind)
|
||||
if module is None:
|
||||
raise unprocessable("Diese Quelle ist nicht registriert.")
|
||||
return module
|
||||
|
||||
|
||||
def accepts_reference(source: str) -> bool:
|
||||
return source == "url" or source in _MODULES
|
||||
+255
-76
@@ -23,25 +23,12 @@ from app.core.errors import Problem, not_found, unprocessable
|
||||
from app.core.settings import get_settings
|
||||
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
|
||||
from app.media.storage import sniff
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceport import CAPABILITIES
|
||||
from app.sourceregistry import SourceModule, get_source, register_source
|
||||
|
||||
|
||||
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
|
||||
if kind == "paperless":
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM connectors WHERE owner_id = %s AND kind = 'paperless'",
|
||||
(owner_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE connectors
|
||||
SET name = %s, settings = %s, secret = %s
|
||||
WHERE id = %s
|
||||
RETURNING id, kind, name, settings, created_at
|
||||
""",
|
||||
(name, Json(settings), secret, existing["id"]),
|
||||
).fetchone()
|
||||
return _public(row)
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO connectors (owner_id, kind, name, settings, secret)
|
||||
@@ -53,6 +40,21 @@ def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, s
|
||||
return _public(row)
|
||||
|
||||
|
||||
def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict:
|
||||
"""Credential or name change on the same instance. The id does not move."""
|
||||
current = _owned(conn, owner_id, connector_id)
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE connectors
|
||||
SET name = %s, settings = %s, secret = %s
|
||||
WHERE id = %s AND owner_id = %s
|
||||
RETURNING id, kind, name, settings, created_at
|
||||
""",
|
||||
(name, Json(settings), secret, current["id"], owner_id),
|
||||
).fetchone()
|
||||
return _public(row)
|
||||
|
||||
|
||||
def list_connectors(conn, owner_id: UUID) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
@@ -76,24 +78,31 @@ def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
|
||||
|
||||
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] == "paperless":
|
||||
count = _paperless_count(row["settings"]["base_url"], row["secret"])
|
||||
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
||||
_imap_login(row["settings"], row["secret"])
|
||||
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
|
||||
module = get_source(row["kind"])
|
||||
if module is None:
|
||||
raise unprocessable("Diese Quelle ist nicht registriert.")
|
||||
return module.probe(row)
|
||||
|
||||
|
||||
def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
||||
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None) -> tuple[list[dict], str]:
|
||||
if connector_id is not None:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "paperless":
|
||||
raise unprocessable("Das ist kein Paperless-Zugang.")
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
|
||||
row = rows[0]
|
||||
base = row["settings"]["base_url"].rstrip("/")
|
||||
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
|
||||
items = []
|
||||
@@ -109,7 +118,7 @@ def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
|
||||
"open_url": f"{base}/documents/{doc.get('id')}/",
|
||||
}
|
||||
)
|
||||
return items
|
||||
return items, str(row["id"])
|
||||
|
||||
|
||||
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
||||
@@ -177,6 +186,7 @@ def _public(row) -> dict:
|
||||
"name": row["name"],
|
||||
"settings": row["settings"],
|
||||
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None),
|
||||
}
|
||||
|
||||
|
||||
@@ -195,19 +205,25 @@ def _paperless_count(base: str, token: str) -> int:
|
||||
return int(payload.get("count") or 0)
|
||||
|
||||
|
||||
def paperless_content(conn, owner_id: UUID, document_id: str) -> tuple[bytes, str, str]:
|
||||
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
|
||||
if not document_id.isdigit() or len(document_id) > 12:
|
||||
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY created_at DESC LIMIT 1
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise unprocessable("Paperless ist für dieses Konto nicht verbunden.")
|
||||
if connector_id is not None:
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "paperless":
|
||||
raise unprocessable("Das ist kein Paperless-Zugang.")
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, settings, secret FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für Paperless ist kein eindeutiger Zugang zugeordnet.")
|
||||
row = rows[0]
|
||||
return _paperless_bytes(row, document_id)
|
||||
|
||||
|
||||
@@ -240,23 +256,28 @@ def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | No
|
||||
"""The stored document. A grantee never uses a connector of their own account."""
|
||||
detail = (ref or {}).get("detail") or {}
|
||||
external_id = str((ref or {}).get("external_id") or "")
|
||||
stored = _uuid(detail.get("connector_id"))
|
||||
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
|
||||
stored_owner = _connector_owner(conn, stored)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
return paperless_content(conn, actor_id, external_id)
|
||||
if not external_id.isdigit() or len(external_id) > 12:
|
||||
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
||||
result = resolve_for_read(conn, actor_id, "paperless", ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
return _paperless_bytes(result.row, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None or not external_id.isdigit() or len(external_id) > 12:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
row = _mediated_paperless(conn, owner_id, stored, element)
|
||||
if row is None:
|
||||
result = resolve_for_read(conn, owner_id, "paperless", ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", stored, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
data, media, name = _paperless_bytes(row, external_id)
|
||||
data, media, name = _paperless_bytes(result.row, external_id)
|
||||
except Problem:
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", row["id"], "unavailable")
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", row["id"], "ok")
|
||||
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "ok")
|
||||
if external_id and external_id in (name or ""):
|
||||
name = "dokument.pdf"
|
||||
return data, media, name or "dokument.pdf"
|
||||
@@ -267,28 +288,31 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
|
||||
detail = (ref or {}).get("detail") or {}
|
||||
if ref is None or ref["source"] != "imap":
|
||||
raise unprocessable("Das ist keine E-Mail.")
|
||||
connector_id = _uuid(detail.get("connector_id"))
|
||||
connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
|
||||
mailbox = str(detail.get("mailbox") or "").strip()
|
||||
uid = _mail_uid(detail.get("uid"))
|
||||
external_id = str(ref["external_id"])
|
||||
stored_owner = _connector_owner(conn, connector_id)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
if connector_id is None or not mailbox or uid is None:
|
||||
result = resolve_for_read(conn, actor_id, "imap", ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
if not mailbox or uid is None:
|
||||
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
return load_imap_original(conn, actor_id, connector_id, mailbox, uid, external_id)
|
||||
return load_imap_original(conn, actor_id, result.row["id"], mailbox, uid, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None or connector_id is None or not mailbox or uid is None:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
row = _connector_for(conn, connector_id, owner_id, "imap")
|
||||
if row is None:
|
||||
result = resolve_for_read(conn, owner_id, "imap", ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
raw = load_imap_original(conn, owner_id, connector_id, mailbox, uid, external_id)
|
||||
raw = load_imap_original(conn, owner_id, result.row["id"], mailbox, uid, external_id)
|
||||
except Problem:
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "ok")
|
||||
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "ok")
|
||||
return raw
|
||||
|
||||
|
||||
@@ -329,23 +353,6 @@ def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
|
||||
).fetchone()
|
||||
|
||||
|
||||
def _mediated_paperless(conn, owner_id: UUID, stored: UUID | None, element: dict):
|
||||
if stored is not None:
|
||||
return _connector_for(conn, stored, owner_id, "paperless")
|
||||
if element["owner_subject_id"] != owner_id:
|
||||
return None
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT id, owner_id, kind, settings, secret
|
||||
FROM connectors
|
||||
WHERE owner_id = %s AND kind = 'paperless'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def _audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
|
||||
audit(
|
||||
conn,
|
||||
@@ -1014,3 +1021,175 @@ def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> l
|
||||
if len(found) > 5:
|
||||
return found
|
||||
return found
|
||||
|
||||
|
||||
def _paperless_config(payload: dict) -> tuple[str, dict, str]:
|
||||
base_url = str(payload.get("base_url") or "").strip()
|
||||
token = str(payload.get("token") or "").strip()
|
||||
name = str(payload.get("name") or "").strip()
|
||||
if not base_url or len(token) < 8:
|
||||
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
|
||||
return name, {"base_url": clean_base_url(base_url)}, token
|
||||
|
||||
|
||||
def _imap_config(payload: dict) -> tuple[str, dict, str]:
|
||||
host = str(payload.get("host") or "").strip()
|
||||
username = str(payload.get("username") or "").strip()
|
||||
password = str(payload.get("password") or "")
|
||||
name = str(payload.get("name") or "").strip()
|
||||
if not host or not username or not password:
|
||||
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
|
||||
if " " in host or len(host) > 200:
|
||||
raise unprocessable("Der Mailserver ist ungültig.")
|
||||
port = payload.get("port") or 993
|
||||
return name, {
|
||||
"host": host,
|
||||
"port": int(port),
|
||||
"username": username,
|
||||
"mailbox": str(payload.get("mailbox") or "INBOX").strip() or "INBOX",
|
||||
"security": payload.get("security") or "ssl",
|
||||
}, password
|
||||
|
||||
|
||||
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
del options
|
||||
items, bound = paperless_documents(conn, owner_id, query, connector_id)
|
||||
return {"items": items, "connector_id": bound}
|
||||
|
||||
|
||||
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
if connector_id is None:
|
||||
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
|
||||
before = options.get("before")
|
||||
return imap_messages(conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before)
|
||||
|
||||
|
||||
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
||||
from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem
|
||||
|
||||
del detail
|
||||
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
|
||||
if resolved.row is None:
|
||||
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
||||
try:
|
||||
data, media, name = pull_paperless_original(
|
||||
resolved.row["settings"]["base_url"].rstrip("/"),
|
||||
resolved.row["secret"],
|
||||
str(external_id),
|
||||
EXPORT_BYTE_LIMIT,
|
||||
)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(data) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
|
||||
return FilePayload(name or f"paperless-{external_id}", media, data, "")
|
||||
|
||||
|
||||
def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
||||
from app.mailpack import identities_match, parse_mail
|
||||
from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem
|
||||
|
||||
resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned)
|
||||
if resolved.row is None:
|
||||
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
||||
mailbox = str(detail.get("mailbox") or "").strip()
|
||||
uid = detail.get("uid")
|
||||
if isinstance(uid, str) and uid.isdigit():
|
||||
uid = int(uid)
|
||||
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
||||
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
try:
|
||||
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(raw) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure(
|
||||
"too_large",
|
||||
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
||||
"Eine gekürzte Vorschau wird nicht als Original übernommen.",
|
||||
)
|
||||
parsed = parse_mail(raw)
|
||||
if not identities_match(parsed.message_id, str(external_id)):
|
||||
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
|
||||
if parsed.missing:
|
||||
names = ", ".join(parsed.missing[:5])
|
||||
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
|
||||
return MailPayload(raw, parsed)
|
||||
|
||||
|
||||
def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
||||
raw = read_linked_mail(conn, actor_id, element, context, ref)
|
||||
return raw, "message/rfc822", "nachricht.eml"
|
||||
|
||||
|
||||
class _PaperlessAdapter:
|
||||
kind = "paperless"
|
||||
|
||||
def capabilities(self) -> dict:
|
||||
return CAPABILITIES[self.kind]
|
||||
|
||||
def probe(self, row) -> dict:
|
||||
count = _paperless_count(row["settings"]["base_url"], row["secret"])
|
||||
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
||||
|
||||
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
|
||||
return paperless_documents(conn, owner_id, query, connector_id)
|
||||
|
||||
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
return _paperless_find(conn, owner_id, connector_id, query, options)
|
||||
|
||||
|
||||
class _ImapAdapter:
|
||||
kind = "imap"
|
||||
|
||||
def capabilities(self) -> dict:
|
||||
return CAPABILITIES[self.kind]
|
||||
|
||||
def probe(self, row) -> dict:
|
||||
_imap_login(row["settings"], row["secret"])
|
||||
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
|
||||
|
||||
def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
||||
return imap_mailboxes(conn, owner_id, connector_id)
|
||||
|
||||
def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict:
|
||||
return imap_messages(conn, owner_id, connector_id, query, mailbox, before)
|
||||
|
||||
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
return _imap_find(conn, owner_id, connector_id, query, options)
|
||||
|
||||
|
||||
def _install_sources() -> None:
|
||||
paperless = _PaperlessAdapter()
|
||||
imap = _ImapAdapter()
|
||||
register_source(SourceModule(
|
||||
kind="paperless",
|
||||
label="Paperless",
|
||||
representation="pdf",
|
||||
capabilities=CAPABILITIES["paperless"],
|
||||
parse_config=_paperless_config,
|
||||
probe=paperless.probe,
|
||||
find=paperless.find,
|
||||
open_original=read_linked_paperless,
|
||||
export_original=_export_paperless,
|
||||
navigate=None,
|
||||
preview=None,
|
||||
))
|
||||
register_source(SourceModule(
|
||||
kind="imap",
|
||||
label="E-Mail",
|
||||
representation="email",
|
||||
capabilities=CAPABILITIES["imap"],
|
||||
parse_config=_imap_config,
|
||||
probe=imap.probe,
|
||||
find=imap.find,
|
||||
open_original=_imap_bytes,
|
||||
export_original=_export_imap,
|
||||
navigate=imap.navigate,
|
||||
preview=lambda conn, owner, connector_id, options: imap_preview(
|
||||
conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0)
|
||||
),
|
||||
))
|
||||
|
||||
|
||||
_install_sources()
|
||||
@@ -0,0 +1,66 @@
|
||||
-- Source instance binding and opaque storage generation.
|
||||
-- Connector ids are not a foreign key: deleting an access must not
|
||||
-- retarget or remove an existing reference.
|
||||
-- The legacy root path is bound in the migrator, not in this file.
|
||||
|
||||
CREATE TABLE storage_instances (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
kind text NOT NULL,
|
||||
generation text NOT NULL,
|
||||
root_path text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT storage_instances_kind CHECK (kind IN ('local')),
|
||||
CONSTRAINT storage_instances_generation UNIQUE (generation),
|
||||
CONSTRAINT storage_instances_generation_len CHECK (char_length(generation) BETWEEN 1 AND 80)
|
||||
);
|
||||
|
||||
ALTER TABLE media_assets
|
||||
ADD COLUMN storage_instance_id uuid REFERENCES storage_instances (id),
|
||||
ADD COLUMN storage_generation text,
|
||||
ADD COLUMN object_key text,
|
||||
ADD COLUMN content_revision text,
|
||||
ADD COLUMN protection_profile text NOT NULL DEFAULT 'legacy-plaintext',
|
||||
ADD COLUMN key_responsibility text NOT NULL DEFAULT 'none',
|
||||
ADD CONSTRAINT media_assets_protection CHECK (
|
||||
protection_profile IN ('legacy-plaintext', 'managed-encrypted', 'client-encrypted')
|
||||
),
|
||||
ADD CONSTRAINT media_assets_key_responsibility CHECK (
|
||||
key_responsibility IN ('none', 'operator', 'client')
|
||||
);
|
||||
|
||||
ALTER TABLE dispatch_snapshots
|
||||
ADD COLUMN storage_instance_id uuid REFERENCES storage_instances (id),
|
||||
ADD COLUMN storage_generation text,
|
||||
ADD COLUMN object_key text,
|
||||
ADD COLUMN content_revision text,
|
||||
ADD COLUMN protection_profile text NOT NULL DEFAULT 'legacy-plaintext',
|
||||
ADD COLUMN key_responsibility text NOT NULL DEFAULT 'none',
|
||||
ADD CONSTRAINT dispatch_snapshots_protection CHECK (
|
||||
protection_profile IN ('legacy-plaintext', 'managed-encrypted', 'client-encrypted')
|
||||
),
|
||||
ADD CONSTRAINT dispatch_snapshots_key_responsibility CHECK (
|
||||
key_responsibility IN ('none', 'operator', 'client')
|
||||
);
|
||||
|
||||
UPDATE media_assets
|
||||
SET storage_generation = 'legacy-local-1',
|
||||
object_key = id::text,
|
||||
content_revision = id::text,
|
||||
protection_profile = 'legacy-plaintext',
|
||||
key_responsibility = 'none'
|
||||
WHERE object_key IS NULL;
|
||||
|
||||
UPDATE dispatch_snapshots
|
||||
SET storage_generation = 'legacy-local-1',
|
||||
object_key = 'dispatch/' || id::text,
|
||||
content_revision = id::text,
|
||||
protection_profile = 'legacy-plaintext',
|
||||
key_responsibility = 'none'
|
||||
WHERE object_key IS NULL;
|
||||
|
||||
ALTER TABLE external_references
|
||||
ADD COLUMN connector_id uuid,
|
||||
ADD COLUMN resolution text NOT NULL DEFAULT 'unresolved',
|
||||
ADD CONSTRAINT external_references_resolution CHECK (
|
||||
resolution IN ('bound', 'ambiguous', 'unresolved', 'none')
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Connector kinds are validated by the registered source modules.
|
||||
-- A database checklist must not grow for every additional source.
|
||||
|
||||
ALTER TABLE connectors DROP CONSTRAINT IF EXISTS connectors_kind;
|
||||
ALTER TABLE external_references DROP CONSTRAINT IF EXISTS external_references_source;
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Third source used only to prove that registration is enough.
|
||||
|
||||
This module is not a product connector and is not imported by the application.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from app.access.policy import mediated_source_owner
|
||||
from app.core.errors import not_found, unprocessable
|
||||
from app.retrieval import FilePayload, RetrievalFailure
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceregistry import SourceModule, register_source, unregister_source
|
||||
from app.sources import ORIGINAL_UNAVAILABLE, _audit_mediated, _connector_owner
|
||||
|
||||
KIND = "sample"
|
||||
|
||||
CATALOG = {
|
||||
"alpha": {
|
||||
"brief": {
|
||||
"id": "brief",
|
||||
"title": "Probenbrief",
|
||||
"text": "Der Probenbrief bleibt lesbar.",
|
||||
},
|
||||
"notiz": {
|
||||
"id": "notiz",
|
||||
"title": "Randnotiz",
|
||||
"text": "Die Randnotiz bleibt in der Quelle.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
CAPABILITIES = {
|
||||
"kind": KIND,
|
||||
"navigate": False,
|
||||
"read_original": True,
|
||||
"read_preview": True,
|
||||
"read_attachment": False,
|
||||
"edit_content": False,
|
||||
"edit_metadata": False,
|
||||
"search": {
|
||||
"query": True,
|
||||
"title": False,
|
||||
"correspondent": False,
|
||||
"document_type": False,
|
||||
"tags": False,
|
||||
"document_range": False,
|
||||
"sort": False,
|
||||
"continuation": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_config(payload: dict) -> tuple[str, dict, str]:
|
||||
config = payload.get("config") or {}
|
||||
catalog = str(config.get("catalog") or "").strip()
|
||||
name = str(payload.get("name") or "").strip() or catalog
|
||||
if catalog not in CATALOG:
|
||||
raise unprocessable("Der Katalog dieser Beispielquelle ist unbekannt.")
|
||||
return name, {"catalog": catalog}, ""
|
||||
|
||||
|
||||
def probe(row) -> dict:
|
||||
items = _items(row)
|
||||
return {"ok": True, "detail": f"Beispielquelle antwortet. {len(items)} Einträge sichtbar."}
|
||||
|
||||
|
||||
def find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
||||
del options
|
||||
row = _connector(conn, owner_id, connector_id)
|
||||
needle = query.casefold()
|
||||
items = list(_items(row).values())
|
||||
if needle:
|
||||
items = [
|
||||
item for item in items
|
||||
if needle in item["title"].casefold() or needle in item["text"].casefold()
|
||||
]
|
||||
return {"items": items, "connector_id": str(row["id"])}
|
||||
|
||||
|
||||
def preview(conn, owner_id: UUID, connector_id: UUID, options: dict) -> dict:
|
||||
row = _connector(conn, owner_id, connector_id)
|
||||
item = _items(row).get(str(options.get("item") or ""))
|
||||
if item is None:
|
||||
raise unprocessable("Dieser Eintrag ist in der Beispielquelle nicht vorhanden.")
|
||||
return {"id": item["id"], "title": item["title"], "text": item["text"], "media": "text"}
|
||||
|
||||
|
||||
def open_original(conn, actor_id: UUID, element: dict, context, ref) -> tuple[bytes, str, str]:
|
||||
external_id = str((ref or {}).get("external_id") or "")
|
||||
stored = ref.get("connector_id") if ref else None
|
||||
stored_owner = _connector_owner(conn, stored)
|
||||
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
|
||||
result = resolve_for_read(conn, actor_id, KIND, ref, None)
|
||||
if result.row is None:
|
||||
raise unprocessable(result.message)
|
||||
return _bytes(result.row, external_id)
|
||||
owner_id = mediated_source_owner(conn, actor_id, element, context)
|
||||
if owner_id is None:
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
result = resolve_for_read(conn, owner_id, KIND, ref, None)
|
||||
if result.row is None:
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, stored, "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
||||
try:
|
||||
data, media, name = _bytes(result.row, external_id)
|
||||
except Exception:
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "unavailable")
|
||||
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
|
||||
_audit_mediated(conn, actor_id, element, context, KIND, result.row["id"], "ok")
|
||||
return data, media, name
|
||||
|
||||
|
||||
def export_original(conn, actor_id: UUID, external_id: str, detail: dict, assigned, ref):
|
||||
del detail
|
||||
result = resolve_for_read(conn, actor_id, KIND, ref, assigned)
|
||||
if result.row is None:
|
||||
return RetrievalFailure(result.code, result.message, result.choices)
|
||||
try:
|
||||
data, media, name = _bytes(result.row, str(external_id))
|
||||
except Exception as exc:
|
||||
return RetrievalFailure("missing", str(getattr(exc, "detail", exc)))
|
||||
return FilePayload(name, media, data, "")
|
||||
|
||||
|
||||
def install_sample_source() -> None:
|
||||
register_source(SourceModule(
|
||||
kind=KIND,
|
||||
label="Beispielquelle",
|
||||
representation="text",
|
||||
capabilities=CAPABILITIES,
|
||||
parse_config=parse_config,
|
||||
probe=probe,
|
||||
find=find,
|
||||
open_original=open_original,
|
||||
export_original=export_original,
|
||||
navigate=None,
|
||||
preview=preview,
|
||||
))
|
||||
|
||||
|
||||
def remove_sample_source() -> None:
|
||||
unregister_source(KIND)
|
||||
|
||||
|
||||
def _connector(conn, owner_id: UUID, connector_id: UUID | None):
|
||||
if connector_id is not None:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, owner_id, kind, name, settings, secret
|
||||
FROM connectors WHERE id = %s AND owner_id = %s
|
||||
""",
|
||||
(connector_id, owner_id),
|
||||
).fetchone()
|
||||
if row is None or row["kind"] != KIND:
|
||||
raise not_found()
|
||||
return row
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, owner_id, kind, name, settings, secret
|
||||
FROM connectors
|
||||
WHERE owner_id = %s AND kind = %s
|
||||
ORDER BY name, id
|
||||
""",
|
||||
(owner_id, KIND),
|
||||
).fetchall()
|
||||
if len(rows) != 1:
|
||||
raise unprocessable("Für diese Quelle ist kein eindeutiger Zugang zugeordnet.")
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _items(row) -> dict:
|
||||
catalog = (row["settings"] or {}).get("catalog")
|
||||
return CATALOG.get(catalog) or {}
|
||||
|
||||
|
||||
def _bytes(row, external_id: str) -> tuple[bytes, str, str]:
|
||||
item = _items(row).get(external_id)
|
||||
if item is None:
|
||||
raise unprocessable("Dieser Eintrag ist in der Beispielquelle nicht vorhanden.")
|
||||
return item["text"].encode("utf-8"), "text/plain", f"{item['id']}.txt"
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Section A: source binding and the storage port, with today's behaviour kept."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import Problem
|
||||
from app.media.port import (
|
||||
LEGACY_GENERATION,
|
||||
LocalAdapter,
|
||||
MemoryAdapter,
|
||||
check_object_key,
|
||||
instance_for_write,
|
||||
publish,
|
||||
read_clear_bytes,
|
||||
)
|
||||
from app.sourcebind import backfill_source_bindings
|
||||
from tests.support import ORIGIN, Actor, register
|
||||
|
||||
PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n"
|
||||
|
||||
|
||||
def _raw(actor: Actor, path: str):
|
||||
return actor.client.get(
|
||||
path,
|
||||
headers={
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": actor.csrf,
|
||||
"X-Scope-Kind": actor.scope["kind"],
|
||||
"X-Scope-Id": actor.scope["id"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _case(actor: Actor) -> dict:
|
||||
return actor.request("POST", "/api/v1/cases", json={"title": "Bestand"}, status=201)
|
||||
|
||||
|
||||
def _paperless(actor: Actor, name: str, token: str) -> str:
|
||||
row = actor.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": name, "base_url": "https://paper.example.test", "token": token},
|
||||
status=201,
|
||||
)
|
||||
return row["id"]
|
||||
|
||||
|
||||
def _link(actor: Actor, case_id: str, connector_id: str | None, external_id: str, title: str) -> str:
|
||||
body = {
|
||||
"title": title,
|
||||
"source": "paperless",
|
||||
"external_id": external_id,
|
||||
"context": {"kind": "case", "id": case_id},
|
||||
}
|
||||
if connector_id:
|
||||
body["detail"] = {"connector_id": connector_id}
|
||||
created = actor.request("POST", "/api/v1/external-refs", json=body, status=201)
|
||||
return created["id"]
|
||||
|
||||
|
||||
def test_storage_adapters_roundtrip_and_reject_escape(tmp_path):
|
||||
payload = b"opaque-bytes"
|
||||
key = str(uuid4())
|
||||
revision = "rev-1"
|
||||
for adapter in (LocalAdapter(tmp_path), MemoryAdapter()):
|
||||
publish(adapter, key, payload)
|
||||
assert adapter.open(key) == payload
|
||||
assert adapter.describe(key)["byte_size"] == len(payload)
|
||||
assert adapter.open(key) == payload
|
||||
assert revision == "rev-1"
|
||||
memory = MemoryAdapter()
|
||||
memory.fail_put = True
|
||||
with pytest.raises(Problem) as failed:
|
||||
publish(memory, str(uuid4()), payload)
|
||||
assert failed.value.code == "storage_unavailable"
|
||||
assert memory._items == {}
|
||||
with pytest.raises(Problem):
|
||||
check_object_key("../secret")
|
||||
with pytest.raises(Problem):
|
||||
check_object_key("/etc/passwd")
|
||||
with pytest.raises(Problem):
|
||||
LocalAdapter(tmp_path).open("dispatch/../../" + str(uuid4()))
|
||||
|
||||
|
||||
def test_changed_root_does_not_retarget_legacy(db, database):
|
||||
legacy = db.execute(
|
||||
"SELECT id, root_path FROM storage_instances WHERE generation = %s",
|
||||
(LEGACY_GENERATION,),
|
||||
).fetchone()
|
||||
key = str(uuid4())
|
||||
original = b"%PDF-legacy-root"
|
||||
publish(LocalAdapter(legacy["root_path"]), key, original)
|
||||
other = Path(database.media_root).parent / f"other-root-{uuid4().hex}"
|
||||
other.mkdir()
|
||||
moved = instance_for_write(db, str(other))
|
||||
assert moved["id"] != legacy["id"]
|
||||
assert moved["generation"] != LEGACY_GENERATION
|
||||
publish(LocalAdapter(moved["root_path"]), key, b"%PDF-other-root")
|
||||
seen = read_clear_bytes(
|
||||
db,
|
||||
instance_id=legacy["id"],
|
||||
generation=LEGACY_GENERATION,
|
||||
object_key=key,
|
||||
media_root=str(other),
|
||||
protection_profile="legacy-plaintext",
|
||||
expected_sha256=None,
|
||||
)
|
||||
assert seen == original
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_existing_file_keeps_id_bytes_and_revision(client, db):
|
||||
register(db, "ada-file@example.test", "Ada File")
|
||||
ada = Actor(client, "ada-file@example.test", "Ada File")
|
||||
ada.login()
|
||||
case = _case(ada)
|
||||
upload = ada.request(
|
||||
"POST",
|
||||
"/api/v1/files",
|
||||
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
|
||||
files={"upload": ("scan.pdf", PDF, "application/pdf")},
|
||||
status=201,
|
||||
)
|
||||
stored = db.execute(
|
||||
"""
|
||||
SELECT a.id, a.object_key, a.content_revision, a.storage_generation, a.protection_profile,
|
||||
s.root_path
|
||||
FROM media_assets a
|
||||
JOIN storage_instances s ON s.id = a.storage_instance_id
|
||||
WHERE a.element_id = %s
|
||||
""",
|
||||
(upload["id"],),
|
||||
).fetchone()
|
||||
assert stored["protection_profile"] == "legacy-plaintext"
|
||||
assert stored["object_key"] == str(stored["id"])
|
||||
assert Path(stored["root_path"]) == Path(client.app.state.settings.media_root).resolve()
|
||||
assert stored["storage_generation"]
|
||||
first = stored["content_revision"]
|
||||
opened = _raw(ada, f"/api/v1/files/{upload['id']}/content")
|
||||
assert opened.status_code == 200
|
||||
assert opened.content == PDF
|
||||
again = db.execute(
|
||||
"SELECT content_revision FROM media_assets WHERE element_id = %s",
|
||||
(upload["id"],),
|
||||
).fetchone()
|
||||
assert again["content_revision"] == first
|
||||
refused = ada.client.patch(
|
||||
f"/api/v1/notes/{upload['id']}",
|
||||
json={"expected_version": 1, "document": {"version": 1, "blocks": [{"type": "paragraph", "text": "nein"}]}},
|
||||
headers={
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": ada.csrf,
|
||||
"X-Scope-Kind": ada.scope["kind"],
|
||||
"X-Scope-Id": ada.scope["id"],
|
||||
},
|
||||
)
|
||||
assert refused.status_code == 422
|
||||
assert refused.json()["code"] == "unsupported"
|
||||
|
||||
|
||||
def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monkeypatch):
|
||||
register(db, "ada-two@example.test", "Ada Two")
|
||||
ada = Actor(client, "ada-two@example.test", "Ada Two")
|
||||
ada.login()
|
||||
case = _case(ada)
|
||||
older = _paperless(ada, "Zweitens", "token-older")
|
||||
newer = _paperless(ada, "Erstens", "token-newer")
|
||||
assert older != newer
|
||||
|
||||
def fake_bytes(row, document_id):
|
||||
return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf"
|
||||
|
||||
monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes)
|
||||
left = _link(ada, case["id"], older, "15", "Altbestand")
|
||||
right = _link(ada, case["id"], newer, "15", "Zweiter Bestand")
|
||||
db.execute("UPDATE connectors SET created_at = now() - interval '2 days' WHERE id = %s", (newer,))
|
||||
db.execute("UPDATE connectors SET created_at = now() WHERE id = %s", (older,))
|
||||
db.commit()
|
||||
assert _raw(ada, f"/api/v1/external-refs/{left}/content").content.startswith(b"%PDF-token-older-15")
|
||||
assert _raw(ada, f"/api/v1/external-refs/{right}/content").content.startswith(b"%PDF-token-newer-15")
|
||||
calls = []
|
||||
|
||||
def fake_get(*args, **kwargs):
|
||||
calls.append(args)
|
||||
return {"results": [{"id": 15, "title": "Treffer", "created": "2024-01-02"}]}
|
||||
|
||||
monkeypatch.setattr("app.sources._paperless_get", fake_get)
|
||||
headers = {
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": ada.csrf,
|
||||
"X-Scope-Kind": ada.scope["kind"],
|
||||
"X-Scope-Id": ada.scope["id"],
|
||||
}
|
||||
refused = ada.client.get("/api/v1/sources/paperless/documents", params={"q": "Treffer"}, headers=headers)
|
||||
assert refused.status_code == 422
|
||||
assert calls == []
|
||||
found = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"q": "Treffer", "connector_id": older},
|
||||
)
|
||||
assert found["items"][0]["title"] == "Treffer"
|
||||
assert found["connector_id"] == older
|
||||
assert found["capabilities"]["search"]["correspondent"] is False
|
||||
assert "correspondent" in found["unsupported"]
|
||||
filtered = ada.client.get(
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"q": "Treffer", "connector_id": older, "correspondent": "Haus"},
|
||||
headers=headers,
|
||||
)
|
||||
assert filtered.status_code == 422
|
||||
assert filtered.json()["code"] == "unsupported"
|
||||
assert "Haus" not in json.dumps(filtered.json())
|
||||
|
||||
|
||||
def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch):
|
||||
register(db, "ada-old@example.test", "Ada Old")
|
||||
ada = Actor(client, "ada-old@example.test", "Ada Old")
|
||||
ada.login()
|
||||
case = _case(ada)
|
||||
loose = _link(ada, case["id"], None, "15", "Ohne Instanz")
|
||||
_paperless(ada, "Alpha", "token-alpha")
|
||||
second = _paperless(ada, "Beta", "token-beta")
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET connector_id = NULL, resolution = 'unresolved', detail = '{}'::jsonb
|
||||
WHERE element_id = %s
|
||||
""",
|
||||
(loose,),
|
||||
)
|
||||
backfill_source_bindings(db)
|
||||
state = db.execute(
|
||||
"SELECT connector_id, resolution FROM external_references WHERE element_id = %s",
|
||||
(loose,),
|
||||
).fetchone()
|
||||
db.commit()
|
||||
assert state["resolution"] == "ambiguous"
|
||||
assert state["connector_id"] is None
|
||||
blocked = _raw(ada, f"/api/v1/external-refs/{loose}/content")
|
||||
assert blocked.status_code == 422
|
||||
assert "eindeutiger Zugang" in blocked.json()["detail"]
|
||||
|
||||
def fake_bytes(row, document_id):
|
||||
del document_id
|
||||
return f"%PDF-{row['secret']}\n".encode(), "application/pdf", "dokument.pdf"
|
||||
|
||||
monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes)
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/external-refs/{loose}/assignment",
|
||||
json={"connector_id": second},
|
||||
)
|
||||
_paperless(ada, "Gamma", "token-gamma")
|
||||
assert _raw(ada, f"/api/v1/external-refs/{loose}/content").content.startswith(b"%PDF-token-beta")
|
||||
|
||||
|
||||
def test_locked_profile_serves_metadata_not_cleartext(client, db):
|
||||
register(db, "ada-lock@example.test", "Ada Lock")
|
||||
ada = Actor(client, "ada-lock@example.test", "Ada Lock")
|
||||
ada.login()
|
||||
case = _case(ada)
|
||||
upload = ada.request(
|
||||
"POST",
|
||||
"/api/v1/files",
|
||||
data={"title": "Geschuetzt", "context_kind": "case", "context_id": case["id"]},
|
||||
files={"upload": ("geheim.pdf", PDF, "application/pdf")},
|
||||
status=201,
|
||||
)
|
||||
dummy = b"cipher-dummy-not-clear"
|
||||
asset = db.execute(
|
||||
"SELECT id, object_key, storage_instance_id FROM media_assets WHERE element_id = %s",
|
||||
(upload["id"],),
|
||||
).fetchone()
|
||||
root = db.execute(
|
||||
"SELECT root_path FROM storage_instances WHERE id = %s",
|
||||
(asset["storage_instance_id"],),
|
||||
).fetchone()["root_path"]
|
||||
path = Path(root) / asset["object_key"]
|
||||
path.write_bytes(dummy)
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE media_assets
|
||||
SET protection_profile = 'client-encrypted',
|
||||
key_responsibility = 'client',
|
||||
sha256 = %s
|
||||
WHERE element_id = %s
|
||||
""",
|
||||
(hashlib.sha256(dummy).hexdigest(), upload["id"]),
|
||||
)
|
||||
db.commit()
|
||||
detail = ada.request("GET", f"/api/v1/cases/{case['id']}")
|
||||
rendered = json.dumps(detail)
|
||||
assert "client-encrypted" in rendered
|
||||
assert "Geschuetzt" in rendered
|
||||
assert dummy.decode() not in rendered
|
||||
opened = _raw(ada, f"/api/v1/files/{upload['id']}/content")
|
||||
assert opened.status_code == 422
|
||||
assert opened.json()["code"] == "unsupported"
|
||||
assert path.read_bytes() == dummy
|
||||
|
||||
|
||||
def test_credential_update_keeps_the_instance(client, db):
|
||||
register(db, "ada-key@example.test", "Ada Key")
|
||||
ada = Actor(client, "ada-key@example.test", "Ada Key")
|
||||
ada.login()
|
||||
first = _paperless(ada, "Archiv", "token-first-1")
|
||||
second = _paperless(ada, "Anderes Archiv", "token-second-2")
|
||||
renewed = ada.request(
|
||||
"PATCH",
|
||||
f"/api/v1/connectors/{first}",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-renewed"},
|
||||
)
|
||||
assert renewed["id"] == first
|
||||
secrets = {
|
||||
str(row["id"]): row["secret"]
|
||||
for row in db.execute("SELECT id, secret FROM connectors").fetchall()
|
||||
}
|
||||
assert secrets[first] == "token-renewed"
|
||||
assert secrets[second] == "token-second-2"
|
||||
caps = ada.request("GET", "/api/v1/sources/capabilities")
|
||||
assert caps["editors"]["note"] is True
|
||||
assert caps["editors"]["markdown_file"] is False
|
||||
assert caps["protection_profiles"]["client-encrypted"] == "locked"
|
||||
paperless = next(item for item in caps["sources"] if item["kind"] == "paperless")
|
||||
assert paperless["search"]["query"] is True
|
||||
assert paperless["edit_content"] is False
|
||||
@@ -230,7 +230,7 @@ def test_dispatch_keeps_the_secured_stand_and_distinct_mail_parts(client, db, mo
|
||||
"Fachlicher Zusatz: die Frist bleibt der 1. Juni.",
|
||||
[("rechnung.pdf", PDF_A), ("lage.pdf", PDF_B), ("konto.pdf", PDF_C), ("lage.pdf", PDF_D)],
|
||||
)
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
ada = Actor(client, "ada-versand@example.test", "Ada Versand")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
|
||||
@@ -141,9 +141,9 @@ def test_handover_package_follows_the_confirmed_selection(client, db, monkeypatc
|
||||
"Der vollständige Brieftext bleibt erhalten.",
|
||||
[("lage.pdf", PDF_A), ("lage.pdf", PDF_B)],
|
||||
)
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
monkeypatch.setattr(
|
||||
"app.retrieval.pull_paperless_original",
|
||||
"app.sources.pull_paperless_original",
|
||||
lambda *args, **kwargs: (PDF_B, "application/pdf", "archiv.pdf"),
|
||||
)
|
||||
ada = Actor(client, "ada-export@example.test", "Ada Export")
|
||||
@@ -491,7 +491,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
|
||||
calls.append(args)
|
||||
return _eml("<stored@example.test>", "darf nicht ersatzweise kommen", [])
|
||||
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", pull)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", pull)
|
||||
context = {"kind": "case", "id": case["id"]}
|
||||
tree = ada.request(
|
||||
"GET",
|
||||
@@ -547,7 +547,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
|
||||
assert {item["name"] for item in pending["issues"][0]["choices"]} == {"Neu", "Weiter"}
|
||||
monkeypatch.setattr("app.retrieval.EXPORT_BYTE_LIMIT", 40)
|
||||
huge = _eml("<offen@example.test>", "GEKÜRZT-VERBOTEN " * 20, [])
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: huge)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: huge)
|
||||
assigned = ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{matter['id']}/export/review",
|
||||
@@ -557,7 +557,7 @@ def test_export_names_gaps_and_hides_unreadable_counterparts(client, db, monkeyp
|
||||
assert "GEKÜRZT-VERBOTEN" not in json.dumps(assigned)
|
||||
wrong = _eml("<andere@example.test>", "FALSCHE-NACHRICHT", [])
|
||||
monkeypatch.setattr("app.retrieval.EXPORT_BYTE_LIMIT", 80 * 1024 * 1024)
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: wrong)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: wrong)
|
||||
identity = ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{matter['id']}/export/review",
|
||||
@@ -609,7 +609,7 @@ def test_nested_chapter_export_keeps_path_and_complete_mail(client, db, monkeypa
|
||||
"Die Antwort bleibt vollständig.",
|
||||
[("lage.pdf", PDF_A), ("konto.pdf", PDF_B)],
|
||||
)
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
ada = Actor(client, "ada-nested-export@example.test", "Ada Verschachtelt")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
@@ -844,7 +844,7 @@ def test_chapter_entries_share_one_numbering_sequence(client, db, monkeypatch):
|
||||
files={"upload": ("drittes.pdf", PDF_C, "application/pdf")},
|
||||
status=201,
|
||||
)
|
||||
monkeypatch.setattr("app.retrieval.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
monkeypatch.setattr("app.sources.pull_rfc822", lambda *args, **kwargs: raw)
|
||||
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
|
||||
exportable = {"binding", "annotation", "attachment", "dispatch_item", "evidence"}
|
||||
selected = [node["id"] for node in _walk(tree) if node["exportable"] and node["kind"] in exportable]
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
"""A third source appears only by registering its module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.sample_source import install_sample_source, remove_sample_source
|
||||
from tests.support import ORIGIN, Actor, document, register
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
TEXT = "Der Probenbrief bleibt lesbar."
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_source():
|
||||
install_sample_source()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
remove_sample_source()
|
||||
|
||||
|
||||
def test_registered_sample_source_links_reads_and_exports(client, db, sample_source):
|
||||
del sample_source
|
||||
ada = Actor(client, "ada-sample@example.test", "Ada Probe")
|
||||
ben = Actor(client, "ben-sample@example.test", "Ben Probe")
|
||||
register(db, ada.email, ada.name)
|
||||
register(db, ben.email, ben.name)
|
||||
ada.login()
|
||||
refused = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "missing", "name": "Unbekannt"},
|
||||
status=422,
|
||||
)
|
||||
assert refused["code"] == "unprocessable"
|
||||
ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "sample", "name": "Ohne Katalog", "config": {"catalog": "kein-katalog"}},
|
||||
status=422,
|
||||
)
|
||||
connector_id = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "sample", "name": "Katalog Alpha", "config": {"catalog": "alpha"}},
|
||||
status=201,
|
||||
)["id"]
|
||||
probed = ada.request("POST", f"/api/v1/connectors/{connector_id}/probe")
|
||||
assert probed["ok"] is True
|
||||
assert "2 Einträge" in probed["detail"]
|
||||
blocked = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/sample/search",
|
||||
params={"q": "Probe", "correspondent": "geheim-wert"},
|
||||
status=422,
|
||||
)
|
||||
assert blocked["code"] == "unsupported"
|
||||
assert "geheim-wert" not in blocked["detail"]
|
||||
ada.request("GET", "/api/v1/sources/sample/navigate", params={"connector_id": connector_id}, status=422)
|
||||
found = ada.request("GET", "/api/v1/sources/sample/search", params={"q": "Proben", "connector_id": connector_id})
|
||||
assert [item["id"] for item in found["items"]] == ["brief"]
|
||||
preview = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/sample/preview",
|
||||
params={"connector_id": connector_id, "item": "brief"},
|
||||
)
|
||||
assert preview["text"] == TEXT
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Probenvorgang"}, status=201)
|
||||
linked = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Probenbrief",
|
||||
"source": "sample",
|
||||
"external_id": "brief",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": connector_id},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
opened = _raw(ada, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
|
||||
assert opened.status_code == 200
|
||||
assert opened.content == TEXT.encode("utf-8")
|
||||
assert opened.headers["content-type"].startswith("text/plain")
|
||||
detail = ada.request("GET", f"/api/v1/matters/{matter['id']}")
|
||||
element = next(item["element"] for item in detail["bindings"] if item["element"]["id"] == linked["id"])
|
||||
assert element["preview"]["media"] == "text"
|
||||
written = ada.request(
|
||||
"PATCH",
|
||||
f"/api/v1/notes/{linked['id']}",
|
||||
json={
|
||||
"expected_version": element["version"],
|
||||
"document": document("Das darf nicht geschrieben werden."),
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
},
|
||||
status=422,
|
||||
)
|
||||
assert written["code"] == "unsupported"
|
||||
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
|
||||
selected = [node["id"] for node in _walk(tree) if node["kind"] == "binding" and node.get("exportable")]
|
||||
body = {"context": {"kind": "matter", "id": matter["id"]}, "selected": selected}
|
||||
review = ada.request("POST", f"/api/v1/matters/{matter['id']}/export/review", json=body)
|
||||
assert review["ready"] is True, review
|
||||
package = _post_zip(ada, matter["id"], body)
|
||||
assert package.status_code == 200, package.text
|
||||
archive = zipfile.ZipFile(io.BytesIO(package.content))
|
||||
assert any(archive.read(name) == TEXT.encode("utf-8") for name in archive.namelist())
|
||||
|
||||
ben.login()
|
||||
ben.request(
|
||||
"GET",
|
||||
"/api/v1/sources/sample/search",
|
||||
params={"connector_id": connector_id, "q": "Proben"},
|
||||
status=404,
|
||||
)
|
||||
hidden = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
|
||||
assert hidden.status_code == 404
|
||||
ada.login()
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{matter['id']}/grants",
|
||||
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": False},
|
||||
status=201,
|
||||
)
|
||||
ben.login()
|
||||
denied = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
|
||||
assert denied.status_code == 422
|
||||
assert denied.json()["detail"] == "Das Original ist über die Freigabe nicht abrufbar."
|
||||
ada.login()
|
||||
grant_id = next(item["id"] for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["grants"])
|
||||
ada.request("DELETE", f"/api/v1/grants/{grant_id}", status=204)
|
||||
ada.request(
|
||||
"POST",
|
||||
f"/api/v1/matters/{matter['id']}/grants",
|
||||
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True},
|
||||
status=201,
|
||||
)
|
||||
ben.login()
|
||||
shared = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
|
||||
assert shared.status_code == 200
|
||||
assert shared.content == TEXT.encode("utf-8")
|
||||
shown = ben.request("GET", f"/api/v1/matters/{matter['id']}")
|
||||
visible = next(item["element"] for item in shown["bindings"] if item["element"]["id"] == linked["id"])
|
||||
assert "connector_id" not in (visible["external"].get("detail") or {})
|
||||
ada.login()
|
||||
grant_id = next(item["id"] for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["grants"])
|
||||
ada.request("DELETE", f"/api/v1/grants/{grant_id}", status=204)
|
||||
ben.login()
|
||||
revoked = _raw(ben, f"/api/v1/external-refs/{linked['id']}/content?context_kind=matter&context_id={matter['id']}")
|
||||
assert revoked.status_code == 404
|
||||
|
||||
|
||||
def test_sample_registration_does_not_edit_generic_modules():
|
||||
forbidden = [
|
||||
"backend/app/api/schemas.py",
|
||||
"backend/app/exporting.py",
|
||||
"backend/app/present.py",
|
||||
"frontend/src/pages/Cases.jsx",
|
||||
"frontend/src/pages/Matters.jsx",
|
||||
"frontend/src/pages/ReadingPane.jsx",
|
||||
"frontend/src/pages/ExportPackage.jsx",
|
||||
"frontend/src/App.jsx",
|
||||
"frontend/src/pages/Sources.jsx",
|
||||
]
|
||||
for relative in forbidden:
|
||||
text = (ROOT / relative).read_text(encoding="utf-8")
|
||||
assert '"sample"' not in text and "'sample'" not in text, relative
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"node",
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
(
|
||||
"import { installSampleSource, removeSampleSource } from './src/sources/sample-source.js';"
|
||||
"import { sourceModules } from './src/sources/registry-core.js';"
|
||||
"installSampleSource();"
|
||||
"if (!sourceModules().some((item) => item.kind === 'sample' && item.representation === 'text')) process.exit(2);"
|
||||
"removeSampleSource();"
|
||||
"if (sourceModules().some((item) => item.kind === 'sample')) process.exit(3);"
|
||||
),
|
||||
],
|
||||
cwd=ROOT / "frontend",
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
|
||||
|
||||
def _raw(actor: Actor, path: str):
|
||||
return actor.client.get(
|
||||
path,
|
||||
headers={
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": actor.csrf,
|
||||
"X-Scope-Kind": actor.scope["kind"],
|
||||
"X-Scope-Id": actor.scope["id"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _post_zip(actor: Actor, matter_id: str, body: dict):
|
||||
return actor.client.post(
|
||||
f"/api/v1/matters/{matter_id}/export",
|
||||
json=body,
|
||||
headers={
|
||||
"Origin": ORIGIN,
|
||||
"X-CSRF-Token": actor.csrf,
|
||||
"Content-Type": "application/json",
|
||||
"X-Scope-Kind": actor.scope["kind"],
|
||||
"X-Scope-Id": actor.scope["id"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _walk(node):
|
||||
yield node
|
||||
for child in node.get("children") or []:
|
||||
yield from _walk(child)
|
||||
Reference in new issue
Block a user