Files
tsumugu/backend/app/api/sources.py
T
LarsandCursor cd239a3ab8
Deploy Development / deploy (push) Successful in 47s
Test Suite / pytest-backend (push) Successful in 6m16s
Test Suite / smoke-dev (push) Successful in 1s
Test Suite / frontend-build (push) Successful in 22s
Zeige PDF-Anlagen in der Mail-Ansicht, damit der Browser sie nicht unter einem zufälligen Namen speichert.
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-23 12:59:07 +02:00

535 lines
19 KiB
Python

"""Connector administration and live Paperless / IMAP lookup."""
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from fastapi.responses import JSONResponse, Response
from psycopg import Connection
from app.access.policy import Ctx, element_rights, get_element, parse_query_context
from app.api.deps import Principal, get_conn, get_principal
from app.api.routes import _created, _ctx_model
from app.api.schemas import ConnectorIn, ExternalIn, SourceAssignmentIn
from app.core.errors import DOWNLOAD_REFUSED, forbidden, not_found, unprocessable, unsupported
from app.mailpack import _inline_images, parse_mail, sanitize_html
from app.remoteimages import RemoteImages
from app.media.storage import content_disposition
from app.ops import create_external
from app.retrieval import RetrievalFailure
from app.sourcebind import assign_reference
from app.sourceport import capability_payload, with_capabilities
from app.sourceregistry import get_source, require_source
from app.sources import (
connector_detail,
delete_connector,
imap_preview,
list_connectors,
owned_connector,
paperless_content,
parse_connector,
probe,
save_connector,
update_connector,
)
_PRIVATE = {
"Cache-Control": "private, no-store",
"Referrer-Policy": "no-referrer",
"X-Content-Type-Options": "nosniff",
}
router = APIRouter()
@router.get("/api/v1/connectors")
def get_connectors(principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
return {"items": list_connectors(conn, principal.account_id)}
@router.get("/api/v1/sources/capabilities")
def source_capabilities(principal: Principal = Depends(get_principal)):
del principal
return capability_payload()
@router.post("/api/v1/connectors", status_code=201)
def post_connector(body: ConnectorIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
name, settings, secret = _connector_parts(body, None)
row = save_connector(conn, principal.account_id, body.kind, name, settings, secret)
return row
@router.get("/api/v1/connectors/{connector_id}")
def get_connector(connector_id: UUID, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
return connector_detail(conn, principal.account_id, connector_id)
@router.patch("/api/v1/connectors/{connector_id}")
def patch_connector(
connector_id: UUID,
body: ConnectorIn,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
current = owned_connector(conn, principal.account_id, connector_id)
if current["kind"] != body.kind:
raise unprocessable("Der Zugang bleibt bei seiner Quellenart.")
name, settings, secret = _connector_parts(body, current)
return update_connector(conn, principal.account_id, connector_id, name, settings, secret)
@router.delete("/api/v1/connectors/{connector_id}", status_code=204)
def remove_connector(connector_id: UUID, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
delete_connector(conn, principal.account_id, connector_id)
@router.post("/api/v1/connectors/{connector_id}/probe")
def probe_connector(connector_id: UUID, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
return probe(conn, principal.account_id, connector_id)
@router.get("/api/v1/external-refs/{element_id}/content")
def view_external_content(
element_id: UUID,
context_kind: str | None = None,
context_id: str | None = None,
save: bool = False,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
if save:
_require_download(conn, principal, element, context)
module = get_source(ref["source"]) if ref is not None else None
if module is None or module.open_original is None:
raise unprocessable("Für diese Quelle ist kein Dateiabruf vorgesehen.")
data, media_type, filename = module.open_original(conn, principal.account_id, element, context, ref)
return Response(
content=data,
media_type=media_type,
headers={
**_PRIVATE,
"Content-Disposition": content_disposition(filename or "dokument.pdf").replace("attachment;", "inline;", 1),
},
)
@router.get("/api/v1/external-refs/{element_id}/mail")
def view_mail(
element_id: UUID,
context_kind: str | None = None,
context_id: str | None = None,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
parsed, _raw = _complete_mail(conn, principal, element, context, ref)
return JSONResponse(
content={
"title": parsed.subject,
"message_from": parsed.sender,
"message_to": parsed.recipient,
"message_cc": parsed.copy,
"text": parsed.text,
"html": parsed.html,
"remote_images": len(parsed.remote_images),
"text_truncated": parsed.text_truncated,
"complete": not parsed.missing,
"notices": parsed.notices,
"attachments": [
{
"index": index,
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
"byte_size": len(item.data),
"inline": item.inline,
}
for index, item in enumerate(parsed.attachments)
],
},
headers={"Cache-Control": "private, no-store", "Referrer-Policy": "no-referrer"},
)
def _require_download(conn, principal: Principal, element, context) -> None:
if not element_rights(conn, principal.account_id, element, context).download:
raise forbidden(DOWNLOAD_REFUSED)
def _opened_ref(conn, principal: Principal, element_id: UUID, context_kind: str | None, context_id: str | None):
element = get_element(conn, element_id)
if element is None or element["kind"] != "external_ref":
raise not_found()
context = parse_query_context(context_kind, context_id)
rights = element_rights(conn, principal.account_id, element, context)
if not rights.read:
raise not_found()
ref = conn.execute(
"""
SELECT source, external_id, detail, connector_id, resolution
FROM external_references WHERE element_id = %s
""",
(element_id,),
).fetchone()
return element, context, ref
def _complete_mail(conn, principal: Principal, element, context, ref):
module = get_source(ref["source"]) if ref is not None else None
if module is None or module.representation != "email":
raise unprocessable("Das ist keine E-Mail.")
opened = module.open_original(conn, principal.account_id, element, context, ref)
if isinstance(opened, RetrievalFailure):
raise unprocessable(opened.message)
raw = opened[0] if isinstance(opened, tuple) else opened
parsed = parse_mail(raw)
if parsed.missing:
names = ", ".join(parsed.missing[:5])
raise unprocessable(f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
return parsed, raw
@router.get("/api/v1/external-refs/{element_id}/mail/html-with-images")
def view_mail_with_images(
element_id: UUID,
context_kind: str | None = None,
context_id: str | None = None,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
"""The formatted mail with its pictures from other servers, loaded by the server on request."""
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
parsed, _raw = _complete_mail(conn, principal, element, context, ref)
loader = RemoteImages()
rendered = sanitize_html(parsed.html_source, _inline_images(parsed.attachments), remote=loader) if parsed.html_source else ""
return JSONResponse(
content={"html": rendered, "loaded": loader.loaded, "failed": loader.failed},
headers={"Cache-Control": "private, no-store", "Referrer-Policy": "no-referrer"},
)
@router.get("/api/v1/external-refs/{element_id}/mail/original")
def download_mail(
element_id: UUID,
context_kind: str | None = None,
context_id: str | None = None,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
_require_download(conn, principal, element, context)
parsed, raw = _complete_mail(conn, principal, element, context, ref)
return _mail_bytes(raw, "message/rfc822", f"{parsed.subject or 'nachricht'}.eml")
@router.get("/api/v1/external-refs/{element_id}/mail/attachments/{index}")
def download_mail_attachment(
element_id: UUID,
index: int,
context_kind: str | None = None,
context_id: str | None = None,
save: bool = False,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id)
if save:
_require_download(conn, principal, element, context)
parsed, _raw = _complete_mail(conn, principal, element, context, ref)
if index < 0 or index >= len(parsed.attachments):
raise not_found()
item = parsed.attachments[index]
name = item.display_name or item.filename or "anlage"
media = (item.media_type or "application/octet-stream").split(";", 1)[0].strip()
if item.data[:1024].lstrip(b"\xef\xbb\xbf\r\n\t ").startswith(b"%PDF"):
media = "application/pdf"
if not name.lower().endswith(".pdf"):
leaf = name.replace("\\", "/").split("/")[-1]
name = name if "." in leaf else f"{name}.pdf"
return _mail_bytes(item.data, media or "application/octet-stream", name, inline=True)
def _mail_bytes(data: bytes, media_type: str, filename: str, inline: bool = False) -> Response:
disposition = content_disposition(filename, inline=inline)
return Response(
content=data,
media_type=media_type,
headers={
**_PRIVATE,
"Content-Disposition": disposition,
},
)
@router.get("/api/v1/sources/paperless/documents")
def search_paperless(
q: str = "",
connector_id: UUID | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return _run_search(
"paperless",
conn,
principal.account_id,
q.strip(),
connector_id,
"",
None,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
)
@router.get("/api/v1/sources/{kind}/facets")
def source_facets(
kind: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
module = require_source(kind)
if module.facets is None:
raise unsupported("Diese Quelle hat keine Auswahllisten.")
return module.facets(conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/paperless/documents/{document_id}/preview")
def preview_paperless_document(
document_id: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
data, media, name = paperless_content(conn, principal.account_id, document_id, connector_id)
return Response(
content=data,
media_type=media,
headers={**_PRIVATE, "Content-Disposition": content_disposition(name).replace("attachment;", "inline;", 1)},
)
@router.get("/api/v1/sources/imap/mailboxes")
def list_mailboxes(
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return _run_navigate("imap", conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/imap/messages")
def search_mail(
connector_id: UUID,
q: str = "",
mailbox: str = "",
before: int | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
uidvalidity: int | None = None,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
if before is not None and before < 1:
raise unprocessable("Die Seitenangabe ist ungültig.")
return _run_search(
"imap",
conn,
principal.account_id,
q.strip(),
connector_id,
mailbox.strip(),
before,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
uidvalidity,
)
@router.get("/api/v1/sources/imap/message")
def read_mail(
connector_id: UUID,
mailbox: str,
uid: int,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
if uid < 1:
raise unprocessable("Die Nachricht ist ungültig.")
return imap_preview(conn, principal.account_id, connector_id, mailbox.strip(), uid)
@router.post("/api/v1/sources/link", status_code=201)
def link_source(body: ExternalIn, principal: Principal = Depends(get_principal), conn: Connection = Depends(get_conn)):
if get_source(body.source) is None:
raise unprocessable("Verknüpft werden registrierte Quellen.")
element = create_external(
conn,
principal.account_id,
principal.scope_id,
body.title,
body.source,
body.external_id,
Ctx(body.context.kind, body.context.id),
_ctx_model(body.placement),
body.chapter_id,
body.dates.document_on,
body.dates.sent_on,
body.dates.received_on,
body.detail.model_dump(mode="json", exclude_none=True) if body.detail else None,
)
return _created(f"/api/v1/elements/{element['id']}", {"id": str(element["id"]), "title": element["title"]})
@router.post("/api/v1/external-refs/{element_id}/assignment")
def assign_source(
element_id: UUID,
body: SourceAssignmentIn,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
assign_reference(conn, principal.account_id, element_id, body.connector_id)
return {"id": str(element_id), "connector_id": str(body.connector_id), "resolution": "bound"}
def _connector_parts(body: ConnectorIn, current: dict | None) -> tuple[str, dict, str]:
"""The registered module validates. Flat fields of the first release are folded into ``config``."""
flat = body.model_dump(exclude={"kind", "name", "config"})
return parse_connector(body.kind, body.name, flat, body.config, current)
_SEARCH_FLAGS = {
"title": "title",
"correspondent": "correspondent",
"document_type": "document_type",
"tag": "tags",
"document_from": "document_range",
"document_to": "document_range",
"sort": "sort",
"cursor": "continuation",
}
def _run_search(kind: str, conn, owner, query: str, connector_id: UUID | None, mailbox: str, before: int | None, filters: dict, uidvalidity: int | None = None) -> dict:
module = require_source(kind)
search = module.capabilities["search"]
refused = [name for name, value in filters.items() if str(value or "").strip() and not search.get(_SEARCH_FLAGS.get(name, name))]
if before is not None and not search.get("continuation"):
refused.append("before")
if refused:
raise unsupported("Diese Suche unterstützt den Filter nicht: " + ", ".join(refused) + ".")
options = {"mailbox": mailbox, "before": before}
if uidvalidity is not None:
options["uidvalidity"] = uidvalidity
options.update({key: value for key, value in filters.items() if str(value or "").strip()})
found = module.find(conn, owner, connector_id, query, options)
bound = str(found.get("connector_id") or connector_id or "")
return with_capabilities(kind, bound, found)
def _run_navigate(kind: str, conn, owner, connector_id: UUID) -> dict:
module = require_source(kind)
if not module.capabilities.get("navigate") or module.navigate is None:
raise unsupported("Diese Quelle hat keine Fachnavigation.")
return {"items": module.navigate(conn, owner, connector_id)}
@router.get("/api/v1/sources/{kind}/search")
def search_source(
kind: str,
q: str = "",
connector_id: UUID | None = None,
mailbox: str = "",
before: int | None = None,
title: str = "",
correspondent: str = "",
document_type: str = "",
tag: str = "",
document_from: str = "",
document_to: str = "",
sort: str = "",
cursor: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
if before is not None and before < 1:
raise unprocessable("Die Seitenangabe ist ungültig.")
return _run_search(
kind,
conn,
principal.account_id,
q.strip(),
connector_id,
mailbox.strip(),
before,
{
"title": title,
"correspondent": correspondent,
"document_type": document_type,
"tag": tag,
"document_from": document_from,
"document_to": document_to,
"sort": sort,
"cursor": cursor,
},
)
@router.get("/api/v1/sources/{kind}/navigate")
def navigate_source(
kind: str,
connector_id: UUID,
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
return _run_navigate(kind, conn, principal.account_id, connector_id)
@router.get("/api/v1/sources/{kind}/preview")
def preview_source(
kind: str,
connector_id: UUID,
mailbox: str = "",
uid: int = 0,
item: str = "",
principal: Principal = Depends(get_principal),
conn: Connection = Depends(get_conn),
):
module = require_source(kind)
if not module.capabilities.get("read_preview") or module.preview is None:
raise unsupported("Diese Quelle hat keine Vorschau vor dem Übernehmen.")
return module.preview(conn, principal.account_id, connector_id, {"mailbox": mailbox, "uid": uid, "item": item})