Files
tsumugu/backend/app/sources.py
T
LarsandCursor b328385561
Deploy Development / deploy (push) Failing after 39s
Lies Paperless-Dateien aus dem Ablageordner, wenn die Vorschau keine anzeigbare Datei liefert.
Eine unbekannte Dokumentnummer wird als solche gemeldet.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-17 07:12:35 +02:00

381 lines
14 KiB
Python

"""Paperless and IMAP connectors. Secrets stay on the server."""
from __future__ import annotations
import imaplib
import json
import ssl
from email.header import decode_header
from email.parser import BytesParser
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen
from uuid import UUID
from psycopg.types.json import Json
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.media.storage import sniff
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
if kind == "paperless":
existing = conn.execute(
"SELECT id FROM connectors WHERE owner_id = %s AND kind = 'paperless'",
(owner_id,),
).fetchone()
if existing:
row = conn.execute(
"""
UPDATE connectors
SET name = %s, settings = %s, secret = %s
WHERE id = %s
RETURNING id, kind, name, settings, created_at
""",
(name, Json(settings), secret, existing["id"]),
).fetchone()
return _public(row)
row = conn.execute(
"""
INSERT INTO connectors (owner_id, kind, name, settings, secret)
VALUES (%s, %s, %s, %s, %s)
RETURNING id, kind, name, settings, created_at
""",
(owner_id, kind, name, Json(settings), secret),
).fetchone()
return _public(row)
def list_connectors(conn, owner_id: UUID) -> list[dict]:
rows = conn.execute(
"""
SELECT id, kind, name, settings, created_at
FROM connectors WHERE owner_id = %s
ORDER BY kind, name
""",
(owner_id,),
).fetchall()
return [_public(row) for row in rows]
def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
row = conn.execute(
"DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id",
(connector_id, owner_id),
).fetchone()
if row is None:
raise not_found()
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = _owned(conn, owner_id, connector_id)
if row["kind"] == "paperless":
count = _paperless_count(row["settings"]["base_url"], row["secret"])
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
_imap_login(row["settings"], row["secret"])
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
row = conn.execute(
"""
SELECT settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY created_at DESC LIMIT 1
""",
(owner_id,),
).fetchone()
if row is None:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
base = row["settings"]["base_url"].rstrip("/")
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
items = []
for doc in payload.get("results") or []:
created = str(doc.get("created") or "")[:10]
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
items.append(
{
"id": str(doc.get("id")),
"title": title[:200],
"document_on": created if len(created) == 10 else None,
"correspondent": _correspondent(doc),
"open_url": f"{base}/documents/{doc.get('id')}/",
}
)
return items
def imap_messages(conn, owner_id: UUID, connector_id: UUID, query: str) -> list[dict]:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
return _imap_list(row["settings"], row["secret"], query)
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = conn.execute(
"SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s",
(connector_id, owner_id),
).fetchone()
if row is None:
raise not_found()
return row
def _public(row) -> dict:
return {
"id": str(row["id"]),
"kind": row["kind"],
"name": row["name"],
"settings": row["settings"],
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
}
def clean_base_url(value: str) -> str:
raw = value.strip()
parsed = urlparse(raw)
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.")
if len(raw) > 300:
raise unprocessable("Die Paperless-Adresse ist zu lang.")
return raw.rstrip("/")
def _paperless_count(base: str, token: str) -> int:
payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"})
return int(payload.get("count") or 0)
def paperless_content(conn, owner_id: UUID, document_id: str) -> tuple[bytes, str, str]:
if not document_id.isdigit() or len(document_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
row = conn.execute(
"""
SELECT settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY created_at DESC LIMIT 1
""",
(owner_id,),
).fetchone()
if row is None:
raise unprocessable("Paperless ist für dieses Konto nicht verbunden.")
base = row["settings"]["base_url"].rstrip("/")
token = row["secret"]
misses = 0
for suffix in ("preview", "download"):
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/")
if fetched is None:
misses += 1
continue
viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id)
if viewable is not None:
return viewable
meta = _paperless_metadata(base, token, document_id)
if meta is None and misses == 2:
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
if meta is not None:
local = _paperless_from_disk(document_id, meta)
if local is not None:
return local
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
def _paperless_file(base: str, token: str, path: str) -> tuple[bytes, str, str] | None:
request = Request(
f"{base}{path}",
headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"},
)
try:
with urlopen(request, timeout=30) as response:
final = urlparse(response.geturl())
allowed = urlparse(base)
if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc:
raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.")
data = response.read(40 * 1024 * 1024 + 1)
if len(data) > 40 * 1024 * 1024:
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip()
return data, media, _filename(response.headers.get("Content-Disposition"))
except HTTPError as exc:
if exc.code == 404:
return None
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
except (URLError, TimeoutError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None:
media = media.split(";")[0].strip().lower()
if not (media.startswith("application/pdf") or media.startswith("image/")):
try:
media = sniff(data)
except Problem:
return None
if media.startswith("application/pdf") or media.startswith("image/"):
return data, media, name or f"paperless-{document_id}.pdf"
return None
def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None:
request = Request(
f"{base}/api/documents/{document_id}/metadata/",
headers={"Authorization": f"Token {token}", "Accept": "application/json"},
)
try:
with urlopen(request, timeout=20) as response:
payload = json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
if exc.code == 404:
return None
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
return payload if isinstance(payload, dict) else None
def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None:
root = get_settings().paperless_media_root.strip()
if not root:
return None
candidates: list[tuple[str, str]] = []
archive_name = meta.get("archive_media_filename")
if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip():
candidates.append(("archive", archive_name))
original_name = meta.get("media_filename")
if isinstance(original_name, str) and original_name.strip():
candidates.append(("originals", original_name))
for kind, relative in candidates:
data = read_paperless_file(root, kind, relative)
if data is None:
continue
viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id)
if viewable is not None:
return viewable
return None
def read_paperless_file(media_root: str, kind: str, relative: str) -> bytes | None:
if kind not in {"archive", "originals"}:
return None
raw = relative.replace("\\", "/").strip()
if not raw or raw.startswith("/") or ".." in Path(raw).parts:
return None
base = (Path(media_root) / "documents" / kind).resolve()
target = (base / raw).resolve()
if target != base and base not in target.parents:
return None
if not target.is_file():
return None
data = target.read_bytes()
if len(data) > 40 * 1024 * 1024:
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
return data
def _filename(disposition: str | None) -> str:
if not disposition:
return ""
marker = "filename="
lower = disposition.lower()
if marker not in lower:
return ""
value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"')
return value.replace("\\", "").replace("/", "")[:180]
def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
url = f"{base}{path}?{urlencode(params)}"
request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"})
try:
with urlopen(request, timeout=12) as response:
return json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
def _correspondent(doc: dict) -> str | None:
value = doc.get("correspondent_name") or doc.get("correspondent")
if isinstance(value, str) and value.strip():
return value.strip()[:200]
return None
def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
host = settings["host"]
port = int(settings["port"])
security = settings.get("security") or "ssl"
try:
if security == "starttls":
client = imaplib.IMAP4(host, port, timeout=12)
client.starttls(ssl.create_default_context())
else:
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12)
client.login(settings["username"], secret)
return client
except imaplib.IMAP4.error as exc:
raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc
except (OSError, TimeoutError) as exc:
raise unprocessable("Das Postfach ist nicht erreichbar.") from exc
def _imap_list(settings: dict, secret: str, query: str) -> list[dict]:
client = _imap_login(settings, secret)
try:
mailbox = settings.get("mailbox") or "INBOX"
status, _ = client.select(mailbox, readonly=True)
if status != "OK":
raise unprocessable("Das Postfach enthält dieses Fach nicht.")
status, data = client.search(None, "ALL")
if status != "OK" or not data or not data[0]:
return []
ids = data[0].split()[-25:]
needle = query.strip().lower()
items = []
for mid in reversed(ids):
status, fetched = client.fetch(mid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE MESSAGE-ID)])")
if status != "OK" or not fetched or not isinstance(fetched[0], tuple):
continue
header = BytesParser().parsebytes(fetched[0][1])
subject = _decode(header.get("Subject")) or "(ohne Betreff)"
sender = _decode(header.get("From"))
if needle and needle not in subject.lower() and needle not in sender.lower():
continue
message_id = (header.get("Message-ID") or f"uid:{mid.decode()}").strip()
items.append(
{
"id": message_id[:200],
"title": subject[:200],
"message_from": sender[:200],
"date": _decode(header.get("Date"))[:80],
}
)
return items
finally:
try:
client.logout()
except imaplib.IMAP4.error:
pass
def _decode(value: str | None) -> str:
if not value:
return ""
parts = []
for text, encoding in decode_header(value):
if isinstance(text, bytes):
parts.append(text.decode(encoding or "utf-8", errors="replace"))
else:
parts.append(text)
return "".join(parts).strip()