Files
tsumugu/backend/app/sources.py
T
LarsandCursor d6b70d8f92
Deploy Development / deploy (push) Successful in 28s
Test Suite / pytest-backend (push) Failing after 2m18s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 10s
Schließe die Connector-Kapselung ab, damit Paperless, IMAP und weitere Quellen über dieselbe Anbindung laufen.
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-19 15:00:50 +02:00

1196 lines
45 KiB
Python

"""Paperless and IMAP connectors. Secrets stay on the server."""
from __future__ import annotations
import base64
import imaplib
import json
import re
import ssl
from email.header import decode_header
from email.parser import BytesParser
from email.utils import parsedate_to_datetime
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen
from uuid import UUID
from psycopg.types.json import Json
from app.access.policy import Ctx, audit, mediated_source_owner
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceport import CAPABILITIES
from app.sourceregistry import SourceModule, get_source, register_source
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
row = conn.execute(
"""
INSERT INTO connectors (owner_id, kind, name, settings, secret)
VALUES (%s, %s, %s, %s, %s)
RETURNING id, kind, name, settings, created_at
""",
(owner_id, kind, name, Json(settings), secret),
).fetchone()
return _public(row)
def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict:
"""Credential or name change on the same instance. The id does not move."""
current = _owned(conn, owner_id, connector_id)
row = conn.execute(
"""
UPDATE connectors
SET name = %s, settings = %s, secret = %s
WHERE id = %s AND owner_id = %s
RETURNING id, kind, name, settings, created_at
""",
(name, Json(settings), secret, current["id"], owner_id),
).fetchone()
return _public(row)
def list_connectors(conn, owner_id: UUID) -> list[dict]:
rows = conn.execute(
"""
SELECT id, kind, name, settings, created_at
FROM connectors WHERE owner_id = %s
ORDER BY kind, name
""",
(owner_id,),
).fetchall()
return [_public(row) for row in rows]
def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
row = conn.execute(
"DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id",
(connector_id, owner_id),
).fetchone()
if row is None:
raise not_found()
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = _owned(conn, owner_id, connector_id)
module = get_source(row["kind"])
if module is None:
raise unprocessable("Diese Quelle ist nicht registriert.")
return module.probe(row)
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None) -> tuple[list[dict], str]:
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if not rows:
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
if len(rows) != 1:
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
row = rows[0]
base = row["settings"]["base_url"].rstrip("/")
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
items = []
for doc in payload.get("results") or []:
created = str(doc.get("created") or "")[:10]
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
items.append(
{
"id": str(doc.get("id")),
"title": title[:200],
"document_on": created if len(created) == 10 else None,
"correspondent": _correspondent(doc),
"open_url": f"{base}/documents/{doc.get('id')}/",
}
)
return items, str(row["id"])
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
client = _imap_login(row["settings"], row["secret"])
try:
return _imap_mailboxes(client)
finally:
_imap_logout(client)
def imap_messages(
conn,
owner_id: UUID,
connector_id: UUID,
query: str,
mailbox: str,
before: int | None,
) -> dict:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
if any(char in mailbox for char in "\r\n\x00"):
raise unprocessable("Der Fachname ist ungültig.")
if len(query) > 200:
raise unprocessable("Die Suche ist zu lang.")
client = _imap_login(row["settings"], row["secret"])
try:
return _imap_page(client, mailbox, query, before)
finally:
_imap_logout(client)
def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
_check_mailbox(mailbox)
if uid < 1:
raise unprocessable("Die Nachricht ist ungültig.")
client = _imap_login(row["settings"], row["secret"])
try:
_select_mailbox(client, mailbox)
return _fetch_message(client, uid, mailbox)
finally:
_imap_logout(client)
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
row = conn.execute(
"SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s",
(connector_id, owner_id),
).fetchone()
if row is None:
raise not_found()
return row
def _public(row) -> dict:
return {
"id": str(row["id"]),
"kind": row["kind"],
"name": row["name"],
"settings": row["settings"],
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
"capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None),
}
def clean_base_url(value: str) -> str:
raw = value.strip()
parsed = urlparse(raw)
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.")
if len(raw) > 300:
raise unprocessable("Die Paperless-Adresse ist zu lang.")
return raw.rstrip("/")
def _paperless_count(base: str, token: str) -> int:
payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"})
return int(payload.get("count") or 0)
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
if not document_id.isdigit() or len(document_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
if connector_id is not None:
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "paperless":
raise unprocessable("Das ist kein Paperless-Zugang.")
else:
rows = conn.execute(
"""
SELECT id, settings, secret FROM connectors
WHERE owner_id = %s AND kind = 'paperless'
ORDER BY name, id
""",
(owner_id,),
).fetchall()
if len(rows) != 1:
raise unprocessable("Für Paperless ist kein eindeutiger Zugang zugeordnet.")
row = rows[0]
return _paperless_bytes(row, document_id)
def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
base = row["settings"]["base_url"].rstrip("/")
token = row["secret"]
misses = 0
for suffix in ("preview", "download"):
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/")
if fetched is None:
misses += 1
continue
viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id)
if viewable is not None:
return viewable
meta = _paperless_metadata(base, token, document_id)
if meta is None and misses == 2:
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
if meta is not None:
local = _paperless_from_disk(document_id, meta)
if local is not None:
return local
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
ORIGINAL_UNAVAILABLE = "Das Original ist über die Freigabe nicht abrufbar."
def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
"""The stored document. A grantee never uses a connector of their own account."""
detail = (ref or {}).get("detail") or {}
external_id = str((ref or {}).get("external_id") or "")
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
stored_owner = _connector_owner(conn, stored)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
if not external_id.isdigit() or len(external_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
result = resolve_for_read(conn, actor_id, "paperless", ref, None)
if result.row is None:
raise unprocessable(result.message)
return _paperless_bytes(result.row, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or not external_id.isdigit() or len(external_id) > 12:
raise unprocessable(ORIGINAL_UNAVAILABLE)
result = resolve_for_read(conn, owner_id, "paperless", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "paperless", stored, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
data, media, name = _paperless_bytes(result.row, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "paperless", result.row["id"], "ok")
if external_id and external_id in (name or ""):
name = "dokument.pdf"
return data, media, name or "dokument.pdf"
def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> bytes:
"""The stored message. Mailbox, UID and connector come only from the saved reference."""
detail = (ref or {}).get("detail") or {}
if ref is None or ref["source"] != "imap":
raise unprocessable("Das ist keine E-Mail.")
connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
mailbox = str(detail.get("mailbox") or "").strip()
uid = _mail_uid(detail.get("uid"))
external_id = str(ref["external_id"])
stored_owner = _connector_owner(conn, connector_id)
if element["owner_subject_id"] == actor_id or stored_owner == actor_id:
result = resolve_for_read(conn, actor_id, "imap", ref, None)
if result.row is None:
raise unprocessable(result.message)
if not mailbox or uid is None:
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
return load_imap_original(conn, actor_id, result.row["id"], mailbox, uid, external_id)
owner_id = mediated_source_owner(conn, actor_id, element, context)
if owner_id is None or connector_id is None or not mailbox or uid is None:
raise unprocessable(ORIGINAL_UNAVAILABLE)
result = resolve_for_read(conn, owner_id, "imap", ref, None)
if result.row is None:
_audit_mediated(conn, actor_id, element, context, "imap", connector_id, "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE)
try:
raw = load_imap_original(conn, owner_id, result.row["id"], mailbox, uid, external_id)
except Problem:
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "unavailable")
raise unprocessable(ORIGINAL_UNAVAILABLE) from None
_audit_mediated(conn, actor_id, element, context, "imap", result.row["id"], "ok")
return raw
def _uuid(value) -> UUID | None:
if isinstance(value, UUID):
return value
try:
return UUID(str(value))
except (TypeError, ValueError):
return None
def _mail_uid(value) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, int) and value >= 1:
return value
if isinstance(value, str) and value.isdigit() and int(value) >= 1:
return int(value)
return None
def _connector_owner(conn, connector_id: UUID | None) -> UUID | None:
if connector_id is None:
return None
row = conn.execute("SELECT owner_id FROM connectors WHERE id = %s", (connector_id,)).fetchone()
return row["owner_id"] if row else None
def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
return conn.execute(
"""
SELECT id, owner_id, kind, settings, secret
FROM connectors
WHERE id = %s AND owner_id = %s AND kind = %s
""",
(connector_id, owner_id, kind),
).fetchone()
def _audit_mediated(conn, actor_id: UUID, element: dict, context: Ctx | None, source: str, connector_id, outcome: str) -> None:
audit(
conn,
actor_id,
element["home_scope_id"],
"source_read_mediated",
"element",
element["id"],
context,
result="ok" if outcome == "ok" else "unavailable",
detail={
"source": source,
"connector_id": str(connector_id) if connector_id else None,
"outcome": outcome,
},
)
if outcome != "ok":
conn.commit()
def _paperless_file(
base: str,
token: str,
path: str,
max_bytes: int = 40 * 1024 * 1024,
) -> tuple[bytes, str, str] | None:
request = Request(
f"{base}{path}",
headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"},
)
try:
with urlopen(request, timeout=30) as response:
final = urlparse(response.geturl())
allowed = urlparse(base)
if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc:
raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.")
data = response.read(max_bytes + 1)
if len(data) > max_bytes:
if max_bytes > 40 * 1024 * 1024:
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip()
return data, media, _filename(response.headers.get("Content-Disposition"))
except HTTPError as exc:
if exc.code == 404:
return None
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
except (URLError, TimeoutError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None:
media = media.split(";")[0].strip().lower()
if not (media.startswith("application/pdf") or media.startswith("image/")):
try:
media = sniff(data)
except Problem:
return None
if media.startswith("application/pdf") or media.startswith("image/"):
return data, media, name or f"paperless-{document_id}.pdf"
return None
def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None:
request = Request(
f"{base}/api/documents/{document_id}/metadata/",
headers={"Authorization": f"Token {token}", "Accept": "application/json"},
)
try:
with urlopen(request, timeout=20) as response:
payload = json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
if exc.code == 404:
return None
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
return payload if isinstance(payload, dict) else None
def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None:
root = get_settings().paperless_media_root.strip()
if not root:
return None
candidates: list[tuple[str, str]] = []
archive_name = meta.get("archive_media_filename")
if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip():
candidates.append(("archive", archive_name))
original_name = meta.get("media_filename")
if isinstance(original_name, str) and original_name.strip():
candidates.append(("originals", original_name))
for kind, relative in candidates:
data = read_paperless_file(root, kind, relative)
if data is None:
continue
viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id)
if viewable is not None:
return viewable
return None
def read_paperless_file(
media_root: str,
kind: str,
relative: str,
max_bytes: int = 40 * 1024 * 1024,
) -> bytes | None:
if kind not in {"archive", "originals"}:
return None
raw = relative.replace("\\", "/").strip()
if not raw or raw.startswith("/") or ".." in Path(raw).parts:
return None
base = (Path(media_root) / "documents" / kind).resolve()
target = (base / raw).resolve()
if target != base and base not in target.parents:
return None
if not target.is_file():
return None
data = target.read_bytes()
if len(data) > max_bytes:
if max_bytes > 40 * 1024 * 1024:
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
return data
def _filename(disposition: str | None) -> str:
if not disposition:
return ""
marker = "filename="
lower = disposition.lower()
if marker not in lower:
return ""
value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"')
return value.replace("\\", "").replace("/", "")[:180]
def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
url = f"{base}{path}?{urlencode(params)}"
request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"})
try:
with urlopen(request, timeout=12) as response:
return json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
if exc.code in {401, 403}:
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
raise unprocessable("Paperless ist nicht erreichbar.") from exc
def _correspondent(doc: dict) -> str | None:
value = doc.get("correspondent_name") or doc.get("correspondent")
if isinstance(value, str) and value.strip():
return value.strip()[:200]
return None
def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
host = settings["host"]
port = int(settings["port"])
security = settings.get("security") or "ssl"
try:
if security == "starttls":
client = imaplib.IMAP4(host, port, timeout=12)
client.starttls(ssl.create_default_context())
else:
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12)
client.login(settings["username"], secret)
return client
except imaplib.IMAP4.error as exc:
raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc
except (OSError, TimeoutError) as exc:
raise unprocessable("Das Postfach ist nicht erreichbar.") from exc
PAGE_SIZE = 30
_LIST_LINE = re.compile(
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
)
def _imap_logout(client: imaplib.IMAP4) -> None:
try:
client.logout()
except imaplib.IMAP4.error:
pass
def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
try:
status, data = client.list()
except imaplib.IMAP4.error as exc:
raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc
if status != "OK":
raise unprocessable("Die Fächer konnten nicht gelesen werden.")
items = []
seen: set[str] = set()
for raw in data or []:
parsed = _parse_list_line(raw)
if parsed is None or parsed["name"] in seen:
continue
seen.add(parsed["name"])
items.append(parsed)
items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold()))
return items
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None) -> dict:
folders = _imap_mailboxes(client)
chosen = mailbox or "INBOX"
match = next((item for item in folders if item["name"] == chosen), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(chosen, readonly=True)
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
status, data = _uid_search(client, query)
if status != "OK" or not data or not data[0]:
return {"mailbox": chosen, "items": [], "next_before": None}
uids = sorted(int(part) for part in data[0].split() if part.isdigit())
if before is not None:
uids = [uid for uid in uids if uid < before]
page = uids[-PAGE_SIZE:]
next_before = page[0] if len(uids) > PAGE_SIZE else None
items = []
for uid in reversed(page):
item = _fetch_header(client, uid, chosen)
if item is not None:
items.append(item)
return {"mailbox": chosen, "items": items, "next_before": next_before}
def _uid_search(client: imaplib.IMAP4, query: str):
needle = " ".join(query.split())
if not needle:
return client.uid("SEARCH", None, "ALL")
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
try:
if needle.isascii():
return client.uid("SEARCH", None, "OR", "SUBJECT", quoted, "FROM", quoted)
encoded = quoted.encode("utf-8")
return client.uid("SEARCH", "CHARSET", "UTF-8", "OR", "SUBJECT", encoded, "FROM", encoded)
except imaplib.IMAP4.error as exc:
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
def _check_mailbox(mailbox: str) -> None:
if any(char in mailbox for char in "\r\n\x00"):
raise unprocessable("Der Fachname ist ungültig.")
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
folders = _imap_mailboxes(client)
match = next((item for item in folders if item["name"] == mailbox), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(mailbox, readonly=True)
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
try:
status, fetched = client.uid(
"FETCH",
str(uid),
"(BODY.PEEK[HEADER.FIELDS (FROM TO CC SUBJECT DATE MESSAGE-ID)])",
)
except imaplib.IMAP4.error:
return None
if status != "OK" or not fetched or not isinstance(fetched[0], tuple):
return None
attachments = []
try:
struct_status, structure = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
if struct_status == "OK":
attachments = _structure_attachments(structure)
except imaplib.IMAP4.error:
attachments = []
header = BytesParser().parsebytes(fetched[0][1])
subject = _decode(header.get("Subject")) or "(ohne Betreff)"
sender = _decode(header.get("From"))
message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip()
return {
"id": message_id[:200],
"uid": uid,
"title": subject[:200],
"message_from": sender[:200],
"message_to": _decode(header.get("To"))[:500],
"message_cc": _decode(header.get("Cc"))[:500],
"direction": _direction(mailbox),
"attachments": attachments,
"document_on": _mail_date(header.get("Date")),
}
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
except imaplib.IMAP4.error as exc:
raise unprocessable("Die Nachricht lässt sich nicht lesen.") from exc
raw = _literal(fetched)
if status != "OK" or raw is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
if len(raw) > 8_000_000:
return _oversized_message(client, uid, mailbox)
message = BytesParser().parsebytes(raw)
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
return {
"uid": uid,
"mailbox": mailbox,
"title": subject[:200],
"message_from": _decode(message.get("From"))[:200],
"message_to": _decode(message.get("To"))[:500],
"message_cc": _decode(message.get("Cc"))[:500],
"direction": _direction(mailbox),
"document_on": _mail_date(message.get("Date")),
"attachments": _message_attachments(message),
"text": (_text := _message_text(message))[:200_000],
"text_truncated": len(_text) > 200_000,
}
_SENT_FOLDER = re.compile(
r"(^|/)(sent|sent items|sent messages|gesendet|gesendete|gesendete elemente|outbox|postausgang)($|/)",
re.IGNORECASE,
)
_INBOX_FOLDER = re.compile(
r"(^|/)(inbox|posteingang|eingang)($|/)",
re.IGNORECASE,
)
def _direction(mailbox: str) -> str | None:
"""Sent folders are outgoing, an inbox is incoming. Other folder names prove nothing."""
folded = mailbox.casefold().replace("\\", "/")
if _SENT_FOLDER.search(folded):
return "out"
if _INBOX_FOLDER.search(folded):
return "in"
return None
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
header = _fetch_header(client, uid, mailbox) or {}
text = _partial_text(client, uid)
if not header and not text:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
return {
"uid": uid,
"mailbox": mailbox,
"title": header.get("title") or "(ohne Betreff)",
"message_from": header.get("message_from") or "",
"message_to": header.get("message_to") or "",
"message_cc": header.get("message_cc") or "",
"direction": header.get("direction", _direction(mailbox)),
"document_on": header.get("document_on"),
"attachments": header.get("attachments") or [],
"text": text,
"text_truncated": True,
}
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
except imaplib.IMAP4.error:
return ""
if status != "OK":
return ""
raw = _literal(fetched)
if not raw:
return ""
sample = raw[:200_000]
try:
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
text = _message_text(message)
except Exception:
text = ""
if not text:
text = sample.decode("utf-8", errors="replace")
return text.strip()[:200_000]
def _literal(fetched) -> bytes | None:
for part in fetched or []:
if isinstance(part, tuple) and len(part) > 1 and isinstance(part[1], (bytes, bytearray)):
return bytes(part[1])
return None
def _structure_attachments(fetched) -> list[dict]:
chunks: list[bytes] = []
for part in fetched or []:
if isinstance(part, tuple):
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
elif isinstance(part, (bytes, bytearray)):
chunks.append(bytes(part))
blob = b" ".join(chunks)
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
items = []
seen: set[str] = set()
for raw in names:
name = raw.decode("utf-8", errors="replace").strip()[:200]
if not name or name in seen:
continue
seen.add(name)
items.append({"filename": name})
if len(items) == 40:
break
return items
def _message_attachments(message) -> list[dict]:
items = []
seen: set[str] = set()
for part in message.walk():
filename = part.get_filename()
if not filename:
continue
name = _decode(filename)[:200]
if not name or name in seen:
continue
seen.add(name)
items.append({"filename": name, "content_type": part.get_content_type()[:120]})
if len(items) == 40:
break
return items
def _message_text(message) -> str:
plain: list[str] = []
html: list[str] = []
parts = message.walk() if message.is_multipart() else [message]
for part in parts:
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
continue
kind = part.get_content_type()
if kind == "text/plain":
plain.append(_part_text(part))
elif kind == "text/html":
html.append(_html_text(_part_text(part)))
text = "\n".join(chunk for chunk in plain if chunk).strip()
if not text:
text = "\n".join(chunk for chunk in html if chunk).strip()
return text
def _part_text(part) -> str:
payload = part.get_payload(decode=True)
if payload is None:
raw = part.get_payload()
return raw.strip() if isinstance(raw, str) else ""
charset = part.get_content_charset() or "utf-8"
return payload.decode(charset, errors="replace").strip()
def _html_text(value: str) -> str:
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
text = re.sub(r"(?i)</p>", "\n", text)
text = re.sub(r"<[^>]+>", " ", text)
return re.sub(r"[ \t]{2,}", " ", text).strip()
def _parse_list_line(raw) -> dict | None:
if not isinstance(raw, (bytes, str)) or not raw:
return None
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
match = _LIST_LINE.match(text.strip())
if match is None:
return None
flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag}
delimiter = _unquote(match.group("delim"))
name = _unquote(match.group("name"))
if not name:
return None
label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name)
depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter)
return {
"name": name,
"label": label,
"delimiter": delimiter,
"depth": depth,
"selectable": "noselect" not in flags,
}
def _unquote(token: str) -> str:
if token == "NIL":
return ""
if len(token) >= 2 and token.startswith('"') and token.endswith('"'):
return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\")
return token
def _decode_utf7(value: str) -> str:
if "&" not in value:
return value
parts: list[str] = []
index = 0
while index < len(value):
if value[index] != "&":
parts.append(value[index])
index += 1
continue
end = value.find("-", index)
if end < 0:
parts.append(value[index:])
break
if end == index + 1:
parts.append("&")
else:
chunk = value[index + 1:end].replace(",", "/")
padding = "=" * ((4 - len(chunk) % 4) % 4)
try:
parts.append(base64.b64decode(chunk + padding).decode("utf-16-be"))
except (ValueError, UnicodeDecodeError):
parts.append(value[index:end + 1])
index = end + 1
return "".join(parts)
def _mail_date(value: str | None) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(value).date().isoformat()
except (TypeError, ValueError, OverflowError):
return None
def _decode(value: str | None) -> str:
if not value:
return ""
parts = []
for text, encoding in decode_header(value):
if isinstance(text, bytes):
parts.append(text.decode(encoding or "utf-8", errors="replace"))
else:
parts.append(text)
return "".join(parts).strip()
def pull_paperless_original(base: str, token: str, document_id: str, max_bytes: int) -> tuple[bytes, str, str]:
"""Original download only. A preview is never returned in its place."""
if not document_id.isdigit() or len(document_id) > 12:
raise unprocessable("Die Paperless-Nummer ist ungültig.")
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/download/", max_bytes)
if fetched is not None:
data, media, name = fetched
return data, media or "application/octet-stream", name or f"paperless-{document_id}"
meta = _paperless_metadata(base, token, document_id)
if meta is None:
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
local = _paperless_original_from_disk(meta, max_bytes)
if local is None:
raise unprocessable(
"Das Paperless-Original ist nicht verfügbar. Eine Vorschau wird nicht als Original übernommen."
)
return local
def _paperless_original_from_disk(meta: dict, max_bytes: int) -> tuple[bytes, str, str] | None:
root = get_settings().paperless_media_root.strip()
original_name = meta.get("media_filename")
if not root or not isinstance(original_name, str) or not original_name.strip():
return None
data = read_paperless_file(root, "originals", original_name, max_bytes)
if data is None:
return None
return data, "application/octet-stream", Path(original_name).name
COMPLETE_MAIL_LIMIT = 80 * 1024 * 1024
def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int, expected_id: str) -> bytes:
"""The referenced message in full. A preview is not returned in its place."""
row = _owned(conn, owner_id, connector_id)
if row["kind"] != "imap":
raise unprocessable("Das ist kein E-Mail-Konto.")
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT)
def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> bytes:
"""Fetch the referenced message. A different message at the old UID is not returned."""
client = _imap_login(settings, secret)
try:
_select_mailbox(client, mailbox)
candidate = _fetch_rfc822(client, uid)
if candidate is not None and len(candidate) > max_bytes:
raise unprocessable(
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen."
)
alternates: list[bytes] = []
if candidate is None or not identities_match(header_message_id(candidate), expected_id):
alternates = _search_rfc822(client, expected_id, max_bytes)
try:
chosen = choose_message(candidate, alternates, expected_id)
except MailIdentityError as exc:
raise unprocessable(exc.message) from exc
if len(chosen) > max_bytes:
raise unprocessable(
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen."
)
return chosen
finally:
_imap_logout(client)
def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
except imaplib.IMAP4.error:
return None
if status != "OK":
return None
return _literal(fetched)
def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> list[bytes]:
token = (expected_id or "").strip()
if not token or token.casefold().startswith("uid:"):
return []
found: list[bytes] = []
for folder in _imap_mailboxes(client):
if not folder["selectable"]:
continue
try:
_select_mailbox(client, folder["name"])
status, data = client.uid("SEARCH", "HEADER", "Message-ID", token)
except (imaplib.IMAP4.error, Problem):
continue
if status != "OK" or not data or not data[0]:
continue
for raw_uid in data[0].split():
try:
uid = int(raw_uid)
except ValueError:
continue
blob = _fetch_rfc822(client, uid)
if blob is None:
continue
if len(blob) > max_bytes:
raise unprocessable(
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen."
)
found.append(blob)
if len(found) > 5:
return found
return found
def _paperless_config(payload: dict) -> tuple[str, dict, str]:
base_url = str(payload.get("base_url") or "").strip()
token = str(payload.get("token") or "").strip()
name = str(payload.get("name") or "").strip()
if not base_url or len(token) < 8:
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
return name, {"base_url": clean_base_url(base_url)}, token
def _imap_config(payload: dict) -> tuple[str, dict, str]:
host = str(payload.get("host") or "").strip()
username = str(payload.get("username") or "").strip()
password = str(payload.get("password") or "")
name = str(payload.get("name") or "").strip()
if not host or not username or not password:
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
if " " in host or len(host) > 200:
raise unprocessable("Der Mailserver ist ungültig.")
port = payload.get("port") or 993
return name, {
"host": host,
"port": int(port),
"username": username,
"mailbox": str(payload.get("mailbox") or "INBOX").strip() or "INBOX",
"security": payload.get("security") or "ssl",
}, password
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
del options
items, bound = paperless_documents(conn, owner_id, query, connector_id)
return {"items": items, "connector_id": bound}
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
if connector_id is None:
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
before = options.get("before")
return imap_messages(conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before)
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem
del detail
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
try:
data, media, name = pull_paperless_original(
resolved.row["settings"]["base_url"].rstrip("/"),
resolved.row["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
except Problem as exc:
return failure_from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
return FilePayload(name or f"paperless-{external_id}", media, data, "")
def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
from app.mailpack import identities_match, parse_mail
from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem
resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
mailbox = str(detail.get("mailbox") or "").strip()
uid = detail.get("uid")
if isinstance(uid, str) and uid.isdigit():
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
try:
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT:
return RetrievalFailure(
"too_large",
"Die vollständige Nachricht überschreitet die Exportgrenze. "
"Eine gekürzte Vorschau wird nicht als Original übernommen.",
)
parsed = parse_mail(raw)
if not identities_match(parsed.message_id, str(external_id)):
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
if parsed.missing:
names = ", ".join(parsed.missing[:5])
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
return MailPayload(raw, parsed)
def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
raw = read_linked_mail(conn, actor_id, element, context, ref)
return raw, "message/rfc822", "nachricht.eml"
class _PaperlessAdapter:
kind = "paperless"
def capabilities(self) -> dict:
return CAPABILITIES[self.kind]
def probe(self, row) -> dict:
count = _paperless_count(row["settings"]["base_url"], row["secret"])
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
return paperless_documents(conn, owner_id, query, connector_id)
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
return _paperless_find(conn, owner_id, connector_id, query, options)
class _ImapAdapter:
kind = "imap"
def capabilities(self) -> dict:
return CAPABILITIES[self.kind]
def probe(self, row) -> dict:
_imap_login(row["settings"], row["secret"])
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
return imap_mailboxes(conn, owner_id, connector_id)
def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict:
return imap_messages(conn, owner_id, connector_id, query, mailbox, before)
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
return _imap_find(conn, owner_id, connector_id, query, options)
def _install_sources() -> None:
paperless = _PaperlessAdapter()
imap = _ImapAdapter()
register_source(SourceModule(
kind="paperless",
label="Paperless",
representation="pdf",
capabilities=CAPABILITIES["paperless"],
parse_config=_paperless_config,
probe=paperless.probe,
find=paperless.find,
open_original=read_linked_paperless,
export_original=_export_paperless,
navigate=None,
preview=None,
))
register_source(SourceModule(
kind="imap",
label="E-Mail",
representation="email",
capabilities=CAPABILITIES["imap"],
parse_config=_imap_config,
probe=imap.probe,
find=imap.find,
open_original=_imap_bytes,
export_original=_export_imap,
navigate=imap.navigate,
preview=lambda conn, owner, connector_id, options: imap_preview(
conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0)
),
))
_install_sources()