"""Paperless and IMAP connectors. Secrets stay on the server.""" from __future__ import annotations import base64 import hashlib import imaplib import json import re import ssl import threading import time from dataclasses import dataclass from email.header import decode_header from email.parser import BytesParser from email.utils import parsedate_to_datetime from html import unescape from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urlparse from urllib.request import Request, urlopen from uuid import UUID from psycopg.types.json import Json from app.access.policy import Ctx from app.core.errors import Problem, not_found, unprocessable from app.core.settings import get_settings from app.mailpack import ( MailIdentityError, choose_message, decode_transfer, header_message_id, header_text, identities_match, parse_mail, sanitize_html, simplified_text, ) from app.media.storage import sniff from app.sourcebind import remember_imap_location, resolve_for_read from app.sourceaccess import ( ORIGINAL_UNAVAILABLE, assert_still_owned, audit_mediated as _audit_mediated, borrow_original, complete_borrow, connector_owner as _connector_owner, release_for_external, ) from app.sourceregistry import ConfigField, SourceModule, get_source, parse_fields, register_source, require_source _PUBLIC_COLUMNS = """ id, kind, name, settings, created_at, secret <> '' AS has_secret, last_probe_at, last_probe_ok, last_probe_detail """ def parse_connector( kind: str, name: str, flat: dict, config: dict | None, current: dict | None = None, ) -> tuple[str, dict, str]: """Validate a connector body with the registered module. ``current`` is the stored row on change.""" module = require_source(kind) name = " ".join(str(name or "").split()) values = {key: value for key, value in flat.items() if value is not None} values.update(config or {}) if module.config_fields: settings, secret = parse_fields( module.config_fields, values, current["settings"] if current else None, current["secret"] if current else "", ) if module.normalize_config is not None: settings = module.normalize_config(settings, secret) return name, settings, secret return module.parse_config({**values, "kind": kind, "name": name, "config": config or {}}) def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict: row = conn.execute( f""" INSERT INTO connectors (owner_id, kind, name, settings, secret) VALUES (%s, %s, %s, %s, %s) RETURNING {_PUBLIC_COLUMNS} """, (owner_id, kind, name, Json(settings), secret), ).fetchone() return _public(row) def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict: """Credential or name change on the same instance. The id does not move.""" current = _owned(conn, owner_id, connector_id) row = conn.execute( f""" UPDATE connectors SET name = %s, settings = %s, secret = %s, last_probe_at = NULL, last_probe_ok = NULL, last_probe_detail = NULL WHERE id = %s AND owner_id = %s RETURNING {_PUBLIC_COLUMNS} """, (name, Json(settings), secret, current["id"], owner_id), ).fetchone() return _public(row) def list_connectors(conn, owner_id: UUID) -> list[dict]: rows = conn.execute( f""" SELECT {_PUBLIC_COLUMNS} FROM connectors WHERE owner_id = %s ORDER BY kind, name """, (owner_id,), ).fetchall() return [_public(row) for row in rows] def connector_detail(conn, owner_id: UUID, connector_id: UUID) -> dict: """One own instance with its use. The secret itself is never part of it.""" row = conn.execute( f"SELECT {_PUBLIC_COLUMNS} FROM connectors WHERE id = %s AND owner_id = %s", (connector_id, owner_id), ).fetchone() if row is None: raise not_found() used = conn.execute( """ SELECT count(*) AS total FROM external_references er JOIN elements e ON e.id = er.element_id WHERE (er.connector_id = %s OR er.detail ->> 'connector_id' = %s) AND e.deleted_at IS NULL AND e.owner_subject_id = %s """, (connector_id, str(connector_id), owner_id), ).fetchone() body = _public(row) body["references"] = int(used["total"]) body["protection_profile"] = "legacy-plaintext" return body def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None: row = conn.execute( "DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id", (connector_id, owner_id), ).fetchone() if row is None: raise not_found() def probe(conn, owner_id: UUID, connector_id: UUID) -> dict: row = _owned(conn, owner_id, connector_id) module = get_source(row["kind"]) if module is None: raise unprocessable("Diese Quelle ist nicht registriert.") release_for_external(conn) try: result = module.probe(row) except Problem as exc: assert_still_owned(conn, row["id"], owner_id) _record_probe(conn, row["id"], False, exc.detail) conn.commit() raise assert_still_owned(conn, row["id"], owner_id) _record_probe(conn, row["id"], bool(result.get("ok", True)), str(result.get("detail") or "")) return result def _record_probe(conn, connector_id: UUID, ok: bool, detail: str) -> None: """Display only. The detail comes from the module and must not carry a secret.""" conn.execute( """ UPDATE connectors SET last_probe_at = now(), last_probe_ok = %s, last_probe_detail = %s WHERE id = %s """, (ok, detail[:300] or None, connector_id), ) PAPERLESS_PAGE_SIZE = 20 _PAPERLESS_MODES = { # Paperless-ngx offers these searches in its own filter bar. The first needs no search index. "title_content": "title_content", "title": "title__icontains", "content": "content__icontains", "index": "query", } _PAPERLESS_TAG_MODES = {"all": "tags__id__all", "any": "tags__id__in", "none": "tags__id__none"} _PAPERLESS_SORTS = {"created", "-created", "added", "-added", "title", "-title", "score", "-score"} def _paperless_choice(value, label: str, allow_none: bool = False) -> str | None: text = str(value or "").strip() if not text: return None if allow_none and text == "none": return text if not text.isdigit() or len(text) > 12: raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.") return text def _paperless_day(value, label: str) -> str | None: text = str(value or "").strip() if not text: return None if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", text): raise unprocessable(f"{label} ist kein Datum.") return text def _paperless_range(options: dict, prefix: str, field: str, label: str, params: dict) -> None: start = _paperless_day(options.get(f"{prefix}_from"), f"{label}, von") end = _paperless_day(options.get(f"{prefix}_to"), f"{label}, bis") if start and end and start > end: raise unprocessable(f"{label}: Der Beginn liegt nach dem Ende.") if start: params[f"{field}__date__gte"] = start if end: params[f"{field}__date__lte"] = end _CUSTOM_KINDS = { "string": "text", "url": "text", "longtext": "text", "integer": "number", "float": "number", "monetary": "number", "date": "date", "boolean": "boolean", "select": "select", } _CUSTOM_OPERATORS = { "text": {"exact", "icontains", "istartswith", "iendswith"}, "number": {"exact", "gt", "gte", "lt", "lte", "range"}, "date": {"exact", "gt", "gte", "lt", "lte", "range"}, "boolean": {"exact"}, "select": {"exact"}, } _CUSTOM_LIMIT = 5 def _custom_number(value, label: str): text = str(value).strip().replace(",", ".") try: number = float(text) except ValueError as exc: raise unprocessable(f"{label}: Der Wert ist keine Zahl.") from exc if number != number or number in (float("inf"), float("-inf")): raise unprocessable(f"{label}: Der Wert ist keine Zahl.") return int(number) if "." not in text else number def _custom_value(kind: str, op: str, value, field: dict): label = field["name"] if op == "exists": if not isinstance(value, bool): raise unprocessable(f"{label}: Gesetzt oder leer ist ungültig.") return value if op == "range": if not isinstance(value, (list, tuple)) or len(value) != 2: raise unprocessable(f"{label}: Der Bereich braucht Anfang und Ende.") low, high = (_custom_value(kind, "exact", item, field) for item in value) if low > high: raise unprocessable(f"{label}: Der Beginn liegt nach dem Ende.") return [low, high] if kind == "text": text = " ".join(str(value or "").split()) if not text or len(text) > 200: raise unprocessable(f"{label}: Der Text fehlt oder ist zu lang.") return text if kind == "number": return _custom_number(value, label) if kind == "date": day = _paperless_day(value, label) if not day: raise unprocessable(f"{label}: Das Datum fehlt.") return day if kind == "boolean": if not isinstance(value, bool): raise unprocessable(f"{label}: Ja oder Nein ist ungültig.") return value allowed = {option["id"] for option in field.get("options") or []} if str(value) not in allowed: raise unprocessable(f"{label}: Das ist keine Auswahl dieses Feldes.") return str(value) def _paperless_custom_query(raw, fields: list[dict]) -> str | None: """Paperless-ngx custom_field_query, built only from fields and operators this instance offers.""" if raw in (None, "", []): return None if isinstance(raw, str): try: raw = json.loads(raw) except json.JSONDecodeError as exc: raise unprocessable("Die Filter der Zusatzfelder sind ungültig.") from exc if not isinstance(raw, list) or len(raw) > _CUSTOM_LIMIT: raise unprocessable(f"Es sind höchstens {_CUSTOM_LIMIT} Zusatzfeld-Filter möglich.") known = {field["id"]: field for field in fields} atoms = [] for row in raw: if not isinstance(row, dict): raise unprocessable("Die Filter der Zusatzfelder sind ungültig.") field = known.get(str(row.get("field") or "")) if field is None: raise unprocessable("Das Zusatzfeld gibt es an diesem Zugang nicht.") kind = _CUSTOM_KINDS[field["type"]] op = str(row.get("op") or "") if op != "exists" and op not in _CUSTOM_OPERATORS[kind]: raise unprocessable(f"{field['name']}: Diese Bedingung gibt es für den Feldtyp nicht.") atoms.append([field["name"], op, _custom_value(kind, op, row.get("value"), field)]) if not atoms: return None return json.dumps(atoms[0] if len(atoms) == 1 else ["AND", atoms], ensure_ascii=False) def _paperless_params(query: str, options: dict | None, custom_fields: list[dict] | None = None) -> dict: """Paperless-ngx document date is the created field, not the upload or modified time.""" options = options or {} params = {"page_size": str(PAPERLESS_PAGE_SIZE)} text = " ".join(str(query or "").split()) mode = str(options.get("mode") or "").strip() or "title_content" if mode not in _PAPERLESS_MODES: raise unprocessable("Diese Suchart gibt es nicht.") if text: params[_PAPERLESS_MODES[mode]] = text title = " ".join(str(options.get("title") or "").split()) if title: if "title__icontains" in params and params["title__icontains"] != title: raise unprocessable("Die Suche nur im Titel und der Titelfilter sind beide gesetzt. Nutze eines davon.") params["title__icontains"] = title for name, field, label in ( ("correspondent", "correspondent", "Der Korrespondent"), ("document_type", "document_type", "Der Dokumenttyp"), ("storage_path", "storage_path", "Der Speicherpfad"), ): chosen = _paperless_choice(options.get(name), label, allow_none=True) if chosen == "none": params[f"{field}__isnull"] = "true" elif chosen: params[f"{field}__id"] = chosen raw_tags = options.get("tag") or options.get("tags") or "" if isinstance(raw_tags, (list, tuple)): parts = [str(item) for item in raw_tags] else: parts = str(raw_tags).split(",") tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen] tag_mode = str(options.get("tag_mode") or "").strip() or "all" if tag_mode not in _PAPERLESS_TAG_MODES: raise unprocessable("Diese Tag-Verknüpfung gibt es nicht.") if tag_ids: params[_PAPERLESS_TAG_MODES[tag_mode]] = ",".join(tag_ids) _paperless_range(options, "document", "created", "Dokumentdatum", params) _paperless_range(options, "added", "added", "Hinzugefügt am", params) custom = _paperless_custom_query(options.get("custom"), custom_fields or []) if custom: params["custom_field_query"] = custom asn = str(options.get("asn") or "").strip() if asn: if not asn.isdigit() or len(asn) > 10: raise unprocessable("Die Archivnummer ist keine Zahl.") params["archive_serial_number"] = asn sort = str(options.get("sort") or "").strip() if sort: if sort not in _PAPERLESS_SORTS: raise unprocessable("Diese Sortierung gibt es nicht.") if sort in {"score", "-score"}: if not text or mode != "index": raise unprocessable("Die Relevanz gilt nur für die Suche im Volltextindex.") # Paperless-ngx 2.20.6 has no ordering field named score. A query without # ordering is the relevance order of the full-text index. else: params["ordering"] = sort cursor = str(options.get("cursor") or "").strip() if cursor: if not cursor.isdigit() or not 1 <= int(cursor) <= 10000: raise unprocessable("Die Seite ist ungültig.") params["page"] = cursor return params def _paperless_row(conn, owner_id: UUID, connector_id: UUID | None): if connector_id is not None: row = _owned(conn, owner_id, connector_id) if row["kind"] != "paperless": raise unprocessable("Das ist kein Paperless-Zugang.") return row rows = conn.execute( """ SELECT id, settings, secret FROM connectors WHERE owner_id = %s AND kind = 'paperless' ORDER BY name, id """, (owner_id,), ).fetchall() if not rows: raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.") if len(rows) != 1: raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.") return rows[0] def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None, options: dict | None = None) -> dict: row = _paperless_row(conn, owner_id, connector_id) base = row["settings"]["base_url"].rstrip("/") release_for_external(conn) names = _paperless_lookup(row["id"], base, row["secret"]) params = _paperless_params(query, options, names["custom_fields"]) # Observed on a real instance: all-tags together with a custom field makes Paperless answer 500. try: payload = _paperless_get(base, row["secret"], "/api/documents/", params) except Problem as problem: clash = "," in params.get("tags__id__all", "") and "custom_field_query" in params if clash and "HTTP 5" in problem.detail: raise unprocessable(f"{problem.detail} Das tritt auf, wenn „Alle gewählten Tags“ mit einem Zusatzfeld zusammentrifft. Wähle „Mindestens eines“ oder lass das Zusatzfeld weg.") from problem raise assert_still_owned(conn, row["id"], owner_id) terms = _search_terms(query, (options or {}).get("title")) items = [] for doc in payload.get("results") or []: created = str(doc.get("created") or "")[:10] added = str(doc.get("added") or "")[:10] title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip() pages = doc.get("page_count") asn = doc.get("archive_serial_number") items.append( { "id": str(doc.get("id")), "title": title[:200], "document_on": created if len(created) == 10 else None, "added_on": added if len(added) == 10 else None, "correspondent": _correspondent(doc, names), "document_type": _named(doc.get("document_type"), names["document_types"]), "storage_path": _named(doc.get("storage_path"), names["storage_paths"]), "tags": [name for name in (_named(tag, names["tags"]) for tag in doc.get("tags") or []) if name][:12], "custom_fields": _custom_display(doc.get("custom_fields"), names["custom_fields"]), "pages": pages if isinstance(pages, int) and pages > 0 else None, "asn": asn if isinstance(asn, int) else None, "snippet": _snippet(doc, terms), "open_url": f"{base}/documents/{doc.get('id')}/", } ) page = int(params.get("page") or 1) count = payload.get("count") if isinstance(count, int): next_page = str(page + 1) if page * PAPERLESS_PAGE_SIZE < count else None else: next_page = str(page + 1) if len(items) == PAPERLESS_PAGE_SIZE else None return { "items": items, "connector_id": str(row["id"]), "next_page": next_page, "previous_page": str(page - 1) if page > 1 else None, "total": count if isinstance(count, int) else None, "incomplete": False, } def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict: row = _paperless_row(conn, owner_id, connector_id) base = row["settings"]["base_url"].rstrip("/") release_for_external(conn) names = _paperless_lookup(row["id"], base, row["secret"]) assert_still_owned(conn, row["id"], owner_id) return { "correspondents": names["correspondents"], "document_types": names["document_types"], "storage_paths": names["storage_paths"], "tags": names["tags"], "custom_fields": names["custom_fields"], "tag_mode": "all", "tag_modes": list(_PAPERLESS_TAG_MODES), "modes": list(_PAPERLESS_MODES), } _THUMBNAIL_MEDIA = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"}) PAPERLESS_SEARCH_PARAMS = { "mode": "mode", "title": "title", "correspondent": "correspondent", "document_type": "document_type", "storage_path": "storage_path", "tag": "tags", "tag_mode": "tag_mode", "document_from": "document_range", "document_to": "document_range", "added_from": "added_range", "added_to": "added_range", "asn": "asn", "custom": "custom_fields", "sort": "sort", "cursor": "continuation", } def paperless_thumbnail(conn, owner_id: UUID, document_id: str, connector_id: UUID) -> tuple[bytes, str]: if not document_id.isdigit() or len(document_id) > 12: raise unprocessable("Die Paperless-Nummer ist ungültig.") row = _paperless_row(conn, owner_id, connector_id) base = row["settings"]["base_url"].rstrip("/") release_for_external(conn) fetched = _paperless_file(base, row["secret"], f"/api/documents/{document_id}/thumb/", 2 * 1024 * 1024) assert_still_owned(conn, row["id"], owner_id) if fetched is None: raise not_found() data, header, _name = fetched return data, _thumbnail_media(data, header) def _thumbnail_media(data: bytes, header: str) -> str: """Raster pictures only, told by their bytes. A vector file could carry markup, a header can be wrong.""" try: media = sniff(data) except Problem: media = "image/gif" if data[:4] == b"GIF8" else "" if media not in _THUMBNAIL_MEDIA: raise unprocessable(f"Paperless hat kein Vorschaubild geliefert (Antworttyp {header or 'unbekannt'}).") return media _LOOKUP_TTL = 60.0 _LOOKUP_LIMIT = 32 _lookup_cache: dict[tuple, tuple[float, dict]] = {} _lookup_lock = threading.Lock() def _paperless_lookup(connector_id, base: str, token: str) -> dict: """Names behind the ids of a result list. Bound to the instance and key that asked, kept for a minute.""" key = (str(connector_id), base, hashlib.sha256(token.encode("utf-8")).hexdigest()) now = time.monotonic() with _lookup_lock: hit = _lookup_cache.get(key) if hit and now - hit[0] < _LOOKUP_TTL: return hit[1] names = { "correspondents": _paperless_names(base, token, "/api/correspondents/"), "document_types": _paperless_names(base, token, "/api/document_types/"), "tags": _paperless_names(base, token, "/api/tags/"), # An instance without storage paths, or a key that may not read them, still searches. "storage_paths": _paperless_names_or_none(base, token, "/api/storage_paths/"), "custom_fields": _paperless_custom_fields(base, token), } with _lookup_lock: if len(_lookup_cache) >= _LOOKUP_LIMIT: _lookup_cache.clear() _lookup_cache[key] = (now, names) return names def _paperless_custom_fields(base: str, token: str) -> list[dict]: """Custom fields the picker can filter on. Types it cannot query, such as document links, are left out.""" found = [] try: for page in range(1, 6): payload = _paperless_get(base, token, "/api/custom_fields/", {"page": str(page), "page_size": "100"}) rows = payload.get("results") or [] for item in rows: kind = item.get("data_type") if item.get("id") is None or kind not in _CUSTOM_KINDS: continue extra = item.get("extra_data") if isinstance(item.get("extra_data"), dict) else {} options = [ {"id": str(option["id"]), "name": str(option.get("label") or option["id"])[:200]} for option in extra.get("select_options") or [] if isinstance(option, dict) and option.get("id") is not None ] found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200], "type": kind, "options": options}) if len(rows) < 100: break except Problem: return [] return found def _paperless_names_or_none(base: str, token: str, path: str) -> list[dict]: try: return _paperless_names(base, token, path) except Problem: return [] _CUSTOM_SHOWN = 8 def _custom_display(values, fields: list[dict]) -> list[dict]: """The custom fields of a hit as name, type and readable value. Empty values and unknown fields are left out.""" if not isinstance(values, list): return [] known = {field["id"]: field for field in fields} shown = [] for entry in values: if not isinstance(entry, dict): continue field = known.get(str(entry.get("field"))) value = entry.get("value") if field is None or value is None or value == "" or isinstance(value, list): continue if field["type"] == "boolean": text = "Ja" if value else "Nein" elif field["type"] == "select": text = next((option["name"] for option in field["options"] if option["id"] == str(value)), "") else: text = str(value) if text: shown.append({"name": field["name"], "type": field["type"], "value": text[:200]}) return shown[:_CUSTOM_SHOWN] def _named(value, entries: list[dict]) -> str | None: if value is None or isinstance(value, bool): return None wanted = str(value) for entry in entries: if entry["id"] == wanted: return entry["name"] return None _TAG_MARKUP = re.compile(r"<[^>]{0,200}>") _SNIPPET_LENGTH = 260 def _search_terms(query: str, title) -> list[str]: words = re.findall(r"[\w-]{3,}", f"{query or ''} {title or ''}".casefold()) return list(dict.fromkeys(words))[:8] def _snippet(doc: dict, terms: list[str]) -> str | None: """A short plain-text excerpt: the index hit when Paperless sends one, else the text around the first match.""" hit = doc.get("__search_hit__") if isinstance(hit, dict): lines = hit.get("highlights") if isinstance(lines, str) and lines.strip(): text = " ".join(unescape(_TAG_MARKUP.sub("", lines)).split()) if text: return text[:_SNIPPET_LENGTH] content = doc.get("content") if not isinstance(content, str): return None text = " ".join(content.split()) if not text: return None folded = text.casefold() at = min((index for index in (folded.find(term) for term in terms) if index >= 0), default=-1) if at < 0: return text[:_SNIPPET_LENGTH] + ("…" if len(text) > _SNIPPET_LENGTH else "") first = max(0, at - 60) if first: first = text.rfind(" ", 0, first) + 1 or first last = first + _SNIPPET_LENGTH return ("…" if first else "") + text[first:last] + ("…" if last < len(text) else "") def _paperless_names(base: str, token: str, path: str) -> list[dict]: found = [] for page in range(1, 21): payload = _paperless_get(base, token, path, {"page": str(page), "page_size": "100"}) rows = payload.get("results") or [] for item in rows: if item.get("id") is None: continue found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200]}) if len(rows) < 100: break return found def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]: row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") release_for_external(conn) client = _imap_login(row["settings"], row["secret"]) try: items = _imap_mailboxes(client) finally: _imap_logout(client) assert_still_owned(conn, row["id"], owner_id) return items def imap_messages( conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None, uidvalidity: int | None = None, ) -> dict: row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") if any(char in mailbox for char in "\r\n\x00"): raise unprocessable("Der Fachname ist ungültig.") if len(query) > 200: raise unprocessable("Die Suche ist zu lang.") release_for_external(conn) client = _imap_login(row["settings"], row["secret"]) try: page = _imap_page(client, mailbox, query, before, uidvalidity) finally: _imap_logout(client) assert_still_owned(conn, row["id"], owner_id) return page def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict: row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") _check_mailbox(mailbox) if uid < 1: raise unprocessable("Die Nachricht ist ungültig.") settings = dict(row["settings"]) secret = row["secret"] connector_id = row["id"] release_for_external(conn) client = _imap_login(settings, secret) try: _select_mailbox(client, mailbox) preview = _fetch_message(client, uid, mailbox) finally: _imap_logout(client) assert_still_owned(conn, connector_id, owner_id) return preview def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict: row = conn.execute( "SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s", (connector_id, owner_id), ).fetchone() if row is None: raise not_found() return row def owned_connector(conn, owner_id: UUID, connector_id: UUID) -> dict: """The stored row of an own instance, secret included. For server-side use only.""" return _owned(conn, owner_id, connector_id) def _public(row) -> dict: return { "id": str(row["id"]), "kind": row["kind"], "name": row["name"], "settings": row["settings"], "created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"), "capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None), "has_secret": bool(row["has_secret"]), "probe": { "at": row["last_probe_at"].strftime("%Y-%m-%dT%H:%M:%SZ"), "ok": bool(row["last_probe_ok"]), "detail": row["last_probe_detail"] or "", } if row["last_probe_at"] else None, } def clean_base_url(value: str) -> str: raw = value.strip() parsed = urlparse(raw) if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password: raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.") if len(raw) > 300: raise unprocessable("Die Paperless-Adresse ist zu lang.") return raw.rstrip("/") def _paperless_count(base: str, token: str) -> int: payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"}) return int(payload.get("count") or 0) def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]: if not document_id.isdigit() or len(document_id) > 12: raise unprocessable("Die Paperless-Nummer ist ungültig.") row = _paperless_row(conn, owner_id, connector_id) release_for_external(conn) fetched = _paperless_bytes(row, document_id) assert_still_owned(conn, row["id"], owner_id) return fetched def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]: base = row["settings"]["base_url"].rstrip("/") token = row["secret"] misses = 0 for suffix in ("preview", "download"): fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/") if fetched is None: misses += 1 continue viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id) if viewable is not None: return viewable meta = _paperless_metadata(base, token, document_id) if meta is None and misses == 2: raise unprocessable("Dieses Dokument gibt es in Paperless nicht.") if meta is not None: local = _paperless_from_disk(document_id, meta) if local is not None: return local raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.") def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]: """The stored document. A grantee never uses a connector of their own account.""" external_id = str((ref or {}).get("external_id") or "") detail = (ref or {}).get("detail") or {} stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id")) owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, stored) == actor_id if not external_id.isdigit() or len(external_id) > 12: if owner_path: raise unprocessable("Die Paperless-Nummer ist ungültig.") raise unprocessable(ORIGINAL_UNAVAILABLE) borrowed = borrow_original(conn, actor_id, element, context, ref, "paperless") data, media, name = complete_borrow(conn, borrowed, lambda row: _paperless_bytes(row, external_id)) if external_id and external_id in (name or ""): name = "dokument.pdf" return data, media, name or "dokument.pdf" def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> bytes: """The stored message. Mailbox, UID and connector come only from the saved reference.""" detail = (ref or {}).get("detail") or {} if ref is None or ref["source"] != "imap": raise unprocessable("Das ist keine E-Mail.") connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None) mailbox = str(detail.get("mailbox") or "").strip() uid = _mail_uid(detail.get("uid")) external_id = str(ref["external_id"]) owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, connector_id) == actor_id if owner_path: if not mailbox or uid is None: raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.") elif connector_id is None or not mailbox or uid is None: raise unprocessable(ORIGINAL_UNAVAILABLE) borrowed = borrow_original(conn, actor_id, element, context, ref, "imap") def fetch(row): return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT) found = complete_borrow(conn, borrowed, fetch) return _keep_moved_mail(conn, element["id"], external_id, mailbox, uid, found) def _uuid(value) -> UUID | None: if isinstance(value, UUID): return value try: return UUID(str(value)) except (TypeError, ValueError): return None def _mail_uid(value) -> int | None: if isinstance(value, bool): return None if isinstance(value, int) and value >= 1: return value if isinstance(value, str) and value.isdigit() and int(value) >= 1: return int(value) return None def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str): return conn.execute( """ SELECT id, owner_id, kind, settings, secret FROM connectors WHERE id = %s AND owner_id = %s AND kind = %s """, (connector_id, owner_id, kind), ).fetchone() def _paperless_file( base: str, token: str, path: str, max_bytes: int = 40 * 1024 * 1024, ) -> tuple[bytes, str, str] | None: request = Request( f"{base}{path}", headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"}, ) try: with urlopen(request, timeout=30) as response: final = urlparse(response.geturl()) allowed = urlparse(base) if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc: raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.") data = response.read(max_bytes + 1) if len(data) > max_bytes: if max_bytes > 40 * 1024 * 1024: raise unprocessable("Die Originaldatei ist für den Export zu groß.") raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.") media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip() return data, media, _filename(response.headers.get("Content-Disposition")) except HTTPError as exc: if exc.code == 404: return None if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc except (URLError, TimeoutError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None: media = media.split(";")[0].strip().lower() if not (media.startswith("application/pdf") or media.startswith("image/")): try: media = sniff(data) except Problem: return None if media.startswith("application/pdf") or media.startswith("image/"): return data, media, name or f"paperless-{document_id}.pdf" return None def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None: request = Request( f"{base}/api/documents/{document_id}/metadata/", headers={"Authorization": f"Token {token}", "Accept": "application/json"}, ) try: with urlopen(request, timeout=20) as response: payload = json.loads(response.read().decode("utf-8")) except HTTPError as exc: if exc.code == 404: return None if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc except (URLError, TimeoutError, json.JSONDecodeError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc return payload if isinstance(payload, dict) else None def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None: root = get_settings().paperless_media_root.strip() if not root: return None candidates: list[tuple[str, str]] = [] archive_name = meta.get("archive_media_filename") if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip(): candidates.append(("archive", archive_name)) original_name = meta.get("media_filename") if isinstance(original_name, str) and original_name.strip(): candidates.append(("originals", original_name)) for kind, relative in candidates: data = read_paperless_file(root, kind, relative) if data is None: continue viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id) if viewable is not None: return viewable return None def read_paperless_file( media_root: str, kind: str, relative: str, max_bytes: int = 40 * 1024 * 1024, ) -> bytes | None: if kind not in {"archive", "originals"}: return None raw = relative.replace("\\", "/").strip() if not raw or raw.startswith("/") or ".." in Path(raw).parts: return None base = (Path(media_root) / "documents" / kind).resolve() target = (base / raw).resolve() if target != base and base not in target.parents: return None if not target.is_file(): return None data = target.read_bytes() if len(data) > max_bytes: if max_bytes > 40 * 1024 * 1024: raise unprocessable("Die Originaldatei ist für den Export zu groß.") raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.") return data def _filename(disposition: str | None) -> str: if not disposition: return "" marker = "filename=" lower = disposition.lower() if marker not in lower: return "" value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"') return value.replace("\\", "").replace("/", "")[:180] def _paperless_get(base: str, token: str, path: str, params: dict) -> dict: url = f"{base}{path}?{urlencode(params)}" request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"}) try: with urlopen(request, timeout=12) as response: return json.loads(response.read().decode("utf-8")) except HTTPError as exc: if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable(f"Paperless hat die Anfrage nicht beantwortet ({_refusal(exc, path)}).") from exc except (URLError, TimeoutError, json.JSONDecodeError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc def _refusal(exc: HTTPError, path: str) -> str: """Status and, when Paperless names one, its reason. The body is the owner's own instance, cut short.""" reason = "" try: body = json.loads(exc.read(2000).decode("utf-8", "replace")) reason = json.dumps(body, ensure_ascii=False)[:240] if isinstance(body, (dict, list)) else "" except (ValueError, OSError): reason = "" return f"HTTP {exc.code} auf {path}" + (f": {reason}" if reason else "") def _correspondent(doc: dict, names: dict | None = None) -> str | None: value = doc.get("correspondent_name") or doc.get("correspondent") if isinstance(value, str) and value.strip(): return value.strip()[:200] if names is not None: return _named(value, names["correspondents"]) return None def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4: host = settings["host"] port = int(settings["port"]) security = settings.get("security") or "ssl" try: if security == "starttls": client = imaplib.IMAP4(host, port, timeout=12) client.starttls(ssl.create_default_context()) else: client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12) client.login(settings["username"], secret) return client except imaplib.IMAP4.error as exc: raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc except (OSError, TimeoutError) as exc: raise unprocessable("Das Postfach ist nicht erreichbar.") from exc PAGE_SIZE = 30 UID_WINDOW = 400 UID_ROUNDS = 8 _LIST_LINE = re.compile( r'^\((?P[^)]*)\)\s+(?P"(?:\\.|[^"])*"|NIL)\s+(?P"(?:\\.|[^"])*"|\S+)\s*$' ) def _imap_logout(client: imaplib.IMAP4) -> None: try: client.logout() except imaplib.IMAP4.error: pass def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]: try: status, data = client.list() except imaplib.IMAP4.error as exc: raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc if status != "OK": raise unprocessable("Die Fächer konnten nicht gelesen werden.") items = [] seen: set[str] = set() for raw in data or []: parsed = _parse_list_line(raw) if parsed is None or parsed["name"] in seen: continue seen.add(parsed["name"]) items.append(parsed) items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold())) return items def _untagged_int(client: imaplib.IMAP4, name: str) -> int | None: values = (getattr(client, "untagged_responses", None) or {}).get(name) or [] if not values: return None raw = values[-1] if isinstance(raw, bytes): raw = raw.decode("ascii", "replace") try: return int(str(raw).strip()) except ValueError: return None def _uids_from(data, low: int | None, high: int | None) -> list[int]: if data is None or isinstance(data, (bytes, str)): raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.") chunks = [] for part in data: if part is None: continue if isinstance(part, bytes): chunks.append(part) elif isinstance(part, str): chunks.append(part.encode("utf-8")) else: raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.") found = [] for piece in b" ".join(chunks).split(): if not piece.isdigit(): continue uid = int(piece) if low is not None and uid < low: continue if high is not None and uid > high: continue found.append(uid) return found def _uid_call(client: imaplib.IMAP4, *args): try: return client.uid(*args) except imaplib.IMAP4.abort as exc: raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc except TimeoutError as exc: raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc except imaplib.IMAP4.error as exc: raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc def _search_status(status) -> None: if status in {"NO", "BAD"}: raise unprocessable("Das Postfach hat die Suche abgelehnt.") if status != "OK": raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.") def _uid_window(client: imaplib.IMAP4, query: str, low: int, high: int) -> list[int]: needle = " ".join(query.split()) uid_range = f"{low}:{high}" if not needle: status, data = _uid_call(client, "SEARCH", None, "UID", uid_range) else: quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"' if needle.isascii(): status, data = _uid_call( client, "SEARCH", None, "UID", uid_range, "OR", "OR", "SUBJECT", quoted, "FROM", quoted, "TEXT", quoted, ) else: encoded = quoted.encode("utf-8") status, data = _uid_call( client, "SEARCH", "CHARSET", "UTF-8", "UID", uid_range, "OR", "OR", "SUBJECT", encoded, "FROM", encoded, "TEXT", encoded, ) _search_status(status) return _uids_from(data, low, high) def _highest_uid(client: imaplib.IMAP4) -> int: uidnext = _untagged_int(client, "UIDNEXT") if uidnext is not None: return max(0, uidnext - 1) status, data = _uid_call(client, "SEARCH", None, "UID", "*") _search_status(status) found = _uids_from(data, None, None) return max(found) if found else 0 def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None, claimed_validity: int | None = None) -> dict: folders = _imap_mailboxes(client) chosen = mailbox or "INBOX" match = next((item for item in folders if item["name"] == chosen), None) if match is None or not match["selectable"]: raise unprocessable("Dieses Fach gibt es in dem Konto nicht.") status, _data = _examine(client, chosen) if status != "OK": raise unprocessable("Dieses Fach lässt sich nicht öffnen.") exists = _untagged_int(client, "EXISTS") validity = _untagged_int(client, "UIDVALIDITY") restarted = False if claimed_validity is not None and validity is not None and int(claimed_validity) != validity: before = None restarted = True high = before - 1 if before is not None else _highest_uid(client) collected = [] attempts = 0 failures = 0 low = 1 rounds = 0 while high >= 1 and len(collected) < PAGE_SIZE and rounds < UID_ROUNDS: low = max(1, high - UID_WINDOW + 1) for uid in sorted(_uid_window(client, query, low, high), reverse=True): if len(collected) >= PAGE_SIZE: break attempts += 1 item = _fetch_header(client, uid, chosen) if item is None: failures += 1 continue collected.append(item) rounds += 1 if len(collected) >= PAGE_SIZE or low == 1: break high = low - 1 if attempts and failures == attempts: raise unprocessable("Die Nachrichtenköpfe dieses Fachs konnten nicht gelesen werden.") incomplete = bool(collected) is False and low > 1 and high >= 1 and rounds >= UID_ROUNDS if len(collected) < PAGE_SIZE and low > 1 and rounds >= UID_ROUNDS: incomplete = True oldest = collected[-1]["uid"] if collected else None if len(collected) >= PAGE_SIZE and oldest and oldest > 1: next_before = oldest elif incomplete: next_before = low else: next_before = None query_text = " ".join(query.split()) contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete) return { "mailbox": chosen, "items": collected, "next_before": next_before, "incomplete": incomplete, "restarted": restarted, "uidvalidity": validity, "exists": exists, "contradictory": contradictory, } def _check_mailbox(mailbox: str) -> None: if any(char in mailbox for char in "\r\n\x00"): raise unprocessable("Der Fachname ist ungültig.") def _mailbox_argument(mailbox: str) -> str: """IMAP atoms stay bare. Everything else is a quoted protocol string.""" _check_mailbox(mailbox) protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol): return protocol escaped = protocol.replace("\\", "\\\\").replace('"', '\\"') return f'"{escaped}"' def _examine(client: imaplib.IMAP4, mailbox: str): argument = _mailbox_argument(mailbox) try: return client.select(argument, readonly=True) except imaplib.IMAP4.abort as exc: raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc except TimeoutError as exc: raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc except imaplib.IMAP4.error as exc: raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None: folders = _imap_mailboxes(client) match = next((item for item in folders if item["name"] == mailbox), None) if match is None or not match["selectable"]: raise unprocessable("Dieses Fach gibt es in dem Konto nicht.") status, _data = _examine(client, mailbox) if status != "OK": raise unprocessable("Dieses Fach lässt sich nicht öffnen.") def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None: try: status, fetched = client.uid( "FETCH", str(uid), "(BODY.PEEK[HEADER.FIELDS (FROM TO CC SUBJECT DATE MESSAGE-ID)])", ) except imaplib.IMAP4.error: return None if status != "OK" or not fetched or not isinstance(fetched[0], tuple): return None attachments = [] try: struct_status, structure = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)") if struct_status == "OK": attachments = _structure_attachments(structure) except imaplib.IMAP4.error: attachments = [] header = BytesParser().parsebytes(fetched[0][1]) subject = _decode(header.get("Subject")) or "(ohne Betreff)" sender = _decode(header.get("From")) message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip() return { "id": message_id[:200], "uid": uid, "title": subject[:200], "message_from": sender[:200], "message_to": _decode(header.get("To"))[:500], "message_cc": _decode(header.get("Cc"))[:500], "direction": _direction(mailbox), "attachments": attachments, "document_on": _mail_date(header.get("Date")), } PREVIEW_BYTE_LIMIT = 8_000_000 PREVIEW_TEXT_LIMIT = 200_000 def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict: try: status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])") except imaplib.IMAP4.error as exc: raise unprocessable("Die Nachricht lässt sich nicht lesen.") from exc raw = _literal(fetched) if status != "OK" or raw is None: raise unprocessable("Die Nachricht lässt sich nicht lesen.") if len(raw) > PREVIEW_BYTE_LIMIT: return _oversized_message(client, uid, mailbox) parsed = parse_mail(raw) return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated) def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict: text = parsed.text[:PREVIEW_TEXT_LIMIT] return { "uid": uid, "mailbox": mailbox, "title": (parsed.subject or "(ohne Betreff)")[:200], "message_from": parsed.sender[:200], "message_to": parsed.recipient[:500], "message_cc": parsed.copy[:500], "direction": _direction(mailbox), "document_on": parsed.document_on, "attachments": [ { "filename": item.display_name or item.filename, "archive_name": item.filename, "content_type": item.media_type, } for item in parsed.attachments[:40] ], "text": text, "html": parsed.html, "text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT, "preview_limited": limited, "notices": parsed.notices, } _SENT_FOLDER = re.compile( r"(^|/)(sent|sent items|sent messages|gesendet|gesendete|gesendete elemente|outbox|postausgang)($|/)", re.IGNORECASE, ) _INBOX_FOLDER = re.compile( r"(^|/)(inbox|posteingang|eingang)($|/)", re.IGNORECASE, ) def _direction(mailbox: str) -> str | None: """Sent folders are outgoing, an inbox is incoming. Other folder names prove nothing.""" folded = mailbox.casefold().replace("\\", "/") if _SENT_FOLDER.search(folded): return "out" if _INBOX_FOLDER.search(folded): return "in" return None def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict: """Read named text parts. A cut-off multipart blob is not a message.""" header = _fetch_header(client, uid, mailbox) or {} structure = _body_structure(client, uid) plain: list[str] = [] html_parts: list[str] = [] truncated = False notices = [ "Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. " "Das Original bleibt die vollständige E-Mail." ] if structure is None: notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.") else: for part in _text_sections(structure): fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT) if fetched is None: notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.") continue truncated = truncated or shortened or part["octets"] > len(fetched) text = decode_transfer(fetched, part["charset"]).strip() if part["media"] == "text/plain": plain.append(text) else: html_parts.append(text) simplified = "\n".join(chunk for chunk in plain if chunk).strip() rendered = sanitize_html("\n".join(html_parts)) if html_parts else "" if not simplified and html_parts: simplified = simplified_text("\n".join(html_parts)) if not header and not simplified and not rendered and structure is None: raise unprocessable("Die Nachricht lässt sich nicht lesen.") attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else []) return { "uid": uid, "mailbox": mailbox, "title": header.get("title") or "(ohne Betreff)", "message_from": header.get("message_from") or "", "message_to": header.get("message_to") or "", "message_cc": header.get("message_cc") or "", "direction": header.get("direction", _direction(mailbox)), "document_on": header.get("document_on"), "attachments": attachments, "text": simplified[:PREVIEW_TEXT_LIMIT], "html": rendered, "text_truncated": True or truncated, "preview_limited": True, "notices": notices, } def _body_structure(client: imaplib.IMAP4, uid: int): try: status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)") except imaplib.IMAP4.error: return None if status != "OK": return None blob = _literal(fetched) if blob is None: chunks: list[bytes] = [] for part in fetched or []: if isinstance(part, tuple): chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray))) elif isinstance(part, (bytes, bytearray)): chunks.append(bytes(part)) blob = b" ".join(chunks) text = blob.decode("utf-8", "replace") start = text.find("(") if start < 0: return None try: node, _end = _parse_imap_list(text, start) except ValueError: return None return node def _parse_imap_list(text: str, index: int): if index >= len(text) or text[index] != "(": raise ValueError(text[index:index + 20]) index += 1 items = [] while index < len(text): char = text[index] if char == ")": return items, index + 1 if char in " \t\r\n": index += 1 continue if char == "(": child, index = _parse_imap_list(text, index) items.append(child) continue if char == '"': token, index = _parse_imap_string(text, index) items.append(token) continue end = index while end < len(text) and text[end] not in " \t\r\n()": end += 1 token = text[index:end] items.append(None if token.upper() == "NIL" else token) index = end raise ValueError("unclosed") def _parse_imap_string(text: str, index: int): index += 1 chars: list[str] = [] while index < len(text): char = text[index] if char == "\\": if index + 1 < len(text): chars.append(text[index + 1]) index += 2 continue if char == '"': return "".join(chars), index + 1 chars.append(char) index += 1 raise ValueError("string") def _text_sections(node, prefix: str = "") -> list[dict]: if not isinstance(node, list) or not node: return [] if isinstance(node[0], list): children = [] for item in node: if isinstance(item, list): children.append(item) else: break found = [] for number, child in enumerate(children, start=1): section = f"{number}" if not prefix else f"{prefix}.{number}" found.extend(_text_sections(child, section)) return found media = str(node[0] or "").upper() subtype = str(node[1] or "").upper() if media == "MESSAGE": return [] if media != "TEXT" or subtype not in {"PLAIN", "HTML"}: return [] disposition, params = _disposition(node) if disposition == "ATTACHMENT": return [] body_params = _param_map(node[2]) if len(node) > 2 else {} octets = _imap_int(node[6]) if len(node) > 6 else 0 return [{ "section": prefix or "1", "media": f"text/{subtype.lower()}", "charset": body_params.get("charset") or "utf-8", "octets": octets, }] def _disposition(node: list) -> tuple[str, dict]: for item in node: if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}: params = _param_map(item[1]) if len(item) > 1 else {} return item[0].upper(), params return "", {} def _param_map(value) -> dict: if not isinstance(value, list): return {} pairs = {} index = 0 while index + 1 < len(value): key = value[index] raw = value[index + 1] if isinstance(key, str): pairs[key.casefold()] = "" if raw is None else str(raw) index += 2 return pairs def _imap_int(value) -> int: try: return int(value) except (TypeError, ValueError): return 0 def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]: spec = f"(BODY.PEEK[{section}]<0.{limit}>)" try: status, fetched = client.uid("FETCH", str(uid), spec) except imaplib.IMAP4.error: return None, False if status != "OK": return None, False raw = _literal(fetched) if not raw: return None, False return raw[:limit], len(raw) >= limit def _structure_names(node) -> list[dict]: names = [] def visit(item): if not isinstance(item, list) or not item: return if isinstance(item[0], list): for child in item: if isinstance(child, list): visit(child) return _disposition_name, params = _disposition(item) body = _param_map(item[2]) if len(item) > 2 else {} raw = params.get("filename") or body.get("name") or "" label = header_text(raw) if label: names.append({"filename": label[:200]}) visit(node) return names[:40] def _literal(fetched) -> bytes | None: for part in fetched or []: if isinstance(part, tuple) and len(part) > 1 and isinstance(part[1], (bytes, bytearray)): return bytes(part[1]) return None def _structure_attachments(fetched) -> list[dict]: chunks: list[bytes] = [] for part in fetched or []: if isinstance(part, tuple): chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray))) elif isinstance(part, (bytes, bytearray)): chunks.append(bytes(part)) blob = b" ".join(chunks) names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE) items = [] for raw in names: name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200] if not name: continue items.append({"filename": name}) if len(items) == 40: break return items def _message_attachments(message) -> list[dict]: items = [] seen: set[str] = set() for part in message.walk(): filename = part.get_filename() if not filename: continue name = _decode(filename)[:200] if not name or name in seen: continue seen.add(name) items.append({"filename": name, "content_type": part.get_content_type()[:120]}) if len(items) == 40: break return items def _message_text(message) -> str: plain: list[str] = [] html: list[str] = [] parts = message.walk() if message.is_multipart() else [message] for part in parts: if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename(): continue kind = part.get_content_type() if kind == "text/plain": plain.append(_part_text(part)) elif kind == "text/html": html.append(_html_text(_part_text(part))) text = "\n".join(chunk for chunk in plain if chunk).strip() if not text: text = "\n".join(chunk for chunk in html if chunk).strip() return text def _part_text(part) -> str: payload = part.get_payload(decode=True) if payload is None: raw = part.get_payload() return raw.strip() if isinstance(raw, str) else "" charset = part.get_content_charset() or "utf-8" return payload.decode(charset, errors="replace").strip() def _html_text(value: str) -> str: text = re.sub(r"(?is)<(script|style)\b.*?>.*?", " ", value) text = re.sub(r"(?i)", "\n", text) text = re.sub(r"(?i)

", "\n", text) text = re.sub(r"<[^>]+>", " ", text) return re.sub(r"[ \t]{2,}", " ", text).strip() def _parse_list_line(raw) -> dict | None: if not isinstance(raw, (bytes, str)) or not raw: return None text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw match = _LIST_LINE.match(text.strip()) if match is None: return None flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag} delimiter = _unquote(match.group("delim")) name = _unquote(match.group("name")) if not name: return None label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name) depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter) return { "name": name, "label": label, "delimiter": delimiter, "depth": depth, "selectable": "noselect" not in flags, } def _unquote(token: str) -> str: if token == "NIL": return "" if len(token) >= 2 and token.startswith('"') and token.endswith('"'): return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\") return token def _encode_utf7(value: str) -> str: """Modified UTF-7 for a mailbox that is not already an ASCII protocol name.""" pieces: list[str] = [] pending: list[str] = [] def flush() -> None: if not pending: return raw = "".join(pending).encode("utf-16-be") token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",") pieces.append(f"&{token}-") pending.clear() for char in value: if char == "&": flush() pieces.append("&-") elif ord(char) < 128: flush() pieces.append(char) else: pending.append(char) flush() return "".join(pieces) def _decode_utf7(value: str) -> str: if "&" not in value: return value parts: list[str] = [] index = 0 while index < len(value): if value[index] != "&": parts.append(value[index]) index += 1 continue end = value.find("-", index) if end < 0: parts.append(value[index:]) break if end == index + 1: parts.append("&") else: chunk = value[index + 1:end].replace(",", "/") padding = "=" * ((4 - len(chunk) % 4) % 4) try: parts.append(base64.b64decode(chunk + padding).decode("utf-16-be")) except (ValueError, UnicodeDecodeError): parts.append(value[index:end + 1]) index = end + 1 return "".join(parts) def _mail_date(value: str | None) -> str | None: if not value: return None try: return parsedate_to_datetime(value).date().isoformat() except (TypeError, ValueError, OverflowError): return None def _decode(value: str | None) -> str: if not value: return "" parts = [] for text, encoding in decode_header(value): if isinstance(text, bytes): parts.append(text.decode(encoding or "utf-8", errors="replace")) else: parts.append(text) return "".join(parts).strip() def pull_paperless_original(base: str, token: str, document_id: str, max_bytes: int) -> tuple[bytes, str, str]: """Original download only. A preview is never returned in its place.""" if not document_id.isdigit() or len(document_id) > 12: raise unprocessable("Die Paperless-Nummer ist ungültig.") fetched = _paperless_file(base, token, f"/api/documents/{document_id}/download/", max_bytes) if fetched is not None: data, media, name = fetched return data, media or "application/octet-stream", name or f"paperless-{document_id}" meta = _paperless_metadata(base, token, document_id) if meta is None: raise unprocessable("Dieses Dokument gibt es in Paperless nicht.") local = _paperless_original_from_disk(meta, max_bytes) if local is None: raise unprocessable( "Das Paperless-Original ist nicht verfügbar. Eine Vorschau wird nicht als Original übernommen." ) return local def _paperless_original_from_disk(meta: dict, max_bytes: int) -> tuple[bytes, str, str] | None: root = get_settings().paperless_media_root.strip() original_name = meta.get("media_filename") if not root or not isinstance(original_name, str) or not original_name.strip(): return None data = read_paperless_file(root, "originals", original_name, max_bytes) if data is None: return None return data, "application/octet-stream", Path(original_name).name COMPLETE_MAIL_LIMIT = 80 * 1024 * 1024 def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int, expected_id: str) -> bytes: """The referenced message in full. A preview is not returned in its place.""" row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") found = pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT) return found.raw if isinstance(found, LocatedMail) else found @dataclass(frozen=True) class LocatedMail: """Bytes plus the folder they were read from. `relocated` is a complete, unique hit.""" raw: bytes mailbox: str uid: int relocated: bool = False @dataclass(frozen=True) class _MailHit: raw: bytes mailbox: str uid: int def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> LocatedMail: """Fetch the referenced message. A different message at the old UID is not returned. The stored folder is tried first. A single Message-ID hit elsewhere comes back marked relocated; the caller stores that place after the access check. """ client = _imap_login(settings, secret) try: _select_mailbox(client, mailbox) candidate = _fetch_rfc822(client, uid) if candidate is not None and len(candidate) > max_bytes: raise unprocessable( "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) if candidate is not None and identities_match(header_message_id(candidate), expected_id): return LocatedMail(candidate, mailbox, uid, relocated=False) hits, capped = _search_rfc822(client, expected_id, max_bytes) try: chosen = choose_message(None, [hit.raw for hit in hits], expected_id) except MailIdentityError as exc: raise unprocessable(exc.message) from exc match = next( hit for hit in hits if hit.raw == chosen and identities_match(header_message_id(hit.raw), expected_id) ) if len(match.raw) > max_bytes: raise unprocessable( "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) moved = not capped and (match.mailbox != mailbox or match.uid != uid) return LocatedMail(match.raw, match.mailbox, match.uid, relocated=moved) finally: _imap_logout(client) def _keep_moved_mail(conn, element_id, external_id: str, mailbox: str, uid: int | None, found) -> bytes: """Return the bytes. A unique new place is written only after the fetch has finished.""" if not isinstance(found, LocatedMail): return found if found.relocated and element_id is not None and uid is not None: remember_imap_location( conn, element_id, external_id, mailbox, uid, found.mailbox, found.uid, ) return found.raw def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None: try: status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])") except imaplib.IMAP4.error: return None if status != "OK": return None return _literal(fetched) def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> tuple[list[_MailHit], bool]: """Search every selectable folder. The second value is true when the scan stopped early.""" token = (expected_id or "").strip() if not token or token.casefold().startswith("uid:"): return [], False found: list[_MailHit] = [] for folder in _imap_mailboxes(client): if not folder["selectable"]: continue try: _select_mailbox(client, folder["name"]) status, data = client.uid("SEARCH", "HEADER", "Message-ID", token) except (imaplib.IMAP4.error, Problem): continue if status != "OK" or not data or not data[0]: continue for raw_uid in data[0].split(): try: uid = int(raw_uid) except ValueError: continue blob = _fetch_rfc822(client, uid) if blob is None: continue if len(blob) > max_bytes: raise unprocessable( "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) found.append(_MailHit(blob, folder["name"], uid)) if len(found) > 5: return found, True return found, False PAPERLESS_FIELDS = ( ConfigField( "base_url", "Adresse", type="url", required=True, placeholder="https://paperless.example", help="Adresse deiner Paperless-Instanz ohne Pfad.", max_length=300, ), ConfigField( "token", "API-Schlüssel", type="password", required=True, secret=True, help="Den Schlüssel erzeugst du in Paperless unter deinem Profil.", max_length=200, ), ) IMAP_FIELDS = ( ConfigField("host", "Server", required=True, placeholder="imap.example", max_length=200), ConfigField("port", "Port", type="number", required=True, default=993, minimum=1, maximum=65535), ConfigField( "security", "Verbindung", type="select", required=True, default="ssl", options=(("ssl", "SSL"), ("starttls", "STARTTLS")), ), ConfigField("username", "Benutzername", required=True, max_length=200), ConfigField("password", "Passwort", type="password", required=True, secret=True, max_length=500), ConfigField( "mailbox", "Standardfach", default="INBOX", help="Dieses Fach öffnet die Auswahl zuerst.", max_length=200, ), ) def _paperless_normalize(settings: dict, secret: str) -> dict: if len(secret.strip()) < 8: raise unprocessable("Der API-Schlüssel ist zu kurz.") return {**settings, "base_url": clean_base_url(settings["base_url"])} def _imap_normalize(settings: dict, secret: str) -> dict: del secret if " " in settings["host"]: raise unprocessable("Der Mailserver ist ungültig.") _check_mailbox(settings.get("mailbox") or "INBOX") return settings def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict: return paperless_documents(conn, owner_id, query, connector_id, options) def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict: if connector_id is None: raise unprocessable("Für dieses Postfach fehlt der Zugang.") before = options.get("before") validity = options.get("uidvalidity") return imap_messages( conn, owner_id, connector_id, query, str(options.get("mailbox") or ""), before, int(validity) if validity else None, ) def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref): from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem del detail resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned) if resolved.row is None: return RetrievalFailure(resolved.code, resolved.message, resolved.choices) base = resolved.row["settings"]["base_url"].rstrip("/") token = resolved.row["secret"] connector_id = resolved.row["id"] owner_id = resolved.row["owner_id"] release_for_external(conn) try: data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT) except Problem as exc: return failure_from_problem(exc) try: assert_still_owned(conn, connector_id, owner_id) except Problem as exc: return failure_from_problem(exc) if len(data) > EXPORT_BYTE_LIMIT: return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.") return FilePayload(name or f"paperless-{external_id}", media, data, "") def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref): from app.mailpack import identities_match, parse_mail from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned) if resolved.row is None: return RetrievalFailure(resolved.code, resolved.message, resolved.choices) mailbox = str(detail.get("mailbox") or "").strip() uid = detail.get("uid") if isinstance(uid, str) and uid.isdigit(): uid = int(uid) if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1: return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.") settings = dict(resolved.row["settings"]) secret = resolved.row["secret"] connector_id = resolved.row["id"] owner_id = resolved.row["owner_id"] release_for_external(conn) try: found = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT) except Problem as exc: return failure_from_problem(exc) try: assert_still_owned(conn, connector_id, owner_id) except Problem as exc: return failure_from_problem(exc) element_id = ref["element_id"] if ref is not None and "element_id" in ref.keys() else None raw = _keep_moved_mail(conn, element_id, str(external_id), mailbox, uid, found) if len(raw) > EXPORT_BYTE_LIMIT: return RetrievalFailure( "too_large", "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen.", ) parsed = parse_mail(raw) if not identities_match(parsed.message_id, str(external_id)): return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.") if parsed.missing: names = ", ".join(parsed.missing[:5]) return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.") return MailPayload(raw, parsed) def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]: raw = read_linked_mail(conn, actor_id, element, context, ref) return raw, "message/rfc822", "nachricht.eml" PAPERLESS_CAPABILITIES = { "kind": "paperless", "navigate": False, "read_original": True, "read_preview": True, "read_attachment": False, "edit_content": False, "edit_metadata": False, "search": { "query": True, "mode": True, "title": True, "correspondent": True, "document_type": True, "storage_path": True, "tags": True, "tag_mode": True, "document_range": True, "added_range": True, "asn": True, "custom_fields": True, "sort": True, "continuation": True, "result_preview": True, }, } IMAP_CAPABILITIES = { "kind": "imap", "navigate": True, "read_original": True, "read_preview": True, "read_attachment": True, "edit_content": False, "edit_metadata": False, "search": { "query": True, "mode": False, "title": False, "correspondent": False, "document_type": False, "storage_path": False, "tags": False, "tag_mode": False, "document_range": False, "added_range": False, "asn": False, "custom_fields": False, "sort": False, "continuation": True, "result_preview": False, }, } class _PaperlessAdapter: kind = "paperless" def capabilities(self) -> dict: return PAPERLESS_CAPABILITIES def probe(self, row) -> dict: count = _paperless_count(row["settings"]["base_url"], row["secret"]) return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."} def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None): found = paperless_documents(conn, owner_id, query, connector_id) return found["items"], found["connector_id"] def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict: return _paperless_find(conn, owner_id, connector_id, query, options) def facets(self, conn, owner_id: UUID, connector_id: UUID) -> dict: return paperless_facets(conn, owner_id, connector_id) class _ImapAdapter: kind = "imap" def capabilities(self) -> dict: return IMAP_CAPABILITIES def probe(self, row) -> dict: _imap_login(row["settings"], row["secret"]) return {"ok": True, "detail": "Das Postfach ist erreichbar."} def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]: return imap_mailboxes(conn, owner_id, connector_id) def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict: return imap_messages(conn, owner_id, connector_id, query, mailbox, before) def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict: return _imap_find(conn, owner_id, connector_id, query, options) def _install_sources() -> None: paperless = _PaperlessAdapter() imap = _ImapAdapter() register_source(SourceModule( kind="paperless", label="Paperless", representation="pdf", capabilities=PAPERLESS_CAPABILITIES, config_fields=PAPERLESS_FIELDS, normalize_config=_paperless_normalize, probe=paperless.probe, find=paperless.find, open_original=read_linked_paperless, export_original=_export_paperless, navigate=None, preview=None, facets=paperless.facets, search_params=PAPERLESS_SEARCH_PARAMS, thumbnail=lambda conn, owner, connector_id, item: paperless_thumbnail(conn, owner, str(item), connector_id), )) register_source(SourceModule( kind="imap", label="E-Mail", representation="email", capabilities=IMAP_CAPABILITIES, config_fields=IMAP_FIELDS, normalize_config=_imap_normalize, probe=imap.probe, find=imap.find, open_original=_imap_bytes, export_original=_export_imap, navigate=imap.navigate, preview=lambda conn, owner, connector_id, options: imap_preview( conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0) ), )) _install_sources()