"""Paperless and IMAP connectors. Secrets stay on the server.""" from __future__ import annotations import base64 import imaplib import json import re import ssl from email.header import decode_header from email.parser import BytesParser from email.utils import parsedate_to_datetime from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urlparse from urllib.request import Request, urlopen from uuid import UUID from psycopg.types.json import Json from app.core.errors import Problem, not_found, unprocessable from app.core.settings import get_settings from app.media.storage import sniff def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict: if kind == "paperless": existing = conn.execute( "SELECT id FROM connectors WHERE owner_id = %s AND kind = 'paperless'", (owner_id,), ).fetchone() if existing: row = conn.execute( """ UPDATE connectors SET name = %s, settings = %s, secret = %s WHERE id = %s RETURNING id, kind, name, settings, created_at """, (name, Json(settings), secret, existing["id"]), ).fetchone() return _public(row) row = conn.execute( """ INSERT INTO connectors (owner_id, kind, name, settings, secret) VALUES (%s, %s, %s, %s, %s) RETURNING id, kind, name, settings, created_at """, (owner_id, kind, name, Json(settings), secret), ).fetchone() return _public(row) def list_connectors(conn, owner_id: UUID) -> list[dict]: rows = conn.execute( """ SELECT id, kind, name, settings, created_at FROM connectors WHERE owner_id = %s ORDER BY kind, name """, (owner_id,), ).fetchall() return [_public(row) for row in rows] def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None: row = conn.execute( "DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id", (connector_id, owner_id), ).fetchone() if row is None: raise not_found() def probe(conn, owner_id: UUID, connector_id: UUID) -> dict: row = _owned(conn, owner_id, connector_id) if row["kind"] == "paperless": count = _paperless_count(row["settings"]["base_url"], row["secret"]) return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."} _imap_login(row["settings"], row["secret"]) return {"ok": True, "detail": "Das Postfach ist erreichbar."} def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]: row = conn.execute( """ SELECT settings, secret FROM connectors WHERE owner_id = %s AND kind = 'paperless' ORDER BY created_at DESC LIMIT 1 """, (owner_id,), ).fetchone() if row is None: raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.") base = row["settings"]["base_url"].rstrip("/") payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"}) items = [] for doc in payload.get("results") or []: created = str(doc.get("created") or "")[:10] title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip() items.append( { "id": str(doc.get("id")), "title": title[:200], "document_on": created if len(created) == 10 else None, "correspondent": _correspondent(doc), "open_url": f"{base}/documents/{doc.get('id')}/", } ) return items def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]: row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") client = _imap_login(row["settings"], row["secret"]) try: return _imap_mailboxes(client) finally: _imap_logout(client) def imap_messages( conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None, ) -> dict: row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") if any(char in mailbox for char in "\r\n\x00"): raise unprocessable("Der Fachname ist ungültig.") if len(query) > 200: raise unprocessable("Die Suche ist zu lang.") client = _imap_login(row["settings"], row["secret"]) try: return _imap_page(client, mailbox, query, before) finally: _imap_logout(client) def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict: row = conn.execute( "SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s", (connector_id, owner_id), ).fetchone() if row is None: raise not_found() return row def _public(row) -> dict: return { "id": str(row["id"]), "kind": row["kind"], "name": row["name"], "settings": row["settings"], "created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"), } def clean_base_url(value: str) -> str: raw = value.strip() parsed = urlparse(raw) if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password: raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.") if len(raw) > 300: raise unprocessable("Die Paperless-Adresse ist zu lang.") return raw.rstrip("/") def _paperless_count(base: str, token: str) -> int: payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"}) return int(payload.get("count") or 0) def paperless_content(conn, owner_id: UUID, document_id: str) -> tuple[bytes, str, str]: if not document_id.isdigit() or len(document_id) > 12: raise unprocessable("Die Paperless-Nummer ist ungültig.") row = conn.execute( """ SELECT settings, secret FROM connectors WHERE owner_id = %s AND kind = 'paperless' ORDER BY created_at DESC LIMIT 1 """, (owner_id,), ).fetchone() if row is None: raise unprocessable("Paperless ist für dieses Konto nicht verbunden.") base = row["settings"]["base_url"].rstrip("/") token = row["secret"] misses = 0 for suffix in ("preview", "download"): fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/") if fetched is None: misses += 1 continue viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id) if viewable is not None: return viewable meta = _paperless_metadata(base, token, document_id) if meta is None and misses == 2: raise unprocessable("Dieses Dokument gibt es in Paperless nicht.") if meta is not None: local = _paperless_from_disk(document_id, meta) if local is not None: return local raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.") def _paperless_file(base: str, token: str, path: str) -> tuple[bytes, str, str] | None: request = Request( f"{base}{path}", headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"}, ) try: with urlopen(request, timeout=30) as response: final = urlparse(response.geturl()) allowed = urlparse(base) if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc: raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.") data = response.read(40 * 1024 * 1024 + 1) if len(data) > 40 * 1024 * 1024: raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.") media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip() return data, media, _filename(response.headers.get("Content-Disposition")) except HTTPError as exc: if exc.code == 404: return None if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc except (URLError, TimeoutError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None: media = media.split(";")[0].strip().lower() if not (media.startswith("application/pdf") or media.startswith("image/")): try: media = sniff(data) except Problem: return None if media.startswith("application/pdf") or media.startswith("image/"): return data, media, name or f"paperless-{document_id}.pdf" return None def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None: request = Request( f"{base}/api/documents/{document_id}/metadata/", headers={"Authorization": f"Token {token}", "Accept": "application/json"}, ) try: with urlopen(request, timeout=20) as response: payload = json.loads(response.read().decode("utf-8")) except HTTPError as exc: if exc.code == 404: return None if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc except (URLError, TimeoutError, json.JSONDecodeError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc return payload if isinstance(payload, dict) else None def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None: root = get_settings().paperless_media_root.strip() if not root: return None candidates: list[tuple[str, str]] = [] archive_name = meta.get("archive_media_filename") if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip(): candidates.append(("archive", archive_name)) original_name = meta.get("media_filename") if isinstance(original_name, str) and original_name.strip(): candidates.append(("originals", original_name)) for kind, relative in candidates: data = read_paperless_file(root, kind, relative) if data is None: continue viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id) if viewable is not None: return viewable return None def read_paperless_file(media_root: str, kind: str, relative: str) -> bytes | None: if kind not in {"archive", "originals"}: return None raw = relative.replace("\\", "/").strip() if not raw or raw.startswith("/") or ".." in Path(raw).parts: return None base = (Path(media_root) / "documents" / kind).resolve() target = (base / raw).resolve() if target != base and base not in target.parents: return None if not target.is_file(): return None data = target.read_bytes() if len(data) > 40 * 1024 * 1024: raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.") return data def _filename(disposition: str | None) -> str: if not disposition: return "" marker = "filename=" lower = disposition.lower() if marker not in lower: return "" value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"') return value.replace("\\", "").replace("/", "")[:180] def _paperless_get(base: str, token: str, path: str, params: dict) -> dict: url = f"{base}{path}?{urlencode(params)}" request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"}) try: with urlopen(request, timeout=12) as response: return json.loads(response.read().decode("utf-8")) except HTTPError as exc: if exc.code in {401, 403}: raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc except (URLError, TimeoutError, json.JSONDecodeError) as exc: raise unprocessable("Paperless ist nicht erreichbar.") from exc def _correspondent(doc: dict) -> str | None: value = doc.get("correspondent_name") or doc.get("correspondent") if isinstance(value, str) and value.strip(): return value.strip()[:200] return None def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4: host = settings["host"] port = int(settings["port"]) security = settings.get("security") or "ssl" try: if security == "starttls": client = imaplib.IMAP4(host, port, timeout=12) client.starttls(ssl.create_default_context()) else: client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12) client.login(settings["username"], secret) return client except imaplib.IMAP4.error as exc: raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc except (OSError, TimeoutError) as exc: raise unprocessable("Das Postfach ist nicht erreichbar.") from exc PAGE_SIZE = 30 _LIST_LINE = re.compile( r'^\((?P[^)]*)\)\s+(?P"(?:\\.|[^"])*"|NIL)\s+(?P"(?:\\.|[^"])*"|\S+)\s*$' ) def _imap_logout(client: imaplib.IMAP4) -> None: try: client.logout() except imaplib.IMAP4.error: pass def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]: try: status, data = client.list() except imaplib.IMAP4.error as exc: raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc if status != "OK": raise unprocessable("Die Fächer konnten nicht gelesen werden.") items = [] seen: set[str] = set() for raw in data or []: parsed = _parse_list_line(raw) if parsed is None or parsed["name"] in seen: continue seen.add(parsed["name"]) items.append(parsed) items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold())) return items def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None) -> dict: folders = _imap_mailboxes(client) chosen = mailbox or "INBOX" match = next((item for item in folders if item["name"] == chosen), None) if match is None or not match["selectable"]: raise unprocessable("Dieses Fach gibt es in dem Konto nicht.") try: status, _ = client.select(chosen, readonly=True) except imaplib.IMAP4.error as exc: raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc if status != "OK": raise unprocessable("Dieses Fach lässt sich nicht öffnen.") status, data = _uid_search(client, query) if status != "OK" or not data or not data[0]: return {"mailbox": chosen, "items": [], "next_before": None} uids = sorted(int(part) for part in data[0].split() if part.isdigit()) if before is not None: uids = [uid for uid in uids if uid < before] page = uids[-PAGE_SIZE:] next_before = page[0] if len(uids) > PAGE_SIZE else None items = [] for uid in reversed(page): item = _fetch_header(client, uid, chosen) if item is not None: items.append(item) return {"mailbox": chosen, "items": items, "next_before": next_before} def _uid_search(client: imaplib.IMAP4, query: str): needle = " ".join(query.split()) if not needle: return client.uid("SEARCH", None, "ALL") quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"' try: if needle.isascii(): return client.uid("SEARCH", None, "OR", "SUBJECT", quoted, "FROM", quoted) encoded = quoted.encode("utf-8") return client.uid("SEARCH", "CHARSET", "UTF-8", "OR", "SUBJECT", encoded, "FROM", encoded) except imaplib.IMAP4.error as exc: raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None: try: status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE MESSAGE-ID)])") except imaplib.IMAP4.error: return None if status != "OK" or not fetched or not isinstance(fetched[0], tuple): return None header = BytesParser().parsebytes(fetched[0][1]) subject = _decode(header.get("Subject")) or "(ohne Betreff)" sender = _decode(header.get("From")) message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip() return { "id": message_id[:200], "uid": uid, "title": subject[:200], "message_from": sender[:200], "document_on": _mail_date(header.get("Date")), } def _parse_list_line(raw) -> dict | None: if not isinstance(raw, (bytes, str)) or not raw: return None text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw match = _LIST_LINE.match(text.strip()) if match is None: return None flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag} delimiter = _unquote(match.group("delim")) name = _unquote(match.group("name")) if not name: return None label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name) depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter) return { "name": name, "label": label, "delimiter": delimiter, "depth": depth, "selectable": "noselect" not in flags, } def _unquote(token: str) -> str: if token == "NIL": return "" if len(token) >= 2 and token.startswith('"') and token.endswith('"'): return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\") return token def _decode_utf7(value: str) -> str: if "&" not in value: return value parts: list[str] = [] index = 0 while index < len(value): if value[index] != "&": parts.append(value[index]) index += 1 continue end = value.find("-", index) if end < 0: parts.append(value[index:]) break if end == index + 1: parts.append("&") else: chunk = value[index + 1:end].replace(",", "/") padding = "=" * ((4 - len(chunk) % 4) % 4) try: parts.append(base64.b64decode(chunk + padding).decode("utf-16-be")) except (ValueError, UnicodeDecodeError): parts.append(value[index:end + 1]) index = end + 1 return "".join(parts) def _mail_date(value: str | None) -> str | None: if not value: return None try: return parsedate_to_datetime(value).date().isoformat() except (TypeError, ValueError, OverflowError): return None def _decode(value: str | None) -> str: if not value: return "" parts = [] for text, encoding in decode_header(value): if isinstance(text, bytes): parts.append(text.decode(encoding or "utf-8", errors="replace")) else: parts.append(text) return "".join(parts).strip()