1385 lines
51 KiB
Python
1385 lines
51 KiB
Python
"""Paperless and IMAP connectors. Secrets stay on the server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import imaplib
|
|
import json
|
|
import re
|
|
import ssl
|
|
from email.header import decode_header
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urlparse
|
|
from urllib.request import Request, urlopen
|
|
from uuid import UUID
|
|
|
|
from psycopg.types.json import Json
|
|
|
|
from app.access.policy import Ctx
|
|
from app.core.errors import Problem, not_found, unprocessable
|
|
from app.core.settings import get_settings
|
|
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
|
|
from app.media.storage import sniff
|
|
from app.sourcebind import resolve_for_read
|
|
from app.sourceaccess import (
|
|
ORIGINAL_UNAVAILABLE,
|
|
assert_still_owned,
|
|
audit_mediated as _audit_mediated,
|
|
borrow_original,
|
|
complete_borrow,
|
|
connector_owner as _connector_owner,
|
|
release_for_external,
|
|
)
|
|
from app.sourceport import CAPABILITIES
|
|
from app.sourceregistry import SourceModule, get_source, register_source
|
|
|
|
|
|
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
|
|
row = conn.execute(
|
|
"""
|
|
INSERT INTO connectors (owner_id, kind, name, settings, secret)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
RETURNING id, kind, name, settings, created_at
|
|
""",
|
|
(owner_id, kind, name, Json(settings), secret),
|
|
).fetchone()
|
|
return _public(row)
|
|
|
|
|
|
def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict:
|
|
"""Credential or name change on the same instance. The id does not move."""
|
|
current = _owned(conn, owner_id, connector_id)
|
|
row = conn.execute(
|
|
"""
|
|
UPDATE connectors
|
|
SET name = %s, settings = %s, secret = %s
|
|
WHERE id = %s AND owner_id = %s
|
|
RETURNING id, kind, name, settings, created_at
|
|
""",
|
|
(name, Json(settings), secret, current["id"], owner_id),
|
|
).fetchone()
|
|
return _public(row)
|
|
|
|
|
|
def list_connectors(conn, owner_id: UUID) -> list[dict]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT id, kind, name, settings, created_at
|
|
FROM connectors WHERE owner_id = %s
|
|
ORDER BY kind, name
|
|
""",
|
|
(owner_id,),
|
|
).fetchall()
|
|
return [_public(row) for row in rows]
|
|
|
|
|
|
def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
|
|
row = conn.execute(
|
|
"DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
|
|
|
|
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
module = get_source(row["kind"])
|
|
if module is None:
|
|
raise unprocessable("Diese Quelle ist nicht registriert.")
|
|
release_for_external(conn)
|
|
result = module.probe(row)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return result
|
|
|
|
|
|
def _paperless_choice(value, label: str) -> str | None:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return None
|
|
if not text.isdigit() or len(text) > 12:
|
|
raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.")
|
|
return text
|
|
|
|
|
|
def _paperless_day(value, label: str) -> str | None:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return None
|
|
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", text):
|
|
raise unprocessable(f"{label} ist kein Datum.")
|
|
return text
|
|
|
|
|
|
def _paperless_params(query: str, options: dict | None) -> dict:
|
|
"""Paperless-ngx document date is the created field, not the upload or modified time."""
|
|
options = options or {}
|
|
params = {"page_size": "20"}
|
|
text = " ".join(str(query or "").split())
|
|
if text:
|
|
params["query"] = text
|
|
title = " ".join(str(options.get("title") or "").split())
|
|
if title:
|
|
params["title__icontains"] = title
|
|
correspondent = _paperless_choice(options.get("correspondent"), "Der Korrespondent")
|
|
if correspondent:
|
|
params["correspondent__id"] = correspondent
|
|
document_type = _paperless_choice(options.get("document_type"), "Der Dokumenttyp")
|
|
if document_type:
|
|
params["document_type__id"] = document_type
|
|
raw_tags = options.get("tag") or options.get("tags") or ""
|
|
if isinstance(raw_tags, (list, tuple)):
|
|
parts = [str(item) for item in raw_tags]
|
|
else:
|
|
parts = str(raw_tags).split(",")
|
|
tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen]
|
|
if tag_ids:
|
|
params["tags__id__all"] = ",".join(tag_ids)
|
|
start = _paperless_day(options.get("document_from"), "Das Dokumentdatum von")
|
|
end = _paperless_day(options.get("document_to"), "Das Dokumentdatum bis")
|
|
if start and end and start > end:
|
|
raise unprocessable("Das Dokumentdatum von liegt nach dem Dokumentdatum bis.")
|
|
if start:
|
|
params["created__date__gte"] = start
|
|
if end:
|
|
params["created__date__lte"] = end
|
|
sort = str(options.get("sort") or "").strip()
|
|
if sort:
|
|
if sort not in {"created", "-created", "title", "-title", "score", "-score"}:
|
|
raise unprocessable("Diese Sortierung gibt es nicht.")
|
|
if sort in {"score", "-score"}:
|
|
if not text:
|
|
raise unprocessable("Die Relevanz gilt zusammen mit einem Suchtext.")
|
|
# Paperless-ngx 2.20.6 has no ordering field named score. A query without
|
|
# ordering is the relevance order of the full-text index.
|
|
else:
|
|
params["ordering"] = sort
|
|
cursor = str(options.get("cursor") or "").strip()
|
|
if cursor:
|
|
if not cursor.isdigit() or not 1 <= int(cursor) <= 10000:
|
|
raise unprocessable("Die Seite ist ungültig.")
|
|
params["page"] = cursor
|
|
return params
|
|
|
|
|
|
def _paperless_row(conn, owner_id: UUID, connector_id: UUID | None):
|
|
if connector_id is not None:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "paperless":
|
|
raise unprocessable("Das ist kein Paperless-Zugang.")
|
|
return row
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT id, settings, secret FROM connectors
|
|
WHERE owner_id = %s AND kind = 'paperless'
|
|
ORDER BY name, id
|
|
""",
|
|
(owner_id,),
|
|
).fetchall()
|
|
if not rows:
|
|
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
|
if len(rows) != 1:
|
|
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
|
|
return rows[0]
|
|
|
|
|
|
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None, options: dict | None = None) -> dict:
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
params = _paperless_params(query, options)
|
|
release_for_external(conn)
|
|
payload = _paperless_get(base, row["secret"], "/api/documents/", params)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
items = []
|
|
for doc in payload.get("results") or []:
|
|
created = str(doc.get("created") or "")[:10]
|
|
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
|
|
items.append(
|
|
{
|
|
"id": str(doc.get("id")),
|
|
"title": title[:200],
|
|
"document_on": created if len(created) == 10 else None,
|
|
"correspondent": _correspondent(doc),
|
|
"open_url": f"{base}/documents/{doc.get('id')}/",
|
|
}
|
|
)
|
|
page = int(params.get("page") or 1)
|
|
count = payload.get("count")
|
|
if isinstance(count, int):
|
|
next_page = str(page + 1) if page * 20 < count else None
|
|
else:
|
|
next_page = str(page + 1) if len(items) == 20 else None
|
|
return {"items": items, "connector_id": str(row["id"]), "next_page": next_page, "incomplete": False}
|
|
|
|
|
|
def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
token = row["secret"]
|
|
release_for_external(conn)
|
|
body = {
|
|
"correspondents": _paperless_names(base, token, "/api/correspondents/"),
|
|
"document_types": _paperless_names(base, token, "/api/document_types/"),
|
|
"tags": _paperless_names(base, token, "/api/tags/"),
|
|
"tag_mode": "all",
|
|
}
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return body
|
|
|
|
|
|
def _paperless_names(base: str, token: str, path: str) -> list[dict]:
|
|
found = []
|
|
for page in range(1, 21):
|
|
payload = _paperless_get(base, token, path, {"page": str(page), "page_size": "100"})
|
|
rows = payload.get("results") or []
|
|
for item in rows:
|
|
if item.get("id") is None:
|
|
continue
|
|
found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200]})
|
|
if len(rows) < 100:
|
|
break
|
|
return found
|
|
|
|
|
|
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
release_for_external(conn)
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
items = _imap_mailboxes(client)
|
|
finally:
|
|
_imap_logout(client)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return items
|
|
|
|
|
|
def imap_messages(
|
|
conn,
|
|
owner_id: UUID,
|
|
connector_id: UUID,
|
|
query: str,
|
|
mailbox: str,
|
|
before: int | None,
|
|
uidvalidity: int | None = None,
|
|
) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
if any(char in mailbox for char in "\r\n\x00"):
|
|
raise unprocessable("Der Fachname ist ungültig.")
|
|
if len(query) > 200:
|
|
raise unprocessable("Die Suche ist zu lang.")
|
|
release_for_external(conn)
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
page = _imap_page(client, mailbox, query, before, uidvalidity)
|
|
finally:
|
|
_imap_logout(client)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return page
|
|
|
|
|
|
def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
_check_mailbox(mailbox)
|
|
if uid < 1:
|
|
raise unprocessable("Die Nachricht ist ungültig.")
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
_select_mailbox(client, mailbox)
|
|
return _fetch_message(client, uid, mailbox)
|
|
finally:
|
|
_imap_logout(client)
|
|
|
|
|
|
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = conn.execute(
|
|
"SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
return row
|
|
|
|
|
|
def _public(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"kind": row["kind"],
|
|
"name": row["name"],
|
|
"settings": row["settings"],
|
|
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None),
|
|
}
|
|
|
|
|
|
def clean_base_url(value: str) -> str:
|
|
raw = value.strip()
|
|
parsed = urlparse(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
|
|
raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.")
|
|
if len(raw) > 300:
|
|
raise unprocessable("Die Paperless-Adresse ist zu lang.")
|
|
return raw.rstrip("/")
|
|
|
|
|
|
def _paperless_count(base: str, token: str) -> int:
|
|
payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"})
|
|
return int(payload.get("count") or 0)
|
|
|
|
|
|
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
release_for_external(conn)
|
|
fetched = _paperless_bytes(row, document_id)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return fetched
|
|
|
|
|
|
def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
token = row["secret"]
|
|
misses = 0
|
|
for suffix in ("preview", "download"):
|
|
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/")
|
|
if fetched is None:
|
|
misses += 1
|
|
continue
|
|
viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
meta = _paperless_metadata(base, token, document_id)
|
|
if meta is None and misses == 2:
|
|
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
|
|
if meta is not None:
|
|
local = _paperless_from_disk(document_id, meta)
|
|
if local is not None:
|
|
return local
|
|
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
|
|
|
|
|
|
def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
|
"""The stored document. A grantee never uses a connector of their own account."""
|
|
external_id = str((ref or {}).get("external_id") or "")
|
|
detail = (ref or {}).get("detail") or {}
|
|
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
|
|
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, stored) == actor_id
|
|
if not external_id.isdigit() or len(external_id) > 12:
|
|
if owner_path:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
|
borrowed = borrow_original(conn, actor_id, element, context, ref, "paperless")
|
|
data, media, name = complete_borrow(conn, borrowed, lambda row: _paperless_bytes(row, external_id))
|
|
if external_id and external_id in (name or ""):
|
|
name = "dokument.pdf"
|
|
return data, media, name or "dokument.pdf"
|
|
|
|
|
|
def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> bytes:
|
|
"""The stored message. Mailbox, UID and connector come only from the saved reference."""
|
|
detail = (ref or {}).get("detail") or {}
|
|
if ref is None or ref["source"] != "imap":
|
|
raise unprocessable("Das ist keine E-Mail.")
|
|
connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
|
|
mailbox = str(detail.get("mailbox") or "").strip()
|
|
uid = _mail_uid(detail.get("uid"))
|
|
external_id = str(ref["external_id"])
|
|
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, connector_id) == actor_id
|
|
if owner_path:
|
|
if not mailbox or uid is None:
|
|
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
|
elif connector_id is None or not mailbox or uid is None:
|
|
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
|
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
|
|
|
|
def fetch(row):
|
|
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
|
|
|
|
return complete_borrow(conn, borrowed, fetch)
|
|
|
|
|
|
def _uuid(value) -> UUID | None:
|
|
if isinstance(value, UUID):
|
|
return value
|
|
try:
|
|
return UUID(str(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _mail_uid(value) -> int | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, int) and value >= 1:
|
|
return value
|
|
if isinstance(value, str) and value.isdigit() and int(value) >= 1:
|
|
return int(value)
|
|
return None
|
|
|
|
|
|
def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
|
|
return conn.execute(
|
|
"""
|
|
SELECT id, owner_id, kind, settings, secret
|
|
FROM connectors
|
|
WHERE id = %s AND owner_id = %s AND kind = %s
|
|
""",
|
|
(connector_id, owner_id, kind),
|
|
).fetchone()
|
|
|
|
|
|
def _paperless_file(
|
|
base: str,
|
|
token: str,
|
|
path: str,
|
|
max_bytes: int = 40 * 1024 * 1024,
|
|
) -> tuple[bytes, str, str] | None:
|
|
request = Request(
|
|
f"{base}{path}",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=30) as response:
|
|
final = urlparse(response.geturl())
|
|
allowed = urlparse(base)
|
|
if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc:
|
|
raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.")
|
|
data = response.read(max_bytes + 1)
|
|
if len(data) > max_bytes:
|
|
if max_bytes > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip()
|
|
return data, media, _filename(response.headers.get("Content-Disposition"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None:
|
|
media = media.split(";")[0].strip().lower()
|
|
if not (media.startswith("application/pdf") or media.startswith("image/")):
|
|
try:
|
|
media = sniff(data)
|
|
except Problem:
|
|
return None
|
|
if media.startswith("application/pdf") or media.startswith("image/"):
|
|
return data, media, name or f"paperless-{document_id}.pdf"
|
|
return None
|
|
|
|
|
|
def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None:
|
|
request = Request(
|
|
f"{base}/api/documents/{document_id}/metadata/",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/json"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=20) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
return payload if isinstance(payload, dict) else None
|
|
|
|
|
|
def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None:
|
|
root = get_settings().paperless_media_root.strip()
|
|
if not root:
|
|
return None
|
|
candidates: list[tuple[str, str]] = []
|
|
archive_name = meta.get("archive_media_filename")
|
|
if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip():
|
|
candidates.append(("archive", archive_name))
|
|
original_name = meta.get("media_filename")
|
|
if isinstance(original_name, str) and original_name.strip():
|
|
candidates.append(("originals", original_name))
|
|
for kind, relative in candidates:
|
|
data = read_paperless_file(root, kind, relative)
|
|
if data is None:
|
|
continue
|
|
viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
return None
|
|
|
|
|
|
def read_paperless_file(
|
|
media_root: str,
|
|
kind: str,
|
|
relative: str,
|
|
max_bytes: int = 40 * 1024 * 1024,
|
|
) -> bytes | None:
|
|
if kind not in {"archive", "originals"}:
|
|
return None
|
|
raw = relative.replace("\\", "/").strip()
|
|
if not raw or raw.startswith("/") or ".." in Path(raw).parts:
|
|
return None
|
|
base = (Path(media_root) / "documents" / kind).resolve()
|
|
target = (base / raw).resolve()
|
|
if target != base and base not in target.parents:
|
|
return None
|
|
if not target.is_file():
|
|
return None
|
|
data = target.read_bytes()
|
|
if len(data) > max_bytes:
|
|
if max_bytes > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
return data
|
|
|
|
|
|
def _filename(disposition: str | None) -> str:
|
|
if not disposition:
|
|
return ""
|
|
marker = "filename="
|
|
lower = disposition.lower()
|
|
if marker not in lower:
|
|
return ""
|
|
value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"')
|
|
return value.replace("\\", "").replace("/", "")[:180]
|
|
|
|
|
|
def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
|
|
url = f"{base}{path}?{urlencode(params)}"
|
|
request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"})
|
|
try:
|
|
with urlopen(request, timeout=12) as response:
|
|
return json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _correspondent(doc: dict) -> str | None:
|
|
value = doc.get("correspondent_name") or doc.get("correspondent")
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()[:200]
|
|
return None
|
|
|
|
|
|
def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
|
|
host = settings["host"]
|
|
port = int(settings["port"])
|
|
security = settings.get("security") or "ssl"
|
|
try:
|
|
if security == "starttls":
|
|
client = imaplib.IMAP4(host, port, timeout=12)
|
|
client.starttls(ssl.create_default_context())
|
|
else:
|
|
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12)
|
|
client.login(settings["username"], secret)
|
|
return client
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc
|
|
except (OSError, TimeoutError) as exc:
|
|
raise unprocessable("Das Postfach ist nicht erreichbar.") from exc
|
|
|
|
|
|
PAGE_SIZE = 30
|
|
UID_WINDOW = 400
|
|
UID_ROUNDS = 8
|
|
_LIST_LINE = re.compile(
|
|
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
|
|
)
|
|
|
|
|
|
def _imap_logout(client: imaplib.IMAP4) -> None:
|
|
try:
|
|
client.logout()
|
|
except imaplib.IMAP4.error:
|
|
pass
|
|
|
|
|
|
def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
|
|
try:
|
|
status, data = client.list()
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.")
|
|
items = []
|
|
seen: set[str] = set()
|
|
for raw in data or []:
|
|
parsed = _parse_list_line(raw)
|
|
if parsed is None or parsed["name"] in seen:
|
|
continue
|
|
seen.add(parsed["name"])
|
|
items.append(parsed)
|
|
items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold()))
|
|
return items
|
|
|
|
|
|
def _untagged_int(client: imaplib.IMAP4, name: str) -> int | None:
|
|
values = (getattr(client, "untagged_responses", None) or {}).get(name) or []
|
|
if not values:
|
|
return None
|
|
raw = values[-1]
|
|
if isinstance(raw, bytes):
|
|
raw = raw.decode("ascii", "replace")
|
|
try:
|
|
return int(str(raw).strip())
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _uids_from(data, low: int | None, high: int | None) -> list[int]:
|
|
if data is None or isinstance(data, (bytes, str)):
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
chunks = []
|
|
for part in data:
|
|
if part is None:
|
|
continue
|
|
if isinstance(part, bytes):
|
|
chunks.append(part)
|
|
elif isinstance(part, str):
|
|
chunks.append(part.encode("utf-8"))
|
|
else:
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
found = []
|
|
for piece in b" ".join(chunks).split():
|
|
if not piece.isdigit():
|
|
continue
|
|
uid = int(piece)
|
|
if low is not None and uid < low:
|
|
continue
|
|
if high is not None and uid > high:
|
|
continue
|
|
found.append(uid)
|
|
return found
|
|
|
|
|
|
def _uid_call(client: imaplib.IMAP4, *args):
|
|
try:
|
|
return client.uid(*args)
|
|
except imaplib.IMAP4.abort as exc:
|
|
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
|
except TimeoutError as exc:
|
|
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
|
|
|
|
|
|
def _search_status(status) -> None:
|
|
if status in {"NO", "BAD"}:
|
|
raise unprocessable("Das Postfach hat die Suche abgelehnt.")
|
|
if status != "OK":
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
|
|
|
|
def _uid_window(client: imaplib.IMAP4, query: str, low: int, high: int) -> list[int]:
|
|
needle = " ".join(query.split())
|
|
uid_range = f"{low}:{high}"
|
|
if not needle:
|
|
status, data = _uid_call(client, "SEARCH", None, "UID", uid_range)
|
|
else:
|
|
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
if needle.isascii():
|
|
status, data = _uid_call(
|
|
client, "SEARCH", None, "UID", uid_range,
|
|
"OR", "OR", "SUBJECT", quoted, "FROM", quoted, "TEXT", quoted,
|
|
)
|
|
else:
|
|
encoded = quoted.encode("utf-8")
|
|
status, data = _uid_call(
|
|
client, "SEARCH", "CHARSET", "UTF-8", "UID", uid_range,
|
|
"OR", "OR", "SUBJECT", encoded, "FROM", encoded, "TEXT", encoded,
|
|
)
|
|
_search_status(status)
|
|
return _uids_from(data, low, high)
|
|
|
|
|
|
def _highest_uid(client: imaplib.IMAP4) -> int:
|
|
uidnext = _untagged_int(client, "UIDNEXT")
|
|
if uidnext is not None:
|
|
return max(0, uidnext - 1)
|
|
status, data = _uid_call(client, "SEARCH", None, "UID", "*")
|
|
_search_status(status)
|
|
found = _uids_from(data, None, None)
|
|
return max(found) if found else 0
|
|
|
|
|
|
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None, claimed_validity: int | None = None) -> dict:
|
|
folders = _imap_mailboxes(client)
|
|
chosen = mailbox or "INBOX"
|
|
match = next((item for item in folders if item["name"] == chosen), None)
|
|
if match is None or not match["selectable"]:
|
|
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
|
try:
|
|
status, _ = client.select(chosen, readonly=True)
|
|
except imaplib.IMAP4.abort as exc:
|
|
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
|
except TimeoutError as exc:
|
|
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
|
validity = _untagged_int(client, "UIDVALIDITY")
|
|
restarted = False
|
|
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
|
|
before = None
|
|
restarted = True
|
|
high = before - 1 if before is not None else _highest_uid(client)
|
|
collected = []
|
|
attempts = 0
|
|
failures = 0
|
|
low = 1
|
|
rounds = 0
|
|
while high >= 1 and len(collected) < PAGE_SIZE and rounds < UID_ROUNDS:
|
|
low = max(1, high - UID_WINDOW + 1)
|
|
for uid in sorted(_uid_window(client, query, low, high), reverse=True):
|
|
if len(collected) >= PAGE_SIZE:
|
|
break
|
|
attempts += 1
|
|
item = _fetch_header(client, uid, chosen)
|
|
if item is None:
|
|
failures += 1
|
|
continue
|
|
collected.append(item)
|
|
rounds += 1
|
|
if len(collected) >= PAGE_SIZE or low == 1:
|
|
break
|
|
high = low - 1
|
|
if attempts and failures == attempts:
|
|
raise unprocessable("Die Nachrichtenköpfe dieses Fachs konnten nicht gelesen werden.")
|
|
incomplete = bool(collected) is False and low > 1 and high >= 1 and rounds >= UID_ROUNDS
|
|
if len(collected) < PAGE_SIZE and low > 1 and rounds >= UID_ROUNDS:
|
|
incomplete = True
|
|
oldest = collected[-1]["uid"] if collected else None
|
|
if len(collected) >= PAGE_SIZE and oldest and oldest > 1:
|
|
next_before = oldest
|
|
elif incomplete:
|
|
next_before = low
|
|
else:
|
|
next_before = None
|
|
return {
|
|
"mailbox": chosen,
|
|
"items": collected,
|
|
"next_before": next_before,
|
|
"incomplete": incomplete,
|
|
"restarted": restarted,
|
|
"uidvalidity": validity,
|
|
}
|
|
|
|
|
|
def _check_mailbox(mailbox: str) -> None:
|
|
if any(char in mailbox for char in "\r\n\x00"):
|
|
raise unprocessable("Der Fachname ist ungültig.")
|
|
|
|
|
|
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
|
|
folders = _imap_mailboxes(client)
|
|
match = next((item for item in folders if item["name"] == mailbox), None)
|
|
if match is None or not match["selectable"]:
|
|
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
|
try:
|
|
status, _ = client.select(mailbox, readonly=True)
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
|
|
|
|
|
def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
|
|
try:
|
|
status, fetched = client.uid(
|
|
"FETCH",
|
|
str(uid),
|
|
"(BODY.PEEK[HEADER.FIELDS (FROM TO CC SUBJECT DATE MESSAGE-ID)])",
|
|
)
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK" or not fetched or not isinstance(fetched[0], tuple):
|
|
return None
|
|
attachments = []
|
|
try:
|
|
struct_status, structure = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
|
|
if struct_status == "OK":
|
|
attachments = _structure_attachments(structure)
|
|
except imaplib.IMAP4.error:
|
|
attachments = []
|
|
header = BytesParser().parsebytes(fetched[0][1])
|
|
subject = _decode(header.get("Subject")) or "(ohne Betreff)"
|
|
sender = _decode(header.get("From"))
|
|
message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip()
|
|
return {
|
|
"id": message_id[:200],
|
|
"uid": uid,
|
|
"title": subject[:200],
|
|
"message_from": sender[:200],
|
|
"message_to": _decode(header.get("To"))[:500],
|
|
"message_cc": _decode(header.get("Cc"))[:500],
|
|
"direction": _direction(mailbox),
|
|
"attachments": attachments,
|
|
"document_on": _mail_date(header.get("Date")),
|
|
}
|
|
|
|
|
|
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.") from exc
|
|
raw = _literal(fetched)
|
|
if status != "OK" or raw is None:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
|
if len(raw) > 8_000_000:
|
|
return _oversized_message(client, uid, mailbox)
|
|
message = BytesParser().parsebytes(raw)
|
|
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
|
|
return {
|
|
"uid": uid,
|
|
"mailbox": mailbox,
|
|
"title": subject[:200],
|
|
"message_from": _decode(message.get("From"))[:200],
|
|
"message_to": _decode(message.get("To"))[:500],
|
|
"message_cc": _decode(message.get("Cc"))[:500],
|
|
"direction": _direction(mailbox),
|
|
"document_on": _mail_date(message.get("Date")),
|
|
"attachments": _message_attachments(message),
|
|
"text": (_text := _message_text(message))[:200_000],
|
|
"text_truncated": len(_text) > 200_000,
|
|
}
|
|
|
|
|
|
_SENT_FOLDER = re.compile(
|
|
r"(^|/)(sent|sent items|sent messages|gesendet|gesendete|gesendete elemente|outbox|postausgang)($|/)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
_INBOX_FOLDER = re.compile(
|
|
r"(^|/)(inbox|posteingang|eingang)($|/)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _direction(mailbox: str) -> str | None:
|
|
"""Sent folders are outgoing, an inbox is incoming. Other folder names prove nothing."""
|
|
folded = mailbox.casefold().replace("\\", "/")
|
|
if _SENT_FOLDER.search(folded):
|
|
return "out"
|
|
if _INBOX_FOLDER.search(folded):
|
|
return "in"
|
|
return None
|
|
|
|
|
|
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
|
header = _fetch_header(client, uid, mailbox) or {}
|
|
text = _partial_text(client, uid)
|
|
if not header and not text:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
|
return {
|
|
"uid": uid,
|
|
"mailbox": mailbox,
|
|
"title": header.get("title") or "(ohne Betreff)",
|
|
"message_from": header.get("message_from") or "",
|
|
"message_to": header.get("message_to") or "",
|
|
"message_cc": header.get("message_cc") or "",
|
|
"direction": header.get("direction", _direction(mailbox)),
|
|
"document_on": header.get("document_on"),
|
|
"attachments": header.get("attachments") or [],
|
|
"text": text,
|
|
"text_truncated": True,
|
|
}
|
|
|
|
|
|
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
|
|
except imaplib.IMAP4.error:
|
|
return ""
|
|
if status != "OK":
|
|
return ""
|
|
raw = _literal(fetched)
|
|
if not raw:
|
|
return ""
|
|
sample = raw[:200_000]
|
|
try:
|
|
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
|
|
text = _message_text(message)
|
|
except Exception:
|
|
text = ""
|
|
if not text:
|
|
text = sample.decode("utf-8", errors="replace")
|
|
return text.strip()[:200_000]
|
|
|
|
|
|
def _literal(fetched) -> bytes | None:
|
|
for part in fetched or []:
|
|
if isinstance(part, tuple) and len(part) > 1 and isinstance(part[1], (bytes, bytearray)):
|
|
return bytes(part[1])
|
|
return None
|
|
|
|
|
|
def _structure_attachments(fetched) -> list[dict]:
|
|
chunks: list[bytes] = []
|
|
for part in fetched or []:
|
|
if isinstance(part, tuple):
|
|
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
|
|
elif isinstance(part, (bytes, bytearray)):
|
|
chunks.append(bytes(part))
|
|
blob = b" ".join(chunks)
|
|
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
|
|
items = []
|
|
seen: set[str] = set()
|
|
for raw in names:
|
|
name = raw.decode("utf-8", errors="replace").strip()[:200]
|
|
if not name or name in seen:
|
|
continue
|
|
seen.add(name)
|
|
items.append({"filename": name})
|
|
if len(items) == 40:
|
|
break
|
|
return items
|
|
|
|
|
|
def _message_attachments(message) -> list[dict]:
|
|
items = []
|
|
seen: set[str] = set()
|
|
for part in message.walk():
|
|
filename = part.get_filename()
|
|
if not filename:
|
|
continue
|
|
name = _decode(filename)[:200]
|
|
if not name or name in seen:
|
|
continue
|
|
seen.add(name)
|
|
items.append({"filename": name, "content_type": part.get_content_type()[:120]})
|
|
if len(items) == 40:
|
|
break
|
|
return items
|
|
|
|
|
|
def _message_text(message) -> str:
|
|
plain: list[str] = []
|
|
html: list[str] = []
|
|
parts = message.walk() if message.is_multipart() else [message]
|
|
for part in parts:
|
|
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
|
|
continue
|
|
kind = part.get_content_type()
|
|
if kind == "text/plain":
|
|
plain.append(_part_text(part))
|
|
elif kind == "text/html":
|
|
html.append(_html_text(_part_text(part)))
|
|
text = "\n".join(chunk for chunk in plain if chunk).strip()
|
|
if not text:
|
|
text = "\n".join(chunk for chunk in html if chunk).strip()
|
|
return text
|
|
|
|
|
|
def _part_text(part) -> str:
|
|
payload = part.get_payload(decode=True)
|
|
if payload is None:
|
|
raw = part.get_payload()
|
|
return raw.strip() if isinstance(raw, str) else ""
|
|
charset = part.get_content_charset() or "utf-8"
|
|
return payload.decode(charset, errors="replace").strip()
|
|
|
|
|
|
def _html_text(value: str) -> str:
|
|
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
|
|
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
|
|
text = re.sub(r"(?i)</p>", "\n", text)
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
return re.sub(r"[ \t]{2,}", " ", text).strip()
|
|
|
|
|
|
def _parse_list_line(raw) -> dict | None:
|
|
if not isinstance(raw, (bytes, str)) or not raw:
|
|
return None
|
|
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
match = _LIST_LINE.match(text.strip())
|
|
if match is None:
|
|
return None
|
|
flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag}
|
|
delimiter = _unquote(match.group("delim"))
|
|
name = _unquote(match.group("name"))
|
|
if not name:
|
|
return None
|
|
label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name)
|
|
depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter)
|
|
return {
|
|
"name": name,
|
|
"label": label,
|
|
"delimiter": delimiter,
|
|
"depth": depth,
|
|
"selectable": "noselect" not in flags,
|
|
}
|
|
|
|
|
|
def _unquote(token: str) -> str:
|
|
if token == "NIL":
|
|
return ""
|
|
if len(token) >= 2 and token.startswith('"') and token.endswith('"'):
|
|
return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\")
|
|
return token
|
|
|
|
|
|
def _decode_utf7(value: str) -> str:
|
|
if "&" not in value:
|
|
return value
|
|
parts: list[str] = []
|
|
index = 0
|
|
while index < len(value):
|
|
if value[index] != "&":
|
|
parts.append(value[index])
|
|
index += 1
|
|
continue
|
|
end = value.find("-", index)
|
|
if end < 0:
|
|
parts.append(value[index:])
|
|
break
|
|
if end == index + 1:
|
|
parts.append("&")
|
|
else:
|
|
chunk = value[index + 1:end].replace(",", "/")
|
|
padding = "=" * ((4 - len(chunk) % 4) % 4)
|
|
try:
|
|
parts.append(base64.b64decode(chunk + padding).decode("utf-16-be"))
|
|
except (ValueError, UnicodeDecodeError):
|
|
parts.append(value[index:end + 1])
|
|
index = end + 1
|
|
return "".join(parts)
|
|
|
|
|
|
def _mail_date(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(value).date().isoformat()
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _decode(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
parts = []
|
|
for text, encoding in decode_header(value):
|
|
if isinstance(text, bytes):
|
|
parts.append(text.decode(encoding or "utf-8", errors="replace"))
|
|
else:
|
|
parts.append(text)
|
|
return "".join(parts).strip()
|
|
|
|
|
|
def pull_paperless_original(base: str, token: str, document_id: str, max_bytes: int) -> tuple[bytes, str, str]:
|
|
"""Original download only. A preview is never returned in its place."""
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/download/", max_bytes)
|
|
if fetched is not None:
|
|
data, media, name = fetched
|
|
return data, media or "application/octet-stream", name or f"paperless-{document_id}"
|
|
meta = _paperless_metadata(base, token, document_id)
|
|
if meta is None:
|
|
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
|
|
local = _paperless_original_from_disk(meta, max_bytes)
|
|
if local is None:
|
|
raise unprocessable(
|
|
"Das Paperless-Original ist nicht verfügbar. Eine Vorschau wird nicht als Original übernommen."
|
|
)
|
|
return local
|
|
|
|
|
|
def _paperless_original_from_disk(meta: dict, max_bytes: int) -> tuple[bytes, str, str] | None:
|
|
root = get_settings().paperless_media_root.strip()
|
|
original_name = meta.get("media_filename")
|
|
if not root or not isinstance(original_name, str) or not original_name.strip():
|
|
return None
|
|
data = read_paperless_file(root, "originals", original_name, max_bytes)
|
|
if data is None:
|
|
return None
|
|
return data, "application/octet-stream", Path(original_name).name
|
|
|
|
|
|
COMPLETE_MAIL_LIMIT = 80 * 1024 * 1024
|
|
|
|
|
|
def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int, expected_id: str) -> bytes:
|
|
"""The referenced message in full. A preview is not returned in its place."""
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT)
|
|
|
|
|
|
def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> bytes:
|
|
"""Fetch the referenced message. A different message at the old UID is not returned."""
|
|
client = _imap_login(settings, secret)
|
|
try:
|
|
_select_mailbox(client, mailbox)
|
|
candidate = _fetch_rfc822(client, uid)
|
|
if candidate is not None and len(candidate) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
alternates: list[bytes] = []
|
|
if candidate is None or not identities_match(header_message_id(candidate), expected_id):
|
|
alternates = _search_rfc822(client, expected_id, max_bytes)
|
|
try:
|
|
chosen = choose_message(candidate, alternates, expected_id)
|
|
except MailIdentityError as exc:
|
|
raise unprocessable(exc.message) from exc
|
|
if len(chosen) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
return chosen
|
|
finally:
|
|
_imap_logout(client)
|
|
|
|
|
|
def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK":
|
|
return None
|
|
return _literal(fetched)
|
|
|
|
|
|
def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> list[bytes]:
|
|
token = (expected_id or "").strip()
|
|
if not token or token.casefold().startswith("uid:"):
|
|
return []
|
|
found: list[bytes] = []
|
|
for folder in _imap_mailboxes(client):
|
|
if not folder["selectable"]:
|
|
continue
|
|
try:
|
|
_select_mailbox(client, folder["name"])
|
|
status, data = client.uid("SEARCH", "HEADER", "Message-ID", token)
|
|
except (imaplib.IMAP4.error, Problem):
|
|
continue
|
|
if status != "OK" or not data or not data[0]:
|
|
continue
|
|
for raw_uid in data[0].split():
|
|
try:
|
|
uid = int(raw_uid)
|
|
except ValueError:
|
|
continue
|
|
blob = _fetch_rfc822(client, uid)
|
|
if blob is None:
|
|
continue
|
|
if len(blob) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
found.append(blob)
|
|
if len(found) > 5:
|
|
return found
|
|
return found
|
|
|
|
|
|
def _paperless_config(payload: dict) -> tuple[str, dict, str]:
|
|
base_url = str(payload.get("base_url") or "").strip()
|
|
token = str(payload.get("token") or "").strip()
|
|
name = str(payload.get("name") or "").strip()
|
|
if not base_url or len(token) < 8:
|
|
raise unprocessable("Paperless braucht eine Adresse und einen API-Schlüssel.")
|
|
return name, {"base_url": clean_base_url(base_url)}, token
|
|
|
|
|
|
def _imap_config(payload: dict) -> tuple[str, dict, str]:
|
|
host = str(payload.get("host") or "").strip()
|
|
username = str(payload.get("username") or "").strip()
|
|
password = str(payload.get("password") or "")
|
|
name = str(payload.get("name") or "").strip()
|
|
if not host or not username or not password:
|
|
raise unprocessable("Ein E-Mail-Konto braucht Server, Benutzername und Passwort.")
|
|
if " " in host or len(host) > 200:
|
|
raise unprocessable("Der Mailserver ist ungültig.")
|
|
port = payload.get("port") or 993
|
|
return name, {
|
|
"host": host,
|
|
"port": int(port),
|
|
"username": username,
|
|
"mailbox": str(payload.get("mailbox") or "INBOX").strip() or "INBOX",
|
|
"security": payload.get("security") or "ssl",
|
|
}, password
|
|
|
|
|
|
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return paperless_documents(conn, owner_id, query, connector_id, options)
|
|
|
|
|
|
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
if connector_id is None:
|
|
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
|
|
before = options.get("before")
|
|
validity = options.get("uidvalidity")
|
|
return imap_messages(
|
|
conn,
|
|
owner_id,
|
|
connector_id,
|
|
query,
|
|
str(options.get("mailbox") or ""),
|
|
before,
|
|
int(validity) if validity else None,
|
|
)
|
|
|
|
|
|
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
|
from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem
|
|
|
|
del detail
|
|
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
|
|
if resolved.row is None:
|
|
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
|
try:
|
|
data, media, name = pull_paperless_original(
|
|
resolved.row["settings"]["base_url"].rstrip("/"),
|
|
resolved.row["secret"],
|
|
str(external_id),
|
|
EXPORT_BYTE_LIMIT,
|
|
)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
if len(data) > EXPORT_BYTE_LIMIT:
|
|
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
|
|
return FilePayload(name or f"paperless-{external_id}", media, data, "")
|
|
|
|
|
|
def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
|
from app.mailpack import identities_match, parse_mail
|
|
from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem
|
|
|
|
resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned)
|
|
if resolved.row is None:
|
|
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
|
mailbox = str(detail.get("mailbox") or "").strip()
|
|
uid = detail.get("uid")
|
|
if isinstance(uid, str) and uid.isdigit():
|
|
uid = int(uid)
|
|
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
|
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
|
try:
|
|
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
if len(raw) > EXPORT_BYTE_LIMIT:
|
|
return RetrievalFailure(
|
|
"too_large",
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen.",
|
|
)
|
|
parsed = parse_mail(raw)
|
|
if not identities_match(parsed.message_id, str(external_id)):
|
|
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
|
|
if parsed.missing:
|
|
names = ", ".join(parsed.missing[:5])
|
|
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
|
|
return MailPayload(raw, parsed)
|
|
|
|
|
|
def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
|
raw = read_linked_mail(conn, actor_id, element, context, ref)
|
|
return raw, "message/rfc822", "nachricht.eml"
|
|
|
|
|
|
class _PaperlessAdapter:
|
|
kind = "paperless"
|
|
|
|
def capabilities(self) -> dict:
|
|
return CAPABILITIES[self.kind]
|
|
|
|
def probe(self, row) -> dict:
|
|
count = _paperless_count(row["settings"]["base_url"], row["secret"])
|
|
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
|
|
|
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
|
|
found = paperless_documents(conn, owner_id, query, connector_id)
|
|
return found["items"], found["connector_id"]
|
|
|
|
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return _paperless_find(conn, owner_id, connector_id, query, options)
|
|
|
|
def facets(self, conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
return paperless_facets(conn, owner_id, connector_id)
|
|
|
|
|
|
class _ImapAdapter:
|
|
kind = "imap"
|
|
|
|
def capabilities(self) -> dict:
|
|
return CAPABILITIES[self.kind]
|
|
|
|
def probe(self, row) -> dict:
|
|
_imap_login(row["settings"], row["secret"])
|
|
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
|
|
|
|
def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
|
return imap_mailboxes(conn, owner_id, connector_id)
|
|
|
|
def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict:
|
|
return imap_messages(conn, owner_id, connector_id, query, mailbox, before)
|
|
|
|
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return _imap_find(conn, owner_id, connector_id, query, options)
|
|
|
|
|
|
def _install_sources() -> None:
|
|
paperless = _PaperlessAdapter()
|
|
imap = _ImapAdapter()
|
|
register_source(SourceModule(
|
|
kind="paperless",
|
|
label="Paperless",
|
|
representation="pdf",
|
|
capabilities=CAPABILITIES["paperless"],
|
|
parse_config=_paperless_config,
|
|
probe=paperless.probe,
|
|
find=paperless.find,
|
|
open_original=read_linked_paperless,
|
|
export_original=_export_paperless,
|
|
navigate=None,
|
|
preview=None,
|
|
facets=paperless.facets,
|
|
))
|
|
register_source(SourceModule(
|
|
kind="imap",
|
|
label="E-Mail",
|
|
representation="email",
|
|
capabilities=CAPABILITIES["imap"],
|
|
parse_config=_imap_config,
|
|
probe=imap.probe,
|
|
find=imap.find,
|
|
open_original=_imap_bytes,
|
|
export_original=_export_imap,
|
|
navigate=imap.navigate,
|
|
preview=lambda conn, owner, connector_id, options: imap_preview(
|
|
conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0)
|
|
),
|
|
))
|
|
|
|
|
|
_install_sources()
|