Standard ist jetzt Titel und Inhalt ohne Index; Titel, Inhalt und Volltextindex sind wählbar. Neue Filter: Speicherpfad, Tag-Verknüpfung (alle, eines, keines), "ohne" bei Korrespondent, Typ und Pfad, Hinzugefügt-Datum und Archivnummer. Treffer zeigen Vorschaubild, Textauszug, Tags, Typ und Seitenzahl; die Vorschaubilder laufen über den Server. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1983 lines
72 KiB
Python
1983 lines
72 KiB
Python
"""Paperless and IMAP connectors. Secrets stay on the server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import imaplib
|
|
import json
|
|
import re
|
|
import ssl
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from email.header import decode_header
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
from html import unescape
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urlparse
|
|
from urllib.request import Request, urlopen
|
|
from uuid import UUID
|
|
|
|
from psycopg.types.json import Json
|
|
|
|
from app.access.policy import Ctx
|
|
from app.core.errors import Problem, not_found, unprocessable
|
|
from app.core.settings import get_settings
|
|
from app.mailpack import (
|
|
MailIdentityError,
|
|
choose_message,
|
|
decode_transfer,
|
|
header_message_id,
|
|
header_text,
|
|
identities_match,
|
|
parse_mail,
|
|
sanitize_html,
|
|
simplified_text,
|
|
)
|
|
from app.media.storage import sniff
|
|
from app.sourcebind import remember_imap_location, resolve_for_read
|
|
from app.sourceaccess import (
|
|
ORIGINAL_UNAVAILABLE,
|
|
assert_still_owned,
|
|
audit_mediated as _audit_mediated,
|
|
borrow_original,
|
|
complete_borrow,
|
|
connector_owner as _connector_owner,
|
|
release_for_external,
|
|
)
|
|
from app.sourceport import CAPABILITIES
|
|
from app.sourceregistry import ConfigField, SourceModule, get_source, parse_fields, register_source, require_source
|
|
|
|
|
|
_PUBLIC_COLUMNS = """
|
|
id, kind, name, settings, created_at,
|
|
secret <> '' AS has_secret,
|
|
last_probe_at, last_probe_ok, last_probe_detail
|
|
"""
|
|
|
|
|
|
def parse_connector(
|
|
kind: str,
|
|
name: str,
|
|
flat: dict,
|
|
config: dict | None,
|
|
current: dict | None = None,
|
|
) -> tuple[str, dict, str]:
|
|
"""Validate a connector body with the registered module. ``current`` is the stored row on change."""
|
|
module = require_source(kind)
|
|
name = " ".join(str(name or "").split())
|
|
values = {key: value for key, value in flat.items() if value is not None}
|
|
values.update(config or {})
|
|
if module.config_fields:
|
|
settings, secret = parse_fields(
|
|
module.config_fields,
|
|
values,
|
|
current["settings"] if current else None,
|
|
current["secret"] if current else "",
|
|
)
|
|
if module.normalize_config is not None:
|
|
settings = module.normalize_config(settings, secret)
|
|
return name, settings, secret
|
|
return module.parse_config({**values, "kind": kind, "name": name, "config": config or {}})
|
|
|
|
|
|
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
|
|
row = conn.execute(
|
|
f"""
|
|
INSERT INTO connectors (owner_id, kind, name, settings, secret)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
RETURNING {_PUBLIC_COLUMNS}
|
|
""",
|
|
(owner_id, kind, name, Json(settings), secret),
|
|
).fetchone()
|
|
return _public(row)
|
|
|
|
|
|
def update_connector(conn, owner_id: UUID, connector_id: UUID, name: str, settings: dict, secret: str) -> dict:
|
|
"""Credential or name change on the same instance. The id does not move."""
|
|
current = _owned(conn, owner_id, connector_id)
|
|
row = conn.execute(
|
|
f"""
|
|
UPDATE connectors
|
|
SET name = %s, settings = %s, secret = %s,
|
|
last_probe_at = NULL, last_probe_ok = NULL, last_probe_detail = NULL
|
|
WHERE id = %s AND owner_id = %s
|
|
RETURNING {_PUBLIC_COLUMNS}
|
|
""",
|
|
(name, Json(settings), secret, current["id"], owner_id),
|
|
).fetchone()
|
|
return _public(row)
|
|
|
|
|
|
def list_connectors(conn, owner_id: UUID) -> list[dict]:
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT {_PUBLIC_COLUMNS}
|
|
FROM connectors WHERE owner_id = %s
|
|
ORDER BY kind, name
|
|
""",
|
|
(owner_id,),
|
|
).fetchall()
|
|
return [_public(row) for row in rows]
|
|
|
|
|
|
def connector_detail(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
"""One own instance with its use. The secret itself is never part of it."""
|
|
row = conn.execute(
|
|
f"SELECT {_PUBLIC_COLUMNS} FROM connectors WHERE id = %s AND owner_id = %s",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
used = conn.execute(
|
|
"""
|
|
SELECT count(*) AS total
|
|
FROM external_references er
|
|
JOIN elements e ON e.id = er.element_id
|
|
WHERE (er.connector_id = %s OR er.detail ->> 'connector_id' = %s)
|
|
AND e.deleted_at IS NULL AND e.owner_subject_id = %s
|
|
""",
|
|
(connector_id, str(connector_id), owner_id),
|
|
).fetchone()
|
|
body = _public(row)
|
|
body["references"] = int(used["total"])
|
|
body["protection_profile"] = "legacy-plaintext"
|
|
return body
|
|
|
|
|
|
def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
|
|
row = conn.execute(
|
|
"DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
|
|
|
|
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
module = get_source(row["kind"])
|
|
if module is None:
|
|
raise unprocessable("Diese Quelle ist nicht registriert.")
|
|
release_for_external(conn)
|
|
try:
|
|
result = module.probe(row)
|
|
except Problem as exc:
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
_record_probe(conn, row["id"], False, exc.detail)
|
|
conn.commit()
|
|
raise
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
_record_probe(conn, row["id"], bool(result.get("ok", True)), str(result.get("detail") or ""))
|
|
return result
|
|
|
|
|
|
def _record_probe(conn, connector_id: UUID, ok: bool, detail: str) -> None:
|
|
"""Display only. The detail comes from the module and must not carry a secret."""
|
|
conn.execute(
|
|
"""
|
|
UPDATE connectors
|
|
SET last_probe_at = now(), last_probe_ok = %s, last_probe_detail = %s
|
|
WHERE id = %s
|
|
""",
|
|
(ok, detail[:300] or None, connector_id),
|
|
)
|
|
|
|
|
|
PAPERLESS_PAGE_SIZE = 20
|
|
_PAPERLESS_MODES = {
|
|
# Paperless-ngx offers these searches in its own filter bar. The first needs no search index.
|
|
"title_content": "title_content",
|
|
"title": "title__icontains",
|
|
"content": "content__icontains",
|
|
"index": "query",
|
|
}
|
|
_PAPERLESS_TAG_MODES = {"all": "tags__id__all", "any": "tags__id__in", "none": "tags__id__none"}
|
|
_PAPERLESS_SORTS = {"created", "-created", "added", "-added", "title", "-title", "score", "-score"}
|
|
|
|
|
|
def _paperless_choice(value, label: str, allow_none: bool = False) -> str | None:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return None
|
|
if allow_none and text == "none":
|
|
return text
|
|
if not text.isdigit() or len(text) > 12:
|
|
raise unprocessable(f"{label} ist keine Auswahl dieser Instanz.")
|
|
return text
|
|
|
|
|
|
def _paperless_day(value, label: str) -> str | None:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return None
|
|
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", text):
|
|
raise unprocessable(f"{label} ist kein Datum.")
|
|
return text
|
|
|
|
|
|
def _paperless_range(options: dict, prefix: str, field: str, label: str, params: dict) -> None:
|
|
start = _paperless_day(options.get(f"{prefix}_from"), f"{label}, von")
|
|
end = _paperless_day(options.get(f"{prefix}_to"), f"{label}, bis")
|
|
if start and end and start > end:
|
|
raise unprocessable(f"{label}: Der Beginn liegt nach dem Ende.")
|
|
if start:
|
|
params[f"{field}__date__gte"] = start
|
|
if end:
|
|
params[f"{field}__date__lte"] = end
|
|
|
|
|
|
def _paperless_params(query: str, options: dict | None) -> dict:
|
|
"""Paperless-ngx document date is the created field, not the upload or modified time."""
|
|
options = options or {}
|
|
params = {"page_size": str(PAPERLESS_PAGE_SIZE)}
|
|
text = " ".join(str(query or "").split())
|
|
mode = str(options.get("mode") or "").strip() or "title_content"
|
|
if mode not in _PAPERLESS_MODES:
|
|
raise unprocessable("Diese Suchart gibt es nicht.")
|
|
if text:
|
|
params[_PAPERLESS_MODES[mode]] = text
|
|
title = " ".join(str(options.get("title") or "").split())
|
|
if title:
|
|
if "title__icontains" in params and params["title__icontains"] != title:
|
|
raise unprocessable("Die Suche nur im Titel und der Titelfilter sind beide gesetzt. Nutze eines davon.")
|
|
params["title__icontains"] = title
|
|
for name, field, label in (
|
|
("correspondent", "correspondent", "Der Korrespondent"),
|
|
("document_type", "document_type", "Der Dokumenttyp"),
|
|
("storage_path", "storage_path", "Der Speicherpfad"),
|
|
):
|
|
chosen = _paperless_choice(options.get(name), label, allow_none=True)
|
|
if chosen == "none":
|
|
params[f"{field}__isnull"] = "true"
|
|
elif chosen:
|
|
params[f"{field}__id"] = chosen
|
|
raw_tags = options.get("tag") or options.get("tags") or ""
|
|
if isinstance(raw_tags, (list, tuple)):
|
|
parts = [str(item) for item in raw_tags]
|
|
else:
|
|
parts = str(raw_tags).split(",")
|
|
tag_ids = [chosen for chosen in (_paperless_choice(part, "Ein Tag") for part in parts) if chosen]
|
|
tag_mode = str(options.get("tag_mode") or "").strip() or "all"
|
|
if tag_mode not in _PAPERLESS_TAG_MODES:
|
|
raise unprocessable("Diese Tag-Verknüpfung gibt es nicht.")
|
|
if tag_ids:
|
|
params[_PAPERLESS_TAG_MODES[tag_mode]] = ",".join(tag_ids)
|
|
_paperless_range(options, "document", "created", "Dokumentdatum", params)
|
|
_paperless_range(options, "added", "added", "Hinzugefügt am", params)
|
|
asn = str(options.get("asn") or "").strip()
|
|
if asn:
|
|
if not asn.isdigit() or len(asn) > 10:
|
|
raise unprocessable("Die Archivnummer ist keine Zahl.")
|
|
params["archive_serial_number"] = asn
|
|
sort = str(options.get("sort") or "").strip()
|
|
if sort:
|
|
if sort not in _PAPERLESS_SORTS:
|
|
raise unprocessable("Diese Sortierung gibt es nicht.")
|
|
if sort in {"score", "-score"}:
|
|
if not text or mode != "index":
|
|
raise unprocessable("Die Relevanz gilt nur für die Suche im Volltextindex.")
|
|
# Paperless-ngx 2.20.6 has no ordering field named score. A query without
|
|
# ordering is the relevance order of the full-text index.
|
|
else:
|
|
params["ordering"] = sort
|
|
cursor = str(options.get("cursor") or "").strip()
|
|
if cursor:
|
|
if not cursor.isdigit() or not 1 <= int(cursor) <= 10000:
|
|
raise unprocessable("Die Seite ist ungültig.")
|
|
params["page"] = cursor
|
|
return params
|
|
|
|
|
|
def _paperless_row(conn, owner_id: UUID, connector_id: UUID | None):
|
|
if connector_id is not None:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "paperless":
|
|
raise unprocessable("Das ist kein Paperless-Zugang.")
|
|
return row
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT id, settings, secret FROM connectors
|
|
WHERE owner_id = %s AND kind = 'paperless'
|
|
ORDER BY name, id
|
|
""",
|
|
(owner_id,),
|
|
).fetchall()
|
|
if not rows:
|
|
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
|
if len(rows) != 1:
|
|
raise unprocessable("Für Paperless ist mehr als ein Zugang vorhanden. Die Suche muss einen Zugang nennen.")
|
|
return rows[0]
|
|
|
|
|
|
def paperless_documents(conn, owner_id: UUID, query: str, connector_id: UUID | None = None, options: dict | None = None) -> dict:
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
params = _paperless_params(query, options)
|
|
release_for_external(conn)
|
|
payload = _paperless_get(base, row["secret"], "/api/documents/", params)
|
|
names = _paperless_lookup(row["id"], base, row["secret"])
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
terms = _search_terms(query, (options or {}).get("title"))
|
|
items = []
|
|
for doc in payload.get("results") or []:
|
|
created = str(doc.get("created") or "")[:10]
|
|
added = str(doc.get("added") or "")[:10]
|
|
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
|
|
pages = doc.get("page_count")
|
|
asn = doc.get("archive_serial_number")
|
|
items.append(
|
|
{
|
|
"id": str(doc.get("id")),
|
|
"title": title[:200],
|
|
"document_on": created if len(created) == 10 else None,
|
|
"added_on": added if len(added) == 10 else None,
|
|
"correspondent": _correspondent(doc, names),
|
|
"document_type": _named(doc.get("document_type"), names["document_types"]),
|
|
"storage_path": _named(doc.get("storage_path"), names["storage_paths"]),
|
|
"tags": [name for name in (_named(tag, names["tags"]) for tag in doc.get("tags") or []) if name][:12],
|
|
"pages": pages if isinstance(pages, int) and pages > 0 else None,
|
|
"asn": asn if isinstance(asn, int) else None,
|
|
"snippet": _snippet(doc, terms),
|
|
"open_url": f"{base}/documents/{doc.get('id')}/",
|
|
}
|
|
)
|
|
page = int(params.get("page") or 1)
|
|
count = payload.get("count")
|
|
if isinstance(count, int):
|
|
next_page = str(page + 1) if page * PAPERLESS_PAGE_SIZE < count else None
|
|
else:
|
|
next_page = str(page + 1) if len(items) == PAPERLESS_PAGE_SIZE else None
|
|
return {
|
|
"items": items,
|
|
"connector_id": str(row["id"]),
|
|
"next_page": next_page,
|
|
"previous_page": str(page - 1) if page > 1 else None,
|
|
"total": count if isinstance(count, int) else None,
|
|
"incomplete": False,
|
|
}
|
|
|
|
|
|
def paperless_facets(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
release_for_external(conn)
|
|
names = _paperless_lookup(row["id"], base, row["secret"])
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return {
|
|
"correspondents": names["correspondents"],
|
|
"document_types": names["document_types"],
|
|
"storage_paths": names["storage_paths"],
|
|
"tags": names["tags"],
|
|
"tag_mode": "all",
|
|
"tag_modes": list(_PAPERLESS_TAG_MODES),
|
|
"modes": list(_PAPERLESS_MODES),
|
|
}
|
|
|
|
|
|
def paperless_thumbnail(conn, owner_id: UUID, document_id: str, connector_id: UUID) -> tuple[bytes, str]:
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
release_for_external(conn)
|
|
fetched = _paperless_file(base, row["secret"], f"/api/documents/{document_id}/thumb/", 2 * 1024 * 1024)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
if fetched is None:
|
|
raise not_found()
|
|
data, media, _name = fetched
|
|
if not media.startswith("image/"):
|
|
raise unprocessable("Paperless hat kein Vorschaubild geliefert.")
|
|
return data, media
|
|
|
|
|
|
_LOOKUP_TTL = 60.0
|
|
_LOOKUP_LIMIT = 32
|
|
_lookup_cache: dict[tuple, tuple[float, dict]] = {}
|
|
_lookup_lock = threading.Lock()
|
|
|
|
|
|
def _paperless_lookup(connector_id, base: str, token: str) -> dict:
|
|
"""Names behind the ids of a result list. Bound to the instance and key that asked, kept for a minute."""
|
|
key = (str(connector_id), base, hashlib.sha256(token.encode("utf-8")).hexdigest())
|
|
now = time.monotonic()
|
|
with _lookup_lock:
|
|
hit = _lookup_cache.get(key)
|
|
if hit and now - hit[0] < _LOOKUP_TTL:
|
|
return hit[1]
|
|
names = {
|
|
"correspondents": _paperless_names(base, token, "/api/correspondents/"),
|
|
"document_types": _paperless_names(base, token, "/api/document_types/"),
|
|
"tags": _paperless_names(base, token, "/api/tags/"),
|
|
# An instance without storage paths, or a key that may not read them, still searches.
|
|
"storage_paths": _paperless_names_or_none(base, token, "/api/storage_paths/"),
|
|
}
|
|
with _lookup_lock:
|
|
if len(_lookup_cache) >= _LOOKUP_LIMIT:
|
|
_lookup_cache.clear()
|
|
_lookup_cache[key] = (now, names)
|
|
return names
|
|
|
|
|
|
def _paperless_names_or_none(base: str, token: str, path: str) -> list[dict]:
|
|
try:
|
|
return _paperless_names(base, token, path)
|
|
except Problem:
|
|
return []
|
|
|
|
|
|
def _named(value, entries: list[dict]) -> str | None:
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
wanted = str(value)
|
|
for entry in entries:
|
|
if entry["id"] == wanted:
|
|
return entry["name"]
|
|
return None
|
|
|
|
|
|
_TAG_MARKUP = re.compile(r"<[^>]{0,200}>")
|
|
_SNIPPET_LENGTH = 260
|
|
|
|
|
|
def _search_terms(query: str, title) -> list[str]:
|
|
words = re.findall(r"[\w-]{3,}", f"{query or ''} {title or ''}".casefold())
|
|
return list(dict.fromkeys(words))[:8]
|
|
|
|
|
|
def _snippet(doc: dict, terms: list[str]) -> str | None:
|
|
"""A short plain-text excerpt: the index hit when Paperless sends one, else the text around the first match."""
|
|
hit = doc.get("__search_hit__")
|
|
if isinstance(hit, dict):
|
|
lines = hit.get("highlights")
|
|
if isinstance(lines, str) and lines.strip():
|
|
text = " ".join(unescape(_TAG_MARKUP.sub("", lines)).split())
|
|
if text:
|
|
return text[:_SNIPPET_LENGTH]
|
|
content = doc.get("content")
|
|
if not isinstance(content, str):
|
|
return None
|
|
text = " ".join(content.split())
|
|
if not text:
|
|
return None
|
|
folded = text.casefold()
|
|
at = min((index for index in (folded.find(term) for term in terms) if index >= 0), default=-1)
|
|
if at < 0:
|
|
return text[:_SNIPPET_LENGTH] + ("…" if len(text) > _SNIPPET_LENGTH else "")
|
|
first = max(0, at - 60)
|
|
if first:
|
|
first = text.rfind(" ", 0, first) + 1 or first
|
|
last = first + _SNIPPET_LENGTH
|
|
return ("…" if first else "") + text[first:last] + ("…" if last < len(text) else "")
|
|
|
|
|
|
def _paperless_names(base: str, token: str, path: str) -> list[dict]:
|
|
found = []
|
|
for page in range(1, 21):
|
|
payload = _paperless_get(base, token, path, {"page": str(page), "page_size": "100"})
|
|
rows = payload.get("results") or []
|
|
for item in rows:
|
|
if item.get("id") is None:
|
|
continue
|
|
found.append({"id": str(item["id"]), "name": str(item.get("name") or item["id"])[:200]})
|
|
if len(rows) < 100:
|
|
break
|
|
return found
|
|
|
|
|
|
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
release_for_external(conn)
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
items = _imap_mailboxes(client)
|
|
finally:
|
|
_imap_logout(client)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return items
|
|
|
|
|
|
def imap_messages(
|
|
conn,
|
|
owner_id: UUID,
|
|
connector_id: UUID,
|
|
query: str,
|
|
mailbox: str,
|
|
before: int | None,
|
|
uidvalidity: int | None = None,
|
|
) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
if any(char in mailbox for char in "\r\n\x00"):
|
|
raise unprocessable("Der Fachname ist ungültig.")
|
|
if len(query) > 200:
|
|
raise unprocessable("Die Suche ist zu lang.")
|
|
release_for_external(conn)
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
page = _imap_page(client, mailbox, query, before, uidvalidity)
|
|
finally:
|
|
_imap_logout(client)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return page
|
|
|
|
|
|
def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
_check_mailbox(mailbox)
|
|
if uid < 1:
|
|
raise unprocessable("Die Nachricht ist ungültig.")
|
|
settings = dict(row["settings"])
|
|
secret = row["secret"]
|
|
connector_id = row["id"]
|
|
release_for_external(conn)
|
|
client = _imap_login(settings, secret)
|
|
try:
|
|
_select_mailbox(client, mailbox)
|
|
preview = _fetch_message(client, uid, mailbox)
|
|
finally:
|
|
_imap_logout(client)
|
|
assert_still_owned(conn, connector_id, owner_id)
|
|
return preview
|
|
|
|
|
|
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = conn.execute(
|
|
"SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
return row
|
|
|
|
|
|
def owned_connector(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
"""The stored row of an own instance, secret included. For server-side use only."""
|
|
return _owned(conn, owner_id, connector_id)
|
|
|
|
|
|
def _public(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"kind": row["kind"],
|
|
"name": row["name"],
|
|
"settings": row["settings"],
|
|
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"capabilities": (get_source(row["kind"]).capabilities if get_source(row["kind"]) else None),
|
|
"has_secret": bool(row["has_secret"]),
|
|
"probe": {
|
|
"at": row["last_probe_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"ok": bool(row["last_probe_ok"]),
|
|
"detail": row["last_probe_detail"] or "",
|
|
}
|
|
if row["last_probe_at"]
|
|
else None,
|
|
}
|
|
|
|
|
|
def clean_base_url(value: str) -> str:
|
|
raw = value.strip()
|
|
parsed = urlparse(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
|
|
raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.")
|
|
if len(raw) > 300:
|
|
raise unprocessable("Die Paperless-Adresse ist zu lang.")
|
|
return raw.rstrip("/")
|
|
|
|
|
|
def _paperless_count(base: str, token: str) -> int:
|
|
payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"})
|
|
return int(payload.get("count") or 0)
|
|
|
|
|
|
def paperless_content(conn, owner_id: UUID, document_id: str, connector_id: UUID | None = None) -> tuple[bytes, str, str]:
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
row = _paperless_row(conn, owner_id, connector_id)
|
|
release_for_external(conn)
|
|
fetched = _paperless_bytes(row, document_id)
|
|
assert_still_owned(conn, row["id"], owner_id)
|
|
return fetched
|
|
|
|
|
|
def _paperless_bytes(row, document_id: str) -> tuple[bytes, str, str]:
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
token = row["secret"]
|
|
misses = 0
|
|
for suffix in ("preview", "download"):
|
|
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/")
|
|
if fetched is None:
|
|
misses += 1
|
|
continue
|
|
viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
meta = _paperless_metadata(base, token, document_id)
|
|
if meta is None and misses == 2:
|
|
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
|
|
if meta is not None:
|
|
local = _paperless_from_disk(document_id, meta)
|
|
if local is not None:
|
|
return local
|
|
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
|
|
|
|
|
|
def read_linked_paperless(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
|
"""The stored document. A grantee never uses a connector of their own account."""
|
|
external_id = str((ref or {}).get("external_id") or "")
|
|
detail = (ref or {}).get("detail") or {}
|
|
stored = _uuid(detail.get("connector_id")) or _uuid((ref or {}).get("connector_id"))
|
|
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, stored) == actor_id
|
|
if not external_id.isdigit() or len(external_id) > 12:
|
|
if owner_path:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
|
borrowed = borrow_original(conn, actor_id, element, context, ref, "paperless")
|
|
data, media, name = complete_borrow(conn, borrowed, lambda row: _paperless_bytes(row, external_id))
|
|
if external_id and external_id in (name or ""):
|
|
name = "dokument.pdf"
|
|
return data, media, name or "dokument.pdf"
|
|
|
|
|
|
def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> bytes:
|
|
"""The stored message. Mailbox, UID and connector come only from the saved reference."""
|
|
detail = (ref or {}).get("detail") or {}
|
|
if ref is None or ref["source"] != "imap":
|
|
raise unprocessable("Das ist keine E-Mail.")
|
|
connector_id = _uuid(detail.get("connector_id")) or _uuid(ref["connector_id"] if "connector_id" in ref.keys() else None)
|
|
mailbox = str(detail.get("mailbox") or "").strip()
|
|
uid = _mail_uid(detail.get("uid"))
|
|
external_id = str(ref["external_id"])
|
|
owner_path = element["owner_subject_id"] == actor_id or _connector_owner(conn, connector_id) == actor_id
|
|
if owner_path:
|
|
if not mailbox or uid is None:
|
|
raise unprocessable("Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
|
elif connector_id is None or not mailbox or uid is None:
|
|
raise unprocessable(ORIGINAL_UNAVAILABLE)
|
|
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
|
|
|
|
def fetch(row):
|
|
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
|
|
|
|
found = complete_borrow(conn, borrowed, fetch)
|
|
return _keep_moved_mail(conn, element["id"], external_id, mailbox, uid, found)
|
|
|
|
|
|
def _uuid(value) -> UUID | None:
|
|
if isinstance(value, UUID):
|
|
return value
|
|
try:
|
|
return UUID(str(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _mail_uid(value) -> int | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, int) and value >= 1:
|
|
return value
|
|
if isinstance(value, str) and value.isdigit() and int(value) >= 1:
|
|
return int(value)
|
|
return None
|
|
|
|
|
|
def _connector_for(conn, connector_id: UUID, owner_id: UUID, kind: str):
|
|
return conn.execute(
|
|
"""
|
|
SELECT id, owner_id, kind, settings, secret
|
|
FROM connectors
|
|
WHERE id = %s AND owner_id = %s AND kind = %s
|
|
""",
|
|
(connector_id, owner_id, kind),
|
|
).fetchone()
|
|
|
|
|
|
def _paperless_file(
|
|
base: str,
|
|
token: str,
|
|
path: str,
|
|
max_bytes: int = 40 * 1024 * 1024,
|
|
) -> tuple[bytes, str, str] | None:
|
|
request = Request(
|
|
f"{base}{path}",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=30) as response:
|
|
final = urlparse(response.geturl())
|
|
allowed = urlparse(base)
|
|
if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc:
|
|
raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.")
|
|
data = response.read(max_bytes + 1)
|
|
if len(data) > max_bytes:
|
|
if max_bytes > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip()
|
|
return data, media, _filename(response.headers.get("Content-Disposition"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None:
|
|
media = media.split(";")[0].strip().lower()
|
|
if not (media.startswith("application/pdf") or media.startswith("image/")):
|
|
try:
|
|
media = sniff(data)
|
|
except Problem:
|
|
return None
|
|
if media.startswith("application/pdf") or media.startswith("image/"):
|
|
return data, media, name or f"paperless-{document_id}.pdf"
|
|
return None
|
|
|
|
|
|
def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None:
|
|
request = Request(
|
|
f"{base}/api/documents/{document_id}/metadata/",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/json"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=20) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
return payload if isinstance(payload, dict) else None
|
|
|
|
|
|
def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None:
|
|
root = get_settings().paperless_media_root.strip()
|
|
if not root:
|
|
return None
|
|
candidates: list[tuple[str, str]] = []
|
|
archive_name = meta.get("archive_media_filename")
|
|
if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip():
|
|
candidates.append(("archive", archive_name))
|
|
original_name = meta.get("media_filename")
|
|
if isinstance(original_name, str) and original_name.strip():
|
|
candidates.append(("originals", original_name))
|
|
for kind, relative in candidates:
|
|
data = read_paperless_file(root, kind, relative)
|
|
if data is None:
|
|
continue
|
|
viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
return None
|
|
|
|
|
|
def read_paperless_file(
|
|
media_root: str,
|
|
kind: str,
|
|
relative: str,
|
|
max_bytes: int = 40 * 1024 * 1024,
|
|
) -> bytes | None:
|
|
if kind not in {"archive", "originals"}:
|
|
return None
|
|
raw = relative.replace("\\", "/").strip()
|
|
if not raw or raw.startswith("/") or ".." in Path(raw).parts:
|
|
return None
|
|
base = (Path(media_root) / "documents" / kind).resolve()
|
|
target = (base / raw).resolve()
|
|
if target != base and base not in target.parents:
|
|
return None
|
|
if not target.is_file():
|
|
return None
|
|
data = target.read_bytes()
|
|
if len(data) > max_bytes:
|
|
if max_bytes > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Originaldatei ist für den Export zu groß.")
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
return data
|
|
|
|
|
|
def _filename(disposition: str | None) -> str:
|
|
if not disposition:
|
|
return ""
|
|
marker = "filename="
|
|
lower = disposition.lower()
|
|
if marker not in lower:
|
|
return ""
|
|
value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"')
|
|
return value.replace("\\", "").replace("/", "")[:180]
|
|
|
|
|
|
def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
|
|
url = f"{base}{path}?{urlencode(params)}"
|
|
request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"})
|
|
try:
|
|
with urlopen(request, timeout=12) as response:
|
|
return json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _correspondent(doc: dict, names: dict | None = None) -> str | None:
|
|
value = doc.get("correspondent_name") or doc.get("correspondent")
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()[:200]
|
|
if names is not None:
|
|
return _named(value, names["correspondents"])
|
|
return None
|
|
|
|
|
|
def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
|
|
host = settings["host"]
|
|
port = int(settings["port"])
|
|
security = settings.get("security") or "ssl"
|
|
try:
|
|
if security == "starttls":
|
|
client = imaplib.IMAP4(host, port, timeout=12)
|
|
client.starttls(ssl.create_default_context())
|
|
else:
|
|
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12)
|
|
client.login(settings["username"], secret)
|
|
return client
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc
|
|
except (OSError, TimeoutError) as exc:
|
|
raise unprocessable("Das Postfach ist nicht erreichbar.") from exc
|
|
|
|
|
|
PAGE_SIZE = 30
|
|
UID_WINDOW = 400
|
|
UID_ROUNDS = 8
|
|
_LIST_LINE = re.compile(
|
|
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
|
|
)
|
|
|
|
|
|
def _imap_logout(client: imaplib.IMAP4) -> None:
|
|
try:
|
|
client.logout()
|
|
except imaplib.IMAP4.error:
|
|
pass
|
|
|
|
|
|
def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
|
|
try:
|
|
status, data = client.list()
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.")
|
|
items = []
|
|
seen: set[str] = set()
|
|
for raw in data or []:
|
|
parsed = _parse_list_line(raw)
|
|
if parsed is None or parsed["name"] in seen:
|
|
continue
|
|
seen.add(parsed["name"])
|
|
items.append(parsed)
|
|
items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold()))
|
|
return items
|
|
|
|
|
|
def _untagged_int(client: imaplib.IMAP4, name: str) -> int | None:
|
|
values = (getattr(client, "untagged_responses", None) or {}).get(name) or []
|
|
if not values:
|
|
return None
|
|
raw = values[-1]
|
|
if isinstance(raw, bytes):
|
|
raw = raw.decode("ascii", "replace")
|
|
try:
|
|
return int(str(raw).strip())
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _uids_from(data, low: int | None, high: int | None) -> list[int]:
|
|
if data is None or isinstance(data, (bytes, str)):
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
chunks = []
|
|
for part in data:
|
|
if part is None:
|
|
continue
|
|
if isinstance(part, bytes):
|
|
chunks.append(part)
|
|
elif isinstance(part, str):
|
|
chunks.append(part.encode("utf-8"))
|
|
else:
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
found = []
|
|
for piece in b" ".join(chunks).split():
|
|
if not piece.isdigit():
|
|
continue
|
|
uid = int(piece)
|
|
if low is not None and uid < low:
|
|
continue
|
|
if high is not None and uid > high:
|
|
continue
|
|
found.append(uid)
|
|
return found
|
|
|
|
|
|
def _uid_call(client: imaplib.IMAP4, *args):
|
|
try:
|
|
return client.uid(*args)
|
|
except imaplib.IMAP4.abort as exc:
|
|
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
|
except TimeoutError as exc:
|
|
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
|
|
|
|
|
|
def _search_status(status) -> None:
|
|
if status in {"NO", "BAD"}:
|
|
raise unprocessable("Das Postfach hat die Suche abgelehnt.")
|
|
if status != "OK":
|
|
raise unprocessable("Das Postfach hat eine unbrauchbare Antwort geliefert.")
|
|
|
|
|
|
def _uid_window(client: imaplib.IMAP4, query: str, low: int, high: int) -> list[int]:
|
|
needle = " ".join(query.split())
|
|
uid_range = f"{low}:{high}"
|
|
if not needle:
|
|
status, data = _uid_call(client, "SEARCH", None, "UID", uid_range)
|
|
else:
|
|
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
if needle.isascii():
|
|
status, data = _uid_call(
|
|
client, "SEARCH", None, "UID", uid_range,
|
|
"OR", "OR", "SUBJECT", quoted, "FROM", quoted, "TEXT", quoted,
|
|
)
|
|
else:
|
|
encoded = quoted.encode("utf-8")
|
|
status, data = _uid_call(
|
|
client, "SEARCH", "CHARSET", "UTF-8", "UID", uid_range,
|
|
"OR", "OR", "SUBJECT", encoded, "FROM", encoded, "TEXT", encoded,
|
|
)
|
|
_search_status(status)
|
|
return _uids_from(data, low, high)
|
|
|
|
|
|
def _highest_uid(client: imaplib.IMAP4) -> int:
|
|
uidnext = _untagged_int(client, "UIDNEXT")
|
|
if uidnext is not None:
|
|
return max(0, uidnext - 1)
|
|
status, data = _uid_call(client, "SEARCH", None, "UID", "*")
|
|
_search_status(status)
|
|
found = _uids_from(data, None, None)
|
|
return max(found) if found else 0
|
|
|
|
|
|
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None, claimed_validity: int | None = None) -> dict:
|
|
folders = _imap_mailboxes(client)
|
|
chosen = mailbox or "INBOX"
|
|
match = next((item for item in folders if item["name"] == chosen), None)
|
|
if match is None or not match["selectable"]:
|
|
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
|
status, _data = _examine(client, chosen)
|
|
if status != "OK":
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
|
exists = _untagged_int(client, "EXISTS")
|
|
validity = _untagged_int(client, "UIDVALIDITY")
|
|
restarted = False
|
|
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
|
|
before = None
|
|
restarted = True
|
|
high = before - 1 if before is not None else _highest_uid(client)
|
|
collected = []
|
|
attempts = 0
|
|
failures = 0
|
|
low = 1
|
|
rounds = 0
|
|
while high >= 1 and len(collected) < PAGE_SIZE and rounds < UID_ROUNDS:
|
|
low = max(1, high - UID_WINDOW + 1)
|
|
for uid in sorted(_uid_window(client, query, low, high), reverse=True):
|
|
if len(collected) >= PAGE_SIZE:
|
|
break
|
|
attempts += 1
|
|
item = _fetch_header(client, uid, chosen)
|
|
if item is None:
|
|
failures += 1
|
|
continue
|
|
collected.append(item)
|
|
rounds += 1
|
|
if len(collected) >= PAGE_SIZE or low == 1:
|
|
break
|
|
high = low - 1
|
|
if attempts and failures == attempts:
|
|
raise unprocessable("Die Nachrichtenköpfe dieses Fachs konnten nicht gelesen werden.")
|
|
incomplete = bool(collected) is False and low > 1 and high >= 1 and rounds >= UID_ROUNDS
|
|
if len(collected) < PAGE_SIZE and low > 1 and rounds >= UID_ROUNDS:
|
|
incomplete = True
|
|
oldest = collected[-1]["uid"] if collected else None
|
|
if len(collected) >= PAGE_SIZE and oldest and oldest > 1:
|
|
next_before = oldest
|
|
elif incomplete:
|
|
next_before = low
|
|
else:
|
|
next_before = None
|
|
query_text = " ".join(query.split())
|
|
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
|
|
return {
|
|
"mailbox": chosen,
|
|
"items": collected,
|
|
"next_before": next_before,
|
|
"incomplete": incomplete,
|
|
"restarted": restarted,
|
|
"uidvalidity": validity,
|
|
"exists": exists,
|
|
"contradictory": contradictory,
|
|
}
|
|
|
|
|
|
def _check_mailbox(mailbox: str) -> None:
|
|
if any(char in mailbox for char in "\r\n\x00"):
|
|
raise unprocessable("Der Fachname ist ungültig.")
|
|
|
|
|
|
def _mailbox_argument(mailbox: str) -> str:
|
|
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
|
|
_check_mailbox(mailbox)
|
|
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
|
|
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
|
|
return protocol
|
|
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
|
|
return f'"{escaped}"'
|
|
|
|
|
|
def _examine(client: imaplib.IMAP4, mailbox: str):
|
|
argument = _mailbox_argument(mailbox)
|
|
try:
|
|
return client.select(argument, readonly=True)
|
|
except imaplib.IMAP4.abort as exc:
|
|
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
|
except TimeoutError as exc:
|
|
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
|
|
|
|
|
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
|
|
folders = _imap_mailboxes(client)
|
|
match = next((item for item in folders if item["name"] == mailbox), None)
|
|
if match is None or not match["selectable"]:
|
|
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
|
status, _data = _examine(client, mailbox)
|
|
if status != "OK":
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
|
|
|
|
|
def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
|
|
try:
|
|
status, fetched = client.uid(
|
|
"FETCH",
|
|
str(uid),
|
|
"(BODY.PEEK[HEADER.FIELDS (FROM TO CC SUBJECT DATE MESSAGE-ID)])",
|
|
)
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK" or not fetched or not isinstance(fetched[0], tuple):
|
|
return None
|
|
attachments = []
|
|
try:
|
|
struct_status, structure = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
|
|
if struct_status == "OK":
|
|
attachments = _structure_attachments(structure)
|
|
except imaplib.IMAP4.error:
|
|
attachments = []
|
|
header = BytesParser().parsebytes(fetched[0][1])
|
|
subject = _decode(header.get("Subject")) or "(ohne Betreff)"
|
|
sender = _decode(header.get("From"))
|
|
message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip()
|
|
return {
|
|
"id": message_id[:200],
|
|
"uid": uid,
|
|
"title": subject[:200],
|
|
"message_from": sender[:200],
|
|
"message_to": _decode(header.get("To"))[:500],
|
|
"message_cc": _decode(header.get("Cc"))[:500],
|
|
"direction": _direction(mailbox),
|
|
"attachments": attachments,
|
|
"document_on": _mail_date(header.get("Date")),
|
|
}
|
|
|
|
|
|
PREVIEW_BYTE_LIMIT = 8_000_000
|
|
PREVIEW_TEXT_LIMIT = 200_000
|
|
|
|
|
|
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.") from exc
|
|
raw = _literal(fetched)
|
|
if status != "OK" or raw is None:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
|
if len(raw) > PREVIEW_BYTE_LIMIT:
|
|
return _oversized_message(client, uid, mailbox)
|
|
parsed = parse_mail(raw)
|
|
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
|
|
|
|
|
|
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
|
|
text = parsed.text[:PREVIEW_TEXT_LIMIT]
|
|
return {
|
|
"uid": uid,
|
|
"mailbox": mailbox,
|
|
"title": (parsed.subject or "(ohne Betreff)")[:200],
|
|
"message_from": parsed.sender[:200],
|
|
"message_to": parsed.recipient[:500],
|
|
"message_cc": parsed.copy[:500],
|
|
"direction": _direction(mailbox),
|
|
"document_on": parsed.document_on,
|
|
"attachments": [
|
|
{
|
|
"filename": item.display_name or item.filename,
|
|
"archive_name": item.filename,
|
|
"content_type": item.media_type,
|
|
}
|
|
for item in parsed.attachments[:40]
|
|
],
|
|
"text": text,
|
|
"html": parsed.html,
|
|
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
|
|
"preview_limited": limited,
|
|
"notices": parsed.notices,
|
|
}
|
|
|
|
|
|
_SENT_FOLDER = re.compile(
|
|
r"(^|/)(sent|sent items|sent messages|gesendet|gesendete|gesendete elemente|outbox|postausgang)($|/)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
_INBOX_FOLDER = re.compile(
|
|
r"(^|/)(inbox|posteingang|eingang)($|/)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _direction(mailbox: str) -> str | None:
|
|
"""Sent folders are outgoing, an inbox is incoming. Other folder names prove nothing."""
|
|
folded = mailbox.casefold().replace("\\", "/")
|
|
if _SENT_FOLDER.search(folded):
|
|
return "out"
|
|
if _INBOX_FOLDER.search(folded):
|
|
return "in"
|
|
return None
|
|
|
|
|
|
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
|
"""Read named text parts. A cut-off multipart blob is not a message."""
|
|
header = _fetch_header(client, uid, mailbox) or {}
|
|
structure = _body_structure(client, uid)
|
|
plain: list[str] = []
|
|
html_parts: list[str] = []
|
|
truncated = False
|
|
notices = [
|
|
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
|
|
"Das Original bleibt die vollständige E-Mail."
|
|
]
|
|
if structure is None:
|
|
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
|
|
else:
|
|
for part in _text_sections(structure):
|
|
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
|
|
if fetched is None:
|
|
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
|
|
continue
|
|
truncated = truncated or shortened or part["octets"] > len(fetched)
|
|
text = decode_transfer(fetched, part["charset"]).strip()
|
|
if part["media"] == "text/plain":
|
|
plain.append(text)
|
|
else:
|
|
html_parts.append(text)
|
|
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
|
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
|
|
if not simplified and html_parts:
|
|
simplified = simplified_text("\n".join(html_parts))
|
|
if not header and not simplified and not rendered and structure is None:
|
|
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
|
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
|
|
return {
|
|
"uid": uid,
|
|
"mailbox": mailbox,
|
|
"title": header.get("title") or "(ohne Betreff)",
|
|
"message_from": header.get("message_from") or "",
|
|
"message_to": header.get("message_to") or "",
|
|
"message_cc": header.get("message_cc") or "",
|
|
"direction": header.get("direction", _direction(mailbox)),
|
|
"document_on": header.get("document_on"),
|
|
"attachments": attachments,
|
|
"text": simplified[:PREVIEW_TEXT_LIMIT],
|
|
"html": rendered,
|
|
"text_truncated": True or truncated,
|
|
"preview_limited": True,
|
|
"notices": notices,
|
|
}
|
|
|
|
|
|
def _body_structure(client: imaplib.IMAP4, uid: int):
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK":
|
|
return None
|
|
blob = _literal(fetched)
|
|
if blob is None:
|
|
chunks: list[bytes] = []
|
|
for part in fetched or []:
|
|
if isinstance(part, tuple):
|
|
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
|
|
elif isinstance(part, (bytes, bytearray)):
|
|
chunks.append(bytes(part))
|
|
blob = b" ".join(chunks)
|
|
text = blob.decode("utf-8", "replace")
|
|
start = text.find("(")
|
|
if start < 0:
|
|
return None
|
|
try:
|
|
node, _end = _parse_imap_list(text, start)
|
|
except ValueError:
|
|
return None
|
|
return node
|
|
|
|
|
|
def _parse_imap_list(text: str, index: int):
|
|
if index >= len(text) or text[index] != "(":
|
|
raise ValueError(text[index:index + 20])
|
|
index += 1
|
|
items = []
|
|
while index < len(text):
|
|
char = text[index]
|
|
if char == ")":
|
|
return items, index + 1
|
|
if char in " \t\r\n":
|
|
index += 1
|
|
continue
|
|
if char == "(":
|
|
child, index = _parse_imap_list(text, index)
|
|
items.append(child)
|
|
continue
|
|
if char == '"':
|
|
token, index = _parse_imap_string(text, index)
|
|
items.append(token)
|
|
continue
|
|
end = index
|
|
while end < len(text) and text[end] not in " \t\r\n()":
|
|
end += 1
|
|
token = text[index:end]
|
|
items.append(None if token.upper() == "NIL" else token)
|
|
index = end
|
|
raise ValueError("unclosed")
|
|
|
|
|
|
def _parse_imap_string(text: str, index: int):
|
|
index += 1
|
|
chars: list[str] = []
|
|
while index < len(text):
|
|
char = text[index]
|
|
if char == "\\":
|
|
if index + 1 < len(text):
|
|
chars.append(text[index + 1])
|
|
index += 2
|
|
continue
|
|
if char == '"':
|
|
return "".join(chars), index + 1
|
|
chars.append(char)
|
|
index += 1
|
|
raise ValueError("string")
|
|
|
|
|
|
def _text_sections(node, prefix: str = "") -> list[dict]:
|
|
if not isinstance(node, list) or not node:
|
|
return []
|
|
if isinstance(node[0], list):
|
|
children = []
|
|
for item in node:
|
|
if isinstance(item, list):
|
|
children.append(item)
|
|
else:
|
|
break
|
|
found = []
|
|
for number, child in enumerate(children, start=1):
|
|
section = f"{number}" if not prefix else f"{prefix}.{number}"
|
|
found.extend(_text_sections(child, section))
|
|
return found
|
|
media = str(node[0] or "").upper()
|
|
subtype = str(node[1] or "").upper()
|
|
if media == "MESSAGE":
|
|
return []
|
|
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
|
|
return []
|
|
disposition, params = _disposition(node)
|
|
if disposition == "ATTACHMENT":
|
|
return []
|
|
body_params = _param_map(node[2]) if len(node) > 2 else {}
|
|
octets = _imap_int(node[6]) if len(node) > 6 else 0
|
|
return [{
|
|
"section": prefix or "1",
|
|
"media": f"text/{subtype.lower()}",
|
|
"charset": body_params.get("charset") or "utf-8",
|
|
"octets": octets,
|
|
}]
|
|
|
|
|
|
def _disposition(node: list) -> tuple[str, dict]:
|
|
for item in node:
|
|
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
|
|
params = _param_map(item[1]) if len(item) > 1 else {}
|
|
return item[0].upper(), params
|
|
return "", {}
|
|
|
|
|
|
def _param_map(value) -> dict:
|
|
if not isinstance(value, list):
|
|
return {}
|
|
pairs = {}
|
|
index = 0
|
|
while index + 1 < len(value):
|
|
key = value[index]
|
|
raw = value[index + 1]
|
|
if isinstance(key, str):
|
|
pairs[key.casefold()] = "" if raw is None else str(raw)
|
|
index += 2
|
|
return pairs
|
|
|
|
|
|
def _imap_int(value) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
|
|
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), spec)
|
|
except imaplib.IMAP4.error:
|
|
return None, False
|
|
if status != "OK":
|
|
return None, False
|
|
raw = _literal(fetched)
|
|
if not raw:
|
|
return None, False
|
|
return raw[:limit], len(raw) >= limit
|
|
|
|
|
|
def _structure_names(node) -> list[dict]:
|
|
names = []
|
|
|
|
def visit(item):
|
|
if not isinstance(item, list) or not item:
|
|
return
|
|
if isinstance(item[0], list):
|
|
for child in item:
|
|
if isinstance(child, list):
|
|
visit(child)
|
|
return
|
|
_disposition_name, params = _disposition(item)
|
|
body = _param_map(item[2]) if len(item) > 2 else {}
|
|
raw = params.get("filename") or body.get("name") or ""
|
|
label = header_text(raw)
|
|
if label:
|
|
names.append({"filename": label[:200]})
|
|
|
|
visit(node)
|
|
return names[:40]
|
|
|
|
|
|
def _literal(fetched) -> bytes | None:
|
|
for part in fetched or []:
|
|
if isinstance(part, tuple) and len(part) > 1 and isinstance(part[1], (bytes, bytearray)):
|
|
return bytes(part[1])
|
|
return None
|
|
|
|
|
|
def _structure_attachments(fetched) -> list[dict]:
|
|
chunks: list[bytes] = []
|
|
for part in fetched or []:
|
|
if isinstance(part, tuple):
|
|
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
|
|
elif isinstance(part, (bytes, bytearray)):
|
|
chunks.append(bytes(part))
|
|
blob = b" ".join(chunks)
|
|
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
|
|
items = []
|
|
for raw in names:
|
|
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
|
|
if not name:
|
|
continue
|
|
items.append({"filename": name})
|
|
if len(items) == 40:
|
|
break
|
|
return items
|
|
|
|
|
|
def _message_attachments(message) -> list[dict]:
|
|
items = []
|
|
seen: set[str] = set()
|
|
for part in message.walk():
|
|
filename = part.get_filename()
|
|
if not filename:
|
|
continue
|
|
name = _decode(filename)[:200]
|
|
if not name or name in seen:
|
|
continue
|
|
seen.add(name)
|
|
items.append({"filename": name, "content_type": part.get_content_type()[:120]})
|
|
if len(items) == 40:
|
|
break
|
|
return items
|
|
|
|
|
|
def _message_text(message) -> str:
|
|
plain: list[str] = []
|
|
html: list[str] = []
|
|
parts = message.walk() if message.is_multipart() else [message]
|
|
for part in parts:
|
|
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
|
|
continue
|
|
kind = part.get_content_type()
|
|
if kind == "text/plain":
|
|
plain.append(_part_text(part))
|
|
elif kind == "text/html":
|
|
html.append(_html_text(_part_text(part)))
|
|
text = "\n".join(chunk for chunk in plain if chunk).strip()
|
|
if not text:
|
|
text = "\n".join(chunk for chunk in html if chunk).strip()
|
|
return text
|
|
|
|
|
|
def _part_text(part) -> str:
|
|
payload = part.get_payload(decode=True)
|
|
if payload is None:
|
|
raw = part.get_payload()
|
|
return raw.strip() if isinstance(raw, str) else ""
|
|
charset = part.get_content_charset() or "utf-8"
|
|
return payload.decode(charset, errors="replace").strip()
|
|
|
|
|
|
def _html_text(value: str) -> str:
|
|
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
|
|
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
|
|
text = re.sub(r"(?i)</p>", "\n", text)
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
return re.sub(r"[ \t]{2,}", " ", text).strip()
|
|
|
|
|
|
def _parse_list_line(raw) -> dict | None:
|
|
if not isinstance(raw, (bytes, str)) or not raw:
|
|
return None
|
|
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
match = _LIST_LINE.match(text.strip())
|
|
if match is None:
|
|
return None
|
|
flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag}
|
|
delimiter = _unquote(match.group("delim"))
|
|
name = _unquote(match.group("name"))
|
|
if not name:
|
|
return None
|
|
label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name)
|
|
depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter)
|
|
return {
|
|
"name": name,
|
|
"label": label,
|
|
"delimiter": delimiter,
|
|
"depth": depth,
|
|
"selectable": "noselect" not in flags,
|
|
}
|
|
|
|
|
|
def _unquote(token: str) -> str:
|
|
if token == "NIL":
|
|
return ""
|
|
if len(token) >= 2 and token.startswith('"') and token.endswith('"'):
|
|
return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\")
|
|
return token
|
|
|
|
|
|
def _encode_utf7(value: str) -> str:
|
|
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
|
|
pieces: list[str] = []
|
|
pending: list[str] = []
|
|
|
|
def flush() -> None:
|
|
if not pending:
|
|
return
|
|
raw = "".join(pending).encode("utf-16-be")
|
|
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
|
|
pieces.append(f"&{token}-")
|
|
pending.clear()
|
|
|
|
for char in value:
|
|
if char == "&":
|
|
flush()
|
|
pieces.append("&-")
|
|
elif ord(char) < 128:
|
|
flush()
|
|
pieces.append(char)
|
|
else:
|
|
pending.append(char)
|
|
flush()
|
|
return "".join(pieces)
|
|
|
|
|
|
def _decode_utf7(value: str) -> str:
|
|
if "&" not in value:
|
|
return value
|
|
parts: list[str] = []
|
|
index = 0
|
|
while index < len(value):
|
|
if value[index] != "&":
|
|
parts.append(value[index])
|
|
index += 1
|
|
continue
|
|
end = value.find("-", index)
|
|
if end < 0:
|
|
parts.append(value[index:])
|
|
break
|
|
if end == index + 1:
|
|
parts.append("&")
|
|
else:
|
|
chunk = value[index + 1:end].replace(",", "/")
|
|
padding = "=" * ((4 - len(chunk) % 4) % 4)
|
|
try:
|
|
parts.append(base64.b64decode(chunk + padding).decode("utf-16-be"))
|
|
except (ValueError, UnicodeDecodeError):
|
|
parts.append(value[index:end + 1])
|
|
index = end + 1
|
|
return "".join(parts)
|
|
|
|
|
|
def _mail_date(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(value).date().isoformat()
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _decode(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
parts = []
|
|
for text, encoding in decode_header(value):
|
|
if isinstance(text, bytes):
|
|
parts.append(text.decode(encoding or "utf-8", errors="replace"))
|
|
else:
|
|
parts.append(text)
|
|
return "".join(parts).strip()
|
|
|
|
|
|
def pull_paperless_original(base: str, token: str, document_id: str, max_bytes: int) -> tuple[bytes, str, str]:
|
|
"""Original download only. A preview is never returned in its place."""
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/download/", max_bytes)
|
|
if fetched is not None:
|
|
data, media, name = fetched
|
|
return data, media or "application/octet-stream", name or f"paperless-{document_id}"
|
|
meta = _paperless_metadata(base, token, document_id)
|
|
if meta is None:
|
|
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
|
|
local = _paperless_original_from_disk(meta, max_bytes)
|
|
if local is None:
|
|
raise unprocessable(
|
|
"Das Paperless-Original ist nicht verfügbar. Eine Vorschau wird nicht als Original übernommen."
|
|
)
|
|
return local
|
|
|
|
|
|
def _paperless_original_from_disk(meta: dict, max_bytes: int) -> tuple[bytes, str, str] | None:
|
|
root = get_settings().paperless_media_root.strip()
|
|
original_name = meta.get("media_filename")
|
|
if not root or not isinstance(original_name, str) or not original_name.strip():
|
|
return None
|
|
data = read_paperless_file(root, "originals", original_name, max_bytes)
|
|
if data is None:
|
|
return None
|
|
return data, "application/octet-stream", Path(original_name).name
|
|
|
|
|
|
COMPLETE_MAIL_LIMIT = 80 * 1024 * 1024
|
|
|
|
|
|
def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: int, expected_id: str) -> bytes:
|
|
"""The referenced message in full. A preview is not returned in its place."""
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
found = pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT)
|
|
return found.raw if isinstance(found, LocatedMail) else found
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LocatedMail:
|
|
"""Bytes plus the folder they were read from. `relocated` is a complete, unique hit."""
|
|
|
|
raw: bytes
|
|
mailbox: str
|
|
uid: int
|
|
relocated: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _MailHit:
|
|
raw: bytes
|
|
mailbox: str
|
|
uid: int
|
|
|
|
|
|
def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> LocatedMail:
|
|
"""Fetch the referenced message. A different message at the old UID is not returned.
|
|
|
|
The stored folder is tried first. A single Message-ID hit elsewhere comes back
|
|
marked relocated; the caller stores that place after the access check.
|
|
"""
|
|
client = _imap_login(settings, secret)
|
|
try:
|
|
_select_mailbox(client, mailbox)
|
|
candidate = _fetch_rfc822(client, uid)
|
|
if candidate is not None and len(candidate) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
if candidate is not None and identities_match(header_message_id(candidate), expected_id):
|
|
return LocatedMail(candidate, mailbox, uid, relocated=False)
|
|
hits, capped = _search_rfc822(client, expected_id, max_bytes)
|
|
try:
|
|
chosen = choose_message(None, [hit.raw for hit in hits], expected_id)
|
|
except MailIdentityError as exc:
|
|
raise unprocessable(exc.message) from exc
|
|
match = next(
|
|
hit for hit in hits
|
|
if hit.raw == chosen and identities_match(header_message_id(hit.raw), expected_id)
|
|
)
|
|
if len(match.raw) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
moved = not capped and (match.mailbox != mailbox or match.uid != uid)
|
|
return LocatedMail(match.raw, match.mailbox, match.uid, relocated=moved)
|
|
finally:
|
|
_imap_logout(client)
|
|
|
|
|
|
def _keep_moved_mail(conn, element_id, external_id: str, mailbox: str, uid: int | None, found) -> bytes:
|
|
"""Return the bytes. A unique new place is written only after the fetch has finished."""
|
|
if not isinstance(found, LocatedMail):
|
|
return found
|
|
if found.relocated and element_id is not None and uid is not None:
|
|
remember_imap_location(
|
|
conn,
|
|
element_id,
|
|
external_id,
|
|
mailbox,
|
|
uid,
|
|
found.mailbox,
|
|
found.uid,
|
|
)
|
|
return found.raw
|
|
|
|
|
|
def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK":
|
|
return None
|
|
return _literal(fetched)
|
|
|
|
|
|
def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> tuple[list[_MailHit], bool]:
|
|
"""Search every selectable folder. The second value is true when the scan stopped early."""
|
|
token = (expected_id or "").strip()
|
|
if not token or token.casefold().startswith("uid:"):
|
|
return [], False
|
|
found: list[_MailHit] = []
|
|
for folder in _imap_mailboxes(client):
|
|
if not folder["selectable"]:
|
|
continue
|
|
try:
|
|
_select_mailbox(client, folder["name"])
|
|
status, data = client.uid("SEARCH", "HEADER", "Message-ID", token)
|
|
except (imaplib.IMAP4.error, Problem):
|
|
continue
|
|
if status != "OK" or not data or not data[0]:
|
|
continue
|
|
for raw_uid in data[0].split():
|
|
try:
|
|
uid = int(raw_uid)
|
|
except ValueError:
|
|
continue
|
|
blob = _fetch_rfc822(client, uid)
|
|
if blob is None:
|
|
continue
|
|
if len(blob) > max_bytes:
|
|
raise unprocessable(
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
|
)
|
|
found.append(_MailHit(blob, folder["name"], uid))
|
|
if len(found) > 5:
|
|
return found, True
|
|
return found, False
|
|
|
|
|
|
PAPERLESS_FIELDS = (
|
|
ConfigField(
|
|
"base_url",
|
|
"Adresse",
|
|
type="url",
|
|
required=True,
|
|
placeholder="https://paperless.example",
|
|
help="Adresse deiner Paperless-Instanz ohne Pfad.",
|
|
max_length=300,
|
|
),
|
|
ConfigField(
|
|
"token",
|
|
"API-Schlüssel",
|
|
type="password",
|
|
required=True,
|
|
secret=True,
|
|
help="Den Schlüssel erzeugst du in Paperless unter deinem Profil.",
|
|
max_length=200,
|
|
),
|
|
)
|
|
|
|
IMAP_FIELDS = (
|
|
ConfigField("host", "Server", required=True, placeholder="imap.example", max_length=200),
|
|
ConfigField("port", "Port", type="number", required=True, default=993, minimum=1, maximum=65535),
|
|
ConfigField(
|
|
"security",
|
|
"Verbindung",
|
|
type="select",
|
|
required=True,
|
|
default="ssl",
|
|
options=(("ssl", "SSL"), ("starttls", "STARTTLS")),
|
|
),
|
|
ConfigField("username", "Benutzername", required=True, max_length=200),
|
|
ConfigField("password", "Passwort", type="password", required=True, secret=True, max_length=500),
|
|
ConfigField(
|
|
"mailbox",
|
|
"Standardfach",
|
|
default="INBOX",
|
|
help="Dieses Fach öffnet die Auswahl zuerst.",
|
|
max_length=200,
|
|
),
|
|
)
|
|
|
|
|
|
def _paperless_normalize(settings: dict, secret: str) -> dict:
|
|
if len(secret.strip()) < 8:
|
|
raise unprocessable("Der API-Schlüssel ist zu kurz.")
|
|
return {**settings, "base_url": clean_base_url(settings["base_url"])}
|
|
|
|
|
|
def _imap_normalize(settings: dict, secret: str) -> dict:
|
|
del secret
|
|
if " " in settings["host"]:
|
|
raise unprocessable("Der Mailserver ist ungültig.")
|
|
_check_mailbox(settings.get("mailbox") or "INBOX")
|
|
return settings
|
|
|
|
|
|
def _paperless_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return paperless_documents(conn, owner_id, query, connector_id, options)
|
|
|
|
|
|
def _imap_find(conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
if connector_id is None:
|
|
raise unprocessable("Für dieses Postfach fehlt der Zugang.")
|
|
before = options.get("before")
|
|
validity = options.get("uidvalidity")
|
|
return imap_messages(
|
|
conn,
|
|
owner_id,
|
|
connector_id,
|
|
query,
|
|
str(options.get("mailbox") or ""),
|
|
before,
|
|
int(validity) if validity else None,
|
|
)
|
|
|
|
|
|
def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
|
from app.retrieval import EXPORT_BYTE_LIMIT, FilePayload, RetrievalFailure, failure_from_problem
|
|
|
|
del detail
|
|
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
|
|
if resolved.row is None:
|
|
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
|
base = resolved.row["settings"]["base_url"].rstrip("/")
|
|
token = resolved.row["secret"]
|
|
connector_id = resolved.row["id"]
|
|
owner_id = resolved.row["owner_id"]
|
|
release_for_external(conn)
|
|
try:
|
|
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
try:
|
|
assert_still_owned(conn, connector_id, owner_id)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
if len(data) > EXPORT_BYTE_LIMIT:
|
|
return RetrievalFailure("too_large", "Die Originaldatei ist für den Export zu groß.")
|
|
return FilePayload(name or f"paperless-{external_id}", media, data, "")
|
|
|
|
|
|
def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: UUID | None, ref):
|
|
from app.mailpack import identities_match, parse_mail
|
|
from app.retrieval import EXPORT_BYTE_LIMIT, MailPayload, RetrievalFailure, failure_from_problem
|
|
|
|
resolved = resolve_for_read(conn, actor_id, "imap", ref, assigned)
|
|
if resolved.row is None:
|
|
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
|
mailbox = str(detail.get("mailbox") or "").strip()
|
|
uid = detail.get("uid")
|
|
if isinstance(uid, str) and uid.isdigit():
|
|
uid = int(uid)
|
|
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
|
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
|
settings = dict(resolved.row["settings"])
|
|
secret = resolved.row["secret"]
|
|
connector_id = resolved.row["id"]
|
|
owner_id = resolved.row["owner_id"]
|
|
release_for_external(conn)
|
|
try:
|
|
found = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
try:
|
|
assert_still_owned(conn, connector_id, owner_id)
|
|
except Problem as exc:
|
|
return failure_from_problem(exc)
|
|
element_id = ref["element_id"] if ref is not None and "element_id" in ref.keys() else None
|
|
raw = _keep_moved_mail(conn, element_id, str(external_id), mailbox, uid, found)
|
|
if len(raw) > EXPORT_BYTE_LIMIT:
|
|
return RetrievalFailure(
|
|
"too_large",
|
|
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
|
"Eine gekürzte Vorschau wird nicht als Original übernommen.",
|
|
)
|
|
parsed = parse_mail(raw)
|
|
if not identities_match(parsed.message_id, str(external_id)):
|
|
return RetrievalFailure("identity", "Die gefundene Nachricht ist nicht die referenzierte E-Mail.")
|
|
if parsed.missing:
|
|
names = ", ".join(parsed.missing[:5])
|
|
return RetrievalFailure("attachment", f"Eine Anlage der E-Mail ließ sich nicht vollständig lesen: {names}.")
|
|
return MailPayload(raw, parsed)
|
|
|
|
|
|
def _imap_bytes(conn, actor_id: UUID, element: dict, context: Ctx | None, ref) -> tuple[bytes, str, str]:
|
|
raw = read_linked_mail(conn, actor_id, element, context, ref)
|
|
return raw, "message/rfc822", "nachricht.eml"
|
|
|
|
|
|
class _PaperlessAdapter:
|
|
kind = "paperless"
|
|
|
|
def capabilities(self) -> dict:
|
|
return CAPABILITIES[self.kind]
|
|
|
|
def probe(self, row) -> dict:
|
|
count = _paperless_count(row["settings"]["base_url"], row["secret"])
|
|
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
|
|
|
def search(self, conn, owner_id: UUID, query: str, connector_id: UUID | None):
|
|
found = paperless_documents(conn, owner_id, query, connector_id)
|
|
return found["items"], found["connector_id"]
|
|
|
|
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return _paperless_find(conn, owner_id, connector_id, query, options)
|
|
|
|
def facets(self, conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
return paperless_facets(conn, owner_id, connector_id)
|
|
|
|
|
|
class _ImapAdapter:
|
|
kind = "imap"
|
|
|
|
def capabilities(self) -> dict:
|
|
return CAPABILITIES[self.kind]
|
|
|
|
def probe(self, row) -> dict:
|
|
_imap_login(row["settings"], row["secret"])
|
|
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
|
|
|
|
def navigate(self, conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
|
return imap_mailboxes(conn, owner_id, connector_id)
|
|
|
|
def search(self, conn, owner_id: UUID, connector_id: UUID, query: str, mailbox: str, before: int | None) -> dict:
|
|
return imap_messages(conn, owner_id, connector_id, query, mailbox, before)
|
|
|
|
def find(self, conn, owner_id: UUID, connector_id: UUID | None, query: str, options: dict) -> dict:
|
|
return _imap_find(conn, owner_id, connector_id, query, options)
|
|
|
|
|
|
def _install_sources() -> None:
|
|
paperless = _PaperlessAdapter()
|
|
imap = _ImapAdapter()
|
|
register_source(SourceModule(
|
|
kind="paperless",
|
|
label="Paperless",
|
|
representation="pdf",
|
|
capabilities=CAPABILITIES["paperless"],
|
|
config_fields=PAPERLESS_FIELDS,
|
|
normalize_config=_paperless_normalize,
|
|
probe=paperless.probe,
|
|
find=paperless.find,
|
|
open_original=read_linked_paperless,
|
|
export_original=_export_paperless,
|
|
navigate=None,
|
|
preview=None,
|
|
facets=paperless.facets,
|
|
))
|
|
register_source(SourceModule(
|
|
kind="imap",
|
|
label="E-Mail",
|
|
representation="email",
|
|
capabilities=CAPABILITIES["imap"],
|
|
config_fields=IMAP_FIELDS,
|
|
normalize_config=_imap_normalize,
|
|
probe=imap.probe,
|
|
find=imap.find,
|
|
open_original=_imap_bytes,
|
|
export_original=_export_imap,
|
|
navigate=imap.navigate,
|
|
preview=lambda conn, owner, connector_id, options: imap_preview(
|
|
conn, owner, connector_id, str(options.get("mailbox") or ""), int(options.get("uid") or 0)
|
|
),
|
|
))
|
|
|
|
|
|
_install_sources()
|