532 lines
19 KiB
Python
532 lines
19 KiB
Python
"""Paperless and IMAP connectors. Secrets stay on the server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import imaplib
|
|
import json
|
|
import re
|
|
import ssl
|
|
from email.header import decode_header
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urlparse
|
|
from urllib.request import Request, urlopen
|
|
from uuid import UUID
|
|
|
|
from psycopg.types.json import Json
|
|
|
|
from app.core.errors import Problem, not_found, unprocessable
|
|
from app.core.settings import get_settings
|
|
from app.media.storage import sniff
|
|
|
|
|
|
def save_connector(conn, owner_id: UUID, kind: str, name: str, settings: dict, secret: str) -> dict:
|
|
if kind == "paperless":
|
|
existing = conn.execute(
|
|
"SELECT id FROM connectors WHERE owner_id = %s AND kind = 'paperless'",
|
|
(owner_id,),
|
|
).fetchone()
|
|
if existing:
|
|
row = conn.execute(
|
|
"""
|
|
UPDATE connectors
|
|
SET name = %s, settings = %s, secret = %s
|
|
WHERE id = %s
|
|
RETURNING id, kind, name, settings, created_at
|
|
""",
|
|
(name, Json(settings), secret, existing["id"]),
|
|
).fetchone()
|
|
return _public(row)
|
|
row = conn.execute(
|
|
"""
|
|
INSERT INTO connectors (owner_id, kind, name, settings, secret)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
RETURNING id, kind, name, settings, created_at
|
|
""",
|
|
(owner_id, kind, name, Json(settings), secret),
|
|
).fetchone()
|
|
return _public(row)
|
|
|
|
|
|
def list_connectors(conn, owner_id: UUID) -> list[dict]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT id, kind, name, settings, created_at
|
|
FROM connectors WHERE owner_id = %s
|
|
ORDER BY kind, name
|
|
""",
|
|
(owner_id,),
|
|
).fetchall()
|
|
return [_public(row) for row in rows]
|
|
|
|
|
|
def delete_connector(conn, owner_id: UUID, connector_id: UUID) -> None:
|
|
row = conn.execute(
|
|
"DELETE FROM connectors WHERE id = %s AND owner_id = %s RETURNING id",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
|
|
|
|
def probe(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] == "paperless":
|
|
count = _paperless_count(row["settings"]["base_url"], row["secret"])
|
|
return {"ok": True, "detail": f"Paperless antwortet. {count} Dokumente sichtbar."}
|
|
_imap_login(row["settings"], row["secret"])
|
|
return {"ok": True, "detail": "Das Postfach ist erreichbar."}
|
|
|
|
|
|
def paperless_documents(conn, owner_id: UUID, query: str) -> list[dict]:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT settings, secret FROM connectors
|
|
WHERE owner_id = %s AND kind = 'paperless'
|
|
ORDER BY created_at DESC LIMIT 1
|
|
""",
|
|
(owner_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
raise unprocessable("Paperless ist noch nicht verbunden. Lege die Quelle unter Quellen an.")
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
payload = _paperless_get(base, row["secret"], "/api/documents/", {"query": query, "page_size": "20"})
|
|
items = []
|
|
for doc in payload.get("results") or []:
|
|
created = str(doc.get("created") or "")[:10]
|
|
title = (doc.get("title") or doc.get("original_file_name") or f"Dokument {doc.get('id')}").strip()
|
|
items.append(
|
|
{
|
|
"id": str(doc.get("id")),
|
|
"title": title[:200],
|
|
"document_on": created if len(created) == 10 else None,
|
|
"correspondent": _correspondent(doc),
|
|
"open_url": f"{base}/documents/{doc.get('id')}/",
|
|
}
|
|
)
|
|
return items
|
|
|
|
|
|
def imap_mailboxes(conn, owner_id: UUID, connector_id: UUID) -> list[dict]:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
return _imap_mailboxes(client)
|
|
finally:
|
|
_imap_logout(client)
|
|
|
|
|
|
def imap_messages(
|
|
conn,
|
|
owner_id: UUID,
|
|
connector_id: UUID,
|
|
query: str,
|
|
mailbox: str,
|
|
before: int | None,
|
|
) -> dict:
|
|
row = _owned(conn, owner_id, connector_id)
|
|
if row["kind"] != "imap":
|
|
raise unprocessable("Das ist kein E-Mail-Konto.")
|
|
if any(char in mailbox for char in "\r\n\x00"):
|
|
raise unprocessable("Der Fachname ist ungültig.")
|
|
if len(query) > 200:
|
|
raise unprocessable("Die Suche ist zu lang.")
|
|
client = _imap_login(row["settings"], row["secret"])
|
|
try:
|
|
return _imap_page(client, mailbox, query, before)
|
|
finally:
|
|
_imap_logout(client)
|
|
|
|
|
|
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
|
row = conn.execute(
|
|
"SELECT id, kind, name, settings, secret FROM connectors WHERE id = %s AND owner_id = %s",
|
|
(connector_id, owner_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
return row
|
|
|
|
|
|
def _public(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"kind": row["kind"],
|
|
"name": row["name"],
|
|
"settings": row["settings"],
|
|
"created_at": row["created_at"].strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
}
|
|
|
|
|
|
def clean_base_url(value: str) -> str:
|
|
raw = value.strip()
|
|
parsed = urlparse(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
|
|
raise unprocessable("Die Paperless-Adresse muss mit http:// oder https:// beginnen.")
|
|
if len(raw) > 300:
|
|
raise unprocessable("Die Paperless-Adresse ist zu lang.")
|
|
return raw.rstrip("/")
|
|
|
|
|
|
def _paperless_count(base: str, token: str) -> int:
|
|
payload = _paperless_get(base, token, "/api/documents/", {"page_size": "1"})
|
|
return int(payload.get("count") or 0)
|
|
|
|
|
|
def paperless_content(conn, owner_id: UUID, document_id: str) -> tuple[bytes, str, str]:
|
|
if not document_id.isdigit() or len(document_id) > 12:
|
|
raise unprocessable("Die Paperless-Nummer ist ungültig.")
|
|
row = conn.execute(
|
|
"""
|
|
SELECT settings, secret FROM connectors
|
|
WHERE owner_id = %s AND kind = 'paperless'
|
|
ORDER BY created_at DESC LIMIT 1
|
|
""",
|
|
(owner_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
raise unprocessable("Paperless ist für dieses Konto nicht verbunden.")
|
|
base = row["settings"]["base_url"].rstrip("/")
|
|
token = row["secret"]
|
|
misses = 0
|
|
for suffix in ("preview", "download"):
|
|
fetched = _paperless_file(base, token, f"/api/documents/{document_id}/{suffix}/")
|
|
if fetched is None:
|
|
misses += 1
|
|
continue
|
|
viewable = _viewable(fetched[0], fetched[1], fetched[2], document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
meta = _paperless_metadata(base, token, document_id)
|
|
if meta is None and misses == 2:
|
|
raise unprocessable("Dieses Dokument gibt es in Paperless nicht.")
|
|
if meta is not None:
|
|
local = _paperless_from_disk(document_id, meta)
|
|
if local is not None:
|
|
return local
|
|
raise unprocessable("Paperless stellt diese Datei nicht zum Ansehen bereit.")
|
|
|
|
|
|
def _paperless_file(base: str, token: str, path: str) -> tuple[bytes, str, str] | None:
|
|
request = Request(
|
|
f"{base}{path}",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/pdf, image/*, */*"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=30) as response:
|
|
final = urlparse(response.geturl())
|
|
allowed = urlparse(base)
|
|
if final.scheme not in {"http", "https"} or final.netloc != allowed.netloc:
|
|
raise unprocessable("Paperless hat auf eine fremde Adresse verwiesen.")
|
|
data = response.read(40 * 1024 * 1024 + 1)
|
|
if len(data) > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
media = (response.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip()
|
|
return data, media, _filename(response.headers.get("Content-Disposition"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _viewable(data: bytes, media: str, name: str, document_id: str) -> tuple[bytes, str, str] | None:
|
|
media = media.split(";")[0].strip().lower()
|
|
if not (media.startswith("application/pdf") or media.startswith("image/")):
|
|
try:
|
|
media = sniff(data)
|
|
except Problem:
|
|
return None
|
|
if media.startswith("application/pdf") or media.startswith("image/"):
|
|
return data, media, name or f"paperless-{document_id}.pdf"
|
|
return None
|
|
|
|
|
|
def _paperless_metadata(base: str, token: str, document_id: str) -> dict | None:
|
|
request = Request(
|
|
f"{base}/api/documents/{document_id}/metadata/",
|
|
headers={"Authorization": f"Token {token}", "Accept": "application/json"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=20) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Datei nicht bereitgestellt.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
return payload if isinstance(payload, dict) else None
|
|
|
|
|
|
def _paperless_from_disk(document_id: str, meta: dict) -> tuple[bytes, str, str] | None:
|
|
root = get_settings().paperless_media_root.strip()
|
|
if not root:
|
|
return None
|
|
candidates: list[tuple[str, str]] = []
|
|
archive_name = meta.get("archive_media_filename")
|
|
if meta.get("has_archive_version") and isinstance(archive_name, str) and archive_name.strip():
|
|
candidates.append(("archive", archive_name))
|
|
original_name = meta.get("media_filename")
|
|
if isinstance(original_name, str) and original_name.strip():
|
|
candidates.append(("originals", original_name))
|
|
for kind, relative in candidates:
|
|
data = read_paperless_file(root, kind, relative)
|
|
if data is None:
|
|
continue
|
|
viewable = _viewable(data, "application/octet-stream", Path(relative).name, document_id)
|
|
if viewable is not None:
|
|
return viewable
|
|
return None
|
|
|
|
|
|
def read_paperless_file(media_root: str, kind: str, relative: str) -> bytes | None:
|
|
if kind not in {"archive", "originals"}:
|
|
return None
|
|
raw = relative.replace("\\", "/").strip()
|
|
if not raw or raw.startswith("/") or ".." in Path(raw).parts:
|
|
return None
|
|
base = (Path(media_root) / "documents" / kind).resolve()
|
|
target = (base / raw).resolve()
|
|
if target != base and base not in target.parents:
|
|
return None
|
|
if not target.is_file():
|
|
return None
|
|
data = target.read_bytes()
|
|
if len(data) > 40 * 1024 * 1024:
|
|
raise unprocessable("Die Datei ist zum Ansehen in Tsumugu zu groß.")
|
|
return data
|
|
|
|
|
|
def _filename(disposition: str | None) -> str:
|
|
if not disposition:
|
|
return ""
|
|
marker = "filename="
|
|
lower = disposition.lower()
|
|
if marker not in lower:
|
|
return ""
|
|
value = disposition[lower.index(marker) + len(marker):].split(";")[0].strip().strip('"')
|
|
return value.replace("\\", "").replace("/", "")[:180]
|
|
|
|
|
|
def _paperless_get(base: str, token: str, path: str, params: dict) -> dict:
|
|
url = f"{base}{path}?{urlencode(params)}"
|
|
request = Request(url, headers={"Authorization": f"Token {token}", "Accept": "application/json"})
|
|
try:
|
|
with urlopen(request, timeout=12) as response:
|
|
return json.loads(response.read().decode("utf-8"))
|
|
except HTTPError as exc:
|
|
if exc.code in {401, 403}:
|
|
raise unprocessable("Paperless hat den Schlüssel abgelehnt.") from exc
|
|
raise unprocessable("Paperless hat die Anfrage nicht beantwortet.") from exc
|
|
except (URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
raise unprocessable("Paperless ist nicht erreichbar.") from exc
|
|
|
|
|
|
def _correspondent(doc: dict) -> str | None:
|
|
value = doc.get("correspondent_name") or doc.get("correspondent")
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()[:200]
|
|
return None
|
|
|
|
|
|
def _imap_login(settings: dict, secret: str) -> imaplib.IMAP4:
|
|
host = settings["host"]
|
|
port = int(settings["port"])
|
|
security = settings.get("security") or "ssl"
|
|
try:
|
|
if security == "starttls":
|
|
client = imaplib.IMAP4(host, port, timeout=12)
|
|
client.starttls(ssl.create_default_context())
|
|
else:
|
|
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=12)
|
|
client.login(settings["username"], secret)
|
|
return client
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach hat die Anmeldung abgelehnt.") from exc
|
|
except (OSError, TimeoutError) as exc:
|
|
raise unprocessable("Das Postfach ist nicht erreichbar.") from exc
|
|
|
|
|
|
PAGE_SIZE = 30
|
|
_LIST_LINE = re.compile(
|
|
r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:\\.|[^"])*"|NIL)\s+(?P<name>"(?:\\.|[^"])*"|\S+)\s*$'
|
|
)
|
|
|
|
|
|
def _imap_logout(client: imaplib.IMAP4) -> None:
|
|
try:
|
|
client.logout()
|
|
except imaplib.IMAP4.error:
|
|
pass
|
|
|
|
|
|
def _imap_mailboxes(client: imaplib.IMAP4) -> list[dict]:
|
|
try:
|
|
status, data = client.list()
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Die Fächer konnten nicht gelesen werden.")
|
|
items = []
|
|
seen: set[str] = set()
|
|
for raw in data or []:
|
|
parsed = _parse_list_line(raw)
|
|
if parsed is None or parsed["name"] in seen:
|
|
continue
|
|
seen.add(parsed["name"])
|
|
items.append(parsed)
|
|
items.sort(key=lambda item: (0 if item["name"].upper() == "INBOX" else 1, item["label"].casefold()))
|
|
return items
|
|
|
|
|
|
def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | None) -> dict:
|
|
folders = _imap_mailboxes(client)
|
|
chosen = mailbox or "INBOX"
|
|
match = next((item for item in folders if item["name"] == chosen), None)
|
|
if match is None or not match["selectable"]:
|
|
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
|
try:
|
|
status, _ = client.select(chosen, readonly=True)
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
|
if status != "OK":
|
|
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
|
status, data = _uid_search(client, query)
|
|
if status != "OK" or not data or not data[0]:
|
|
return {"mailbox": chosen, "items": [], "next_before": None}
|
|
uids = sorted(int(part) for part in data[0].split() if part.isdigit())
|
|
if before is not None:
|
|
uids = [uid for uid in uids if uid < before]
|
|
page = uids[-PAGE_SIZE:]
|
|
next_before = page[0] if len(uids) > PAGE_SIZE else None
|
|
items = []
|
|
for uid in reversed(page):
|
|
item = _fetch_header(client, uid, chosen)
|
|
if item is not None:
|
|
items.append(item)
|
|
return {"mailbox": chosen, "items": items, "next_before": next_before}
|
|
|
|
|
|
def _uid_search(client: imaplib.IMAP4, query: str):
|
|
needle = " ".join(query.split())
|
|
if not needle:
|
|
return client.uid("SEARCH", None, "ALL")
|
|
quoted = '"' + needle.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
try:
|
|
if needle.isascii():
|
|
return client.uid("SEARCH", None, "OR", "SUBJECT", quoted, "FROM", quoted)
|
|
encoded = quoted.encode("utf-8")
|
|
return client.uid("SEARCH", "CHARSET", "UTF-8", "OR", "SUBJECT", encoded, "FROM", encoded)
|
|
except imaplib.IMAP4.error as exc:
|
|
raise unprocessable("Das Postfach kann diese Suche nicht ausführen. Blättere im Fach oder kürze den Suchtext.") from exc
|
|
|
|
|
|
def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
|
|
try:
|
|
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE MESSAGE-ID)])")
|
|
except imaplib.IMAP4.error:
|
|
return None
|
|
if status != "OK" or not fetched or not isinstance(fetched[0], tuple):
|
|
return None
|
|
header = BytesParser().parsebytes(fetched[0][1])
|
|
subject = _decode(header.get("Subject")) or "(ohne Betreff)"
|
|
sender = _decode(header.get("From"))
|
|
message_id = (header.get("Message-ID") or f"uid:{mailbox}:{uid}").strip()
|
|
return {
|
|
"id": message_id[:200],
|
|
"uid": uid,
|
|
"title": subject[:200],
|
|
"message_from": sender[:200],
|
|
"document_on": _mail_date(header.get("Date")),
|
|
}
|
|
|
|
|
|
def _parse_list_line(raw) -> dict | None:
|
|
if not isinstance(raw, (bytes, str)) or not raw:
|
|
return None
|
|
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
match = _LIST_LINE.match(text.strip())
|
|
if match is None:
|
|
return None
|
|
flags = {flag.lstrip("\\").casefold() for flag in match.group("flags").split() if flag}
|
|
delimiter = _unquote(match.group("delim"))
|
|
name = _unquote(match.group("name"))
|
|
if not name:
|
|
return None
|
|
label = "Posteingang" if name.upper() == "INBOX" else _decode_utf7(name)
|
|
depth = 0 if not delimiter or name.upper() == "INBOX" else name.count(delimiter)
|
|
return {
|
|
"name": name,
|
|
"label": label,
|
|
"delimiter": delimiter,
|
|
"depth": depth,
|
|
"selectable": "noselect" not in flags,
|
|
}
|
|
|
|
|
|
def _unquote(token: str) -> str:
|
|
if token == "NIL":
|
|
return ""
|
|
if len(token) >= 2 and token.startswith('"') and token.endswith('"'):
|
|
return token[1:-1].replace(r"\"", '"').replace(r"\\", "\\")
|
|
return token
|
|
|
|
|
|
def _decode_utf7(value: str) -> str:
|
|
if "&" not in value:
|
|
return value
|
|
parts: list[str] = []
|
|
index = 0
|
|
while index < len(value):
|
|
if value[index] != "&":
|
|
parts.append(value[index])
|
|
index += 1
|
|
continue
|
|
end = value.find("-", index)
|
|
if end < 0:
|
|
parts.append(value[index:])
|
|
break
|
|
if end == index + 1:
|
|
parts.append("&")
|
|
else:
|
|
chunk = value[index + 1:end].replace(",", "/")
|
|
padding = "=" * ((4 - len(chunk) % 4) % 4)
|
|
try:
|
|
parts.append(base64.b64decode(chunk + padding).decode("utf-16-be"))
|
|
except (ValueError, UnicodeDecodeError):
|
|
parts.append(value[index:end + 1])
|
|
index = end + 1
|
|
return "".join(parts)
|
|
|
|
|
|
def _mail_date(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(value).date().isoformat()
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _decode(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
parts = []
|
|
for text, encoding in decode_header(value):
|
|
if isinstance(text, bytes):
|
|
parts.append(text.decode(encoding or "utf-8", errors="replace"))
|
|
else:
|
|
parts.append(text)
|
|
return "".join(parts).strip()
|