Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Deploy Development / deploy (push) Successful in 39s
Test Suite / pytest-backend (push) Successful in 2m29s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 20s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-20 08:47:24 +02:00
1 parent 34c4084a7c
commit c29bd29f34
29 files changed
+2562 -410

No files matched your search

+326 -57
View File
@@ -21,7 +21,17 @@ from psycopg.types.json import Json
from app.access.policy import Ctx
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.mailpack import (
MailIdentityError,
choose_message,
decode_transfer,
header_message_id,
header_text,
identities_match,
parse_mail,
sanitize_html,
simplified_text,
)
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceaccess import (
@@ -291,12 +301,18 @@ def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: in
_check_mailbox(mailbox)
if uid < 1:
raise unprocessable("Die Nachricht ist ungültig.")
client = _imap_login(row["settings"], row["secret"])
settings = dict(row["settings"])
secret = row["secret"]
connector_id = row["id"]
release_for_external(conn)
client = _imap_login(settings, secret)
try:
_select_mailbox(client, mailbox)
return _fetch_message(client, uid, mailbox)
preview = _fetch_message(client, uid, mailbox)
finally:
_imap_logout(client)
assert_still_owned(conn, connector_id, owner_id)
return preview
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
@@ -402,7 +418,7 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
def fetch(row):
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
return complete_borrow(conn, borrowed, fetch)
@@ -726,16 +742,10 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
match = next((item for item in folders if item["name"] == chosen), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(chosen, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, chosen)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
exists = _untagged_int(client, "EXISTS")
validity = _untagged_int(client, "UIDVALIDITY")
restarted = False
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
@@ -774,6 +784,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
next_before = low
else:
next_before = None
query_text = " ".join(query.split())
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
return {
"mailbox": chosen,
"items": collected,
@@ -781,6 +793,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
"incomplete": incomplete,
"restarted": restarted,
"uidvalidity": validity,
"exists": exists,
"contradictory": contradictory,
}
@@ -789,15 +803,34 @@ def _check_mailbox(mailbox: str) -> None:
raise unprocessable("Der Fachname ist ungültig.")
def _mailbox_argument(mailbox: str) -> str:
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
_check_mailbox(mailbox)
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
return protocol
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _examine(client: imaplib.IMAP4, mailbox: str):
argument = _mailbox_argument(mailbox)
try:
return client.select(argument, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
folders = _imap_mailboxes(client)
match = next((item for item in folders if item["name"] == mailbox), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(mailbox, readonly=True)
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, mailbox)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
@@ -837,6 +870,10 @@ def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
}
PREVIEW_BYTE_LIMIT = 8_000_000
PREVIEW_TEXT_LIMIT = 200_000
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
@@ -845,22 +882,36 @@ def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
raw = _literal(fetched)
if status != "OK" or raw is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
if len(raw) > 8_000_000:
if len(raw) > PREVIEW_BYTE_LIMIT:
return _oversized_message(client, uid, mailbox)
message = BytesParser().parsebytes(raw)
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
parsed = parse_mail(raw)
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
text = parsed.text[:PREVIEW_TEXT_LIMIT]
return {
"uid": uid,
"mailbox": mailbox,
"title": subject[:200],
"message_from": _decode(message.get("From"))[:200],
"message_to": _decode(message.get("To"))[:500],
"message_cc": _decode(message.get("Cc"))[:500],
"title": (parsed.subject or "(ohne Betreff)")[:200],
"message_from": parsed.sender[:200],
"message_to": parsed.recipient[:500],
"message_cc": parsed.copy[:500],
"direction": _direction(mailbox),
"document_on": _mail_date(message.get("Date")),
"attachments": _message_attachments(message),
"text": (_text := _message_text(message))[:200_000],
"text_truncated": len(_text) > 200_000,
"document_on": parsed.document_on,
"attachments": [
{
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
}
for item in parsed.attachments[:40]
],
"text": text,
"html": parsed.html,
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
"preview_limited": limited,
"notices": parsed.notices,
}
@@ -887,10 +938,37 @@ def _direction(mailbox: str) -> str | None:
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"""Read named text parts. A cut-off multipart blob is not a message."""
header = _fetch_header(client, uid, mailbox) or {}
text = _partial_text(client, uid)
if not header and not text:
structure = _body_structure(client, uid)
plain: list[str] = []
html_parts: list[str] = []
truncated = False
notices = [
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
"Das Original bleibt die vollständige E-Mail."
]
if structure is None:
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
else:
for part in _text_sections(structure):
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
if fetched is None:
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
continue
truncated = truncated or shortened or part["octets"] > len(fetched)
text = decode_transfer(fetched, part["charset"]).strip()
if part["media"] == "text/plain":
plain.append(text)
else:
html_parts.append(text)
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
if not simplified and html_parts:
simplified = simplified_text("\n".join(html_parts))
if not header and not simplified and not rendered and structure is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
return {
"uid": uid,
"mailbox": mailbox,
@@ -900,31 +978,185 @@ def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"message_cc": header.get("message_cc") or "",
"direction": header.get("direction", _direction(mailbox)),
"document_on": header.get("document_on"),
"attachments": header.get("attachments") or [],
"text": text,
"text_truncated": True,
"attachments": attachments,
"text": simplified[:PREVIEW_TEXT_LIMIT],
"html": rendered,
"text_truncated": True or truncated,
"preview_limited": True,
"notices": notices,
}
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
def _body_structure(client: imaplib.IMAP4, uid: int):
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
except imaplib.IMAP4.error:
return ""
return None
if status != "OK":
return ""
return None
blob = _literal(fetched)
if blob is None:
chunks: list[bytes] = []
for part in fetched or []:
if isinstance(part, tuple):
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
elif isinstance(part, (bytes, bytearray)):
chunks.append(bytes(part))
blob = b" ".join(chunks)
text = blob.decode("utf-8", "replace")
start = text.find("(")
if start < 0:
return None
try:
node, _end = _parse_imap_list(text, start)
except ValueError:
return None
return node
def _parse_imap_list(text: str, index: int):
if index >= len(text) or text[index] != "(":
raise ValueError(text[index:index + 20])
index += 1
items = []
while index < len(text):
char = text[index]
if char == ")":
return items, index + 1
if char in " \t\r\n":
index += 1
continue
if char == "(":
child, index = _parse_imap_list(text, index)
items.append(child)
continue
if char == '"':
token, index = _parse_imap_string(text, index)
items.append(token)
continue
end = index
while end < len(text) and text[end] not in " \t\r\n()":
end += 1
token = text[index:end]
items.append(None if token.upper() == "NIL" else token)
index = end
raise ValueError("unclosed")
def _parse_imap_string(text: str, index: int):
index += 1
chars: list[str] = []
while index < len(text):
char = text[index]
if char == "\\":
if index + 1 < len(text):
chars.append(text[index + 1])
index += 2
continue
if char == '"':
return "".join(chars), index + 1
chars.append(char)
index += 1
raise ValueError("string")
def _text_sections(node, prefix: str = "") -> list[dict]:
if not isinstance(node, list) or not node:
return []
if isinstance(node[0], list):
children = []
for item in node:
if isinstance(item, list):
children.append(item)
else:
break
found = []
for number, child in enumerate(children, start=1):
section = f"{number}" if not prefix else f"{prefix}.{number}"
found.extend(_text_sections(child, section))
return found
media = str(node[0] or "").upper()
subtype = str(node[1] or "").upper()
if media == "MESSAGE":
return []
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
return []
disposition, params = _disposition(node)
if disposition == "ATTACHMENT":
return []
body_params = _param_map(node[2]) if len(node) > 2 else {}
octets = _imap_int(node[6]) if len(node) > 6 else 0
return [{
"section": prefix or "1",
"media": f"text/{subtype.lower()}",
"charset": body_params.get("charset") or "utf-8",
"octets": octets,
}]
def _disposition(node: list) -> tuple[str, dict]:
for item in node:
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
params = _param_map(item[1]) if len(item) > 1 else {}
return item[0].upper(), params
return "", {}
def _param_map(value) -> dict:
if not isinstance(value, list):
return {}
pairs = {}
index = 0
while index + 1 < len(value):
key = value[index]
raw = value[index + 1]
if isinstance(key, str):
pairs[key.casefold()] = "" if raw is None else str(raw)
index += 2
return pairs
def _imap_int(value) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
try:
status, fetched = client.uid("FETCH", str(uid), spec)
except imaplib.IMAP4.error:
return None, False
if status != "OK":
return None, False
raw = _literal(fetched)
if not raw:
return ""
sample = raw[:200_000]
try:
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
text = _message_text(message)
except Exception:
text = ""
if not text:
text = sample.decode("utf-8", errors="replace")
return text.strip()[:200_000]
return None, False
return raw[:limit], len(raw) >= limit
def _structure_names(node) -> list[dict]:
names = []
def visit(item):
if not isinstance(item, list) or not item:
return
if isinstance(item[0], list):
for child in item:
if isinstance(child, list):
visit(child)
return
_disposition_name, params = _disposition(item)
body = _param_map(item[2]) if len(item) > 2 else {}
raw = params.get("filename") or body.get("name") or ""
label = header_text(raw)
if label:
names.append({"filename": label[:200]})
visit(node)
return names[:40]
def _literal(fetched) -> bytes | None:
@@ -944,12 +1176,10 @@ def _structure_attachments(fetched) -> list[dict]:
blob = b" ".join(chunks)
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
items = []
seen: set[str] = set()
for raw in names:
name = raw.decode("utf-8", errors="replace").strip()[:200]
if not name or name in seen:
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
if not name:
continue
seen.add(name)
items.append({"filename": name})
if len(items) == 40:
break
@@ -1039,6 +1269,32 @@ def _unquote(token: str) -> str:
return token
def _encode_utf7(value: str) -> str:
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
pieces: list[str] = []
pending: list[str] = []
def flush() -> None:
if not pending:
return
raw = "".join(pending).encode("utf-16-be")
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
pieces.append(f"&{token}-")
pending.clear()
for char in value:
if char == "&":
flush()
pieces.append("&-")
elif ord(char) < 128:
flush()
pieces.append(char)
else:
pending.append(char)
flush()
return "".join(pieces)
def _decode_utf7(value: str) -> str:
if "&" not in value:
return value
@@ -1255,13 +1511,17 @@ def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assi
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
base = resolved.row["settings"]["base_url"].rstrip("/")
token = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
data, media, name = pull_paperless_original(
resolved.row["settings"]["base_url"].rstrip("/"),
resolved.row["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
@@ -1282,8 +1542,17 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
settings = dict(resolved.row["settings"])
secret = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT: