Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
34c4084a7c
commit
c29bd29f34
29 files changed
+2562
-410
No files matched your search
+326
-57
@@ -21,7 +21,17 @@ from psycopg.types.json import Json
|
||||
from app.access.policy import Ctx
|
||||
from app.core.errors import Problem, not_found, unprocessable
|
||||
from app.core.settings import get_settings
|
||||
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
|
||||
from app.mailpack import (
|
||||
MailIdentityError,
|
||||
choose_message,
|
||||
decode_transfer,
|
||||
header_message_id,
|
||||
header_text,
|
||||
identities_match,
|
||||
parse_mail,
|
||||
sanitize_html,
|
||||
simplified_text,
|
||||
)
|
||||
from app.media.storage import sniff
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceaccess import (
|
||||
@@ -291,12 +301,18 @@ def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: in
|
||||
_check_mailbox(mailbox)
|
||||
if uid < 1:
|
||||
raise unprocessable("Die Nachricht ist ungültig.")
|
||||
client = _imap_login(row["settings"], row["secret"])
|
||||
settings = dict(row["settings"])
|
||||
secret = row["secret"]
|
||||
connector_id = row["id"]
|
||||
release_for_external(conn)
|
||||
client = _imap_login(settings, secret)
|
||||
try:
|
||||
_select_mailbox(client, mailbox)
|
||||
return _fetch_message(client, uid, mailbox)
|
||||
preview = _fetch_message(client, uid, mailbox)
|
||||
finally:
|
||||
_imap_logout(client)
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
return preview
|
||||
|
||||
|
||||
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
@@ -402,7 +418,7 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
|
||||
|
||||
def fetch(row):
|
||||
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
|
||||
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
|
||||
|
||||
return complete_borrow(conn, borrowed, fetch)
|
||||
|
||||
@@ -726,16 +742,10 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
match = next((item for item in folders if item["name"] == chosen), None)
|
||||
if match is None or not match["selectable"]:
|
||||
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
||||
try:
|
||||
status, _ = client.select(chosen, readonly=True)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
status, _data = _examine(client, chosen)
|
||||
if status != "OK":
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
||||
exists = _untagged_int(client, "EXISTS")
|
||||
validity = _untagged_int(client, "UIDVALIDITY")
|
||||
restarted = False
|
||||
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
|
||||
@@ -774,6 +784,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
next_before = low
|
||||
else:
|
||||
next_before = None
|
||||
query_text = " ".join(query.split())
|
||||
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
|
||||
return {
|
||||
"mailbox": chosen,
|
||||
"items": collected,
|
||||
@@ -781,6 +793,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
"incomplete": incomplete,
|
||||
"restarted": restarted,
|
||||
"uidvalidity": validity,
|
||||
"exists": exists,
|
||||
"contradictory": contradictory,
|
||||
}
|
||||
|
||||
|
||||
@@ -789,15 +803,34 @@ def _check_mailbox(mailbox: str) -> None:
|
||||
raise unprocessable("Der Fachname ist ungültig.")
|
||||
|
||||
|
||||
def _mailbox_argument(mailbox: str) -> str:
|
||||
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
|
||||
_check_mailbox(mailbox)
|
||||
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
|
||||
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
|
||||
return protocol
|
||||
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def _examine(client: imaplib.IMAP4, mailbox: str):
|
||||
argument = _mailbox_argument(mailbox)
|
||||
try:
|
||||
return client.select(argument, readonly=True)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
|
||||
|
||||
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
|
||||
folders = _imap_mailboxes(client)
|
||||
match = next((item for item in folders if item["name"] == mailbox), None)
|
||||
if match is None or not match["selectable"]:
|
||||
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
||||
try:
|
||||
status, _ = client.select(mailbox, readonly=True)
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
status, _data = _examine(client, mailbox)
|
||||
if status != "OK":
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
||||
|
||||
@@ -837,6 +870,10 @@ def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
PREVIEW_BYTE_LIMIT = 8_000_000
|
||||
PREVIEW_TEXT_LIMIT = 200_000
|
||||
|
||||
|
||||
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
||||
@@ -845,22 +882,36 @@ def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
raw = _literal(fetched)
|
||||
if status != "OK" or raw is None:
|
||||
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
||||
if len(raw) > 8_000_000:
|
||||
if len(raw) > PREVIEW_BYTE_LIMIT:
|
||||
return _oversized_message(client, uid, mailbox)
|
||||
message = BytesParser().parsebytes(raw)
|
||||
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
|
||||
parsed = parse_mail(raw)
|
||||
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
|
||||
|
||||
|
||||
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
|
||||
text = parsed.text[:PREVIEW_TEXT_LIMIT]
|
||||
return {
|
||||
"uid": uid,
|
||||
"mailbox": mailbox,
|
||||
"title": subject[:200],
|
||||
"message_from": _decode(message.get("From"))[:200],
|
||||
"message_to": _decode(message.get("To"))[:500],
|
||||
"message_cc": _decode(message.get("Cc"))[:500],
|
||||
"title": (parsed.subject or "(ohne Betreff)")[:200],
|
||||
"message_from": parsed.sender[:200],
|
||||
"message_to": parsed.recipient[:500],
|
||||
"message_cc": parsed.copy[:500],
|
||||
"direction": _direction(mailbox),
|
||||
"document_on": _mail_date(message.get("Date")),
|
||||
"attachments": _message_attachments(message),
|
||||
"text": (_text := _message_text(message))[:200_000],
|
||||
"text_truncated": len(_text) > 200_000,
|
||||
"document_on": parsed.document_on,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": item.display_name or item.filename,
|
||||
"archive_name": item.filename,
|
||||
"content_type": item.media_type,
|
||||
}
|
||||
for item in parsed.attachments[:40]
|
||||
],
|
||||
"text": text,
|
||||
"html": parsed.html,
|
||||
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
|
||||
"preview_limited": limited,
|
||||
"notices": parsed.notices,
|
||||
}
|
||||
|
||||
|
||||
@@ -887,10 +938,37 @@ def _direction(mailbox: str) -> str | None:
|
||||
|
||||
|
||||
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
"""Read named text parts. A cut-off multipart blob is not a message."""
|
||||
header = _fetch_header(client, uid, mailbox) or {}
|
||||
text = _partial_text(client, uid)
|
||||
if not header and not text:
|
||||
structure = _body_structure(client, uid)
|
||||
plain: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
truncated = False
|
||||
notices = [
|
||||
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
|
||||
"Das Original bleibt die vollständige E-Mail."
|
||||
]
|
||||
if structure is None:
|
||||
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
|
||||
else:
|
||||
for part in _text_sections(structure):
|
||||
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
|
||||
if fetched is None:
|
||||
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
|
||||
continue
|
||||
truncated = truncated or shortened or part["octets"] > len(fetched)
|
||||
text = decode_transfer(fetched, part["charset"]).strip()
|
||||
if part["media"] == "text/plain":
|
||||
plain.append(text)
|
||||
else:
|
||||
html_parts.append(text)
|
||||
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
|
||||
if not simplified and html_parts:
|
||||
simplified = simplified_text("\n".join(html_parts))
|
||||
if not header and not simplified and not rendered and structure is None:
|
||||
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
||||
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
|
||||
return {
|
||||
"uid": uid,
|
||||
"mailbox": mailbox,
|
||||
@@ -900,31 +978,185 @@ def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
"message_cc": header.get("message_cc") or "",
|
||||
"direction": header.get("direction", _direction(mailbox)),
|
||||
"document_on": header.get("document_on"),
|
||||
"attachments": header.get("attachments") or [],
|
||||
"text": text,
|
||||
"text_truncated": True,
|
||||
"attachments": attachments,
|
||||
"text": simplified[:PREVIEW_TEXT_LIMIT],
|
||||
"html": rendered,
|
||||
"text_truncated": True or truncated,
|
||||
"preview_limited": True,
|
||||
"notices": notices,
|
||||
}
|
||||
|
||||
|
||||
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
|
||||
def _body_structure(client: imaplib.IMAP4, uid: int):
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
|
||||
except imaplib.IMAP4.error:
|
||||
return ""
|
||||
return None
|
||||
if status != "OK":
|
||||
return ""
|
||||
return None
|
||||
blob = _literal(fetched)
|
||||
if blob is None:
|
||||
chunks: list[bytes] = []
|
||||
for part in fetched or []:
|
||||
if isinstance(part, tuple):
|
||||
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
|
||||
elif isinstance(part, (bytes, bytearray)):
|
||||
chunks.append(bytes(part))
|
||||
blob = b" ".join(chunks)
|
||||
text = blob.decode("utf-8", "replace")
|
||||
start = text.find("(")
|
||||
if start < 0:
|
||||
return None
|
||||
try:
|
||||
node, _end = _parse_imap_list(text, start)
|
||||
except ValueError:
|
||||
return None
|
||||
return node
|
||||
|
||||
|
||||
def _parse_imap_list(text: str, index: int):
|
||||
if index >= len(text) or text[index] != "(":
|
||||
raise ValueError(text[index:index + 20])
|
||||
index += 1
|
||||
items = []
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == ")":
|
||||
return items, index + 1
|
||||
if char in " \t\r\n":
|
||||
index += 1
|
||||
continue
|
||||
if char == "(":
|
||||
child, index = _parse_imap_list(text, index)
|
||||
items.append(child)
|
||||
continue
|
||||
if char == '"':
|
||||
token, index = _parse_imap_string(text, index)
|
||||
items.append(token)
|
||||
continue
|
||||
end = index
|
||||
while end < len(text) and text[end] not in " \t\r\n()":
|
||||
end += 1
|
||||
token = text[index:end]
|
||||
items.append(None if token.upper() == "NIL" else token)
|
||||
index = end
|
||||
raise ValueError("unclosed")
|
||||
|
||||
|
||||
def _parse_imap_string(text: str, index: int):
|
||||
index += 1
|
||||
chars: list[str] = []
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == "\\":
|
||||
if index + 1 < len(text):
|
||||
chars.append(text[index + 1])
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
return "".join(chars), index + 1
|
||||
chars.append(char)
|
||||
index += 1
|
||||
raise ValueError("string")
|
||||
|
||||
|
||||
def _text_sections(node, prefix: str = "") -> list[dict]:
|
||||
if not isinstance(node, list) or not node:
|
||||
return []
|
||||
if isinstance(node[0], list):
|
||||
children = []
|
||||
for item in node:
|
||||
if isinstance(item, list):
|
||||
children.append(item)
|
||||
else:
|
||||
break
|
||||
found = []
|
||||
for number, child in enumerate(children, start=1):
|
||||
section = f"{number}" if not prefix else f"{prefix}.{number}"
|
||||
found.extend(_text_sections(child, section))
|
||||
return found
|
||||
media = str(node[0] or "").upper()
|
||||
subtype = str(node[1] or "").upper()
|
||||
if media == "MESSAGE":
|
||||
return []
|
||||
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
|
||||
return []
|
||||
disposition, params = _disposition(node)
|
||||
if disposition == "ATTACHMENT":
|
||||
return []
|
||||
body_params = _param_map(node[2]) if len(node) > 2 else {}
|
||||
octets = _imap_int(node[6]) if len(node) > 6 else 0
|
||||
return [{
|
||||
"section": prefix or "1",
|
||||
"media": f"text/{subtype.lower()}",
|
||||
"charset": body_params.get("charset") or "utf-8",
|
||||
"octets": octets,
|
||||
}]
|
||||
|
||||
|
||||
def _disposition(node: list) -> tuple[str, dict]:
|
||||
for item in node:
|
||||
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
|
||||
params = _param_map(item[1]) if len(item) > 1 else {}
|
||||
return item[0].upper(), params
|
||||
return "", {}
|
||||
|
||||
|
||||
def _param_map(value) -> dict:
|
||||
if not isinstance(value, list):
|
||||
return {}
|
||||
pairs = {}
|
||||
index = 0
|
||||
while index + 1 < len(value):
|
||||
key = value[index]
|
||||
raw = value[index + 1]
|
||||
if isinstance(key, str):
|
||||
pairs[key.casefold()] = "" if raw is None else str(raw)
|
||||
index += 2
|
||||
return pairs
|
||||
|
||||
|
||||
def _imap_int(value) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
|
||||
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), spec)
|
||||
except imaplib.IMAP4.error:
|
||||
return None, False
|
||||
if status != "OK":
|
||||
return None, False
|
||||
raw = _literal(fetched)
|
||||
if not raw:
|
||||
return ""
|
||||
sample = raw[:200_000]
|
||||
try:
|
||||
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
|
||||
text = _message_text(message)
|
||||
except Exception:
|
||||
text = ""
|
||||
if not text:
|
||||
text = sample.decode("utf-8", errors="replace")
|
||||
return text.strip()[:200_000]
|
||||
return None, False
|
||||
return raw[:limit], len(raw) >= limit
|
||||
|
||||
|
||||
def _structure_names(node) -> list[dict]:
|
||||
names = []
|
||||
|
||||
def visit(item):
|
||||
if not isinstance(item, list) or not item:
|
||||
return
|
||||
if isinstance(item[0], list):
|
||||
for child in item:
|
||||
if isinstance(child, list):
|
||||
visit(child)
|
||||
return
|
||||
_disposition_name, params = _disposition(item)
|
||||
body = _param_map(item[2]) if len(item) > 2 else {}
|
||||
raw = params.get("filename") or body.get("name") or ""
|
||||
label = header_text(raw)
|
||||
if label:
|
||||
names.append({"filename": label[:200]})
|
||||
|
||||
visit(node)
|
||||
return names[:40]
|
||||
|
||||
|
||||
def _literal(fetched) -> bytes | None:
|
||||
@@ -944,12 +1176,10 @@ def _structure_attachments(fetched) -> list[dict]:
|
||||
blob = b" ".join(chunks)
|
||||
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
|
||||
items = []
|
||||
seen: set[str] = set()
|
||||
for raw in names:
|
||||
name = raw.decode("utf-8", errors="replace").strip()[:200]
|
||||
if not name or name in seen:
|
||||
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
|
||||
if not name:
|
||||
continue
|
||||
seen.add(name)
|
||||
items.append({"filename": name})
|
||||
if len(items) == 40:
|
||||
break
|
||||
@@ -1039,6 +1269,32 @@ def _unquote(token: str) -> str:
|
||||
return token
|
||||
|
||||
|
||||
def _encode_utf7(value: str) -> str:
|
||||
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
|
||||
pieces: list[str] = []
|
||||
pending: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
if not pending:
|
||||
return
|
||||
raw = "".join(pending).encode("utf-16-be")
|
||||
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
|
||||
pieces.append(f"&{token}-")
|
||||
pending.clear()
|
||||
|
||||
for char in value:
|
||||
if char == "&":
|
||||
flush()
|
||||
pieces.append("&-")
|
||||
elif ord(char) < 128:
|
||||
flush()
|
||||
pieces.append(char)
|
||||
else:
|
||||
pending.append(char)
|
||||
flush()
|
||||
return "".join(pieces)
|
||||
|
||||
|
||||
def _decode_utf7(value: str) -> str:
|
||||
if "&" not in value:
|
||||
return value
|
||||
@@ -1255,13 +1511,17 @@ def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assi
|
||||
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
|
||||
if resolved.row is None:
|
||||
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
||||
base = resolved.row["settings"]["base_url"].rstrip("/")
|
||||
token = resolved.row["secret"]
|
||||
connector_id = resolved.row["id"]
|
||||
owner_id = resolved.row["owner_id"]
|
||||
release_for_external(conn)
|
||||
try:
|
||||
data, media, name = pull_paperless_original(
|
||||
resolved.row["settings"]["base_url"].rstrip("/"),
|
||||
resolved.row["secret"],
|
||||
str(external_id),
|
||||
EXPORT_BYTE_LIMIT,
|
||||
)
|
||||
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
try:
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(data) > EXPORT_BYTE_LIMIT:
|
||||
@@ -1282,8 +1542,17 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
|
||||
uid = int(uid)
|
||||
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
||||
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
settings = dict(resolved.row["settings"])
|
||||
secret = resolved.row["secret"]
|
||||
connector_id = resolved.row["id"]
|
||||
owner_id = resolved.row["owner_id"]
|
||||
release_for_external(conn)
|
||||
try:
|
||||
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
try:
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(raw) > EXPORT_BYTE_LIMIT:
|
||||
|
||||
Reference in new issue
Block a user