Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Deploy Development / deploy (push) Successful in 39s
Test Suite / pytest-backend (push) Successful in 2m29s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 20s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-20 08:47:24 +02:00
1 parent 34c4084a7c
commit c29bd29f34
29 files changed
+2562 -410

No files matched your search

+7 -3
View File
@@ -124,14 +124,18 @@ def view_mail(
"message_to": parsed.recipient,
"message_cc": parsed.copy,
"text": parsed.text,
"text_truncated": False,
"complete": True,
"html": parsed.html,
"text_truncated": parsed.text_truncated,
"complete": not parsed.missing,
"notices": parsed.notices,
"attachments": [
{
"index": index,
"filename": item.filename,
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
"byte_size": len(item.data),
"inline": item.inline,
}
for index, item in enumerate(parsed.attachments)
],
+222 -49
View File
@@ -211,11 +211,36 @@ def record_dispatch(
raise unprocessable("Dieselbe Anlage ist mehrfach im Versand genannt.")
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
host = _bound_element(conn, actor, host_id, page, scope)
bump_version(conn, "elements", host_id, expected)
known = {row["attachment_element_id"] for row in _uses(conn, scope, host_id)}
if any(item not in known for item in attachment_ids):
raise unprocessable("Mitversendet werden nur Anlagen dieses Schreibens.")
counterpart_text = _counterpart(counterpart)
edition_plans = []
if capture_editions:
edition_plans.append(_plan_edition(conn, actor, host, "host", 0, page, historical_gap, media_root, cleanup))
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
position_of = {item: index for index, item in enumerate(attachment_ids)}
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
for index, row in enumerate(ordered, start=1):
element = _readable(conn, actor, row["attachment_element_id"], page)
if element is None:
raise not_found()
edition_plans.append(
_plan_edition(conn, actor, element, "attachment", index, page, historical_gap, media_root, cleanup)
)
evidence_plans = []
for item in evidence:
element = _readable(conn, actor, item["element_id"], page)
if element is None:
raise not_found()
evidence_plans.append(
_plan_evidence(
conn, actor, element, item["role"], item.get("fold") or "keep", page, media_root, cleanup, container_row["home_scope_id"]
)
)
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
host = _bound_element(conn, actor, host_id, page, scope)
bump_version(conn, "elements", host_id, expected)
dispatch = conn.execute(
"""
INSERT INTO dispatches (
@@ -237,37 +262,11 @@ def record_dispatch(
actor,
),
).fetchone()
states = []
if capture_editions:
states.append(_capture_item(conn, actor, dispatch, host, "host", 0, page, historical_gap, media_root, cleanup))
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
position_of = {item: index for index, item in enumerate(attachment_ids)}
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
for index, row in enumerate(ordered, start=1):
element = _readable(conn, actor, row["attachment_element_id"], page)
if element is None:
raise not_found()
states.append(
_capture_item(conn, actor, dispatch, element, "attachment", index, page, historical_gap, media_root, cleanup)
)
evidence_rows = []
for item in evidence:
evidence_rows.append(
_add_evidence(
conn,
actor,
container_row,
page,
scope,
dispatch,
item["element_id"],
item["role"],
item.get("fold") or "keep",
media_root,
cleanup,
log=False,
)
)
states = [_write_edition(conn, dispatch, plan) for plan in edition_plans]
evidence_rows = [
_write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=False)
for plan in evidence_plans
]
completeness = _completeness(states) if capture_editions else "secured"
conn.execute(
"UPDATE dispatches SET completeness = %s, updated_at = now() WHERE id = %s",
@@ -292,11 +291,16 @@ def add_dispatch_evidence(
) -> dict:
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
dispatch = _dispatch_in(conn, dispatch_id, scope)
_bump_dispatch(conn, dispatch_id, expected)
row = _add_evidence(
conn, actor, container_row, page, scope, dispatch, element_id, role, fold, media_root, cleanup, log=True
element = _readable(conn, actor, element_id, page)
if element is None:
raise not_found()
plan = _plan_evidence(
conn, actor, element, role, fold, page, media_root, cleanup, container_row["home_scope_id"], dispatch_id
)
return row
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
dispatch = _dispatch_in(conn, dispatch_id, scope)
_bump_dispatch(conn, dispatch_id, expected)
return _write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=True)
def detach_dispatch_evidence(
@@ -760,7 +764,7 @@ def _identify_evidence(conn, actor, dispatch, element, page, media_root, cleanup
if isinstance(loaded, RetrievalFailure):
return False, loaded.message, None, None, [], None
if isinstance(loaded, MailPayload):
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded)
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded) if dispatch is not None else []
payload = FilePayload("nachweis.eml", "message/rfc822", loaded.raw, sha256(loaded.raw))
return True, None, loaded.parsed.message_id or None, payload.sha256, comparisons, payload
payload = FilePayload(loaded.filename, loaded.media_type, loaded.data, sha256(loaded.data))
@@ -931,15 +935,179 @@ def _working_bytes(conn, actor, element, media_root: str):
return None, "Für diesen Inhalt ist keine sicherbare Fassung vorgesehen.", None, None, None
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
snapshot_id = uuid4()
def _stage_snapshot(conn, media_root: str, data: bytes, cleanup: list) -> dict:
"""Publish bytes only while the domain transaction is idle. The row comes later."""
from app.sourceaccess import release_for_external
instance = instance_for_write(conn, media_root)
snapshot_id = uuid4()
object_key = f"dispatch/{snapshot_id}"
content_revision = str(uuid4())
adapter = adapter_for(instance)
publish(adapter, object_key, payload.data)
release_for_external(conn)
publish(adapter, object_key, data)
cleanup.append(StoredHandle(adapter, object_key))
digest = sha256(payload.data)
return {
"id": snapshot_id,
"instance": instance,
"object_key": object_key,
"content_revision": str(uuid4()),
}
def _plan_edition(conn, actor, element, role, position, page, historical_gap, media_root, cleanup) -> dict:
dates = _dates(conn, element["id"])
payload = None
gap_reason = None
note_document = None
external_source = None
external_id = None
if historical_gap:
state = "gap"
gap_reason = _HISTORICAL_GAP
else:
payload, gap_reason, note_document, external_source, external_id = _working_bytes(
conn, actor, element, media_root
)
state = "snapshot" if payload is not None else "gap"
staged = _stage_snapshot(conn, media_root, payload.data, cleanup) if payload is not None else None
return {
"element": element,
"role": role,
"position": position,
"dates": dates,
"state": state,
"payload": payload,
"gap_reason": gap_reason,
"note_document": note_document,
"external_source": external_source,
"external_id": external_id,
"staged": staged,
}
def _write_edition(conn, dispatch, plan: dict) -> str:
payload = plan["payload"]
filename = payload.filename if payload is not None else None
content_type = payload.media_type if payload is not None else None
digest = sha256(payload.data) if payload is not None else None
byte_size = len(payload.data) if payload is not None else None
item = conn.execute(
"""
INSERT INTO dispatch_items (
dispatch_id, role, position, element_id, title, document_on, filename,
content_type, byte_size, content_sha256, source_state, gap_reason,
note_document, external_source, external_id
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
dispatch["id"],
plan["role"],
plan["position"],
plan["element"]["id"],
plan["element"]["title"],
plan["dates"]["document_on"],
filename,
content_type,
byte_size,
digest,
plan["state"],
plan["gap_reason"],
Json(plan["note_document"]) if plan["note_document"] is not None else None,
plan["external_source"],
plan["external_id"],
),
).fetchone()
if plan["staged"] is not None and payload is not None:
_record_snapshot(conn, dispatch["home_scope_id"], dispatch["recorded_by"], payload, plan["staged"], dispatch_item_id=item["id"])
return plan["state"]
def _plan_evidence(conn, actor, element, role, fold, page, media_root, cleanup, scope_id, dispatch_id=None) -> dict:
if role not in EVIDENCE_ROLES:
raise unprocessable("Die Rolle des Nachweises ist nicht vorgesehen.")
if fold not in {"keep", "evidence"}:
raise unprocessable("Die Darstellung des Nachweises ist nicht festgelegt.")
if dispatch_id is not None and conn.execute(
"SELECT id FROM dispatch_evidence WHERE dispatch_id = %s AND element_id = %s",
(dispatch_id, element["id"]),
).fetchone():
raise unprocessable("Dieser Nachweis ist dem Versand bereits zugeordnet.")
dispatch = {"id": dispatch_id} if dispatch_id is not None else None
identified, gap_reason, message_id, digest, comparisons, snapshot = _identify_evidence(
conn, actor, dispatch, element, page, media_root, cleanup, scope_id
)
staged = _stage_snapshot(conn, media_root, snapshot.data, cleanup) if snapshot is not None else None
return {
"element": element,
"role": role,
"fold": fold,
"identified": identified,
"gap_reason": gap_reason,
"message_id": message_id,
"digest": digest,
"comparisons": comparisons,
"snapshot": snapshot,
"staged": staged,
}
def _write_evidence(conn, actor, container_row, page, scope, dispatch, plan: dict, log: bool) -> dict:
snapshot = plan["snapshot"]
comparisons = plan["comparisons"]
if snapshot is not None and snapshot.media_type == "message/rfc822":
from app.mailpack import parse_mail
comparisons = _compare_mail(conn, actor, dispatch["id"], page, MailPayload(snapshot.data, parse_mail(snapshot.data)))
proof = conn.execute(
"""
INSERT INTO dispatch_evidence (
dispatch_id, element_id, role, source_state, gap_reason, message_id,
identity_sha256, comparisons, manual, created_by
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, true, %s)
RETURNING *
""",
(
dispatch["id"],
plan["element"]["id"],
plan["role"],
"identified" if plan["identified"] else "gap",
plan["gap_reason"],
plan["message_id"],
plan["digest"],
Json(comparisons),
actor,
),
).fetchone()
if plan["staged"] is not None and snapshot is not None:
_record_snapshot(
conn,
container_row["home_scope_id"],
actor,
snapshot,
plan["staged"],
evidence_id=proof["id"],
)
if plan["fold"] == "evidence":
_set_binding_fold(conn, scope, plan["element"]["id"], "evidence")
if log:
_log(conn, dispatch["id"], actor, f"Nachweis ergänzt: {EVIDENCE_ROLES[plan['role']]}")
audit(conn, actor, container_row["home_scope_id"], "dispatch_evidence", "element", plan["element"]["id"], page)
return {
"id": str(proof["id"]),
"element_id": str(plan["element"]["id"]),
"role": plan["role"],
"role_label": EVIDENCE_ROLES[plan["role"]],
"source_state": proof["source_state"],
"gap_reason": plan["gap_reason"],
"manual": True,
"comparisons": comparisons,
}
def _record_snapshot(conn, scope_id, actor, payload: FilePayload, staged: dict, dispatch_item_id=None, evidence_id=None) -> UUID:
conn.execute(
"""
INSERT INTO dispatch_snapshots (
@@ -951,7 +1119,7 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'none')
""",
(
snapshot_id,
staged["id"],
dispatch_item_id,
evidence_id,
scope_id,
@@ -959,15 +1127,20 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
display_filename(payload.filename),
payload.media_type or "application/octet-stream",
len(payload.data),
digest,
instance["id"],
instance["generation"],
object_key,
content_revision,
sha256(payload.data),
staged["instance"]["id"],
staged["instance"]["generation"],
staged["object_key"],
staged["content_revision"],
LEGACY_PROFILE,
),
)
return snapshot_id
return staged["id"]
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
staged = _stage_snapshot(conn, media_root, payload.data, cleanup)
return _record_snapshot(conn, scope_id, actor, payload, staged, dispatch_item_id, evidence_id)
def _snapshot_file(conn, media_root: str, dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> FilePayload:
+32 -2
View File
@@ -695,9 +695,18 @@ def _emit_one(node, prefix, add, entries, loaded, notice_for, generated: str) ->
links = [("EML-Original", eml)]
try:
parsed = payload.parsed if isinstance(payload, MailPayload) else parse_mail(raw)
reading = _mail_reading_html(parsed)
if reading is not None:
reading_path = add(prefix + ["lesefassung.html"], reading)
links.append(("Formatierte Lesefassung", reading_path))
else:
notice = "\n".join(part for part in [notice, "Eine formatierte Lesefassung liegt für diese Nachricht nicht vor. Die vereinfachte Textfassung steht in der Übersicht."] if part)
for attachment in parsed.attachments:
label = attachment.display_name or attachment.filename
path = add(prefix + ["anlagen", attachment.filename], attachment.data)
links.append((attachment.filename, path))
links.append((label, path))
if parsed.notices:
notice = "\n".join(part for part in [notice, *parsed.notices] if part)
except Exception:
links.append(("Hinweis", eml))
entries.append(_entry(node, eml, links, notice or None))
@@ -1223,6 +1232,21 @@ def _note_markdown(node: ExportNode, body: str, generated: str) -> str:
return "\n".join(lines)
def _mail_reading_html(parsed) -> bytes | None:
if not getattr(parsed, "html", ""):
return None
page = (
"<!DOCTYPE html><html lang=\"de\"><head><meta charset=\"utf-8\">"
"<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'none'; img-src data:; style-src 'none'; script-src 'none'\">"
"<title>Lesefassung</title></head><body>"
f"{parsed.html}"
"<p>Vereinfachte Textfassung und das unveränderte EML-Original stehen daneben. "
"Die Ansicht ist nicht pixelgleich zur Postfachdarstellung.</p>"
"</body></html>"
)
return page.encode("utf-8")
def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attachments, notice: str, generated: str) -> str:
parsed = mail.parsed
lines = [f"# {node.title}", "", "Lesefassung. Das unveränderte Original ist die EML-Datei.", ""]
@@ -1234,7 +1258,13 @@ def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attac
if parsed.copy:
lines.append(f"- Kopie: {parsed.copy}")
lines.append(f"- [EML-Original]({_href(_relative(text_path, eml_path))})")
lines.extend(["", parsed.text or "Der Text steht im EML-Original.", ""])
lines.extend([
"",
"Vereinfachte Textfassung. Sie ist nicht die formatierte Ansicht und nicht das EML-Original.",
"",
parsed.text or "Der Text steht im EML-Original.",
"",
])
if attachments:
lines.append("## Anlagen")
lines.append("")
+311 -66
View File
@@ -1,16 +1,32 @@
"""Pure mail identity and attachment extraction for a complete export.
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
The same tree feeds preview, reading, attachment download and export.
"""
from __future__ import annotations
import hashlib
import html
import re
from dataclasses import dataclass, field
from email import policy
from email.header import decode_header, make_header
from email.message import EmailMessage
from email.parser import BytesParser
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
_UID_PREFIX = "uid:"
_POLICY = policy.default
_ALLOWED_TAGS = {
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
}
_VOID_TAGS = {"br", "img"}
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
class MailIdentityError(Exception):
@@ -25,6 +41,9 @@ class AttachmentBytes:
media_type: str
data: bytes
sha256: str
display_name: str = ""
content_id: str = ""
inline: bool = False
@dataclass
@@ -37,10 +56,14 @@ class ParsedMail:
text: str
attachments: list[AttachmentBytes] = field(default_factory=list)
missing: list[str] = field(default_factory=list)
html: str = ""
notices: list[str] = field(default_factory=list)
text_truncated: bool = False
document_on: str | None = None
def header_message_id(raw: bytes) -> str:
message = BytesParser().parsebytes(raw)
message = BytesParser(policy=_POLICY).parsebytes(raw)
return str(message.get("Message-ID") or "").strip()
@@ -69,67 +92,322 @@ def choose_message(candidate: bytes | None, alternates: list[bytes], expected: s
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
def header_text(value) -> str:
"""Decode encoded-words. A raw MIME header is not a display name."""
if value is None:
return ""
raw = str(value).replace("\r", " ").replace("\n", " ")
try:
text = str(make_header(decode_header(raw)))
except (UnicodeError, LookupError, ValueError):
text = raw
return re.sub(r"\s+", " ", text).strip()
def parse_mail(raw: bytes) -> ParsedMail:
message = BytesParser().parsebytes(raw)
message = BytesParser(policy=_POLICY).parsebytes(raw)
attachments: list[AttachmentBytes] = []
missing: list[str] = []
notices: list[str] = []
used: set[str] = set()
plain: list[str] = []
html_parts: list[str] = []
index = 0
for part in message.walk():
if part.is_multipart():
for kind, part in _walk(message):
if kind == "message":
index += 1
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
data = _message_bytes(part)
if data is None:
missing.append(label)
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label, used)
attachments.append(_attachment(part, name, label, data, inline=False))
continue
filename = part.get_filename()
disposition = (part.get_content_disposition() or "").lower()
if not filename and disposition != "attachment":
media = part.get_content_type()
filename = part.get_filename()
content_id = _content_id(part)
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
if body:
text = _decoded_text(part)
if text is None:
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
continue
if media == "text/plain":
plain.append(text)
else:
html_parts.append(text)
continue
if not filename and disposition != "attachment" and not content_id:
continue
index += 1
label = _header_text(filename) if filename else f"anlage-{index}.bin"
payload = part.get_payload(decode=True)
if not isinstance(payload, (bytes, bytearray)):
label = header_text(filename) if filename else f"anlage-{index}.bin"
data = _leaf_bytes(part)
if data is None:
missing.append(label or f"Anlage {index}")
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label or f"anlage-{index}.bin", used)
data = bytes(payload)
attachments.append(
AttachmentBytes(
filename=name,
media_type=(part.get_content_type() or "application/octet-stream")[:120],
data=data,
sha256=hashlib.sha256(data).hexdigest(),
)
)
text = _body_text(message)
inline = disposition == "inline" or bool(content_id)
attachments.append(_attachment(part, name, label or name, data, inline=inline))
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = ""
if html_parts:
rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments))
if not simplified:
simplified = simplified_text("\n".join(html_parts))
if missing:
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
return ParsedMail(
message_id=str(message.get("Message-ID") or "").strip(),
subject=_header_text(message.get("Subject")) or "(ohne Betreff)",
sender=_header_text(message.get("From")),
recipient=_header_text(message.get("To")),
copy=_header_text(message.get("Cc")),
text=text,
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
sender=header_text(message.get("From")),
recipient=header_text(message.get("To")),
copy=header_text(message.get("Cc")),
text=simplified,
attachments=attachments,
missing=missing,
html=rendered,
notices=notices,
document_on=_message_date(message.get("Date")),
)
def decode_transfer(payload: bytes, charset: str | None) -> str:
return bytes(payload).decode(charset or "utf-8", errors="replace")
def simplified_text(value: str) -> str:
"""A recognizable plain reading. It is not the formatted view."""
class Collector(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style"}:
self.skip += 1
return
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
self.parts.append("\n")
elif name in {"td", "th"}:
self.parts.append("\t")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style"} and self.skip:
self.skip -= 1
return
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
self.parts.append("\n")
def handle_data(self, data):
if not self.skip:
self.parts.append(data)
collector = Collector()
try:
collector.feed(value)
collector.close()
except Exception:
return ""
text = "".join(collector.parts)
text = text.replace("\r", "")
text = re.sub(r"[ \t]+\n", "\n", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def sanitize_html(value: str, images: dict[str, str] | None = None) -> str:
"""Keep a formatted reading. Scripts, active content and remote loads stay out."""
images = images or {}
class Cleaner(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
self.skip += 1
return
if self.skip or name not in _ALLOWED_TAGS:
return
rendered = _clean_attrs(name, attrs, images)
if rendered is None:
return
self.parts.append(f"<{name}{rendered}>")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
if self.skip:
self.skip -= 1
return
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
return
self.parts.append(f"</{name}>")
def handle_data(self, data):
if not self.skip:
self.parts.append(html.escape(data))
def handle_entityref(self, name):
if not self.skip:
self.parts.append(f"&{name};")
def handle_charref(self, name):
if not self.skip:
self.parts.append(f"&#{name};")
cleaner = Cleaner()
try:
cleaner.feed(value)
cleaner.close()
except Exception:
return ""
return "".join(cleaner.parts).strip()
def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None:
kept: list[str] = []
for key, value in attrs:
name = (key or "").lower()
if name.startswith("on") or name in {"style", "srcset"}:
continue
if tag == "a" and name == "href":
target = (value or "").strip()
if not _SAFE_LINK.match(target):
continue
kept.append(f' href="{html.escape(target, quote=True)}"')
elif tag == "img" and name == "src":
source = (value or "").strip()
if source.lower().startswith("cid:"):
token = source[4:].strip().strip("<>")
source = images.get(token.casefold(), "")
if not _DATA_IMAGE.match(source):
continue
kept.append(f' src="{source}"')
elif tag == "img" and name == "alt":
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
kept.append(f" {name}=\"{int(value)}\"")
if tag == "img" and not any(item.startswith(" src=") for item in kept):
return None
if tag == "a":
kept.append(' rel="noreferrer"')
return "".join(kept)
def _walk(message: EmailMessage):
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
yield "message", message
return
if message.is_multipart():
for child in message.iter_parts():
yield from _walk(child)
return
yield "leaf", message
def _decoded_text(part: EmailMessage) -> str | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if not isinstance(payload, (bytes, bytearray)):
raw = part.get_payload()
return raw.strip() if isinstance(raw, str) else None
charset = part.get_content_charset() or "utf-8"
try:
return decode_transfer(bytes(payload), charset).strip()
except LookupError:
return decode_transfer(bytes(payload), "utf-8").strip()
def _leaf_bytes(part: EmailMessage) -> bytes | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if isinstance(payload, (bytes, bytearray)):
return bytes(payload)
return None
def _message_bytes(part: EmailMessage) -> bytes | None:
payload = part.get_payload()
if isinstance(payload, list) and payload:
inner = payload[0]
if isinstance(inner, EmailMessage):
return inner.as_bytes()
data = _leaf_bytes(part)
return data or None
def _content_id(part: EmailMessage) -> str:
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
return token
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
return AttachmentBytes(
filename=filename,
media_type=(part.get_content_type() or "application/octet-stream")[:120],
data=data,
sha256=hashlib.sha256(data).hexdigest(),
display_name=label or filename,
content_id=_content_id(part),
inline=inline,
)
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
import base64
images: dict[str, str] = {}
for item in attachments:
if not item.content_id or not item.media_type.startswith("image/"):
continue
if len(item.data) > 1_500_000:
continue
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
if subtype == "jpg":
subtype = "jpeg"
if subtype not in {"png", "jpeg", "gif", "webp"}:
continue
encoded = base64.b64encode(item.data).decode("ascii")
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
return images
def _message_date(value) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(str(value)).date().isoformat()
except (TypeError, ValueError, OverflowError):
return None
def _bare(value: str) -> str:
return value.strip().strip("<>").strip().casefold()
def _header_text(value) -> str:
if value is None:
return ""
text = str(value).replace("\r", " ").replace("\n", " ")
return re.sub(r"\s+", " ", text).strip()
def _unique_filename(name: str, used: set[str]) -> str:
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
cleaned = cleaned[:180]
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
if cleaned not in used:
used.add(cleaned)
return cleaned
stem, dot, ext = cleaned.rpartition(".")
if not dot:
if not dot or not stem:
stem, ext = cleaned, ""
else:
ext = f".{ext}"
@@ -140,36 +418,3 @@ def _unique_filename(name: str, used: set[str]) -> str:
used.add(candidate)
return candidate
number += 1
def _body_text(message) -> str:
plain: list[str] = []
html: list[str] = []
parts = message.walk() if message.is_multipart() else [message]
for part in parts:
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
continue
kind = part.get_content_type()
payload = part.get_payload(decode=True)
if isinstance(payload, (bytes, bytearray)):
charset = part.get_content_charset() or "utf-8"
text = bytes(payload).decode(charset, errors="replace").strip()
else:
raw = part.get_payload()
text = raw.strip() if isinstance(raw, str) else ""
if kind == "text/plain":
plain.append(text)
elif kind == "text/html":
html.append(_html_text(text))
body = "\n".join(chunk for chunk in plain if chunk).strip()
if not body:
body = "\n".join(chunk for chunk in html if chunk).strip()
return body
def _html_text(value: str) -> str:
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
text = re.sub(r"(?i)</p>", "\n", text)
text = re.sub(r"<[^>]+>", " ", text)
return re.sub(r"[ \t]{2,}", " ", text).strip()
+5 -1
View File
@@ -1028,14 +1028,18 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
content_type = sniff(data)
target, rights_context = _placement(context, placement)
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "file")
asset_id = uuid4()
content_revision = str(uuid4())
instance = instance_for_write(conn, media_root)
object_key = str(asset_id)
adapter = adapter_for(instance)
from app.sourceaccess import release_for_external
release_for_external(conn)
publish(adapter, object_key, data)
rollback_files.append(StoredHandle(adapter, object_key))
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "file")
digest = sha256(data)
conn.execute(
"""
+326 -57
View File
@@ -21,7 +21,17 @@ from psycopg.types.json import Json
from app.access.policy import Ctx
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.mailpack import (
MailIdentityError,
choose_message,
decode_transfer,
header_message_id,
header_text,
identities_match,
parse_mail,
sanitize_html,
simplified_text,
)
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceaccess import (
@@ -291,12 +301,18 @@ def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: in
_check_mailbox(mailbox)
if uid < 1:
raise unprocessable("Die Nachricht ist ungültig.")
client = _imap_login(row["settings"], row["secret"])
settings = dict(row["settings"])
secret = row["secret"]
connector_id = row["id"]
release_for_external(conn)
client = _imap_login(settings, secret)
try:
_select_mailbox(client, mailbox)
return _fetch_message(client, uid, mailbox)
preview = _fetch_message(client, uid, mailbox)
finally:
_imap_logout(client)
assert_still_owned(conn, connector_id, owner_id)
return preview
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
@@ -402,7 +418,7 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
def fetch(row):
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
return complete_borrow(conn, borrowed, fetch)
@@ -726,16 +742,10 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
match = next((item for item in folders if item["name"] == chosen), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(chosen, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, chosen)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
exists = _untagged_int(client, "EXISTS")
validity = _untagged_int(client, "UIDVALIDITY")
restarted = False
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
@@ -774,6 +784,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
next_before = low
else:
next_before = None
query_text = " ".join(query.split())
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
return {
"mailbox": chosen,
"items": collected,
@@ -781,6 +793,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
"incomplete": incomplete,
"restarted": restarted,
"uidvalidity": validity,
"exists": exists,
"contradictory": contradictory,
}
@@ -789,15 +803,34 @@ def _check_mailbox(mailbox: str) -> None:
raise unprocessable("Der Fachname ist ungültig.")
def _mailbox_argument(mailbox: str) -> str:
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
_check_mailbox(mailbox)
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
return protocol
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _examine(client: imaplib.IMAP4, mailbox: str):
argument = _mailbox_argument(mailbox)
try:
return client.select(argument, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
folders = _imap_mailboxes(client)
match = next((item for item in folders if item["name"] == mailbox), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(mailbox, readonly=True)
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, mailbox)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
@@ -837,6 +870,10 @@ def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
}
PREVIEW_BYTE_LIMIT = 8_000_000
PREVIEW_TEXT_LIMIT = 200_000
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
@@ -845,22 +882,36 @@ def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
raw = _literal(fetched)
if status != "OK" or raw is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
if len(raw) > 8_000_000:
if len(raw) > PREVIEW_BYTE_LIMIT:
return _oversized_message(client, uid, mailbox)
message = BytesParser().parsebytes(raw)
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
parsed = parse_mail(raw)
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
text = parsed.text[:PREVIEW_TEXT_LIMIT]
return {
"uid": uid,
"mailbox": mailbox,
"title": subject[:200],
"message_from": _decode(message.get("From"))[:200],
"message_to": _decode(message.get("To"))[:500],
"message_cc": _decode(message.get("Cc"))[:500],
"title": (parsed.subject or "(ohne Betreff)")[:200],
"message_from": parsed.sender[:200],
"message_to": parsed.recipient[:500],
"message_cc": parsed.copy[:500],
"direction": _direction(mailbox),
"document_on": _mail_date(message.get("Date")),
"attachments": _message_attachments(message),
"text": (_text := _message_text(message))[:200_000],
"text_truncated": len(_text) > 200_000,
"document_on": parsed.document_on,
"attachments": [
{
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
}
for item in parsed.attachments[:40]
],
"text": text,
"html": parsed.html,
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
"preview_limited": limited,
"notices": parsed.notices,
}
@@ -887,10 +938,37 @@ def _direction(mailbox: str) -> str | None:
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"""Read named text parts. A cut-off multipart blob is not a message."""
header = _fetch_header(client, uid, mailbox) or {}
text = _partial_text(client, uid)
if not header and not text:
structure = _body_structure(client, uid)
plain: list[str] = []
html_parts: list[str] = []
truncated = False
notices = [
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
"Das Original bleibt die vollständige E-Mail."
]
if structure is None:
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
else:
for part in _text_sections(structure):
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
if fetched is None:
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
continue
truncated = truncated or shortened or part["octets"] > len(fetched)
text = decode_transfer(fetched, part["charset"]).strip()
if part["media"] == "text/plain":
plain.append(text)
else:
html_parts.append(text)
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
if not simplified and html_parts:
simplified = simplified_text("\n".join(html_parts))
if not header and not simplified and not rendered and structure is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
return {
"uid": uid,
"mailbox": mailbox,
@@ -900,31 +978,185 @@ def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"message_cc": header.get("message_cc") or "",
"direction": header.get("direction", _direction(mailbox)),
"document_on": header.get("document_on"),
"attachments": header.get("attachments") or [],
"text": text,
"text_truncated": True,
"attachments": attachments,
"text": simplified[:PREVIEW_TEXT_LIMIT],
"html": rendered,
"text_truncated": True or truncated,
"preview_limited": True,
"notices": notices,
}
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
def _body_structure(client: imaplib.IMAP4, uid: int):
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
except imaplib.IMAP4.error:
return ""
return None
if status != "OK":
return ""
return None
blob = _literal(fetched)
if blob is None:
chunks: list[bytes] = []
for part in fetched or []:
if isinstance(part, tuple):
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
elif isinstance(part, (bytes, bytearray)):
chunks.append(bytes(part))
blob = b" ".join(chunks)
text = blob.decode("utf-8", "replace")
start = text.find("(")
if start < 0:
return None
try:
node, _end = _parse_imap_list(text, start)
except ValueError:
return None
return node
def _parse_imap_list(text: str, index: int):
if index >= len(text) or text[index] != "(":
raise ValueError(text[index:index + 20])
index += 1
items = []
while index < len(text):
char = text[index]
if char == ")":
return items, index + 1
if char in " \t\r\n":
index += 1
continue
if char == "(":
child, index = _parse_imap_list(text, index)
items.append(child)
continue
if char == '"':
token, index = _parse_imap_string(text, index)
items.append(token)
continue
end = index
while end < len(text) and text[end] not in " \t\r\n()":
end += 1
token = text[index:end]
items.append(None if token.upper() == "NIL" else token)
index = end
raise ValueError("unclosed")
def _parse_imap_string(text: str, index: int):
index += 1
chars: list[str] = []
while index < len(text):
char = text[index]
if char == "\\":
if index + 1 < len(text):
chars.append(text[index + 1])
index += 2
continue
if char == '"':
return "".join(chars), index + 1
chars.append(char)
index += 1
raise ValueError("string")
def _text_sections(node, prefix: str = "") -> list[dict]:
if not isinstance(node, list) or not node:
return []
if isinstance(node[0], list):
children = []
for item in node:
if isinstance(item, list):
children.append(item)
else:
break
found = []
for number, child in enumerate(children, start=1):
section = f"{number}" if not prefix else f"{prefix}.{number}"
found.extend(_text_sections(child, section))
return found
media = str(node[0] or "").upper()
subtype = str(node[1] or "").upper()
if media == "MESSAGE":
return []
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
return []
disposition, params = _disposition(node)
if disposition == "ATTACHMENT":
return []
body_params = _param_map(node[2]) if len(node) > 2 else {}
octets = _imap_int(node[6]) if len(node) > 6 else 0
return [{
"section": prefix or "1",
"media": f"text/{subtype.lower()}",
"charset": body_params.get("charset") or "utf-8",
"octets": octets,
}]
def _disposition(node: list) -> tuple[str, dict]:
for item in node:
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
params = _param_map(item[1]) if len(item) > 1 else {}
return item[0].upper(), params
return "", {}
def _param_map(value) -> dict:
if not isinstance(value, list):
return {}
pairs = {}
index = 0
while index + 1 < len(value):
key = value[index]
raw = value[index + 1]
if isinstance(key, str):
pairs[key.casefold()] = "" if raw is None else str(raw)
index += 2
return pairs
def _imap_int(value) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
try:
status, fetched = client.uid("FETCH", str(uid), spec)
except imaplib.IMAP4.error:
return None, False
if status != "OK":
return None, False
raw = _literal(fetched)
if not raw:
return ""
sample = raw[:200_000]
try:
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
text = _message_text(message)
except Exception:
text = ""
if not text:
text = sample.decode("utf-8", errors="replace")
return text.strip()[:200_000]
return None, False
return raw[:limit], len(raw) >= limit
def _structure_names(node) -> list[dict]:
names = []
def visit(item):
if not isinstance(item, list) or not item:
return
if isinstance(item[0], list):
for child in item:
if isinstance(child, list):
visit(child)
return
_disposition_name, params = _disposition(item)
body = _param_map(item[2]) if len(item) > 2 else {}
raw = params.get("filename") or body.get("name") or ""
label = header_text(raw)
if label:
names.append({"filename": label[:200]})
visit(node)
return names[:40]
def _literal(fetched) -> bytes | None:
@@ -944,12 +1176,10 @@ def _structure_attachments(fetched) -> list[dict]:
blob = b" ".join(chunks)
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
items = []
seen: set[str] = set()
for raw in names:
name = raw.decode("utf-8", errors="replace").strip()[:200]
if not name or name in seen:
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
if not name:
continue
seen.add(name)
items.append({"filename": name})
if len(items) == 40:
break
@@ -1039,6 +1269,32 @@ def _unquote(token: str) -> str:
return token
def _encode_utf7(value: str) -> str:
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
pieces: list[str] = []
pending: list[str] = []
def flush() -> None:
if not pending:
return
raw = "".join(pending).encode("utf-16-be")
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
pieces.append(f"&{token}-")
pending.clear()
for char in value:
if char == "&":
flush()
pieces.append("&-")
elif ord(char) < 128:
flush()
pieces.append(char)
else:
pending.append(char)
flush()
return "".join(pieces)
def _decode_utf7(value: str) -> str:
if "&" not in value:
return value
@@ -1255,13 +1511,17 @@ def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assi
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
base = resolved.row["settings"]["base_url"].rstrip("/")
token = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
data, media, name = pull_paperless_original(
resolved.row["settings"]["base_url"].rstrip("/"),
resolved.row["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
@@ -1282,8 +1542,17 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
settings = dict(resolved.row["settings"])
secret = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT: