Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
34c4084a7c
commit
c29bd29f34
29 files changed
+2562
-410
No files matched your search
@@ -124,14 +124,18 @@ def view_mail(
|
||||
"message_to": parsed.recipient,
|
||||
"message_cc": parsed.copy,
|
||||
"text": parsed.text,
|
||||
"text_truncated": False,
|
||||
"complete": True,
|
||||
"html": parsed.html,
|
||||
"text_truncated": parsed.text_truncated,
|
||||
"complete": not parsed.missing,
|
||||
"notices": parsed.notices,
|
||||
"attachments": [
|
||||
{
|
||||
"index": index,
|
||||
"filename": item.filename,
|
||||
"filename": item.display_name or item.filename,
|
||||
"archive_name": item.filename,
|
||||
"content_type": item.media_type,
|
||||
"byte_size": len(item.data),
|
||||
"inline": item.inline,
|
||||
}
|
||||
for index, item in enumerate(parsed.attachments)
|
||||
],
|
||||
|
||||
+222
-49
@@ -211,11 +211,36 @@ def record_dispatch(
|
||||
raise unprocessable("Dieselbe Anlage ist mehrfach im Versand genannt.")
|
||||
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
|
||||
host = _bound_element(conn, actor, host_id, page, scope)
|
||||
bump_version(conn, "elements", host_id, expected)
|
||||
known = {row["attachment_element_id"] for row in _uses(conn, scope, host_id)}
|
||||
if any(item not in known for item in attachment_ids):
|
||||
raise unprocessable("Mitversendet werden nur Anlagen dieses Schreibens.")
|
||||
counterpart_text = _counterpart(counterpart)
|
||||
edition_plans = []
|
||||
if capture_editions:
|
||||
edition_plans.append(_plan_edition(conn, actor, host, "host", 0, page, historical_gap, media_root, cleanup))
|
||||
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
|
||||
position_of = {item: index for index, item in enumerate(attachment_ids)}
|
||||
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
|
||||
for index, row in enumerate(ordered, start=1):
|
||||
element = _readable(conn, actor, row["attachment_element_id"], page)
|
||||
if element is None:
|
||||
raise not_found()
|
||||
edition_plans.append(
|
||||
_plan_edition(conn, actor, element, "attachment", index, page, historical_gap, media_root, cleanup)
|
||||
)
|
||||
evidence_plans = []
|
||||
for item in evidence:
|
||||
element = _readable(conn, actor, item["element_id"], page)
|
||||
if element is None:
|
||||
raise not_found()
|
||||
evidence_plans.append(
|
||||
_plan_evidence(
|
||||
conn, actor, element, item["role"], item.get("fold") or "keep", page, media_root, cleanup, container_row["home_scope_id"]
|
||||
)
|
||||
)
|
||||
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
|
||||
host = _bound_element(conn, actor, host_id, page, scope)
|
||||
bump_version(conn, "elements", host_id, expected)
|
||||
dispatch = conn.execute(
|
||||
"""
|
||||
INSERT INTO dispatches (
|
||||
@@ -237,37 +262,11 @@ def record_dispatch(
|
||||
actor,
|
||||
),
|
||||
).fetchone()
|
||||
states = []
|
||||
if capture_editions:
|
||||
states.append(_capture_item(conn, actor, dispatch, host, "host", 0, page, historical_gap, media_root, cleanup))
|
||||
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
|
||||
position_of = {item: index for index, item in enumerate(attachment_ids)}
|
||||
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
|
||||
for index, row in enumerate(ordered, start=1):
|
||||
element = _readable(conn, actor, row["attachment_element_id"], page)
|
||||
if element is None:
|
||||
raise not_found()
|
||||
states.append(
|
||||
_capture_item(conn, actor, dispatch, element, "attachment", index, page, historical_gap, media_root, cleanup)
|
||||
)
|
||||
evidence_rows = []
|
||||
for item in evidence:
|
||||
evidence_rows.append(
|
||||
_add_evidence(
|
||||
conn,
|
||||
actor,
|
||||
container_row,
|
||||
page,
|
||||
scope,
|
||||
dispatch,
|
||||
item["element_id"],
|
||||
item["role"],
|
||||
item.get("fold") or "keep",
|
||||
media_root,
|
||||
cleanup,
|
||||
log=False,
|
||||
)
|
||||
)
|
||||
states = [_write_edition(conn, dispatch, plan) for plan in edition_plans]
|
||||
evidence_rows = [
|
||||
_write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=False)
|
||||
for plan in evidence_plans
|
||||
]
|
||||
completeness = _completeness(states) if capture_editions else "secured"
|
||||
conn.execute(
|
||||
"UPDATE dispatches SET completeness = %s, updated_at = now() WHERE id = %s",
|
||||
@@ -292,11 +291,16 @@ def add_dispatch_evidence(
|
||||
) -> dict:
|
||||
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
|
||||
dispatch = _dispatch_in(conn, dispatch_id, scope)
|
||||
_bump_dispatch(conn, dispatch_id, expected)
|
||||
row = _add_evidence(
|
||||
conn, actor, container_row, page, scope, dispatch, element_id, role, fold, media_root, cleanup, log=True
|
||||
element = _readable(conn, actor, element_id, page)
|
||||
if element is None:
|
||||
raise not_found()
|
||||
plan = _plan_evidence(
|
||||
conn, actor, element, role, fold, page, media_root, cleanup, container_row["home_scope_id"], dispatch_id
|
||||
)
|
||||
return row
|
||||
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
|
||||
dispatch = _dispatch_in(conn, dispatch_id, scope)
|
||||
_bump_dispatch(conn, dispatch_id, expected)
|
||||
return _write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=True)
|
||||
|
||||
|
||||
def detach_dispatch_evidence(
|
||||
@@ -760,7 +764,7 @@ def _identify_evidence(conn, actor, dispatch, element, page, media_root, cleanup
|
||||
if isinstance(loaded, RetrievalFailure):
|
||||
return False, loaded.message, None, None, [], None
|
||||
if isinstance(loaded, MailPayload):
|
||||
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded)
|
||||
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded) if dispatch is not None else []
|
||||
payload = FilePayload("nachweis.eml", "message/rfc822", loaded.raw, sha256(loaded.raw))
|
||||
return True, None, loaded.parsed.message_id or None, payload.sha256, comparisons, payload
|
||||
payload = FilePayload(loaded.filename, loaded.media_type, loaded.data, sha256(loaded.data))
|
||||
@@ -931,15 +935,179 @@ def _working_bytes(conn, actor, element, media_root: str):
|
||||
return None, "Für diesen Inhalt ist keine sicherbare Fassung vorgesehen.", None, None, None
|
||||
|
||||
|
||||
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
|
||||
snapshot_id = uuid4()
|
||||
def _stage_snapshot(conn, media_root: str, data: bytes, cleanup: list) -> dict:
|
||||
"""Publish bytes only while the domain transaction is idle. The row comes later."""
|
||||
from app.sourceaccess import release_for_external
|
||||
|
||||
instance = instance_for_write(conn, media_root)
|
||||
snapshot_id = uuid4()
|
||||
object_key = f"dispatch/{snapshot_id}"
|
||||
content_revision = str(uuid4())
|
||||
adapter = adapter_for(instance)
|
||||
publish(adapter, object_key, payload.data)
|
||||
release_for_external(conn)
|
||||
publish(adapter, object_key, data)
|
||||
cleanup.append(StoredHandle(adapter, object_key))
|
||||
digest = sha256(payload.data)
|
||||
return {
|
||||
"id": snapshot_id,
|
||||
"instance": instance,
|
||||
"object_key": object_key,
|
||||
"content_revision": str(uuid4()),
|
||||
}
|
||||
|
||||
|
||||
def _plan_edition(conn, actor, element, role, position, page, historical_gap, media_root, cleanup) -> dict:
|
||||
dates = _dates(conn, element["id"])
|
||||
payload = None
|
||||
gap_reason = None
|
||||
note_document = None
|
||||
external_source = None
|
||||
external_id = None
|
||||
if historical_gap:
|
||||
state = "gap"
|
||||
gap_reason = _HISTORICAL_GAP
|
||||
else:
|
||||
payload, gap_reason, note_document, external_source, external_id = _working_bytes(
|
||||
conn, actor, element, media_root
|
||||
)
|
||||
state = "snapshot" if payload is not None else "gap"
|
||||
staged = _stage_snapshot(conn, media_root, payload.data, cleanup) if payload is not None else None
|
||||
return {
|
||||
"element": element,
|
||||
"role": role,
|
||||
"position": position,
|
||||
"dates": dates,
|
||||
"state": state,
|
||||
"payload": payload,
|
||||
"gap_reason": gap_reason,
|
||||
"note_document": note_document,
|
||||
"external_source": external_source,
|
||||
"external_id": external_id,
|
||||
"staged": staged,
|
||||
}
|
||||
|
||||
|
||||
def _write_edition(conn, dispatch, plan: dict) -> str:
|
||||
payload = plan["payload"]
|
||||
filename = payload.filename if payload is not None else None
|
||||
content_type = payload.media_type if payload is not None else None
|
||||
digest = sha256(payload.data) if payload is not None else None
|
||||
byte_size = len(payload.data) if payload is not None else None
|
||||
item = conn.execute(
|
||||
"""
|
||||
INSERT INTO dispatch_items (
|
||||
dispatch_id, role, position, element_id, title, document_on, filename,
|
||||
content_type, byte_size, content_sha256, source_state, gap_reason,
|
||||
note_document, external_source, external_id
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
dispatch["id"],
|
||||
plan["role"],
|
||||
plan["position"],
|
||||
plan["element"]["id"],
|
||||
plan["element"]["title"],
|
||||
plan["dates"]["document_on"],
|
||||
filename,
|
||||
content_type,
|
||||
byte_size,
|
||||
digest,
|
||||
plan["state"],
|
||||
plan["gap_reason"],
|
||||
Json(plan["note_document"]) if plan["note_document"] is not None else None,
|
||||
plan["external_source"],
|
||||
plan["external_id"],
|
||||
),
|
||||
).fetchone()
|
||||
if plan["staged"] is not None and payload is not None:
|
||||
_record_snapshot(conn, dispatch["home_scope_id"], dispatch["recorded_by"], payload, plan["staged"], dispatch_item_id=item["id"])
|
||||
return plan["state"]
|
||||
|
||||
|
||||
def _plan_evidence(conn, actor, element, role, fold, page, media_root, cleanup, scope_id, dispatch_id=None) -> dict:
|
||||
if role not in EVIDENCE_ROLES:
|
||||
raise unprocessable("Die Rolle des Nachweises ist nicht vorgesehen.")
|
||||
if fold not in {"keep", "evidence"}:
|
||||
raise unprocessable("Die Darstellung des Nachweises ist nicht festgelegt.")
|
||||
if dispatch_id is not None and conn.execute(
|
||||
"SELECT id FROM dispatch_evidence WHERE dispatch_id = %s AND element_id = %s",
|
||||
(dispatch_id, element["id"]),
|
||||
).fetchone():
|
||||
raise unprocessable("Dieser Nachweis ist dem Versand bereits zugeordnet.")
|
||||
dispatch = {"id": dispatch_id} if dispatch_id is not None else None
|
||||
identified, gap_reason, message_id, digest, comparisons, snapshot = _identify_evidence(
|
||||
conn, actor, dispatch, element, page, media_root, cleanup, scope_id
|
||||
)
|
||||
staged = _stage_snapshot(conn, media_root, snapshot.data, cleanup) if snapshot is not None else None
|
||||
return {
|
||||
"element": element,
|
||||
"role": role,
|
||||
"fold": fold,
|
||||
"identified": identified,
|
||||
"gap_reason": gap_reason,
|
||||
"message_id": message_id,
|
||||
"digest": digest,
|
||||
"comparisons": comparisons,
|
||||
"snapshot": snapshot,
|
||||
"staged": staged,
|
||||
}
|
||||
|
||||
|
||||
def _write_evidence(conn, actor, container_row, page, scope, dispatch, plan: dict, log: bool) -> dict:
|
||||
snapshot = plan["snapshot"]
|
||||
comparisons = plan["comparisons"]
|
||||
if snapshot is not None and snapshot.media_type == "message/rfc822":
|
||||
from app.mailpack import parse_mail
|
||||
|
||||
comparisons = _compare_mail(conn, actor, dispatch["id"], page, MailPayload(snapshot.data, parse_mail(snapshot.data)))
|
||||
proof = conn.execute(
|
||||
"""
|
||||
INSERT INTO dispatch_evidence (
|
||||
dispatch_id, element_id, role, source_state, gap_reason, message_id,
|
||||
identity_sha256, comparisons, manual, created_by
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, true, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
dispatch["id"],
|
||||
plan["element"]["id"],
|
||||
plan["role"],
|
||||
"identified" if plan["identified"] else "gap",
|
||||
plan["gap_reason"],
|
||||
plan["message_id"],
|
||||
plan["digest"],
|
||||
Json(comparisons),
|
||||
actor,
|
||||
),
|
||||
).fetchone()
|
||||
if plan["staged"] is not None and snapshot is not None:
|
||||
_record_snapshot(
|
||||
conn,
|
||||
container_row["home_scope_id"],
|
||||
actor,
|
||||
snapshot,
|
||||
plan["staged"],
|
||||
evidence_id=proof["id"],
|
||||
)
|
||||
if plan["fold"] == "evidence":
|
||||
_set_binding_fold(conn, scope, plan["element"]["id"], "evidence")
|
||||
if log:
|
||||
_log(conn, dispatch["id"], actor, f"Nachweis ergänzt: {EVIDENCE_ROLES[plan['role']]}")
|
||||
audit(conn, actor, container_row["home_scope_id"], "dispatch_evidence", "element", plan["element"]["id"], page)
|
||||
return {
|
||||
"id": str(proof["id"]),
|
||||
"element_id": str(plan["element"]["id"]),
|
||||
"role": plan["role"],
|
||||
"role_label": EVIDENCE_ROLES[plan["role"]],
|
||||
"source_state": proof["source_state"],
|
||||
"gap_reason": plan["gap_reason"],
|
||||
"manual": True,
|
||||
"comparisons": comparisons,
|
||||
}
|
||||
|
||||
|
||||
def _record_snapshot(conn, scope_id, actor, payload: FilePayload, staged: dict, dispatch_item_id=None, evidence_id=None) -> UUID:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO dispatch_snapshots (
|
||||
@@ -951,7 +1119,7 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'none')
|
||||
""",
|
||||
(
|
||||
snapshot_id,
|
||||
staged["id"],
|
||||
dispatch_item_id,
|
||||
evidence_id,
|
||||
scope_id,
|
||||
@@ -959,15 +1127,20 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
|
||||
display_filename(payload.filename),
|
||||
payload.media_type or "application/octet-stream",
|
||||
len(payload.data),
|
||||
digest,
|
||||
instance["id"],
|
||||
instance["generation"],
|
||||
object_key,
|
||||
content_revision,
|
||||
sha256(payload.data),
|
||||
staged["instance"]["id"],
|
||||
staged["instance"]["generation"],
|
||||
staged["object_key"],
|
||||
staged["content_revision"],
|
||||
LEGACY_PROFILE,
|
||||
),
|
||||
)
|
||||
return snapshot_id
|
||||
return staged["id"]
|
||||
|
||||
|
||||
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
|
||||
staged = _stage_snapshot(conn, media_root, payload.data, cleanup)
|
||||
return _record_snapshot(conn, scope_id, actor, payload, staged, dispatch_item_id, evidence_id)
|
||||
|
||||
|
||||
def _snapshot_file(conn, media_root: str, dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> FilePayload:
|
||||
|
||||
@@ -695,9 +695,18 @@ def _emit_one(node, prefix, add, entries, loaded, notice_for, generated: str) ->
|
||||
links = [("EML-Original", eml)]
|
||||
try:
|
||||
parsed = payload.parsed if isinstance(payload, MailPayload) else parse_mail(raw)
|
||||
reading = _mail_reading_html(parsed)
|
||||
if reading is not None:
|
||||
reading_path = add(prefix + ["lesefassung.html"], reading)
|
||||
links.append(("Formatierte Lesefassung", reading_path))
|
||||
else:
|
||||
notice = "\n".join(part for part in [notice, "Eine formatierte Lesefassung liegt für diese Nachricht nicht vor. Die vereinfachte Textfassung steht in der Übersicht."] if part)
|
||||
for attachment in parsed.attachments:
|
||||
label = attachment.display_name or attachment.filename
|
||||
path = add(prefix + ["anlagen", attachment.filename], attachment.data)
|
||||
links.append((attachment.filename, path))
|
||||
links.append((label, path))
|
||||
if parsed.notices:
|
||||
notice = "\n".join(part for part in [notice, *parsed.notices] if part)
|
||||
except Exception:
|
||||
links.append(("Hinweis", eml))
|
||||
entries.append(_entry(node, eml, links, notice or None))
|
||||
@@ -1223,6 +1232,21 @@ def _note_markdown(node: ExportNode, body: str, generated: str) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _mail_reading_html(parsed) -> bytes | None:
|
||||
if not getattr(parsed, "html", ""):
|
||||
return None
|
||||
page = (
|
||||
"<!DOCTYPE html><html lang=\"de\"><head><meta charset=\"utf-8\">"
|
||||
"<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'none'; img-src data:; style-src 'none'; script-src 'none'\">"
|
||||
"<title>Lesefassung</title></head><body>"
|
||||
f"{parsed.html}"
|
||||
"<p>Vereinfachte Textfassung und das unveränderte EML-Original stehen daneben. "
|
||||
"Die Ansicht ist nicht pixelgleich zur Postfachdarstellung.</p>"
|
||||
"</body></html>"
|
||||
)
|
||||
return page.encode("utf-8")
|
||||
|
||||
|
||||
def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attachments, notice: str, generated: str) -> str:
|
||||
parsed = mail.parsed
|
||||
lines = [f"# {node.title}", "", "Lesefassung. Das unveränderte Original ist die EML-Datei.", ""]
|
||||
@@ -1234,7 +1258,13 @@ def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attac
|
||||
if parsed.copy:
|
||||
lines.append(f"- Kopie: {parsed.copy}")
|
||||
lines.append(f"- [EML-Original]({_href(_relative(text_path, eml_path))})")
|
||||
lines.extend(["", parsed.text or "Der Text steht im EML-Original.", ""])
|
||||
lines.extend([
|
||||
"",
|
||||
"Vereinfachte Textfassung. Sie ist nicht die formatierte Ansicht und nicht das EML-Original.",
|
||||
"",
|
||||
parsed.text or "Der Text steht im EML-Original.",
|
||||
"",
|
||||
])
|
||||
if attachments:
|
||||
lines.append("## Anlagen")
|
||||
lines.append("")
|
||||
|
||||
+311
-66
@@ -1,16 +1,32 @@
|
||||
"""Pure mail identity and attachment extraction for a complete export.
|
||||
|
||||
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
|
||||
The same tree feeds preview, reading, attachment download and export.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from email import policy
|
||||
from email.header import decode_header, make_header
|
||||
from email.message import EmailMessage
|
||||
from email.parser import BytesParser
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
|
||||
_UID_PREFIX = "uid:"
|
||||
_POLICY = policy.default
|
||||
_ALLOWED_TAGS = {
|
||||
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
|
||||
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
|
||||
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
|
||||
}
|
||||
_VOID_TAGS = {"br", "img"}
|
||||
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
|
||||
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
|
||||
|
||||
|
||||
class MailIdentityError(Exception):
|
||||
@@ -25,6 +41,9 @@ class AttachmentBytes:
|
||||
media_type: str
|
||||
data: bytes
|
||||
sha256: str
|
||||
display_name: str = ""
|
||||
content_id: str = ""
|
||||
inline: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -37,10 +56,14 @@ class ParsedMail:
|
||||
text: str
|
||||
attachments: list[AttachmentBytes] = field(default_factory=list)
|
||||
missing: list[str] = field(default_factory=list)
|
||||
html: str = ""
|
||||
notices: list[str] = field(default_factory=list)
|
||||
text_truncated: bool = False
|
||||
document_on: str | None = None
|
||||
|
||||
|
||||
def header_message_id(raw: bytes) -> str:
|
||||
message = BytesParser().parsebytes(raw)
|
||||
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
||||
return str(message.get("Message-ID") or "").strip()
|
||||
|
||||
|
||||
@@ -69,67 +92,322 @@ def choose_message(candidate: bytes | None, alternates: list[bytes], expected: s
|
||||
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
||||
|
||||
|
||||
def header_text(value) -> str:
|
||||
"""Decode encoded-words. A raw MIME header is not a display name."""
|
||||
if value is None:
|
||||
return ""
|
||||
raw = str(value).replace("\r", " ").replace("\n", " ")
|
||||
try:
|
||||
text = str(make_header(decode_header(raw)))
|
||||
except (UnicodeError, LookupError, ValueError):
|
||||
text = raw
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def parse_mail(raw: bytes) -> ParsedMail:
|
||||
message = BytesParser().parsebytes(raw)
|
||||
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
||||
attachments: list[AttachmentBytes] = []
|
||||
missing: list[str] = []
|
||||
notices: list[str] = []
|
||||
used: set[str] = set()
|
||||
plain: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
index = 0
|
||||
for part in message.walk():
|
||||
if part.is_multipart():
|
||||
for kind, part in _walk(message):
|
||||
if kind == "message":
|
||||
index += 1
|
||||
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
|
||||
data = _message_bytes(part)
|
||||
if data is None:
|
||||
missing.append(label)
|
||||
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
|
||||
continue
|
||||
name = _unique_filename(label, used)
|
||||
attachments.append(_attachment(part, name, label, data, inline=False))
|
||||
continue
|
||||
filename = part.get_filename()
|
||||
disposition = (part.get_content_disposition() or "").lower()
|
||||
if not filename and disposition != "attachment":
|
||||
media = part.get_content_type()
|
||||
filename = part.get_filename()
|
||||
content_id = _content_id(part)
|
||||
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
|
||||
if body:
|
||||
text = _decoded_text(part)
|
||||
if text is None:
|
||||
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
|
||||
continue
|
||||
if media == "text/plain":
|
||||
plain.append(text)
|
||||
else:
|
||||
html_parts.append(text)
|
||||
continue
|
||||
if not filename and disposition != "attachment" and not content_id:
|
||||
continue
|
||||
index += 1
|
||||
label = _header_text(filename) if filename else f"anlage-{index}.bin"
|
||||
payload = part.get_payload(decode=True)
|
||||
if not isinstance(payload, (bytes, bytearray)):
|
||||
label = header_text(filename) if filename else f"anlage-{index}.bin"
|
||||
data = _leaf_bytes(part)
|
||||
if data is None:
|
||||
missing.append(label or f"Anlage {index}")
|
||||
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
|
||||
continue
|
||||
name = _unique_filename(label or f"anlage-{index}.bin", used)
|
||||
data = bytes(payload)
|
||||
attachments.append(
|
||||
AttachmentBytes(
|
||||
filename=name,
|
||||
media_type=(part.get_content_type() or "application/octet-stream")[:120],
|
||||
data=data,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
)
|
||||
)
|
||||
text = _body_text(message)
|
||||
inline = disposition == "inline" or bool(content_id)
|
||||
attachments.append(_attachment(part, name, label or name, data, inline=inline))
|
||||
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
rendered = ""
|
||||
if html_parts:
|
||||
rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments))
|
||||
if not simplified:
|
||||
simplified = simplified_text("\n".join(html_parts))
|
||||
if missing:
|
||||
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
|
||||
return ParsedMail(
|
||||
message_id=str(message.get("Message-ID") or "").strip(),
|
||||
subject=_header_text(message.get("Subject")) or "(ohne Betreff)",
|
||||
sender=_header_text(message.get("From")),
|
||||
recipient=_header_text(message.get("To")),
|
||||
copy=_header_text(message.get("Cc")),
|
||||
text=text,
|
||||
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
|
||||
sender=header_text(message.get("From")),
|
||||
recipient=header_text(message.get("To")),
|
||||
copy=header_text(message.get("Cc")),
|
||||
text=simplified,
|
||||
attachments=attachments,
|
||||
missing=missing,
|
||||
html=rendered,
|
||||
notices=notices,
|
||||
document_on=_message_date(message.get("Date")),
|
||||
)
|
||||
|
||||
|
||||
def decode_transfer(payload: bytes, charset: str | None) -> str:
|
||||
return bytes(payload).decode(charset or "utf-8", errors="replace")
|
||||
|
||||
|
||||
def simplified_text(value: str) -> str:
|
||||
"""A recognizable plain reading. It is not the formatted view."""
|
||||
class Collector(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts: list[str] = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style"}:
|
||||
self.skip += 1
|
||||
return
|
||||
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
|
||||
self.parts.append("\n")
|
||||
elif name in {"td", "th"}:
|
||||
self.parts.append("\t")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style"} and self.skip:
|
||||
self.skip -= 1
|
||||
return
|
||||
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||
self.parts.append("\n")
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.skip:
|
||||
self.parts.append(data)
|
||||
|
||||
collector = Collector()
|
||||
try:
|
||||
collector.feed(value)
|
||||
collector.close()
|
||||
except Exception:
|
||||
return ""
|
||||
text = "".join(collector.parts)
|
||||
text = text.replace("\r", "")
|
||||
text = re.sub(r"[ \t]+\n", "\n", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def sanitize_html(value: str, images: dict[str, str] | None = None) -> str:
|
||||
"""Keep a formatted reading. Scripts, active content and remote loads stay out."""
|
||||
images = images or {}
|
||||
|
||||
class Cleaner(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts: list[str] = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
||||
self.skip += 1
|
||||
return
|
||||
if self.skip or name not in _ALLOWED_TAGS:
|
||||
return
|
||||
rendered = _clean_attrs(name, attrs, images)
|
||||
if rendered is None:
|
||||
return
|
||||
self.parts.append(f"<{name}{rendered}>")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
||||
if self.skip:
|
||||
self.skip -= 1
|
||||
return
|
||||
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
|
||||
return
|
||||
self.parts.append(f"</{name}>")
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.skip:
|
||||
self.parts.append(html.escape(data))
|
||||
|
||||
def handle_entityref(self, name):
|
||||
if not self.skip:
|
||||
self.parts.append(f"&{name};")
|
||||
|
||||
def handle_charref(self, name):
|
||||
if not self.skip:
|
||||
self.parts.append(f"&#{name};")
|
||||
|
||||
cleaner = Cleaner()
|
||||
try:
|
||||
cleaner.feed(value)
|
||||
cleaner.close()
|
||||
except Exception:
|
||||
return ""
|
||||
return "".join(cleaner.parts).strip()
|
||||
|
||||
|
||||
def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None:
|
||||
kept: list[str] = []
|
||||
for key, value in attrs:
|
||||
name = (key or "").lower()
|
||||
if name.startswith("on") or name in {"style", "srcset"}:
|
||||
continue
|
||||
if tag == "a" and name == "href":
|
||||
target = (value or "").strip()
|
||||
if not _SAFE_LINK.match(target):
|
||||
continue
|
||||
kept.append(f' href="{html.escape(target, quote=True)}"')
|
||||
elif tag == "img" and name == "src":
|
||||
source = (value or "").strip()
|
||||
if source.lower().startswith("cid:"):
|
||||
token = source[4:].strip().strip("<>")
|
||||
source = images.get(token.casefold(), "")
|
||||
if not _DATA_IMAGE.match(source):
|
||||
continue
|
||||
kept.append(f' src="{source}"')
|
||||
elif tag == "img" and name == "alt":
|
||||
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
|
||||
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
|
||||
kept.append(f" {name}=\"{int(value)}\"")
|
||||
if tag == "img" and not any(item.startswith(" src=") for item in kept):
|
||||
return None
|
||||
if tag == "a":
|
||||
kept.append(' rel="noreferrer"')
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def _walk(message: EmailMessage):
|
||||
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
|
||||
yield "message", message
|
||||
return
|
||||
if message.is_multipart():
|
||||
for child in message.iter_parts():
|
||||
yield from _walk(child)
|
||||
return
|
||||
yield "leaf", message
|
||||
|
||||
|
||||
def _decoded_text(part: EmailMessage) -> str | None:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(payload, (bytes, bytearray)):
|
||||
raw = part.get_payload()
|
||||
return raw.strip() if isinstance(raw, str) else None
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
return decode_transfer(bytes(payload), charset).strip()
|
||||
except LookupError:
|
||||
return decode_transfer(bytes(payload), "utf-8").strip()
|
||||
|
||||
|
||||
def _leaf_bytes(part: EmailMessage) -> bytes | None:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(payload, (bytes, bytearray)):
|
||||
return bytes(payload)
|
||||
return None
|
||||
|
||||
|
||||
def _message_bytes(part: EmailMessage) -> bytes | None:
|
||||
payload = part.get_payload()
|
||||
if isinstance(payload, list) and payload:
|
||||
inner = payload[0]
|
||||
if isinstance(inner, EmailMessage):
|
||||
return inner.as_bytes()
|
||||
data = _leaf_bytes(part)
|
||||
return data or None
|
||||
|
||||
|
||||
def _content_id(part: EmailMessage) -> str:
|
||||
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
|
||||
return token
|
||||
|
||||
|
||||
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
|
||||
return AttachmentBytes(
|
||||
filename=filename,
|
||||
media_type=(part.get_content_type() or "application/octet-stream")[:120],
|
||||
data=data,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
display_name=label or filename,
|
||||
content_id=_content_id(part),
|
||||
inline=inline,
|
||||
)
|
||||
|
||||
|
||||
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
|
||||
import base64
|
||||
|
||||
images: dict[str, str] = {}
|
||||
for item in attachments:
|
||||
if not item.content_id or not item.media_type.startswith("image/"):
|
||||
continue
|
||||
if len(item.data) > 1_500_000:
|
||||
continue
|
||||
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
|
||||
if subtype == "jpg":
|
||||
subtype = "jpeg"
|
||||
if subtype not in {"png", "jpeg", "gif", "webp"}:
|
||||
continue
|
||||
encoded = base64.b64encode(item.data).decode("ascii")
|
||||
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
|
||||
return images
|
||||
|
||||
|
||||
def _message_date(value) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return parsedate_to_datetime(str(value)).date().isoformat()
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _bare(value: str) -> str:
|
||||
return value.strip().strip("<>").strip().casefold()
|
||||
|
||||
|
||||
def _header_text(value) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
text = str(value).replace("\r", " ").replace("\n", " ")
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _unique_filename(name: str, used: set[str]) -> str:
|
||||
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
|
||||
cleaned = cleaned[:180]
|
||||
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
|
||||
if cleaned not in used:
|
||||
used.add(cleaned)
|
||||
return cleaned
|
||||
stem, dot, ext = cleaned.rpartition(".")
|
||||
if not dot:
|
||||
if not dot or not stem:
|
||||
stem, ext = cleaned, ""
|
||||
else:
|
||||
ext = f".{ext}"
|
||||
@@ -140,36 +418,3 @@ def _unique_filename(name: str, used: set[str]) -> str:
|
||||
used.add(candidate)
|
||||
return candidate
|
||||
number += 1
|
||||
|
||||
|
||||
def _body_text(message) -> str:
|
||||
plain: list[str] = []
|
||||
html: list[str] = []
|
||||
parts = message.walk() if message.is_multipart() else [message]
|
||||
for part in parts:
|
||||
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
|
||||
continue
|
||||
kind = part.get_content_type()
|
||||
payload = part.get_payload(decode=True)
|
||||
if isinstance(payload, (bytes, bytearray)):
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
text = bytes(payload).decode(charset, errors="replace").strip()
|
||||
else:
|
||||
raw = part.get_payload()
|
||||
text = raw.strip() if isinstance(raw, str) else ""
|
||||
if kind == "text/plain":
|
||||
plain.append(text)
|
||||
elif kind == "text/html":
|
||||
html.append(_html_text(text))
|
||||
body = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
if not body:
|
||||
body = "\n".join(chunk for chunk in html if chunk).strip()
|
||||
return body
|
||||
|
||||
|
||||
def _html_text(value: str) -> str:
|
||||
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
|
||||
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
|
||||
text = re.sub(r"(?i)</p>", "\n", text)
|
||||
text = re.sub(r"<[^>]+>", " ", text)
|
||||
return re.sub(r"[ \t]{2,}", " ", text).strip()
|
||||
+5
-1
@@ -1028,14 +1028,18 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
|
||||
content_type = sniff(data)
|
||||
target, rights_context = _placement(context, placement)
|
||||
require_container_edit(conn, actor, target, rights_context)
|
||||
element = _new_element(conn, actor, scope_id, title, "file")
|
||||
asset_id = uuid4()
|
||||
content_revision = str(uuid4())
|
||||
instance = instance_for_write(conn, media_root)
|
||||
object_key = str(asset_id)
|
||||
adapter = adapter_for(instance)
|
||||
from app.sourceaccess import release_for_external
|
||||
|
||||
release_for_external(conn)
|
||||
publish(adapter, object_key, data)
|
||||
rollback_files.append(StoredHandle(adapter, object_key))
|
||||
require_container_edit(conn, actor, target, rights_context)
|
||||
element = _new_element(conn, actor, scope_id, title, "file")
|
||||
digest = sha256(data)
|
||||
conn.execute(
|
||||
"""
|
||||
|
||||
+326
-57
@@ -21,7 +21,17 @@ from psycopg.types.json import Json
|
||||
from app.access.policy import Ctx
|
||||
from app.core.errors import Problem, not_found, unprocessable
|
||||
from app.core.settings import get_settings
|
||||
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
|
||||
from app.mailpack import (
|
||||
MailIdentityError,
|
||||
choose_message,
|
||||
decode_transfer,
|
||||
header_message_id,
|
||||
header_text,
|
||||
identities_match,
|
||||
parse_mail,
|
||||
sanitize_html,
|
||||
simplified_text,
|
||||
)
|
||||
from app.media.storage import sniff
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourceaccess import (
|
||||
@@ -291,12 +301,18 @@ def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: in
|
||||
_check_mailbox(mailbox)
|
||||
if uid < 1:
|
||||
raise unprocessable("Die Nachricht ist ungültig.")
|
||||
client = _imap_login(row["settings"], row["secret"])
|
||||
settings = dict(row["settings"])
|
||||
secret = row["secret"]
|
||||
connector_id = row["id"]
|
||||
release_for_external(conn)
|
||||
client = _imap_login(settings, secret)
|
||||
try:
|
||||
_select_mailbox(client, mailbox)
|
||||
return _fetch_message(client, uid, mailbox)
|
||||
preview = _fetch_message(client, uid, mailbox)
|
||||
finally:
|
||||
_imap_logout(client)
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
return preview
|
||||
|
||||
|
||||
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
|
||||
@@ -402,7 +418,7 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
|
||||
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
|
||||
|
||||
def fetch(row):
|
||||
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
|
||||
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
|
||||
|
||||
return complete_borrow(conn, borrowed, fetch)
|
||||
|
||||
@@ -726,16 +742,10 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
match = next((item for item in folders if item["name"] == chosen), None)
|
||||
if match is None or not match["selectable"]:
|
||||
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
||||
try:
|
||||
status, _ = client.select(chosen, readonly=True)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
status, _data = _examine(client, chosen)
|
||||
if status != "OK":
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
||||
exists = _untagged_int(client, "EXISTS")
|
||||
validity = _untagged_int(client, "UIDVALIDITY")
|
||||
restarted = False
|
||||
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
|
||||
@@ -774,6 +784,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
next_before = low
|
||||
else:
|
||||
next_before = None
|
||||
query_text = " ".join(query.split())
|
||||
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
|
||||
return {
|
||||
"mailbox": chosen,
|
||||
"items": collected,
|
||||
@@ -781,6 +793,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
|
||||
"incomplete": incomplete,
|
||||
"restarted": restarted,
|
||||
"uidvalidity": validity,
|
||||
"exists": exists,
|
||||
"contradictory": contradictory,
|
||||
}
|
||||
|
||||
|
||||
@@ -789,15 +803,34 @@ def _check_mailbox(mailbox: str) -> None:
|
||||
raise unprocessable("Der Fachname ist ungültig.")
|
||||
|
||||
|
||||
def _mailbox_argument(mailbox: str) -> str:
|
||||
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
|
||||
_check_mailbox(mailbox)
|
||||
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
|
||||
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
|
||||
return protocol
|
||||
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def _examine(client: imaplib.IMAP4, mailbox: str):
|
||||
argument = _mailbox_argument(mailbox)
|
||||
try:
|
||||
return client.select(argument, readonly=True)
|
||||
except imaplib.IMAP4.abort as exc:
|
||||
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
|
||||
except TimeoutError as exc:
|
||||
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
|
||||
|
||||
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
|
||||
folders = _imap_mailboxes(client)
|
||||
match = next((item for item in folders if item["name"] == mailbox), None)
|
||||
if match is None or not match["selectable"]:
|
||||
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
|
||||
try:
|
||||
status, _ = client.select(mailbox, readonly=True)
|
||||
except imaplib.IMAP4.error as exc:
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
|
||||
status, _data = _examine(client, mailbox)
|
||||
if status != "OK":
|
||||
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
|
||||
|
||||
@@ -837,6 +870,10 @@ def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
PREVIEW_BYTE_LIMIT = 8_000_000
|
||||
PREVIEW_TEXT_LIMIT = 200_000
|
||||
|
||||
|
||||
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
||||
@@ -845,22 +882,36 @@ def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
raw = _literal(fetched)
|
||||
if status != "OK" or raw is None:
|
||||
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
||||
if len(raw) > 8_000_000:
|
||||
if len(raw) > PREVIEW_BYTE_LIMIT:
|
||||
return _oversized_message(client, uid, mailbox)
|
||||
message = BytesParser().parsebytes(raw)
|
||||
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
|
||||
parsed = parse_mail(raw)
|
||||
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
|
||||
|
||||
|
||||
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
|
||||
text = parsed.text[:PREVIEW_TEXT_LIMIT]
|
||||
return {
|
||||
"uid": uid,
|
||||
"mailbox": mailbox,
|
||||
"title": subject[:200],
|
||||
"message_from": _decode(message.get("From"))[:200],
|
||||
"message_to": _decode(message.get("To"))[:500],
|
||||
"message_cc": _decode(message.get("Cc"))[:500],
|
||||
"title": (parsed.subject or "(ohne Betreff)")[:200],
|
||||
"message_from": parsed.sender[:200],
|
||||
"message_to": parsed.recipient[:500],
|
||||
"message_cc": parsed.copy[:500],
|
||||
"direction": _direction(mailbox),
|
||||
"document_on": _mail_date(message.get("Date")),
|
||||
"attachments": _message_attachments(message),
|
||||
"text": (_text := _message_text(message))[:200_000],
|
||||
"text_truncated": len(_text) > 200_000,
|
||||
"document_on": parsed.document_on,
|
||||
"attachments": [
|
||||
{
|
||||
"filename": item.display_name or item.filename,
|
||||
"archive_name": item.filename,
|
||||
"content_type": item.media_type,
|
||||
}
|
||||
for item in parsed.attachments[:40]
|
||||
],
|
||||
"text": text,
|
||||
"html": parsed.html,
|
||||
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
|
||||
"preview_limited": limited,
|
||||
"notices": parsed.notices,
|
||||
}
|
||||
|
||||
|
||||
@@ -887,10 +938,37 @@ def _direction(mailbox: str) -> str | None:
|
||||
|
||||
|
||||
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
"""Read named text parts. A cut-off multipart blob is not a message."""
|
||||
header = _fetch_header(client, uid, mailbox) or {}
|
||||
text = _partial_text(client, uid)
|
||||
if not header and not text:
|
||||
structure = _body_structure(client, uid)
|
||||
plain: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
truncated = False
|
||||
notices = [
|
||||
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
|
||||
"Das Original bleibt die vollständige E-Mail."
|
||||
]
|
||||
if structure is None:
|
||||
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
|
||||
else:
|
||||
for part in _text_sections(structure):
|
||||
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
|
||||
if fetched is None:
|
||||
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
|
||||
continue
|
||||
truncated = truncated or shortened or part["octets"] > len(fetched)
|
||||
text = decode_transfer(fetched, part["charset"]).strip()
|
||||
if part["media"] == "text/plain":
|
||||
plain.append(text)
|
||||
else:
|
||||
html_parts.append(text)
|
||||
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
|
||||
if not simplified and html_parts:
|
||||
simplified = simplified_text("\n".join(html_parts))
|
||||
if not header and not simplified and not rendered and structure is None:
|
||||
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
|
||||
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
|
||||
return {
|
||||
"uid": uid,
|
||||
"mailbox": mailbox,
|
||||
@@ -900,31 +978,185 @@ def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
|
||||
"message_cc": header.get("message_cc") or "",
|
||||
"direction": header.get("direction", _direction(mailbox)),
|
||||
"document_on": header.get("document_on"),
|
||||
"attachments": header.get("attachments") or [],
|
||||
"text": text,
|
||||
"text_truncated": True,
|
||||
"attachments": attachments,
|
||||
"text": simplified[:PREVIEW_TEXT_LIMIT],
|
||||
"html": rendered,
|
||||
"text_truncated": True or truncated,
|
||||
"preview_limited": True,
|
||||
"notices": notices,
|
||||
}
|
||||
|
||||
|
||||
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
|
||||
def _body_structure(client: imaplib.IMAP4, uid: int):
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
|
||||
except imaplib.IMAP4.error:
|
||||
return ""
|
||||
return None
|
||||
if status != "OK":
|
||||
return ""
|
||||
return None
|
||||
blob = _literal(fetched)
|
||||
if blob is None:
|
||||
chunks: list[bytes] = []
|
||||
for part in fetched or []:
|
||||
if isinstance(part, tuple):
|
||||
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
|
||||
elif isinstance(part, (bytes, bytearray)):
|
||||
chunks.append(bytes(part))
|
||||
blob = b" ".join(chunks)
|
||||
text = blob.decode("utf-8", "replace")
|
||||
start = text.find("(")
|
||||
if start < 0:
|
||||
return None
|
||||
try:
|
||||
node, _end = _parse_imap_list(text, start)
|
||||
except ValueError:
|
||||
return None
|
||||
return node
|
||||
|
||||
|
||||
def _parse_imap_list(text: str, index: int):
|
||||
if index >= len(text) or text[index] != "(":
|
||||
raise ValueError(text[index:index + 20])
|
||||
index += 1
|
||||
items = []
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == ")":
|
||||
return items, index + 1
|
||||
if char in " \t\r\n":
|
||||
index += 1
|
||||
continue
|
||||
if char == "(":
|
||||
child, index = _parse_imap_list(text, index)
|
||||
items.append(child)
|
||||
continue
|
||||
if char == '"':
|
||||
token, index = _parse_imap_string(text, index)
|
||||
items.append(token)
|
||||
continue
|
||||
end = index
|
||||
while end < len(text) and text[end] not in " \t\r\n()":
|
||||
end += 1
|
||||
token = text[index:end]
|
||||
items.append(None if token.upper() == "NIL" else token)
|
||||
index = end
|
||||
raise ValueError("unclosed")
|
||||
|
||||
|
||||
def _parse_imap_string(text: str, index: int):
|
||||
index += 1
|
||||
chars: list[str] = []
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == "\\":
|
||||
if index + 1 < len(text):
|
||||
chars.append(text[index + 1])
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
return "".join(chars), index + 1
|
||||
chars.append(char)
|
||||
index += 1
|
||||
raise ValueError("string")
|
||||
|
||||
|
||||
def _text_sections(node, prefix: str = "") -> list[dict]:
|
||||
if not isinstance(node, list) or not node:
|
||||
return []
|
||||
if isinstance(node[0], list):
|
||||
children = []
|
||||
for item in node:
|
||||
if isinstance(item, list):
|
||||
children.append(item)
|
||||
else:
|
||||
break
|
||||
found = []
|
||||
for number, child in enumerate(children, start=1):
|
||||
section = f"{number}" if not prefix else f"{prefix}.{number}"
|
||||
found.extend(_text_sections(child, section))
|
||||
return found
|
||||
media = str(node[0] or "").upper()
|
||||
subtype = str(node[1] or "").upper()
|
||||
if media == "MESSAGE":
|
||||
return []
|
||||
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
|
||||
return []
|
||||
disposition, params = _disposition(node)
|
||||
if disposition == "ATTACHMENT":
|
||||
return []
|
||||
body_params = _param_map(node[2]) if len(node) > 2 else {}
|
||||
octets = _imap_int(node[6]) if len(node) > 6 else 0
|
||||
return [{
|
||||
"section": prefix or "1",
|
||||
"media": f"text/{subtype.lower()}",
|
||||
"charset": body_params.get("charset") or "utf-8",
|
||||
"octets": octets,
|
||||
}]
|
||||
|
||||
|
||||
def _disposition(node: list) -> tuple[str, dict]:
|
||||
for item in node:
|
||||
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
|
||||
params = _param_map(item[1]) if len(item) > 1 else {}
|
||||
return item[0].upper(), params
|
||||
return "", {}
|
||||
|
||||
|
||||
def _param_map(value) -> dict:
|
||||
if not isinstance(value, list):
|
||||
return {}
|
||||
pairs = {}
|
||||
index = 0
|
||||
while index + 1 < len(value):
|
||||
key = value[index]
|
||||
raw = value[index + 1]
|
||||
if isinstance(key, str):
|
||||
pairs[key.casefold()] = "" if raw is None else str(raw)
|
||||
index += 2
|
||||
return pairs
|
||||
|
||||
|
||||
def _imap_int(value) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
|
||||
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), spec)
|
||||
except imaplib.IMAP4.error:
|
||||
return None, False
|
||||
if status != "OK":
|
||||
return None, False
|
||||
raw = _literal(fetched)
|
||||
if not raw:
|
||||
return ""
|
||||
sample = raw[:200_000]
|
||||
try:
|
||||
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
|
||||
text = _message_text(message)
|
||||
except Exception:
|
||||
text = ""
|
||||
if not text:
|
||||
text = sample.decode("utf-8", errors="replace")
|
||||
return text.strip()[:200_000]
|
||||
return None, False
|
||||
return raw[:limit], len(raw) >= limit
|
||||
|
||||
|
||||
def _structure_names(node) -> list[dict]:
|
||||
names = []
|
||||
|
||||
def visit(item):
|
||||
if not isinstance(item, list) or not item:
|
||||
return
|
||||
if isinstance(item[0], list):
|
||||
for child in item:
|
||||
if isinstance(child, list):
|
||||
visit(child)
|
||||
return
|
||||
_disposition_name, params = _disposition(item)
|
||||
body = _param_map(item[2]) if len(item) > 2 else {}
|
||||
raw = params.get("filename") or body.get("name") or ""
|
||||
label = header_text(raw)
|
||||
if label:
|
||||
names.append({"filename": label[:200]})
|
||||
|
||||
visit(node)
|
||||
return names[:40]
|
||||
|
||||
|
||||
def _literal(fetched) -> bytes | None:
|
||||
@@ -944,12 +1176,10 @@ def _structure_attachments(fetched) -> list[dict]:
|
||||
blob = b" ".join(chunks)
|
||||
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
|
||||
items = []
|
||||
seen: set[str] = set()
|
||||
for raw in names:
|
||||
name = raw.decode("utf-8", errors="replace").strip()[:200]
|
||||
if not name or name in seen:
|
||||
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
|
||||
if not name:
|
||||
continue
|
||||
seen.add(name)
|
||||
items.append({"filename": name})
|
||||
if len(items) == 40:
|
||||
break
|
||||
@@ -1039,6 +1269,32 @@ def _unquote(token: str) -> str:
|
||||
return token
|
||||
|
||||
|
||||
def _encode_utf7(value: str) -> str:
|
||||
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
|
||||
pieces: list[str] = []
|
||||
pending: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
if not pending:
|
||||
return
|
||||
raw = "".join(pending).encode("utf-16-be")
|
||||
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
|
||||
pieces.append(f"&{token}-")
|
||||
pending.clear()
|
||||
|
||||
for char in value:
|
||||
if char == "&":
|
||||
flush()
|
||||
pieces.append("&-")
|
||||
elif ord(char) < 128:
|
||||
flush()
|
||||
pieces.append(char)
|
||||
else:
|
||||
pending.append(char)
|
||||
flush()
|
||||
return "".join(pieces)
|
||||
|
||||
|
||||
def _decode_utf7(value: str) -> str:
|
||||
if "&" not in value:
|
||||
return value
|
||||
@@ -1255,13 +1511,17 @@ def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assi
|
||||
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
|
||||
if resolved.row is None:
|
||||
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
|
||||
base = resolved.row["settings"]["base_url"].rstrip("/")
|
||||
token = resolved.row["secret"]
|
||||
connector_id = resolved.row["id"]
|
||||
owner_id = resolved.row["owner_id"]
|
||||
release_for_external(conn)
|
||||
try:
|
||||
data, media, name = pull_paperless_original(
|
||||
resolved.row["settings"]["base_url"].rstrip("/"),
|
||||
resolved.row["secret"],
|
||||
str(external_id),
|
||||
EXPORT_BYTE_LIMIT,
|
||||
)
|
||||
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
try:
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(data) > EXPORT_BYTE_LIMIT:
|
||||
@@ -1282,8 +1542,17 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
|
||||
uid = int(uid)
|
||||
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
|
||||
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
|
||||
settings = dict(resolved.row["settings"])
|
||||
secret = resolved.row["secret"]
|
||||
connector_id = resolved.row["id"]
|
||||
owner_id = resolved.row["owner_id"]
|
||||
release_for_external(conn)
|
||||
try:
|
||||
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
try:
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
if len(raw) > EXPORT_BYTE_LIMIT:
|
||||
|
||||
Reference in new issue
Block a user