Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Deploy Development / deploy (push) Successful in 39s
Test Suite / pytest-backend (push) Successful in 2m29s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 20s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-20 08:47:24 +02:00
1 parent 34c4084a7c
commit c29bd29f34
29 files changed
+2562 -410

No files matched your search

+7 -3
View File
@@ -124,14 +124,18 @@ def view_mail(
"message_to": parsed.recipient,
"message_cc": parsed.copy,
"text": parsed.text,
"text_truncated": False,
"complete": True,
"html": parsed.html,
"text_truncated": parsed.text_truncated,
"complete": not parsed.missing,
"notices": parsed.notices,
"attachments": [
{
"index": index,
"filename": item.filename,
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
"byte_size": len(item.data),
"inline": item.inline,
}
for index, item in enumerate(parsed.attachments)
],
+222 -49
View File
@@ -211,11 +211,36 @@ def record_dispatch(
raise unprocessable("Dieselbe Anlage ist mehrfach im Versand genannt.")
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
host = _bound_element(conn, actor, host_id, page, scope)
bump_version(conn, "elements", host_id, expected)
known = {row["attachment_element_id"] for row in _uses(conn, scope, host_id)}
if any(item not in known for item in attachment_ids):
raise unprocessable("Mitversendet werden nur Anlagen dieses Schreibens.")
counterpart_text = _counterpart(counterpart)
edition_plans = []
if capture_editions:
edition_plans.append(_plan_edition(conn, actor, host, "host", 0, page, historical_gap, media_root, cleanup))
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
position_of = {item: index for index, item in enumerate(attachment_ids)}
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
for index, row in enumerate(ordered, start=1):
element = _readable(conn, actor, row["attachment_element_id"], page)
if element is None:
raise not_found()
edition_plans.append(
_plan_edition(conn, actor, element, "attachment", index, page, historical_gap, media_root, cleanup)
)
evidence_plans = []
for item in evidence:
element = _readable(conn, actor, item["element_id"], page)
if element is None:
raise not_found()
evidence_plans.append(
_plan_evidence(
conn, actor, element, item["role"], item.get("fold") or "keep", page, media_root, cleanup, container_row["home_scope_id"]
)
)
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
host = _bound_element(conn, actor, host_id, page, scope)
bump_version(conn, "elements", host_id, expected)
dispatch = conn.execute(
"""
INSERT INTO dispatches (
@@ -237,37 +262,11 @@ def record_dispatch(
actor,
),
).fetchone()
states = []
if capture_editions:
states.append(_capture_item(conn, actor, dispatch, host, "host", 0, page, historical_gap, media_root, cleanup))
ordered = [row for row in _uses(conn, scope, host_id) if row["attachment_element_id"] in set(attachment_ids)]
position_of = {item: index for index, item in enumerate(attachment_ids)}
ordered.sort(key=lambda row: position_of[row["attachment_element_id"]])
for index, row in enumerate(ordered, start=1):
element = _readable(conn, actor, row["attachment_element_id"], page)
if element is None:
raise not_found()
states.append(
_capture_item(conn, actor, dispatch, element, "attachment", index, page, historical_gap, media_root, cleanup)
)
evidence_rows = []
for item in evidence:
evidence_rows.append(
_add_evidence(
conn,
actor,
container_row,
page,
scope,
dispatch,
item["element_id"],
item["role"],
item.get("fold") or "keep",
media_root,
cleanup,
log=False,
)
)
states = [_write_edition(conn, dispatch, plan) for plan in edition_plans]
evidence_rows = [
_write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=False)
for plan in evidence_plans
]
completeness = _completeness(states) if capture_editions else "secured"
conn.execute(
"UPDATE dispatches SET completeness = %s, updated_at = now() WHERE id = %s",
@@ -292,11 +291,16 @@ def add_dispatch_evidence(
) -> dict:
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
dispatch = _dispatch_in(conn, dispatch_id, scope)
_bump_dispatch(conn, dispatch_id, expected)
row = _add_evidence(
conn, actor, container_row, page, scope, dispatch, element_id, role, fold, media_root, cleanup, log=True
element = _readable(conn, actor, element_id, page)
if element is None:
raise not_found()
plan = _plan_evidence(
conn, actor, element, role, fold, page, media_root, cleanup, container_row["home_scope_id"], dispatch_id
)
return row
container_row, page, scope = open_container(conn, actor, context, placement, "edit")
dispatch = _dispatch_in(conn, dispatch_id, scope)
_bump_dispatch(conn, dispatch_id, expected)
return _write_evidence(conn, actor, container_row, page, scope, dispatch, plan, log=True)
def detach_dispatch_evidence(
@@ -760,7 +764,7 @@ def _identify_evidence(conn, actor, dispatch, element, page, media_root, cleanup
if isinstance(loaded, RetrievalFailure):
return False, loaded.message, None, None, [], None
if isinstance(loaded, MailPayload):
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded)
comparisons = _compare_mail(conn, actor, dispatch["id"], page, loaded) if dispatch is not None else []
payload = FilePayload("nachweis.eml", "message/rfc822", loaded.raw, sha256(loaded.raw))
return True, None, loaded.parsed.message_id or None, payload.sha256, comparisons, payload
payload = FilePayload(loaded.filename, loaded.media_type, loaded.data, sha256(loaded.data))
@@ -931,15 +935,179 @@ def _working_bytes(conn, actor, element, media_root: str):
return None, "Für diesen Inhalt ist keine sicherbare Fassung vorgesehen.", None, None, None
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
snapshot_id = uuid4()
def _stage_snapshot(conn, media_root: str, data: bytes, cleanup: list) -> dict:
"""Publish bytes only while the domain transaction is idle. The row comes later."""
from app.sourceaccess import release_for_external
instance = instance_for_write(conn, media_root)
snapshot_id = uuid4()
object_key = f"dispatch/{snapshot_id}"
content_revision = str(uuid4())
adapter = adapter_for(instance)
publish(adapter, object_key, payload.data)
release_for_external(conn)
publish(adapter, object_key, data)
cleanup.append(StoredHandle(adapter, object_key))
digest = sha256(payload.data)
return {
"id": snapshot_id,
"instance": instance,
"object_key": object_key,
"content_revision": str(uuid4()),
}
def _plan_edition(conn, actor, element, role, position, page, historical_gap, media_root, cleanup) -> dict:
dates = _dates(conn, element["id"])
payload = None
gap_reason = None
note_document = None
external_source = None
external_id = None
if historical_gap:
state = "gap"
gap_reason = _HISTORICAL_GAP
else:
payload, gap_reason, note_document, external_source, external_id = _working_bytes(
conn, actor, element, media_root
)
state = "snapshot" if payload is not None else "gap"
staged = _stage_snapshot(conn, media_root, payload.data, cleanup) if payload is not None else None
return {
"element": element,
"role": role,
"position": position,
"dates": dates,
"state": state,
"payload": payload,
"gap_reason": gap_reason,
"note_document": note_document,
"external_source": external_source,
"external_id": external_id,
"staged": staged,
}
def _write_edition(conn, dispatch, plan: dict) -> str:
payload = plan["payload"]
filename = payload.filename if payload is not None else None
content_type = payload.media_type if payload is not None else None
digest = sha256(payload.data) if payload is not None else None
byte_size = len(payload.data) if payload is not None else None
item = conn.execute(
"""
INSERT INTO dispatch_items (
dispatch_id, role, position, element_id, title, document_on, filename,
content_type, byte_size, content_sha256, source_state, gap_reason,
note_document, external_source, external_id
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
dispatch["id"],
plan["role"],
plan["position"],
plan["element"]["id"],
plan["element"]["title"],
plan["dates"]["document_on"],
filename,
content_type,
byte_size,
digest,
plan["state"],
plan["gap_reason"],
Json(plan["note_document"]) if plan["note_document"] is not None else None,
plan["external_source"],
plan["external_id"],
),
).fetchone()
if plan["staged"] is not None and payload is not None:
_record_snapshot(conn, dispatch["home_scope_id"], dispatch["recorded_by"], payload, plan["staged"], dispatch_item_id=item["id"])
return plan["state"]
def _plan_evidence(conn, actor, element, role, fold, page, media_root, cleanup, scope_id, dispatch_id=None) -> dict:
if role not in EVIDENCE_ROLES:
raise unprocessable("Die Rolle des Nachweises ist nicht vorgesehen.")
if fold not in {"keep", "evidence"}:
raise unprocessable("Die Darstellung des Nachweises ist nicht festgelegt.")
if dispatch_id is not None and conn.execute(
"SELECT id FROM dispatch_evidence WHERE dispatch_id = %s AND element_id = %s",
(dispatch_id, element["id"]),
).fetchone():
raise unprocessable("Dieser Nachweis ist dem Versand bereits zugeordnet.")
dispatch = {"id": dispatch_id} if dispatch_id is not None else None
identified, gap_reason, message_id, digest, comparisons, snapshot = _identify_evidence(
conn, actor, dispatch, element, page, media_root, cleanup, scope_id
)
staged = _stage_snapshot(conn, media_root, snapshot.data, cleanup) if snapshot is not None else None
return {
"element": element,
"role": role,
"fold": fold,
"identified": identified,
"gap_reason": gap_reason,
"message_id": message_id,
"digest": digest,
"comparisons": comparisons,
"snapshot": snapshot,
"staged": staged,
}
def _write_evidence(conn, actor, container_row, page, scope, dispatch, plan: dict, log: bool) -> dict:
snapshot = plan["snapshot"]
comparisons = plan["comparisons"]
if snapshot is not None and snapshot.media_type == "message/rfc822":
from app.mailpack import parse_mail
comparisons = _compare_mail(conn, actor, dispatch["id"], page, MailPayload(snapshot.data, parse_mail(snapshot.data)))
proof = conn.execute(
"""
INSERT INTO dispatch_evidence (
dispatch_id, element_id, role, source_state, gap_reason, message_id,
identity_sha256, comparisons, manual, created_by
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, true, %s)
RETURNING *
""",
(
dispatch["id"],
plan["element"]["id"],
plan["role"],
"identified" if plan["identified"] else "gap",
plan["gap_reason"],
plan["message_id"],
plan["digest"],
Json(comparisons),
actor,
),
).fetchone()
if plan["staged"] is not None and snapshot is not None:
_record_snapshot(
conn,
container_row["home_scope_id"],
actor,
snapshot,
plan["staged"],
evidence_id=proof["id"],
)
if plan["fold"] == "evidence":
_set_binding_fold(conn, scope, plan["element"]["id"], "evidence")
if log:
_log(conn, dispatch["id"], actor, f"Nachweis ergänzt: {EVIDENCE_ROLES[plan['role']]}")
audit(conn, actor, container_row["home_scope_id"], "dispatch_evidence", "element", plan["element"]["id"], page)
return {
"id": str(proof["id"]),
"element_id": str(plan["element"]["id"]),
"role": plan["role"],
"role_label": EVIDENCE_ROLES[plan["role"]],
"source_state": proof["source_state"],
"gap_reason": plan["gap_reason"],
"manual": True,
"comparisons": comparisons,
}
def _record_snapshot(conn, scope_id, actor, payload: FilePayload, staged: dict, dispatch_item_id=None, evidence_id=None) -> UUID:
conn.execute(
"""
INSERT INTO dispatch_snapshots (
@@ -951,7 +1119,7 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'none')
""",
(
snapshot_id,
staged["id"],
dispatch_item_id,
evidence_id,
scope_id,
@@ -959,15 +1127,20 @@ def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str
display_filename(payload.filename),
payload.media_type or "application/octet-stream",
len(payload.data),
digest,
instance["id"],
instance["generation"],
object_key,
content_revision,
sha256(payload.data),
staged["instance"]["id"],
staged["instance"]["generation"],
staged["object_key"],
staged["content_revision"],
LEGACY_PROFILE,
),
)
return snapshot_id
return staged["id"]
def _store_snapshot(conn, scope_id, actor, payload: FilePayload, media_root: str, cleanup: list[Path], dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> UUID:
staged = _stage_snapshot(conn, media_root, payload.data, cleanup)
return _record_snapshot(conn, scope_id, actor, payload, staged, dispatch_item_id, evidence_id)
def _snapshot_file(conn, media_root: str, dispatch_item_id: UUID | None = None, evidence_id: UUID | None = None) -> FilePayload:
+32 -2
View File
@@ -695,9 +695,18 @@ def _emit_one(node, prefix, add, entries, loaded, notice_for, generated: str) ->
links = [("EML-Original", eml)]
try:
parsed = payload.parsed if isinstance(payload, MailPayload) else parse_mail(raw)
reading = _mail_reading_html(parsed)
if reading is not None:
reading_path = add(prefix + ["lesefassung.html"], reading)
links.append(("Formatierte Lesefassung", reading_path))
else:
notice = "\n".join(part for part in [notice, "Eine formatierte Lesefassung liegt für diese Nachricht nicht vor. Die vereinfachte Textfassung steht in der Übersicht."] if part)
for attachment in parsed.attachments:
label = attachment.display_name or attachment.filename
path = add(prefix + ["anlagen", attachment.filename], attachment.data)
links.append((attachment.filename, path))
links.append((label, path))
if parsed.notices:
notice = "\n".join(part for part in [notice, *parsed.notices] if part)
except Exception:
links.append(("Hinweis", eml))
entries.append(_entry(node, eml, links, notice or None))
@@ -1223,6 +1232,21 @@ def _note_markdown(node: ExportNode, body: str, generated: str) -> str:
return "\n".join(lines)
def _mail_reading_html(parsed) -> bytes | None:
if not getattr(parsed, "html", ""):
return None
page = (
"<!DOCTYPE html><html lang=\"de\"><head><meta charset=\"utf-8\">"
"<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'none'; img-src data:; style-src 'none'; script-src 'none'\">"
"<title>Lesefassung</title></head><body>"
f"{parsed.html}"
"<p>Vereinfachte Textfassung und das unveränderte EML-Original stehen daneben. "
"Die Ansicht ist nicht pixelgleich zur Postfachdarstellung.</p>"
"</body></html>"
)
return page.encode("utf-8")
def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attachments, notice: str, generated: str) -> str:
parsed = mail.parsed
lines = [f"# {node.title}", "", "Lesefassung. Das unveränderte Original ist die EML-Datei.", ""]
@@ -1234,7 +1258,13 @@ def _mail_markdown(node, mail: MailPayload, text_path: str, eml_path: str, attac
if parsed.copy:
lines.append(f"- Kopie: {parsed.copy}")
lines.append(f"- [EML-Original]({_href(_relative(text_path, eml_path))})")
lines.extend(["", parsed.text or "Der Text steht im EML-Original.", ""])
lines.extend([
"",
"Vereinfachte Textfassung. Sie ist nicht die formatierte Ansicht und nicht das EML-Original.",
"",
parsed.text or "Der Text steht im EML-Original.",
"",
])
if attachments:
lines.append("## Anlagen")
lines.append("")
+311 -66
View File
@@ -1,16 +1,32 @@
"""Pure mail identity and attachment extraction for a complete export.
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
The same tree feeds preview, reading, attachment download and export.
"""
from __future__ import annotations
import hashlib
import html
import re
from dataclasses import dataclass, field
from email import policy
from email.header import decode_header, make_header
from email.message import EmailMessage
from email.parser import BytesParser
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
_UID_PREFIX = "uid:"
_POLICY = policy.default
_ALLOWED_TAGS = {
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
}
_VOID_TAGS = {"br", "img"}
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
class MailIdentityError(Exception):
@@ -25,6 +41,9 @@ class AttachmentBytes:
media_type: str
data: bytes
sha256: str
display_name: str = ""
content_id: str = ""
inline: bool = False
@dataclass
@@ -37,10 +56,14 @@ class ParsedMail:
text: str
attachments: list[AttachmentBytes] = field(default_factory=list)
missing: list[str] = field(default_factory=list)
html: str = ""
notices: list[str] = field(default_factory=list)
text_truncated: bool = False
document_on: str | None = None
def header_message_id(raw: bytes) -> str:
message = BytesParser().parsebytes(raw)
message = BytesParser(policy=_POLICY).parsebytes(raw)
return str(message.get("Message-ID") or "").strip()
@@ -69,67 +92,322 @@ def choose_message(candidate: bytes | None, alternates: list[bytes], expected: s
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
def header_text(value) -> str:
"""Decode encoded-words. A raw MIME header is not a display name."""
if value is None:
return ""
raw = str(value).replace("\r", " ").replace("\n", " ")
try:
text = str(make_header(decode_header(raw)))
except (UnicodeError, LookupError, ValueError):
text = raw
return re.sub(r"\s+", " ", text).strip()
def parse_mail(raw: bytes) -> ParsedMail:
message = BytesParser().parsebytes(raw)
message = BytesParser(policy=_POLICY).parsebytes(raw)
attachments: list[AttachmentBytes] = []
missing: list[str] = []
notices: list[str] = []
used: set[str] = set()
plain: list[str] = []
html_parts: list[str] = []
index = 0
for part in message.walk():
if part.is_multipart():
for kind, part in _walk(message):
if kind == "message":
index += 1
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
data = _message_bytes(part)
if data is None:
missing.append(label)
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label, used)
attachments.append(_attachment(part, name, label, data, inline=False))
continue
filename = part.get_filename()
disposition = (part.get_content_disposition() or "").lower()
if not filename and disposition != "attachment":
media = part.get_content_type()
filename = part.get_filename()
content_id = _content_id(part)
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
if body:
text = _decoded_text(part)
if text is None:
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
continue
if media == "text/plain":
plain.append(text)
else:
html_parts.append(text)
continue
if not filename and disposition != "attachment" and not content_id:
continue
index += 1
label = _header_text(filename) if filename else f"anlage-{index}.bin"
payload = part.get_payload(decode=True)
if not isinstance(payload, (bytes, bytearray)):
label = header_text(filename) if filename else f"anlage-{index}.bin"
data = _leaf_bytes(part)
if data is None:
missing.append(label or f"Anlage {index}")
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label or f"anlage-{index}.bin", used)
data = bytes(payload)
attachments.append(
AttachmentBytes(
filename=name,
media_type=(part.get_content_type() or "application/octet-stream")[:120],
data=data,
sha256=hashlib.sha256(data).hexdigest(),
)
)
text = _body_text(message)
inline = disposition == "inline" or bool(content_id)
attachments.append(_attachment(part, name, label or name, data, inline=inline))
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = ""
if html_parts:
rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments))
if not simplified:
simplified = simplified_text("\n".join(html_parts))
if missing:
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
return ParsedMail(
message_id=str(message.get("Message-ID") or "").strip(),
subject=_header_text(message.get("Subject")) or "(ohne Betreff)",
sender=_header_text(message.get("From")),
recipient=_header_text(message.get("To")),
copy=_header_text(message.get("Cc")),
text=text,
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
sender=header_text(message.get("From")),
recipient=header_text(message.get("To")),
copy=header_text(message.get("Cc")),
text=simplified,
attachments=attachments,
missing=missing,
html=rendered,
notices=notices,
document_on=_message_date(message.get("Date")),
)
def decode_transfer(payload: bytes, charset: str | None) -> str:
return bytes(payload).decode(charset or "utf-8", errors="replace")
def simplified_text(value: str) -> str:
"""A recognizable plain reading. It is not the formatted view."""
class Collector(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style"}:
self.skip += 1
return
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
self.parts.append("\n")
elif name in {"td", "th"}:
self.parts.append("\t")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style"} and self.skip:
self.skip -= 1
return
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
self.parts.append("\n")
def handle_data(self, data):
if not self.skip:
self.parts.append(data)
collector = Collector()
try:
collector.feed(value)
collector.close()
except Exception:
return ""
text = "".join(collector.parts)
text = text.replace("\r", "")
text = re.sub(r"[ \t]+\n", "\n", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def sanitize_html(value: str, images: dict[str, str] | None = None) -> str:
"""Keep a formatted reading. Scripts, active content and remote loads stay out."""
images = images or {}
class Cleaner(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
self.skip += 1
return
if self.skip or name not in _ALLOWED_TAGS:
return
rendered = _clean_attrs(name, attrs, images)
if rendered is None:
return
self.parts.append(f"<{name}{rendered}>")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
if self.skip:
self.skip -= 1
return
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
return
self.parts.append(f"</{name}>")
def handle_data(self, data):
if not self.skip:
self.parts.append(html.escape(data))
def handle_entityref(self, name):
if not self.skip:
self.parts.append(f"&{name};")
def handle_charref(self, name):
if not self.skip:
self.parts.append(f"&#{name};")
cleaner = Cleaner()
try:
cleaner.feed(value)
cleaner.close()
except Exception:
return ""
return "".join(cleaner.parts).strip()
def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None:
kept: list[str] = []
for key, value in attrs:
name = (key or "").lower()
if name.startswith("on") or name in {"style", "srcset"}:
continue
if tag == "a" and name == "href":
target = (value or "").strip()
if not _SAFE_LINK.match(target):
continue
kept.append(f' href="{html.escape(target, quote=True)}"')
elif tag == "img" and name == "src":
source = (value or "").strip()
if source.lower().startswith("cid:"):
token = source[4:].strip().strip("<>")
source = images.get(token.casefold(), "")
if not _DATA_IMAGE.match(source):
continue
kept.append(f' src="{source}"')
elif tag == "img" and name == "alt":
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
kept.append(f" {name}=\"{int(value)}\"")
if tag == "img" and not any(item.startswith(" src=") for item in kept):
return None
if tag == "a":
kept.append(' rel="noreferrer"')
return "".join(kept)
def _walk(message: EmailMessage):
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
yield "message", message
return
if message.is_multipart():
for child in message.iter_parts():
yield from _walk(child)
return
yield "leaf", message
def _decoded_text(part: EmailMessage) -> str | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if not isinstance(payload, (bytes, bytearray)):
raw = part.get_payload()
return raw.strip() if isinstance(raw, str) else None
charset = part.get_content_charset() or "utf-8"
try:
return decode_transfer(bytes(payload), charset).strip()
except LookupError:
return decode_transfer(bytes(payload), "utf-8").strip()
def _leaf_bytes(part: EmailMessage) -> bytes | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if isinstance(payload, (bytes, bytearray)):
return bytes(payload)
return None
def _message_bytes(part: EmailMessage) -> bytes | None:
payload = part.get_payload()
if isinstance(payload, list) and payload:
inner = payload[0]
if isinstance(inner, EmailMessage):
return inner.as_bytes()
data = _leaf_bytes(part)
return data or None
def _content_id(part: EmailMessage) -> str:
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
return token
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
return AttachmentBytes(
filename=filename,
media_type=(part.get_content_type() or "application/octet-stream")[:120],
data=data,
sha256=hashlib.sha256(data).hexdigest(),
display_name=label or filename,
content_id=_content_id(part),
inline=inline,
)
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
import base64
images: dict[str, str] = {}
for item in attachments:
if not item.content_id or not item.media_type.startswith("image/"):
continue
if len(item.data) > 1_500_000:
continue
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
if subtype == "jpg":
subtype = "jpeg"
if subtype not in {"png", "jpeg", "gif", "webp"}:
continue
encoded = base64.b64encode(item.data).decode("ascii")
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
return images
def _message_date(value) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(str(value)).date().isoformat()
except (TypeError, ValueError, OverflowError):
return None
def _bare(value: str) -> str:
return value.strip().strip("<>").strip().casefold()
def _header_text(value) -> str:
if value is None:
return ""
text = str(value).replace("\r", " ").replace("\n", " ")
return re.sub(r"\s+", " ", text).strip()
def _unique_filename(name: str, used: set[str]) -> str:
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
cleaned = cleaned[:180]
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
if cleaned not in used:
used.add(cleaned)
return cleaned
stem, dot, ext = cleaned.rpartition(".")
if not dot:
if not dot or not stem:
stem, ext = cleaned, ""
else:
ext = f".{ext}"
@@ -140,36 +418,3 @@ def _unique_filename(name: str, used: set[str]) -> str:
used.add(candidate)
return candidate
number += 1
def _body_text(message) -> str:
plain: list[str] = []
html: list[str] = []
parts = message.walk() if message.is_multipart() else [message]
for part in parts:
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
continue
kind = part.get_content_type()
payload = part.get_payload(decode=True)
if isinstance(payload, (bytes, bytearray)):
charset = part.get_content_charset() or "utf-8"
text = bytes(payload).decode(charset, errors="replace").strip()
else:
raw = part.get_payload()
text = raw.strip() if isinstance(raw, str) else ""
if kind == "text/plain":
plain.append(text)
elif kind == "text/html":
html.append(_html_text(text))
body = "\n".join(chunk for chunk in plain if chunk).strip()
if not body:
body = "\n".join(chunk for chunk in html if chunk).strip()
return body
def _html_text(value: str) -> str:
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
text = re.sub(r"(?i)</p>", "\n", text)
text = re.sub(r"<[^>]+>", " ", text)
return re.sub(r"[ \t]{2,}", " ", text).strip()
+5 -1
View File
@@ -1028,14 +1028,18 @@ def create_file(conn, actor: UUID, scope_id: UUID, title: str, filename: str, da
content_type = sniff(data)
target, rights_context = _placement(context, placement)
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "file")
asset_id = uuid4()
content_revision = str(uuid4())
instance = instance_for_write(conn, media_root)
object_key = str(asset_id)
adapter = adapter_for(instance)
from app.sourceaccess import release_for_external
release_for_external(conn)
publish(adapter, object_key, data)
rollback_files.append(StoredHandle(adapter, object_key))
require_container_edit(conn, actor, target, rights_context)
element = _new_element(conn, actor, scope_id, title, "file")
digest = sha256(data)
conn.execute(
"""
+326 -57
View File
@@ -21,7 +21,17 @@ from psycopg.types.json import Json
from app.access.policy import Ctx
from app.core.errors import Problem, not_found, unprocessable
from app.core.settings import get_settings
from app.mailpack import MailIdentityError, choose_message, identities_match, header_message_id
from app.mailpack import (
MailIdentityError,
choose_message,
decode_transfer,
header_message_id,
header_text,
identities_match,
parse_mail,
sanitize_html,
simplified_text,
)
from app.media.storage import sniff
from app.sourcebind import resolve_for_read
from app.sourceaccess import (
@@ -291,12 +301,18 @@ def imap_preview(conn, owner_id: UUID, connector_id: UUID, mailbox: str, uid: in
_check_mailbox(mailbox)
if uid < 1:
raise unprocessable("Die Nachricht ist ungültig.")
client = _imap_login(row["settings"], row["secret"])
settings = dict(row["settings"])
secret = row["secret"]
connector_id = row["id"]
release_for_external(conn)
client = _imap_login(settings, secret)
try:
_select_mailbox(client, mailbox)
return _fetch_message(client, uid, mailbox)
preview = _fetch_message(client, uid, mailbox)
finally:
_imap_logout(client)
assert_still_owned(conn, connector_id, owner_id)
return preview
def _owned(conn, owner_id: UUID, connector_id: UUID) -> dict:
@@ -402,7 +418,7 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
borrowed = borrow_original(conn, actor_id, element, context, ref, "imap")
def fetch(row):
return load_imap_original(conn, borrowed.owner_id, row["id"], mailbox, uid, external_id)
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
return complete_borrow(conn, borrowed, fetch)
@@ -726,16 +742,10 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
match = next((item for item in folders if item["name"] == chosen), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(chosen, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, chosen)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
exists = _untagged_int(client, "EXISTS")
validity = _untagged_int(client, "UIDVALIDITY")
restarted = False
if claimed_validity is not None and validity is not None and int(claimed_validity) != validity:
@@ -774,6 +784,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
next_before = low
else:
next_before = None
query_text = " ".join(query.split())
contradictory = bool(exists and exists > 0 and not collected and not query_text and not incomplete)
return {
"mailbox": chosen,
"items": collected,
@@ -781,6 +793,8 @@ def _imap_page(client: imaplib.IMAP4, mailbox: str, query: str, before: int | No
"incomplete": incomplete,
"restarted": restarted,
"uidvalidity": validity,
"exists": exists,
"contradictory": contradictory,
}
@@ -789,15 +803,34 @@ def _check_mailbox(mailbox: str) -> None:
raise unprocessable("Der Fachname ist ungültig.")
def _mailbox_argument(mailbox: str) -> str:
"""IMAP atoms stay bare. Everything else is a quoted protocol string."""
_check_mailbox(mailbox)
protocol = _encode_utf7(mailbox) if any(ord(char) > 127 for char in mailbox) else mailbox
if re.fullmatch(r"[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+", protocol):
return protocol
escaped = protocol.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _examine(client: imaplib.IMAP4, mailbox: str):
argument = _mailbox_argument(mailbox)
try:
return client.select(argument, readonly=True)
except imaplib.IMAP4.abort as exc:
raise unprocessable("Die Verbindung zum Postfach ist abgebrochen.") from exc
except TimeoutError as exc:
raise unprocessable("Das Postfach hat nicht rechtzeitig geantwortet.") from exc
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
def _select_mailbox(client: imaplib.IMAP4, mailbox: str) -> None:
folders = _imap_mailboxes(client)
match = next((item for item in folders if item["name"] == mailbox), None)
if match is None or not match["selectable"]:
raise unprocessable("Dieses Fach gibt es in dem Konto nicht.")
try:
status, _ = client.select(mailbox, readonly=True)
except imaplib.IMAP4.error as exc:
raise unprocessable("Dieses Fach lässt sich nicht öffnen.") from exc
status, _data = _examine(client, mailbox)
if status != "OK":
raise unprocessable("Dieses Fach lässt sich nicht öffnen.")
@@ -837,6 +870,10 @@ def _fetch_header(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict | None:
}
PREVIEW_BYTE_LIMIT = 8_000_000
PREVIEW_TEXT_LIMIT = 200_000
def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
@@ -845,22 +882,36 @@ def _fetch_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
raw = _literal(fetched)
if status != "OK" or raw is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
if len(raw) > 8_000_000:
if len(raw) > PREVIEW_BYTE_LIMIT:
return _oversized_message(client, uid, mailbox)
message = BytesParser().parsebytes(raw)
subject = _decode(message.get("Subject")) or "(ohne Betreff)"
parsed = parse_mail(raw)
return _preview_from_mail(uid, mailbox, parsed, limited=parsed.text_truncated)
def _preview_from_mail(uid: int, mailbox: str, parsed, limited: bool) -> dict:
text = parsed.text[:PREVIEW_TEXT_LIMIT]
return {
"uid": uid,
"mailbox": mailbox,
"title": subject[:200],
"message_from": _decode(message.get("From"))[:200],
"message_to": _decode(message.get("To"))[:500],
"message_cc": _decode(message.get("Cc"))[:500],
"title": (parsed.subject or "(ohne Betreff)")[:200],
"message_from": parsed.sender[:200],
"message_to": parsed.recipient[:500],
"message_cc": parsed.copy[:500],
"direction": _direction(mailbox),
"document_on": _mail_date(message.get("Date")),
"attachments": _message_attachments(message),
"text": (_text := _message_text(message))[:200_000],
"text_truncated": len(_text) > 200_000,
"document_on": parsed.document_on,
"attachments": [
{
"filename": item.display_name or item.filename,
"archive_name": item.filename,
"content_type": item.media_type,
}
for item in parsed.attachments[:40]
],
"text": text,
"html": parsed.html,
"text_truncated": limited or len(parsed.text) > PREVIEW_TEXT_LIMIT,
"preview_limited": limited,
"notices": parsed.notices,
}
@@ -887,10 +938,37 @@ def _direction(mailbox: str) -> str | None:
def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"""Read named text parts. A cut-off multipart blob is not a message."""
header = _fetch_header(client, uid, mailbox) or {}
text = _partial_text(client, uid)
if not header and not text:
structure = _body_structure(client, uid)
plain: list[str] = []
html_parts: list[str] = []
truncated = False
notices = [
"Die Vorschau ist begrenzt, weil die Nachricht sehr groß ist. "
"Das Original bleibt die vollständige E-Mail."
]
if structure is None:
notices.append("Die MIME-Struktur ließ sich nicht lesen. Rohfragmente werden nicht als Text gezeigt.")
else:
for part in _text_sections(structure):
fetched, shortened = _fetch_section(client, uid, part["section"], PREVIEW_TEXT_LIMIT)
if fetched is None:
notices.append("Ein Textteil der großen Nachricht ließ sich nicht lesen.")
continue
truncated = truncated or shortened or part["octets"] > len(fetched)
text = decode_transfer(fetched, part["charset"]).strip()
if part["media"] == "text/plain":
plain.append(text)
else:
html_parts.append(text)
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = sanitize_html("\n".join(html_parts)) if html_parts else ""
if not simplified and html_parts:
simplified = simplified_text("\n".join(html_parts))
if not header and not simplified and not rendered and structure is None:
raise unprocessable("Die Nachricht lässt sich nicht lesen.")
attachments = header.get("attachments") or (_structure_names(structure) if structure is not None else [])
return {
"uid": uid,
"mailbox": mailbox,
@@ -900,31 +978,185 @@ def _oversized_message(client: imaplib.IMAP4, uid: int, mailbox: str) -> dict:
"message_cc": header.get("message_cc") or "",
"direction": header.get("direction", _direction(mailbox)),
"document_on": header.get("document_on"),
"attachments": header.get("attachments") or [],
"text": text,
"text_truncated": True,
"attachments": attachments,
"text": simplified[:PREVIEW_TEXT_LIMIT],
"html": rendered,
"text_truncated": True or truncated,
"preview_limited": True,
"notices": notices,
}
def _partial_text(client: imaplib.IMAP4, uid: int) -> str:
def _body_structure(client: imaplib.IMAP4, uid: int):
try:
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[TEXT]<0.200000>)")
status, fetched = client.uid("FETCH", str(uid), "(BODYSTRUCTURE)")
except imaplib.IMAP4.error:
return ""
return None
if status != "OK":
return ""
return None
blob = _literal(fetched)
if blob is None:
chunks: list[bytes] = []
for part in fetched or []:
if isinstance(part, tuple):
chunks.extend(bytes(item) for item in part if isinstance(item, (bytes, bytearray)))
elif isinstance(part, (bytes, bytearray)):
chunks.append(bytes(part))
blob = b" ".join(chunks)
text = blob.decode("utf-8", "replace")
start = text.find("(")
if start < 0:
return None
try:
node, _end = _parse_imap_list(text, start)
except ValueError:
return None
return node
def _parse_imap_list(text: str, index: int):
if index >= len(text) or text[index] != "(":
raise ValueError(text[index:index + 20])
index += 1
items = []
while index < len(text):
char = text[index]
if char == ")":
return items, index + 1
if char in " \t\r\n":
index += 1
continue
if char == "(":
child, index = _parse_imap_list(text, index)
items.append(child)
continue
if char == '"':
token, index = _parse_imap_string(text, index)
items.append(token)
continue
end = index
while end < len(text) and text[end] not in " \t\r\n()":
end += 1
token = text[index:end]
items.append(None if token.upper() == "NIL" else token)
index = end
raise ValueError("unclosed")
def _parse_imap_string(text: str, index: int):
index += 1
chars: list[str] = []
while index < len(text):
char = text[index]
if char == "\\":
if index + 1 < len(text):
chars.append(text[index + 1])
index += 2
continue
if char == '"':
return "".join(chars), index + 1
chars.append(char)
index += 1
raise ValueError("string")
def _text_sections(node, prefix: str = "") -> list[dict]:
if not isinstance(node, list) or not node:
return []
if isinstance(node[0], list):
children = []
for item in node:
if isinstance(item, list):
children.append(item)
else:
break
found = []
for number, child in enumerate(children, start=1):
section = f"{number}" if not prefix else f"{prefix}.{number}"
found.extend(_text_sections(child, section))
return found
media = str(node[0] or "").upper()
subtype = str(node[1] or "").upper()
if media == "MESSAGE":
return []
if media != "TEXT" or subtype not in {"PLAIN", "HTML"}:
return []
disposition, params = _disposition(node)
if disposition == "ATTACHMENT":
return []
body_params = _param_map(node[2]) if len(node) > 2 else {}
octets = _imap_int(node[6]) if len(node) > 6 else 0
return [{
"section": prefix or "1",
"media": f"text/{subtype.lower()}",
"charset": body_params.get("charset") or "utf-8",
"octets": octets,
}]
def _disposition(node: list) -> tuple[str, dict]:
for item in node:
if isinstance(item, list) and item and isinstance(item[0], str) and item[0].upper() in {"ATTACHMENT", "INLINE"}:
params = _param_map(item[1]) if len(item) > 1 else {}
return item[0].upper(), params
return "", {}
def _param_map(value) -> dict:
if not isinstance(value, list):
return {}
pairs = {}
index = 0
while index + 1 < len(value):
key = value[index]
raw = value[index + 1]
if isinstance(key, str):
pairs[key.casefold()] = "" if raw is None else str(raw)
index += 2
return pairs
def _imap_int(value) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _fetch_section(client: imaplib.IMAP4, uid: int, section: str, limit: int) -> tuple[bytes | None, bool]:
spec = f"(BODY.PEEK[{section}]<0.{limit}>)"
try:
status, fetched = client.uid("FETCH", str(uid), spec)
except imaplib.IMAP4.error:
return None, False
if status != "OK":
return None, False
raw = _literal(fetched)
if not raw:
return ""
sample = raw[:200_000]
try:
message = BytesParser().parsebytes(b"Subject: x\r\n\r\n" + sample)
text = _message_text(message)
except Exception:
text = ""
if not text:
text = sample.decode("utf-8", errors="replace")
return text.strip()[:200_000]
return None, False
return raw[:limit], len(raw) >= limit
def _structure_names(node) -> list[dict]:
names = []
def visit(item):
if not isinstance(item, list) or not item:
return
if isinstance(item[0], list):
for child in item:
if isinstance(child, list):
visit(child)
return
_disposition_name, params = _disposition(item)
body = _param_map(item[2]) if len(item) > 2 else {}
raw = params.get("filename") or body.get("name") or ""
label = header_text(raw)
if label:
names.append({"filename": label[:200]})
visit(node)
return names[:40]
def _literal(fetched) -> bytes | None:
@@ -944,12 +1176,10 @@ def _structure_attachments(fetched) -> list[dict]:
blob = b" ".join(chunks)
names = re.findall(br'FILENAME"\s+"([^"\r\n]+)"', blob, flags=re.IGNORECASE)
items = []
seen: set[str] = set()
for raw in names:
name = raw.decode("utf-8", errors="replace").strip()[:200]
if not name or name in seen:
name = header_text(raw.decode("utf-8", errors="replace")).strip()[:200]
if not name:
continue
seen.add(name)
items.append({"filename": name})
if len(items) == 40:
break
@@ -1039,6 +1269,32 @@ def _unquote(token: str) -> str:
return token
def _encode_utf7(value: str) -> str:
"""Modified UTF-7 for a mailbox that is not already an ASCII protocol name."""
pieces: list[str] = []
pending: list[str] = []
def flush() -> None:
if not pending:
return
raw = "".join(pending).encode("utf-16-be")
token = base64.b64encode(raw).decode("ascii").rstrip("=").replace("/", ",")
pieces.append(f"&{token}-")
pending.clear()
for char in value:
if char == "&":
flush()
pieces.append("&-")
elif ord(char) < 128:
flush()
pieces.append(char)
else:
pending.append(char)
flush()
return "".join(pieces)
def _decode_utf7(value: str) -> str:
if "&" not in value:
return value
@@ -1255,13 +1511,17 @@ def _export_paperless(conn, actor_id: UUID, external_id: str, detail: dict, assi
resolved = resolve_for_read(conn, actor_id, "paperless", ref, assigned)
if resolved.row is None:
return RetrievalFailure(resolved.code, resolved.message, resolved.choices)
base = resolved.row["settings"]["base_url"].rstrip("/")
token = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
data, media, name = pull_paperless_original(
resolved.row["settings"]["base_url"].rstrip("/"),
resolved.row["secret"],
str(external_id),
EXPORT_BYTE_LIMIT,
)
data, media, name = pull_paperless_original(base, token, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(data) > EXPORT_BYTE_LIMIT:
@@ -1282,8 +1542,17 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
uid = int(uid)
if not mailbox or isinstance(uid, bool) or not isinstance(uid, int) or uid < 1:
return RetrievalFailure("identity", "Für diese E-Mail fehlt die Zuordnung zu Fach und Nachricht.")
settings = dict(resolved.row["settings"])
secret = resolved.row["secret"]
connector_id = resolved.row["id"]
owner_id = resolved.row["owner_id"]
release_for_external(conn)
try:
raw = pull_rfc822(resolved.row["settings"], resolved.row["secret"], mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
except Problem as exc:
return failure_from_problem(exc)
try:
assert_still_owned(conn, connector_id, owner_id)
except Problem as exc:
return failure_from_problem(exc)
if len(raw) > EXPORT_BYTE_LIMIT:
+469
View File
@@ -0,0 +1,469 @@
"""Corrections K1–K3 and K7. Synthetic only; no live mailbox or Paperless."""
from __future__ import annotations
import hashlib
import imaplib
import psycopg
import pytest
from email.message import EmailMessage
import app.sources as sources
from app.core.errors import unprocessable
from app.mailpack import parse_mail
from tests.support import Actor, register
def test_mailbox_names_are_quoted_protocol_strings():
client = imaplib.IMAP4.__new__(imaplib.IMAP4)
client.sent = []
client.state = "AUTH"
client._encoding = "ascii"
client.tagpre = b"T"
client.tagnum = 1
client.tagged_commands = {}
client.untagged_responses = {}
client.is_readonly = True
client.literal = None
client.debug = 0
client._mesg = lambda *args: None
client._log = lambda *args: None
client.send = lambda data: client.sent.append(data)
value = "Sent Items"
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T1 EXAMINE "Sent Items"\r\n'
value = 'Ordner "A"\\B'
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T2 EXAMINE "Ordner \\"A\\"\\\\B"\r\n'
assert sources._mailbox_argument("INBOX") == "INBOX"
with pytest.raises(Exception):
sources._mailbox_argument("Sent\nItems")
def test_parse_mail_keeps_names_html_and_attached_messages():
inner = EmailMessage()
inner["Subject"] = "Innen"
inner.set_content("innen bleibt drin")
inner.add_attachment(b"innen-pdf", maintype="application", subtype="pdf", filename="innen.pdf")
outer = EmailMessage()
outer["From"] = "Ada <ada@example.test>"
outer["To"] = "Ben <ben@example.test>"
outer["Subject"] = "=?utf-8?q?Aussen?="
outer.set_content("aussen bleibt aussen")
outer.add_attachment(b"%PDF-same", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(b"%PDF-other", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(inner, subtype="rfc822", filename="weitergeleitet.eml")
html = (
"<p>A &amp; B</p><table><tr><td>Zelle</td><td>Zwei</td></tr></table>"
"<script>alert(1)</script><img src=\"https://evil.test/a.png\">"
"<a href=\"javascript:alert(1)\">x</a><img src=\"cid:logo@x\">"
)
mixed = EmailMessage()
mixed["Subject"] = "HTML"
mixed.set_content("nur text")
mixed.add_alternative(html, subtype="html")
logo = b"\x89PNG\r\n"
mixed.add_attachment(logo, maintype="image", subtype="png", filename="logo.png", cid="<logo@x>")
encoded = EmailMessage()
encoded["Subject"] = "Namen"
encoded.set_content("datei")
encoded.add_attachment(
b"pdf",
maintype="application",
subtype="pdf",
filename="=?utf-8?q?Buero_Uebergabe=2Epdf?=",
)
parsed = parse_mail(outer.as_bytes())
assert parsed.subject == "Aussen"
assert parsed.text == "aussen bleibt aussen"
assert "innen" not in parsed.text
names = [item.display_name for item in parsed.attachments]
assert "weitergeleitet.eml" in names
assert names.count("lage.pdf") == 2
archives = [item.filename for item in parsed.attachments]
assert len(set(archives)) == len(archives)
attached = next(item for item in parsed.attachments if item.display_name == "weitergeleitet.eml")
assert b"innen bleibt drin" in attached.data or b"Innen" in attached.data
assert attached.sha256 == hashlib.sha256(attached.data).hexdigest()
view = parse_mail(mixed.as_bytes())
assert "<table>" in view.html
assert "<script" not in view.html
assert "https://evil.test" not in view.html
assert "javascript:" not in view.html
assert "data:image/png;base64," in view.html
assert "A & B" in view.html or "A &amp; B" in view.html
assert "Zelle" in view.html
named = parse_mail(encoded.as_bytes())
assert named.attachments[0].display_name == "Buero Uebergabe.pdf"
assert "=?utf-8?" not in named.attachments[0].display_name
assert named.attachments[0].data == b"pdf"
class QuotedIMAP:
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"logged"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "Sent Items"']
def select(self, mailbox, readonly=False):
assert readonly is True
assert mailbox == '"Sent Items"'
self.untagged_responses = {
"EXISTS": [b"4200"],
"UIDVALIDITY": [b"8"],
"UIDNEXT": [b"2"],
}
return "OK", [b"4200"]
def uid(self, command, *args):
if command == "SEARCH":
return "OK", [b""]
header = b"From: Ada <ada@example.test>\r\nSubject: Leer\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_a_filled_folder_with_spaces_is_not_reported_empty(client, db, monkeypatch):
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
ada = Actor(client, "ada-sent@example.test", "Ada Gesendet")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
page = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account["id"], "mailbox": "Sent Items"},
)
assert page["items"] == []
assert page["contradictory"] is True
assert page["exists"] == 4200
assert "leer" not in str(page).casefold()
def _multipart(text: str, name: str, payload: bytes) -> bytes:
message = EmailMessage()
message["From"] = "Ada <ada@example.test>"
message["To"] = "Ben <ben@example.test>"
message["Subject"] = "Grenze"
message["Message-ID"] = "<grenze@example.test>"
message.set_content(text)
message.add_attachment(payload, maintype="application", subtype="pdf", filename=name)
return message.as_bytes()
class LimitIMAP:
raw = b""
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"ok"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
def select(self, mailbox, readonly=False):
self.untagged_responses = {"UIDVALIDITY": [b"1"], "UIDNEXT": [b"3"], "EXISTS": [b"1"]}
return "OK", [b"1"]
def uid(self, command, *args):
spec = " ".join(str(part) for part in args[1:])
if "BODY.PEEK[]" in spec and "<" not in spec:
return "OK", [(b"rfc822", type(self).raw)]
if "BODYSTRUCTURE" in spec:
structure = (
b'(("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 12 1 NIL NIL NIL NIL)'
b'("APPLICATION" "PDF" NIL NIL NIL "BASE64" 20 NIL '
b'("ATTACHMENT" ("FILENAME" "=?utf-8?q?Buero_Uebergabe=2Epdf?=")) NIL NIL)'
b' "MIXED" ("BOUNDARY" "b") NIL NIL NIL)'
)
return "OK", [(b"body", structure)]
if "BODY.PEEK[1]" in spec:
return "OK", [(b"part", b"gleicher text")]
if "BODY.PEEK[2]" in spec:
return "OK", [(b"part", b"nicht text")]
header = (
b"From: Ada <ada@example.test>\r\nTo: Ben <ben@example.test>\r\n"
b"Subject: Grenze\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
)
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_preview_below_and_above_the_limit_uses_the_same_parts(client, db, monkeypatch):
payload = b"%PDF-plan"
small = _multipart("gleicher text", "Buero Uebergabe.pdf", payload)
LimitIMAP.raw = small + b"X" * 50
monkeypatch.setattr(sources, "PREVIEW_BYTE_LIMIT", len(small))
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", LimitIMAP)
ada = Actor(client, "ada-limit@example.test", "Ada Grenze")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
under = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
LimitIMAP.raw = small + b"Y"
over = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
assert "gleicher text" in under["text"]
assert "gleicher text" in over["text"]
assert under["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert over["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert "Content-Transfer-Encoding" not in over["text"]
assert "BASE64" not in over["text"].upper()
assert over["preview_limited"] is True
def test_upload_and_mail_fetch_leave_the_domain_transaction(client, db, monkeypatch):
seen = []
import app.api.routes as routes
import app.ops as ops
from app.media.port import publish as real_publish
def watching(adapter, key, data):
seen.append(("publish", ops_conn[0].info.transaction_status))
return real_publish(adapter, key, data)
ops_conn = [None]
real_create = ops.create_file
def wrapped_create(conn, *args, **kwargs):
ops_conn[0] = conn
return real_create(conn, *args, **kwargs)
monkeypatch.setattr(ops, "publish", watching)
monkeypatch.setattr(routes, "create_file", wrapped_create)
ada = Actor(client, "ada-idle@example.test", "Ada Leerlauf")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-1.4\n", "application/pdf")},
status=201,
)
assert seen[0][1] == psycopg.pq.TransactionStatus.IDLE
import app.sourceaccess as sourceaccess
phase = {"external": False, "status": None}
real_release = sourceaccess.release_for_external
def spy_release(conn):
real_release(conn)
phase["external"] = True
phase["status"] = conn.info.transaction_status
def spy_owned(*args, **kwargs):
if phase["external"]:
raise AssertionError("Datenbankzugriff im externen Abschnitt")
return real_owned(*args, **kwargs)
def spy_pull(*args, **kwargs):
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
return _multipart("brief", "a.pdf", b"%PDF")
real_owned = sources._owned
monkeypatch.setattr(sourceaccess, "release_for_external", spy_release)
monkeypatch.setattr(sources, "_owned", spy_owned)
monkeypatch.setattr(sources, "pull_rfc822", spy_pull)
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Brief",
"source": "imap",
"external_id": "<grenze@example.test>",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": account["id"], "mailbox": "Sent Items", "uid": 4},
},
status=201,
)
opened = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
assert opened["text"]
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
def test_failed_conclusion_keeps_no_partial_file(client, db, monkeypatch):
import app.ops as ops
def fail(*args, **kwargs):
raise unprocessable("Abschluss fehlgeschlagen")
monkeypatch.setattr(ops, "_new_element", fail)
ada = Actor(client, "ada-fail@example.test", "Ada Abschluss")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-FAIL-CONCLUSION\n", "application/pdf")},
status=422,
)
stored = db.execute("SELECT count(*) AS n FROM elements WHERE title = 'Scan'").fetchone()
assert stored["n"] == 0
from pathlib import Path
from app.core.settings import get_settings
root = Path(get_settings().media_root)
left = [path for path in root.rglob("*") if path.is_file() and b"PDF-FAIL-CONCLUSION" in path.read_bytes()]
assert left == []
def test_paperless_filters_change_the_hits(client, db, monkeypatch):
catalog = []
for number in range(1, 46):
catalog.append({
"id": number,
"title": "Vertrag" if number % 5 == 0 else f"Schrift {number}",
"content": "nurinhalt" if number == 41 else f"text {number}",
"correspondent": 4 if number % 2 == 0 else 9,
"document_type": 8 if number < 30 else 1,
"tags": [2, 5] if number % 3 == 0 else [2],
"created": "2024-06-01" if number > 20 else "2023-01-01",
})
def fake_get(base, token, path, params):
if path != "/api/documents/":
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
matched = []
for item in catalog:
query = params.get("query") or ""
title = params.get("title__icontains") or ""
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
continue
if title and title.casefold() not in item["title"].casefold():
continue
if params.get("correspondent__id") and str(item["correspondent"]) != params["correspondent__id"]:
continue
if params.get("document_type__id") and str(item["document_type"]) != params["document_type__id"]:
continue
if params.get("tags__id__all"):
needed = {int(part) for part in params["tags__id__all"].split(",") if part}
if not needed.issubset(set(item["tags"])):
continue
start = params.get("created__date__gte")
end = params.get("created__date__lte")
if start and item["created"] < start:
continue
if end and item["created"] > end:
continue
matched.append(item)
page = int(params.get("page") or 1)
start_at = (page - 1) * 20
rows = matched[start_at:start_at + 20]
return {
"count": len(matched),
"results": [{"id": item["id"], "title": item["title"], "created": item["created"]} for item in rows],
}
monkeypatch.setattr(sources, "_paperless_get", fake_get)
ada = Actor(client, "ada-sense@example.test", "Ada Sinn")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
content = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "q": "nurinhalt"},
)
assert [item["id"] for item in content["items"]] == ["41"]
title = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "title": "Vertrag"},
)
assert title["items"]
assert all("Vertrag" in item["title"] for item in title["items"])
assert all(item["id"] != "41" for item in title["items"])
combined = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={
"connector_id": account["id"],
"correspondent": "4",
"document_type": "8",
"tag": "2,5",
"document_from": "2023-01-01",
"document_to": "2024-12-31",
},
)
plain = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "cursor": "2"},
)
assert [item["id"] for item in combined["items"]] == ["6", "12", "18", "24"]
assert [item["id"] for item in plain["items"]][0] == "21"
+5 -2
View File
@@ -180,9 +180,12 @@ def test_foreign_mailbox_stays_hidden(client, db, monkeypatch):
class HugeIMAP(FakeIMAP):
def uid(self, command, *args):
spec = " ".join(str(part) for part in args[1:]) if len(args) > 1 else ""
if command == "FETCH" and "BODY.PEEK[]" in spec:
if command == "FETCH" and "BODY.PEEK[]" in spec and "<" not in spec:
return "OK", [(b"rfc822", b"x" * 8_000_001)]
if "BODY.PEEK[TEXT]" in spec:
if "BODYSTRUCTURE" in spec:
structure = b'("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 40 1 NIL NIL NIL NIL)'
return "OK", [(b"body", structure)]
if "BODY.PEEK[1]" in spec:
return "OK", [(b"text", "Kurzer Auszug der grossen Nachricht.".encode())]
return super().uid(command, *args)