Files
tsumugu/backend/app/mailpack.py
T
StommerandClaude Sonnet 5.5 a3f328832e
Deploy Development / deploy (push) Successful in 41s
Test Suite / pytest-backend (push) Successful in 5m32s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 21s
Lade Bilder aus E-Mails auf Wunsch über den Server nach und verkleinere den Kopf der Leseansicht auf dem Handy.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 16:47:24 +02:00

475 lines
18 KiB
Python

"""Pure mail identity and attachment extraction for a complete export.
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
The same tree feeds preview, reading, attachment download and export.
"""
from __future__ import annotations
import hashlib
import html
import re
from dataclasses import dataclass, field
from email import policy
from email.header import decode_header, make_header
from email.message import EmailMessage
from email.parser import BytesParser
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
_UID_PREFIX = "uid:"
_POLICY = policy.default
_ALLOWED_TAGS = {
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
"font", "center", "hr", "small", "sub", "sup", "s", "strike", "caption", "tfoot", "section", "article",
}
_VOID_TAGS = {"br", "img", "hr"}
# Formatting a mail may keep. Nothing here loads anything, positions an element or runs code.
_STYLE_PROPS = {
"color", "background-color", "font-family", "font-size", "font-weight", "font-style", "text-align",
"text-decoration", "line-height", "vertical-align", "white-space", "letter-spacing",
"margin", "margin-top", "margin-bottom", "margin-left", "margin-right",
"padding", "padding-top", "padding-bottom", "padding-left", "padding-right",
"border", "border-top", "border-bottom", "border-left", "border-right", "border-color", "border-width",
"border-style", "border-collapse", "width", "max-width", "height", "text-indent", "list-style-type",
}
_STYLE_VALUE = re.compile(r"^[#%a-z0-9 .,()\-+'\"]{1,100}$", re.IGNORECASE)
_STYLE_DENIED = re.compile(r"url|expression|var\(|attr\(|image|@|javascript|behavior|\\", re.IGNORECASE)
_COLOR = re.compile(r"^(#[0-9a-f]{3,8}|[a-z]{3,20}|rgb\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*\))$", re.IGNORECASE)
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
class MailIdentityError(Exception):
def __init__(self, message: str) -> None:
super().__init__(message)
self.message = message
@dataclass(frozen=True)
class AttachmentBytes:
filename: str
media_type: str
data: bytes
sha256: str
display_name: str = ""
content_id: str = ""
inline: bool = False
@dataclass
class ParsedMail:
message_id: str
subject: str
sender: str
recipient: str
copy: str
text: str
attachments: list[AttachmentBytes] = field(default_factory=list)
missing: list[str] = field(default_factory=list)
html: str = ""
html_source: str = ""
remote_images: list[str] = field(default_factory=list)
notices: list[str] = field(default_factory=list)
text_truncated: bool = False
document_on: str | None = None
def header_message_id(raw: bytes) -> str:
message = BytesParser(policy=_POLICY).parsebytes(raw)
return str(message.get("Message-ID") or "").strip()
def identities_match(found: str | None, expected: str | None) -> bool:
"""A stored uid fallback matches only a message that still has no Message-ID."""
expected_text = (expected or "").strip()
found_text = (found or "").strip()
if expected_text.casefold().startswith(_UID_PREFIX):
return found_text == ""
if not expected_text or not found_text:
return False
return _bare(expected_text) == _bare(found_text)
def choose_message(candidate: bytes | None, alternates: list[bytes], expected: str) -> bytes:
"""Keep the stored message when it still matches. Otherwise accept one unambiguous hit."""
if candidate is not None and identities_match(header_message_id(candidate), expected):
return candidate
if (expected or "").strip().casefold().startswith(_UID_PREFIX):
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
matches = [raw for raw in alternates if identities_match(header_message_id(raw), expected)]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
raise MailIdentityError("Die Nachrichtenidentität ist mehrdeutig. Es wurde keine E-Mail übernommen.")
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
def header_text(value) -> str:
"""Decode encoded-words. A raw MIME header is not a display name."""
if value is None:
return ""
raw = str(value).replace("\r", " ").replace("\n", " ")
try:
text = str(make_header(decode_header(raw)))
except (UnicodeError, LookupError, ValueError):
text = raw
return re.sub(r"\s+", " ", text).strip()
def parse_mail(raw: bytes) -> ParsedMail:
message = BytesParser(policy=_POLICY).parsebytes(raw)
attachments: list[AttachmentBytes] = []
missing: list[str] = []
notices: list[str] = []
used: set[str] = set()
plain: list[str] = []
html_parts: list[str] = []
index = 0
for kind, part in _walk(message):
if kind == "message":
index += 1
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
data = _message_bytes(part)
if data is None:
missing.append(label)
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label, used)
attachments.append(_attachment(part, name, label, data, inline=False))
continue
disposition = (part.get_content_disposition() or "").lower()
media = part.get_content_type()
filename = part.get_filename()
content_id = _content_id(part)
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
if body:
text = _decoded_text(part)
if text is None:
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
continue
if media == "text/plain":
plain.append(text)
else:
html_parts.append(text)
continue
if not filename and disposition != "attachment" and not content_id:
continue
index += 1
label = header_text(filename) if filename else f"anlage-{index}.bin"
data = _leaf_bytes(part)
if data is None:
missing.append(label or f"Anlage {index}")
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
continue
name = _unique_filename(label or f"anlage-{index}.bin", used)
inline = disposition == "inline" or bool(content_id)
attachments.append(_attachment(part, name, label or name, data, inline=inline))
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
rendered = ""
if html_parts:
source = "\n".join(html_parts)
remote_found: list[str] = []
rendered = sanitize_html(source, _inline_images(attachments), found=remote_found)
if not simplified:
simplified = simplified_text("\n".join(html_parts))
if missing:
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
return ParsedMail(
message_id=str(message.get("Message-ID") or "").strip(),
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
sender=header_text(message.get("From")),
recipient=header_text(message.get("To")),
copy=header_text(message.get("Cc")),
text=simplified,
attachments=attachments,
missing=missing,
html=rendered,
html_source=source if html_parts else "",
remote_images=remote_found if html_parts else [],
notices=notices,
document_on=_message_date(message.get("Date")),
)
def decode_transfer(payload: bytes, charset: str | None) -> str:
return bytes(payload).decode(charset or "utf-8", errors="replace")
def simplified_text(value: str) -> str:
"""A recognizable plain reading. It is not the formatted view."""
class Collector(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style"}:
self.skip += 1
return
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
self.parts.append("\n")
elif name in {"td", "th"}:
self.parts.append("\t")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style"} and self.skip:
self.skip -= 1
return
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
self.parts.append("\n")
def handle_data(self, data):
if not self.skip:
self.parts.append(data)
collector = Collector()
try:
collector.feed(value)
collector.close()
except Exception:
return ""
text = "".join(collector.parts)
text = text.replace("\r", "")
text = re.sub(r"[ \t]+\n", "\n", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def sanitize_html(value: str, images: dict[str, str] | None = None, *, remote=None, found: list[str] | None = None) -> str:
"""Keep a formatted reading. Scripts and active content stay out.
Pictures from other servers stay out too, unless `remote` turns an address into an inline picture.
`found` collects the addresses that were met."""
images = images or {}
class Cleaner(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.parts: list[str] = []
self.skip = 0
def handle_starttag(self, tag, attrs):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
self.skip += 1
return
if self.skip or name not in _ALLOWED_TAGS:
return
rendered = _clean_attrs(name, attrs, images, remote, found)
if rendered is None:
return
self.parts.append(f"<{name}{rendered}>")
def handle_endtag(self, tag):
name = tag.lower()
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
if self.skip:
self.skip -= 1
return
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
return
self.parts.append(f"</{name}>")
def handle_data(self, data):
if not self.skip:
self.parts.append(html.escape(data))
def handle_entityref(self, name):
if not self.skip:
self.parts.append(f"&{name};")
def handle_charref(self, name):
if not self.skip:
self.parts.append(f"&#{name};")
cleaner = Cleaner()
try:
cleaner.feed(value)
cleaner.close()
except Exception:
return ""
return "".join(cleaner.parts).strip()
def _safe_style(value: str) -> str:
kept = []
for declaration in (value or "").split(";"):
prop, _, raw = declaration.partition(":")
prop, raw = prop.strip().lower(), raw.strip().replace("!important", "").strip()
if prop not in _STYLE_PROPS or not raw or _STYLE_DENIED.search(raw) or not _STYLE_VALUE.match(raw):
continue
kept.append(f"{prop}: {raw}")
return "; ".join(kept)
def _clean_attrs(tag: str, attrs, images: dict[str, str], remote=None, found: list[str] | None = None) -> str | None:
kept: list[str] = []
for key, value in attrs:
name = (key or "").lower()
if name.startswith("on") or name == "srcset":
continue
if name == "style":
style = _safe_style(value or "")
if style:
kept.append(f' style="{html.escape(style, quote=True)}"')
continue
if name in {"bgcolor", "color"} and _COLOR.match((value or "").strip()):
kept.append(f' {name}="{html.escape(value.strip(), quote=True)}"')
continue
if name in {"width", "height"} and re.fullmatch(r"\d{1,4}%?", (value or "").strip()):
kept.append(f' {name}="{value.strip()}"')
continue
if name in {"border", "cellpadding", "cellspacing", "size"} and str(value or "").strip().isdigit():
kept.append(f' {name}="{int(value)}"')
continue
if name in {"align", "valign"} and (value or "").strip().lower() in {"left", "right", "center", "justify", "top", "middle", "bottom"}:
kept.append(f' {name}="{value.strip().lower()}"')
continue
if tag == "a" and name == "href":
target = (value or "").strip()
if not _SAFE_LINK.match(target):
continue
kept.append(f' href="{html.escape(target, quote=True)}" title="{html.escape(target, quote=True)}"')
elif tag == "img" and name == "src":
source = (value or "").strip()
if source.lower().startswith("cid:"):
token = source[4:].strip().strip("<>")
source = images.get(token.casefold(), "")
elif re.match(r"https://", source, re.IGNORECASE):
if found is not None and source not in found:
found.append(source)
source = remote(source) if remote is not None else ""
if not _DATA_IMAGE.match(source):
continue
kept.append(f' src="{source}"')
elif tag == "img" and name == "alt":
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
kept.append(f" {name}=\"{int(value)}\"")
if tag == "img" and not any(item.startswith(" src=") for item in kept):
return None
if tag == "a":
kept.append(' rel="noreferrer noopener" target="_blank"')
return "".join(kept)
def _walk(message: EmailMessage):
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
yield "message", message
return
if message.is_multipart():
for child in message.iter_parts():
yield from _walk(child)
return
yield "leaf", message
def _decoded_text(part: EmailMessage) -> str | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if not isinstance(payload, (bytes, bytearray)):
raw = part.get_payload()
return raw.strip() if isinstance(raw, str) else None
charset = part.get_content_charset() or "utf-8"
try:
return decode_transfer(bytes(payload), charset).strip()
except LookupError:
return decode_transfer(bytes(payload), "utf-8").strip()
def _leaf_bytes(part: EmailMessage) -> bytes | None:
try:
payload = part.get_payload(decode=True)
except Exception:
return None
if isinstance(payload, (bytes, bytearray)):
return bytes(payload)
return None
def _message_bytes(part: EmailMessage) -> bytes | None:
payload = part.get_payload()
if isinstance(payload, list) and payload:
inner = payload[0]
if isinstance(inner, EmailMessage):
return inner.as_bytes()
data = _leaf_bytes(part)
return data or None
def _content_id(part: EmailMessage) -> str:
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
return token
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
return AttachmentBytes(
filename=filename,
media_type=(part.get_content_type() or "application/octet-stream")[:120],
data=data,
sha256=hashlib.sha256(data).hexdigest(),
display_name=label or filename,
content_id=_content_id(part),
inline=inline,
)
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
import base64
images: dict[str, str] = {}
for item in attachments:
if not item.content_id or not item.media_type.startswith("image/"):
continue
if len(item.data) > 1_500_000:
continue
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
if subtype == "jpg":
subtype = "jpeg"
if subtype not in {"png", "jpeg", "gif", "webp"}:
continue
encoded = base64.b64encode(item.data).decode("ascii")
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
return images
def _message_date(value) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(str(value)).date().isoformat()
except (TypeError, ValueError, OverflowError):
return None
def _bare(value: str) -> str:
return value.strip().strip("<>").strip().casefold()
def _unique_filename(name: str, used: set[str]) -> str:
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
if cleaned not in used:
used.add(cleaned)
return cleaned
stem, dot, ext = cleaned.rpartition(".")
if not dot or not stem:
stem, ext = cleaned, ""
else:
ext = f".{ext}"
number = 2
while True:
candidate = f"{stem} ({number}){ext}"
if candidate not in used:
used.add(candidate)
return candidate
number += 1