519 lines
19 KiB
Python
519 lines
19 KiB
Python
"""Pure mail identity and attachment extraction for a complete export.
|
|
|
|
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
|
|
The same tree feeds preview, reading, attachment download and export.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import html
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from email import policy
|
|
from email.header import decode_header, make_header
|
|
from email.message import EmailMessage
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
from html.parser import HTMLParser
|
|
|
|
_UID_PREFIX = "uid:"
|
|
_POLICY = policy.default
|
|
_ALLOWED_TAGS = {
|
|
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
|
|
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
|
|
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
|
|
"font", "center", "hr", "small", "sub", "sup", "s", "strike", "caption", "tfoot", "section", "article",
|
|
}
|
|
_VOID_TAGS = {"br", "img", "hr"}
|
|
# Formatting a mail may keep. Nothing here loads anything, positions an element or runs code.
|
|
_STYLE_PROPS = {
|
|
"color", "background-color", "font-family", "font-size", "font-weight", "font-style", "text-align",
|
|
"text-decoration", "line-height", "vertical-align", "white-space", "letter-spacing",
|
|
"margin", "margin-top", "margin-bottom", "margin-left", "margin-right",
|
|
"padding", "padding-top", "padding-bottom", "padding-left", "padding-right",
|
|
"border", "border-top", "border-bottom", "border-left", "border-right", "border-color", "border-width",
|
|
"border-style", "border-collapse", "width", "max-width", "height", "text-indent", "list-style-type",
|
|
}
|
|
_STYLE_VALUE = re.compile(r"^[#%a-z0-9 .,()\-+'\"]{1,100}$", re.IGNORECASE)
|
|
_STYLE_DENIED = re.compile(r"url|expression|var\(|attr\(|image|@|javascript|behavior|\\", re.IGNORECASE)
|
|
_COLOR = re.compile(r"^(#[0-9a-f]{3,8}|[a-z]{3,20}|rgb\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*\))$", re.IGNORECASE)
|
|
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
|
|
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
|
|
|
|
|
|
class MailIdentityError(Exception):
|
|
def __init__(self, message: str) -> None:
|
|
super().__init__(message)
|
|
self.message = message
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AttachmentBytes:
|
|
filename: str
|
|
media_type: str
|
|
data: bytes
|
|
sha256: str
|
|
display_name: str = ""
|
|
content_id: str = ""
|
|
inline: bool = False
|
|
|
|
|
|
@dataclass
|
|
class ParsedMail:
|
|
message_id: str
|
|
subject: str
|
|
sender: str
|
|
recipient: str
|
|
copy: str
|
|
text: str
|
|
attachments: list[AttachmentBytes] = field(default_factory=list)
|
|
missing: list[str] = field(default_factory=list)
|
|
html: str = ""
|
|
html_source: str = ""
|
|
remote_images: list[str] = field(default_factory=list)
|
|
notices: list[str] = field(default_factory=list)
|
|
text_truncated: bool = False
|
|
document_on: str | None = None
|
|
|
|
|
|
def header_message_id(raw: bytes) -> str:
|
|
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
|
return str(message.get("Message-ID") or "").strip()
|
|
|
|
|
|
def identities_match(found: str | None, expected: str | None) -> bool:
|
|
"""A stored uid fallback matches only a message that still has no Message-ID."""
|
|
expected_text = (expected or "").strip()
|
|
found_text = (found or "").strip()
|
|
if expected_text.casefold().startswith(_UID_PREFIX):
|
|
return found_text == ""
|
|
if not expected_text or not found_text:
|
|
return False
|
|
return _bare(expected_text) == _bare(found_text)
|
|
|
|
|
|
def choose_message(candidate: bytes | None, alternates: list[bytes], expected: str) -> bytes:
|
|
"""Keep the stored message when it still matches. Otherwise accept one unambiguous hit."""
|
|
if candidate is not None and identities_match(header_message_id(candidate), expected):
|
|
return candidate
|
|
if (expected or "").strip().casefold().startswith(_UID_PREFIX):
|
|
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
|
matches = [raw for raw in alternates if identities_match(header_message_id(raw), expected)]
|
|
if len(matches) == 1:
|
|
return matches[0]
|
|
if len(matches) > 1:
|
|
raise MailIdentityError("Die Nachrichtenidentität ist mehrdeutig. Es wurde keine E-Mail übernommen.")
|
|
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
|
|
|
|
|
def header_text(value) -> str:
|
|
"""Decode encoded-words. A raw MIME header is not a display name."""
|
|
if value is None:
|
|
return ""
|
|
raw = str(value).replace("\r", " ").replace("\n", " ")
|
|
try:
|
|
text = str(make_header(decode_header(raw)))
|
|
except (UnicodeError, LookupError, ValueError):
|
|
text = raw
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def parse_mail(raw: bytes) -> ParsedMail:
|
|
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
|
attachments: list[AttachmentBytes] = []
|
|
missing: list[str] = []
|
|
notices: list[str] = []
|
|
used: set[str] = set()
|
|
plain: list[str] = []
|
|
html_parts: list[str] = []
|
|
index = 0
|
|
for kind, part in _walk(message):
|
|
if kind == "message":
|
|
index += 1
|
|
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
|
|
data = _message_bytes(part)
|
|
if data is None:
|
|
missing.append(label)
|
|
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
|
|
continue
|
|
name = _unique_filename(label, used)
|
|
attachments.append(_attachment(part, name, label, data, inline=False))
|
|
continue
|
|
disposition = (part.get_content_disposition() or "").lower()
|
|
media = part.get_content_type()
|
|
filename = part.get_filename()
|
|
content_id = _content_id(part)
|
|
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
|
|
if body:
|
|
text = _decoded_text(part)
|
|
if text is None:
|
|
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
|
|
continue
|
|
if media == "text/plain":
|
|
plain.append(text)
|
|
else:
|
|
html_parts.append(text)
|
|
continue
|
|
if not filename and disposition != "attachment" and not content_id:
|
|
continue
|
|
index += 1
|
|
label = header_text(filename) if filename else f"anlage-{index}.bin"
|
|
data = _leaf_bytes(part)
|
|
if data is None:
|
|
missing.append(label or f"Anlage {index}")
|
|
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
|
|
continue
|
|
data = _unwrap_pdf_payload(data, media, label)
|
|
if _is_pdf(data):
|
|
media = "application/pdf"
|
|
label = _pdf_filename(label)
|
|
name = _unique_filename(label or f"anlage-{index}.bin", used)
|
|
inline = disposition == "inline" or bool(content_id)
|
|
attachments.append(_attachment(part, name, label or name, data, inline=inline, media=media))
|
|
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
|
rendered = ""
|
|
if html_parts:
|
|
source = "\n".join(html_parts)
|
|
remote_found: list[str] = []
|
|
rendered = sanitize_html(source, _inline_images(attachments), found=remote_found)
|
|
if not simplified:
|
|
simplified = simplified_text("\n".join(html_parts))
|
|
if missing:
|
|
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
|
|
return ParsedMail(
|
|
message_id=str(message.get("Message-ID") or "").strip(),
|
|
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
|
|
sender=header_text(message.get("From")),
|
|
recipient=header_text(message.get("To")),
|
|
copy=header_text(message.get("Cc")),
|
|
text=simplified,
|
|
attachments=attachments,
|
|
missing=missing,
|
|
html=rendered,
|
|
html_source=source if html_parts else "",
|
|
remote_images=remote_found if html_parts else [],
|
|
notices=notices,
|
|
document_on=_message_date(message.get("Date")),
|
|
)
|
|
|
|
|
|
def decode_transfer(payload: bytes, charset: str | None) -> str:
|
|
return bytes(payload).decode(charset or "utf-8", errors="replace")
|
|
|
|
|
|
def simplified_text(value: str) -> str:
|
|
"""A recognizable plain reading. It is not the formatted view."""
|
|
class Collector(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts: list[str] = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
name = tag.lower()
|
|
if name in {"script", "style"}:
|
|
self.skip += 1
|
|
return
|
|
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
|
|
self.parts.append("\n")
|
|
elif name in {"td", "th"}:
|
|
self.parts.append("\t")
|
|
|
|
def handle_endtag(self, tag):
|
|
name = tag.lower()
|
|
if name in {"script", "style"} and self.skip:
|
|
self.skip -= 1
|
|
return
|
|
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
|
|
self.parts.append("\n")
|
|
|
|
def handle_data(self, data):
|
|
if not self.skip:
|
|
self.parts.append(data)
|
|
|
|
collector = Collector()
|
|
try:
|
|
collector.feed(value)
|
|
collector.close()
|
|
except Exception:
|
|
return ""
|
|
text = "".join(collector.parts)
|
|
text = text.replace("\r", "")
|
|
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
def sanitize_html(value: str, images: dict[str, str] | None = None, *, remote=None, found: list[str] | None = None) -> str:
|
|
"""Keep a formatted reading. Scripts and active content stay out.
|
|
|
|
Pictures from other servers stay out too, unless `remote` turns an address into an inline picture.
|
|
`found` collects the addresses that were met."""
|
|
images = images or {}
|
|
|
|
class Cleaner(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts: list[str] = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
name = tag.lower()
|
|
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
|
self.skip += 1
|
|
return
|
|
if self.skip or name not in _ALLOWED_TAGS:
|
|
return
|
|
rendered = _clean_attrs(name, attrs, images, remote, found)
|
|
if rendered is None:
|
|
return
|
|
self.parts.append(f"<{name}{rendered}>")
|
|
|
|
def handle_endtag(self, tag):
|
|
name = tag.lower()
|
|
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
|
if self.skip:
|
|
self.skip -= 1
|
|
return
|
|
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
|
|
return
|
|
self.parts.append(f"</{name}>")
|
|
|
|
def handle_data(self, data):
|
|
if not self.skip:
|
|
self.parts.append(html.escape(data))
|
|
|
|
def handle_entityref(self, name):
|
|
if not self.skip:
|
|
self.parts.append(f"&{name};")
|
|
|
|
def handle_charref(self, name):
|
|
if not self.skip:
|
|
self.parts.append(f"&#{name};")
|
|
|
|
cleaner = Cleaner()
|
|
try:
|
|
cleaner.feed(value)
|
|
cleaner.close()
|
|
except Exception:
|
|
return ""
|
|
return "".join(cleaner.parts).strip()
|
|
|
|
|
|
def _safe_style(value: str) -> str:
|
|
kept = []
|
|
for declaration in (value or "").split(";"):
|
|
prop, _, raw = declaration.partition(":")
|
|
prop, raw = prop.strip().lower(), raw.strip().replace("!important", "").strip()
|
|
if prop not in _STYLE_PROPS or not raw or _STYLE_DENIED.search(raw) or not _STYLE_VALUE.match(raw):
|
|
continue
|
|
kept.append(f"{prop}: {raw}")
|
|
return "; ".join(kept)
|
|
|
|
|
|
def _clean_attrs(tag: str, attrs, images: dict[str, str], remote=None, found: list[str] | None = None) -> str | None:
|
|
kept: list[str] = []
|
|
for key, value in attrs:
|
|
name = (key or "").lower()
|
|
if name.startswith("on") or name == "srcset":
|
|
continue
|
|
if name == "style":
|
|
style = _safe_style(value or "")
|
|
if style:
|
|
kept.append(f' style="{html.escape(style, quote=True)}"')
|
|
continue
|
|
if name in {"bgcolor", "color"} and _COLOR.match((value or "").strip()):
|
|
kept.append(f' {name}="{html.escape(value.strip(), quote=True)}"')
|
|
continue
|
|
if name in {"width", "height"} and re.fullmatch(r"\d{1,4}%?", (value or "").strip()):
|
|
kept.append(f' {name}="{value.strip()}"')
|
|
continue
|
|
if name in {"border", "cellpadding", "cellspacing", "size"} and str(value or "").strip().isdigit():
|
|
kept.append(f' {name}="{int(value)}"')
|
|
continue
|
|
if name in {"align", "valign"} and (value or "").strip().lower() in {"left", "right", "center", "justify", "top", "middle", "bottom"}:
|
|
kept.append(f' {name}="{value.strip().lower()}"')
|
|
continue
|
|
if tag == "a" and name == "href":
|
|
target = (value or "").strip()
|
|
if not _SAFE_LINK.match(target):
|
|
continue
|
|
kept.append(f' href="{html.escape(target, quote=True)}" title="{html.escape(target, quote=True)}"')
|
|
elif tag == "img" and name == "src":
|
|
source = (value or "").strip()
|
|
if source.lower().startswith("cid:"):
|
|
token = source[4:].strip().strip("<>")
|
|
source = images.get(token.casefold(), "")
|
|
elif re.match(r"https://", source, re.IGNORECASE):
|
|
if found is not None and source not in found:
|
|
found.append(source)
|
|
source = remote(source) if remote is not None else ""
|
|
if not _DATA_IMAGE.match(source):
|
|
continue
|
|
kept.append(f' src="{source}"')
|
|
elif tag == "img" and name == "alt":
|
|
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
|
|
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
|
|
kept.append(f" {name}=\"{int(value)}\"")
|
|
if tag == "img" and not any(item.startswith(" src=") for item in kept):
|
|
return None
|
|
if tag == "a":
|
|
kept.append(' rel="noreferrer noopener" target="_blank"')
|
|
return "".join(kept)
|
|
|
|
|
|
def _walk(message: EmailMessage):
|
|
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
|
|
yield "message", message
|
|
return
|
|
if message.is_multipart():
|
|
for child in message.iter_parts():
|
|
yield from _walk(child)
|
|
return
|
|
yield "leaf", message
|
|
|
|
|
|
def _decoded_text(part: EmailMessage) -> str | None:
|
|
try:
|
|
payload = part.get_payload(decode=True)
|
|
except Exception:
|
|
return None
|
|
if not isinstance(payload, (bytes, bytearray)):
|
|
raw = part.get_payload()
|
|
return raw.strip() if isinstance(raw, str) else None
|
|
charset = part.get_content_charset() or "utf-8"
|
|
try:
|
|
return decode_transfer(bytes(payload), charset).strip()
|
|
except LookupError:
|
|
return decode_transfer(bytes(payload), "utf-8").strip()
|
|
|
|
|
|
def _leaf_bytes(part: EmailMessage) -> bytes | None:
|
|
try:
|
|
payload = part.get_payload(decode=True)
|
|
except Exception:
|
|
return None
|
|
if isinstance(payload, (bytes, bytearray)):
|
|
return bytes(payload)
|
|
return None
|
|
|
|
|
|
def _message_bytes(part: EmailMessage) -> bytes | None:
|
|
payload = part.get_payload()
|
|
if isinstance(payload, list) and payload:
|
|
inner = payload[0]
|
|
if isinstance(inner, EmailMessage):
|
|
return inner.as_bytes()
|
|
data = _leaf_bytes(part)
|
|
return data or None
|
|
|
|
|
|
def _content_id(part: EmailMessage) -> str:
|
|
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
|
|
return token
|
|
|
|
|
|
def _is_pdf(data: bytes) -> bool:
|
|
return data[:1024].lstrip(b"\xef\xbb\xbf\r\n\t ").startswith(b"%PDF")
|
|
|
|
|
|
def _unwrap_pdf_payload(data: bytes, media: str, name: str) -> bytes:
|
|
"""A PDF that arrived as undecoded Base64 is still the PDF, not the transfer text."""
|
|
if _is_pdf(data):
|
|
return data
|
|
declared = (media or "").split(";", 1)[0].strip().lower()
|
|
hinted = "pdf" in declared or (name or "").lower().endswith(".pdf") or declared in {
|
|
"application/octet-stream",
|
|
"application/x-octet-stream",
|
|
"binary/octet-stream",
|
|
}
|
|
if not hinted:
|
|
return data
|
|
stripped = b"".join(data.split())
|
|
if len(stripped) < 16 or re.fullmatch(rb"[A-Za-z0-9+/=]+", stripped) is None:
|
|
return data
|
|
try:
|
|
decoded = base64.b64decode(stripped, validate=True)
|
|
except Exception:
|
|
return data
|
|
return decoded if _is_pdf(decoded) else data
|
|
|
|
|
|
def _pdf_filename(name: str) -> str:
|
|
text = (name or "").strip() or "anlage"
|
|
if text.lower().endswith(".pdf"):
|
|
return text
|
|
if text.lower().endswith(".bin"):
|
|
text = text[:-4].rstrip(" .") or "anlage"
|
|
leaf = text.replace("\\", "/").split("/")[-1]
|
|
if "." in leaf:
|
|
return text
|
|
return f"{text}.pdf"
|
|
|
|
|
|
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool, media: str | None = None) -> AttachmentBytes:
|
|
resolved = (media or part.get_content_type() or "application/octet-stream").split(";", 1)[0].strip()
|
|
return AttachmentBytes(
|
|
filename=filename,
|
|
media_type=(resolved or "application/octet-stream")[:120],
|
|
data=data,
|
|
sha256=hashlib.sha256(data).hexdigest(),
|
|
display_name=label or filename,
|
|
content_id=_content_id(part),
|
|
inline=inline,
|
|
)
|
|
|
|
|
|
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
|
|
import base64
|
|
|
|
images: dict[str, str] = {}
|
|
for item in attachments:
|
|
if not item.content_id or not item.media_type.startswith("image/"):
|
|
continue
|
|
if len(item.data) > 1_500_000:
|
|
continue
|
|
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
|
|
if subtype == "jpg":
|
|
subtype = "jpeg"
|
|
if subtype not in {"png", "jpeg", "gif", "webp"}:
|
|
continue
|
|
encoded = base64.b64encode(item.data).decode("ascii")
|
|
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
|
|
return images
|
|
|
|
|
|
def _message_date(value) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(str(value)).date().isoformat()
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _bare(value: str) -> str:
|
|
return value.strip().strip("<>").strip().casefold()
|
|
|
|
|
|
def _unique_filename(name: str, used: set[str]) -> str:
|
|
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
|
|
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
|
|
if cleaned not in used:
|
|
used.add(cleaned)
|
|
return cleaned
|
|
stem, dot, ext = cleaned.rpartition(".")
|
|
if not dot or not stem:
|
|
stem, ext = cleaned, ""
|
|
else:
|
|
ext = f".{ext}"
|
|
number = 2
|
|
while True:
|
|
candidate = f"{stem} ({number}){ext}"
|
|
if candidate not in used:
|
|
used.add(candidate)
|
|
return candidate
|
|
number += 1
|