421 lines
14 KiB
Python
421 lines
14 KiB
Python
"""Pure mail identity and attachment extraction for a complete export.
|
|
|
|
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
|
|
The same tree feeds preview, reading, attachment download and export.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import html
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from email import policy
|
|
from email.header import decode_header, make_header
|
|
from email.message import EmailMessage
|
|
from email.parser import BytesParser
|
|
from email.utils import parsedate_to_datetime
|
|
from html.parser import HTMLParser
|
|
|
|
_UID_PREFIX = "uid:"
|
|
_POLICY = policy.default
|
|
_ALLOWED_TAGS = {
|
|
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
|
|
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
|
|
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
|
|
}
|
|
_VOID_TAGS = {"br", "img"}
|
|
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
|
|
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
|
|
|
|
|
|
class MailIdentityError(Exception):
|
|
def __init__(self, message: str) -> None:
|
|
super().__init__(message)
|
|
self.message = message
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AttachmentBytes:
|
|
filename: str
|
|
media_type: str
|
|
data: bytes
|
|
sha256: str
|
|
display_name: str = ""
|
|
content_id: str = ""
|
|
inline: bool = False
|
|
|
|
|
|
@dataclass
|
|
class ParsedMail:
|
|
message_id: str
|
|
subject: str
|
|
sender: str
|
|
recipient: str
|
|
copy: str
|
|
text: str
|
|
attachments: list[AttachmentBytes] = field(default_factory=list)
|
|
missing: list[str] = field(default_factory=list)
|
|
html: str = ""
|
|
notices: list[str] = field(default_factory=list)
|
|
text_truncated: bool = False
|
|
document_on: str | None = None
|
|
|
|
|
|
def header_message_id(raw: bytes) -> str:
|
|
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
|
return str(message.get("Message-ID") or "").strip()
|
|
|
|
|
|
def identities_match(found: str | None, expected: str | None) -> bool:
|
|
"""A stored uid fallback matches only a message that still has no Message-ID."""
|
|
expected_text = (expected or "").strip()
|
|
found_text = (found or "").strip()
|
|
if expected_text.casefold().startswith(_UID_PREFIX):
|
|
return found_text == ""
|
|
if not expected_text or not found_text:
|
|
return False
|
|
return _bare(expected_text) == _bare(found_text)
|
|
|
|
|
|
def choose_message(candidate: bytes | None, alternates: list[bytes], expected: str) -> bytes:
|
|
"""Keep the stored message when it still matches. Otherwise accept one unambiguous hit."""
|
|
if candidate is not None and identities_match(header_message_id(candidate), expected):
|
|
return candidate
|
|
if (expected or "").strip().casefold().startswith(_UID_PREFIX):
|
|
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
|
matches = [raw for raw in alternates if identities_match(header_message_id(raw), expected)]
|
|
if len(matches) == 1:
|
|
return matches[0]
|
|
if len(matches) > 1:
|
|
raise MailIdentityError("Die Nachrichtenidentität ist mehrdeutig. Es wurde keine E-Mail übernommen.")
|
|
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
|
|
|
|
|
def header_text(value) -> str:
|
|
"""Decode encoded-words. A raw MIME header is not a display name."""
|
|
if value is None:
|
|
return ""
|
|
raw = str(value).replace("\r", " ").replace("\n", " ")
|
|
try:
|
|
text = str(make_header(decode_header(raw)))
|
|
except (UnicodeError, LookupError, ValueError):
|
|
text = raw
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def parse_mail(raw: bytes) -> ParsedMail:
|
|
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
|
attachments: list[AttachmentBytes] = []
|
|
missing: list[str] = []
|
|
notices: list[str] = []
|
|
used: set[str] = set()
|
|
plain: list[str] = []
|
|
html_parts: list[str] = []
|
|
index = 0
|
|
for kind, part in _walk(message):
|
|
if kind == "message":
|
|
index += 1
|
|
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
|
|
data = _message_bytes(part)
|
|
if data is None:
|
|
missing.append(label)
|
|
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
|
|
continue
|
|
name = _unique_filename(label, used)
|
|
attachments.append(_attachment(part, name, label, data, inline=False))
|
|
continue
|
|
disposition = (part.get_content_disposition() or "").lower()
|
|
media = part.get_content_type()
|
|
filename = part.get_filename()
|
|
content_id = _content_id(part)
|
|
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
|
|
if body:
|
|
text = _decoded_text(part)
|
|
if text is None:
|
|
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
|
|
continue
|
|
if media == "text/plain":
|
|
plain.append(text)
|
|
else:
|
|
html_parts.append(text)
|
|
continue
|
|
if not filename and disposition != "attachment" and not content_id:
|
|
continue
|
|
index += 1
|
|
label = header_text(filename) if filename else f"anlage-{index}.bin"
|
|
data = _leaf_bytes(part)
|
|
if data is None:
|
|
missing.append(label or f"Anlage {index}")
|
|
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
|
|
continue
|
|
name = _unique_filename(label or f"anlage-{index}.bin", used)
|
|
inline = disposition == "inline" or bool(content_id)
|
|
attachments.append(_attachment(part, name, label or name, data, inline=inline))
|
|
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
|
rendered = ""
|
|
if html_parts:
|
|
rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments))
|
|
if not simplified:
|
|
simplified = simplified_text("\n".join(html_parts))
|
|
if missing:
|
|
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
|
|
return ParsedMail(
|
|
message_id=str(message.get("Message-ID") or "").strip(),
|
|
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
|
|
sender=header_text(message.get("From")),
|
|
recipient=header_text(message.get("To")),
|
|
copy=header_text(message.get("Cc")),
|
|
text=simplified,
|
|
attachments=attachments,
|
|
missing=missing,
|
|
html=rendered,
|
|
notices=notices,
|
|
document_on=_message_date(message.get("Date")),
|
|
)
|
|
|
|
|
|
def decode_transfer(payload: bytes, charset: str | None) -> str:
|
|
return bytes(payload).decode(charset or "utf-8", errors="replace")
|
|
|
|
|
|
def simplified_text(value: str) -> str:
|
|
"""A recognizable plain reading. It is not the formatted view."""
|
|
class Collector(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts: list[str] = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
name = tag.lower()
|
|
if name in {"script", "style"}:
|
|
self.skip += 1
|
|
return
|
|
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
|
|
self.parts.append("\n")
|
|
elif name in {"td", "th"}:
|
|
self.parts.append("\t")
|
|
|
|
def handle_endtag(self, tag):
|
|
name = tag.lower()
|
|
if name in {"script", "style"} and self.skip:
|
|
self.skip -= 1
|
|
return
|
|
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
|
|
self.parts.append("\n")
|
|
|
|
def handle_data(self, data):
|
|
if not self.skip:
|
|
self.parts.append(data)
|
|
|
|
collector = Collector()
|
|
try:
|
|
collector.feed(value)
|
|
collector.close()
|
|
except Exception:
|
|
return ""
|
|
text = "".join(collector.parts)
|
|
text = text.replace("\r", "")
|
|
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
def sanitize_html(value: str, images: dict[str, str] | None = None) -> str:
|
|
"""Keep a formatted reading. Scripts, active content and remote loads stay out."""
|
|
images = images or {}
|
|
|
|
class Cleaner(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts: list[str] = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
name = tag.lower()
|
|
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
|
self.skip += 1
|
|
return
|
|
if self.skip or name not in _ALLOWED_TAGS:
|
|
return
|
|
rendered = _clean_attrs(name, attrs, images)
|
|
if rendered is None:
|
|
return
|
|
self.parts.append(f"<{name}{rendered}>")
|
|
|
|
def handle_endtag(self, tag):
|
|
name = tag.lower()
|
|
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
|
if self.skip:
|
|
self.skip -= 1
|
|
return
|
|
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
|
|
return
|
|
self.parts.append(f"</{name}>")
|
|
|
|
def handle_data(self, data):
|
|
if not self.skip:
|
|
self.parts.append(html.escape(data))
|
|
|
|
def handle_entityref(self, name):
|
|
if not self.skip:
|
|
self.parts.append(f"&{name};")
|
|
|
|
def handle_charref(self, name):
|
|
if not self.skip:
|
|
self.parts.append(f"&#{name};")
|
|
|
|
cleaner = Cleaner()
|
|
try:
|
|
cleaner.feed(value)
|
|
cleaner.close()
|
|
except Exception:
|
|
return ""
|
|
return "".join(cleaner.parts).strip()
|
|
|
|
|
|
def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None:
|
|
kept: list[str] = []
|
|
for key, value in attrs:
|
|
name = (key or "").lower()
|
|
if name.startswith("on") or name in {"style", "srcset"}:
|
|
continue
|
|
if tag == "a" and name == "href":
|
|
target = (value or "").strip()
|
|
if not _SAFE_LINK.match(target):
|
|
continue
|
|
kept.append(f' href="{html.escape(target, quote=True)}"')
|
|
elif tag == "img" and name == "src":
|
|
source = (value or "").strip()
|
|
if source.lower().startswith("cid:"):
|
|
token = source[4:].strip().strip("<>")
|
|
source = images.get(token.casefold(), "")
|
|
if not _DATA_IMAGE.match(source):
|
|
continue
|
|
kept.append(f' src="{source}"')
|
|
elif tag == "img" and name == "alt":
|
|
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
|
|
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
|
|
kept.append(f" {name}=\"{int(value)}\"")
|
|
if tag == "img" and not any(item.startswith(" src=") for item in kept):
|
|
return None
|
|
if tag == "a":
|
|
kept.append(' rel="noreferrer"')
|
|
return "".join(kept)
|
|
|
|
|
|
def _walk(message: EmailMessage):
|
|
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
|
|
yield "message", message
|
|
return
|
|
if message.is_multipart():
|
|
for child in message.iter_parts():
|
|
yield from _walk(child)
|
|
return
|
|
yield "leaf", message
|
|
|
|
|
|
def _decoded_text(part: EmailMessage) -> str | None:
|
|
try:
|
|
payload = part.get_payload(decode=True)
|
|
except Exception:
|
|
return None
|
|
if not isinstance(payload, (bytes, bytearray)):
|
|
raw = part.get_payload()
|
|
return raw.strip() if isinstance(raw, str) else None
|
|
charset = part.get_content_charset() or "utf-8"
|
|
try:
|
|
return decode_transfer(bytes(payload), charset).strip()
|
|
except LookupError:
|
|
return decode_transfer(bytes(payload), "utf-8").strip()
|
|
|
|
|
|
def _leaf_bytes(part: EmailMessage) -> bytes | None:
|
|
try:
|
|
payload = part.get_payload(decode=True)
|
|
except Exception:
|
|
return None
|
|
if isinstance(payload, (bytes, bytearray)):
|
|
return bytes(payload)
|
|
return None
|
|
|
|
|
|
def _message_bytes(part: EmailMessage) -> bytes | None:
|
|
payload = part.get_payload()
|
|
if isinstance(payload, list) and payload:
|
|
inner = payload[0]
|
|
if isinstance(inner, EmailMessage):
|
|
return inner.as_bytes()
|
|
data = _leaf_bytes(part)
|
|
return data or None
|
|
|
|
|
|
def _content_id(part: EmailMessage) -> str:
|
|
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
|
|
return token
|
|
|
|
|
|
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
|
|
return AttachmentBytes(
|
|
filename=filename,
|
|
media_type=(part.get_content_type() or "application/octet-stream")[:120],
|
|
data=data,
|
|
sha256=hashlib.sha256(data).hexdigest(),
|
|
display_name=label or filename,
|
|
content_id=_content_id(part),
|
|
inline=inline,
|
|
)
|
|
|
|
|
|
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
|
|
import base64
|
|
|
|
images: dict[str, str] = {}
|
|
for item in attachments:
|
|
if not item.content_id or not item.media_type.startswith("image/"):
|
|
continue
|
|
if len(item.data) > 1_500_000:
|
|
continue
|
|
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
|
|
if subtype == "jpg":
|
|
subtype = "jpeg"
|
|
if subtype not in {"png", "jpeg", "gif", "webp"}:
|
|
continue
|
|
encoded = base64.b64encode(item.data).decode("ascii")
|
|
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
|
|
return images
|
|
|
|
|
|
def _message_date(value) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(str(value)).date().isoformat()
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _bare(value: str) -> str:
|
|
return value.strip().strip("<>").strip().casefold()
|
|
|
|
|
|
def _unique_filename(name: str, used: set[str]) -> str:
|
|
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
|
|
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
|
|
if cleaned not in used:
|
|
used.add(cleaned)
|
|
return cleaned
|
|
stem, dot, ext = cleaned.rpartition(".")
|
|
if not dot or not stem:
|
|
stem, ext = cleaned, ""
|
|
else:
|
|
ext = f".{ext}"
|
|
number = 2
|
|
while True:
|
|
candidate = f"{stem} ({number}){ext}"
|
|
if candidate not in used:
|
|
used.add(candidate)
|
|
return candidate
|
|
number += 1
|