Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
34c4084a7c
commit
c29bd29f34
29 files changed
+2562
-410
No files matched your search
+311
-66
@@ -1,16 +1,32 @@
|
||||
"""Pure mail identity and attachment extraction for a complete export.
|
||||
|
||||
A preview cutoff does not belong here. Duplicate attachment names stay distinct.
|
||||
The same tree feeds preview, reading, attachment download and export.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from email import policy
|
||||
from email.header import decode_header, make_header
|
||||
from email.message import EmailMessage
|
||||
from email.parser import BytesParser
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
|
||||
_UID_PREFIX = "uid:"
|
||||
_POLICY = policy.default
|
||||
_ALLOWED_TAGS = {
|
||||
"a", "p", "br", "div", "span", "strong", "em", "b", "i", "u",
|
||||
"ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
|
||||
"blockquote", "pre", "code", "h1", "h2", "h3", "h4", "h5", "h6", "img",
|
||||
}
|
||||
_VOID_TAGS = {"br", "img"}
|
||||
_SAFE_LINK = re.compile(r"^(https?://|mailto:)", re.IGNORECASE)
|
||||
_DATA_IMAGE = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,[a-z0-9+/=\s]+$", re.IGNORECASE)
|
||||
|
||||
|
||||
class MailIdentityError(Exception):
|
||||
@@ -25,6 +41,9 @@ class AttachmentBytes:
|
||||
media_type: str
|
||||
data: bytes
|
||||
sha256: str
|
||||
display_name: str = ""
|
||||
content_id: str = ""
|
||||
inline: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -37,10 +56,14 @@ class ParsedMail:
|
||||
text: str
|
||||
attachments: list[AttachmentBytes] = field(default_factory=list)
|
||||
missing: list[str] = field(default_factory=list)
|
||||
html: str = ""
|
||||
notices: list[str] = field(default_factory=list)
|
||||
text_truncated: bool = False
|
||||
document_on: str | None = None
|
||||
|
||||
|
||||
def header_message_id(raw: bytes) -> str:
|
||||
message = BytesParser().parsebytes(raw)
|
||||
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
||||
return str(message.get("Message-ID") or "").strip()
|
||||
|
||||
|
||||
@@ -69,67 +92,322 @@ def choose_message(candidate: bytes | None, alternates: list[bytes], expected: s
|
||||
raise MailIdentityError("Die gespeicherte Nachricht lässt sich nicht eindeutig wiederfinden.")
|
||||
|
||||
|
||||
def header_text(value) -> str:
|
||||
"""Decode encoded-words. A raw MIME header is not a display name."""
|
||||
if value is None:
|
||||
return ""
|
||||
raw = str(value).replace("\r", " ").replace("\n", " ")
|
||||
try:
|
||||
text = str(make_header(decode_header(raw)))
|
||||
except (UnicodeError, LookupError, ValueError):
|
||||
text = raw
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def parse_mail(raw: bytes) -> ParsedMail:
|
||||
message = BytesParser().parsebytes(raw)
|
||||
message = BytesParser(policy=_POLICY).parsebytes(raw)
|
||||
attachments: list[AttachmentBytes] = []
|
||||
missing: list[str] = []
|
||||
notices: list[str] = []
|
||||
used: set[str] = set()
|
||||
plain: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
index = 0
|
||||
for part in message.walk():
|
||||
if part.is_multipart():
|
||||
for kind, part in _walk(message):
|
||||
if kind == "message":
|
||||
index += 1
|
||||
label = header_text(part.get_filename()) or f"nachricht-{index}.eml"
|
||||
data = _message_bytes(part)
|
||||
if data is None:
|
||||
missing.append(label)
|
||||
notices.append(f"Die angehängte Nachricht {label} ließ sich nicht vollständig lesen.")
|
||||
continue
|
||||
name = _unique_filename(label, used)
|
||||
attachments.append(_attachment(part, name, label, data, inline=False))
|
||||
continue
|
||||
filename = part.get_filename()
|
||||
disposition = (part.get_content_disposition() or "").lower()
|
||||
if not filename and disposition != "attachment":
|
||||
media = part.get_content_type()
|
||||
filename = part.get_filename()
|
||||
content_id = _content_id(part)
|
||||
body = media in {"text/plain", "text/html"} and disposition != "attachment" and not filename
|
||||
if body:
|
||||
text = _decoded_text(part)
|
||||
if text is None:
|
||||
notices.append("Ein Textteil ließ sich nicht sicher dekodieren.")
|
||||
continue
|
||||
if media == "text/plain":
|
||||
plain.append(text)
|
||||
else:
|
||||
html_parts.append(text)
|
||||
continue
|
||||
if not filename and disposition != "attachment" and not content_id:
|
||||
continue
|
||||
index += 1
|
||||
label = _header_text(filename) if filename else f"anlage-{index}.bin"
|
||||
payload = part.get_payload(decode=True)
|
||||
if not isinstance(payload, (bytes, bytearray)):
|
||||
label = header_text(filename) if filename else f"anlage-{index}.bin"
|
||||
data = _leaf_bytes(part)
|
||||
if data is None:
|
||||
missing.append(label or f"Anlage {index}")
|
||||
notices.append(f"Die Anlage {label or index} ließ sich nicht vollständig lesen.")
|
||||
continue
|
||||
name = _unique_filename(label or f"anlage-{index}.bin", used)
|
||||
data = bytes(payload)
|
||||
attachments.append(
|
||||
AttachmentBytes(
|
||||
filename=name,
|
||||
media_type=(part.get_content_type() or "application/octet-stream")[:120],
|
||||
data=data,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
)
|
||||
)
|
||||
text = _body_text(message)
|
||||
inline = disposition == "inline" or bool(content_id)
|
||||
attachments.append(_attachment(part, name, label or name, data, inline=inline))
|
||||
simplified = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
rendered = ""
|
||||
if html_parts:
|
||||
rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments))
|
||||
if not simplified:
|
||||
simplified = simplified_text("\n".join(html_parts))
|
||||
if missing:
|
||||
notices.append("Mindestens eine Anlage fehlt in der Aufbereitung.")
|
||||
return ParsedMail(
|
||||
message_id=str(message.get("Message-ID") or "").strip(),
|
||||
subject=_header_text(message.get("Subject")) or "(ohne Betreff)",
|
||||
sender=_header_text(message.get("From")),
|
||||
recipient=_header_text(message.get("To")),
|
||||
copy=_header_text(message.get("Cc")),
|
||||
text=text,
|
||||
subject=header_text(message.get("Subject")) or "(ohne Betreff)",
|
||||
sender=header_text(message.get("From")),
|
||||
recipient=header_text(message.get("To")),
|
||||
copy=header_text(message.get("Cc")),
|
||||
text=simplified,
|
||||
attachments=attachments,
|
||||
missing=missing,
|
||||
html=rendered,
|
||||
notices=notices,
|
||||
document_on=_message_date(message.get("Date")),
|
||||
)
|
||||
|
||||
|
||||
def decode_transfer(payload: bytes, charset: str | None) -> str:
|
||||
return bytes(payload).decode(charset or "utf-8", errors="replace")
|
||||
|
||||
|
||||
def simplified_text(value: str) -> str:
|
||||
"""A recognizable plain reading. It is not the formatted view."""
|
||||
class Collector(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts: list[str] = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style"}:
|
||||
self.skip += 1
|
||||
return
|
||||
if name in {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"}:
|
||||
self.parts.append("\n")
|
||||
elif name in {"td", "th"}:
|
||||
self.parts.append("\t")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style"} and self.skip:
|
||||
self.skip -= 1
|
||||
return
|
||||
if name in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||
self.parts.append("\n")
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.skip:
|
||||
self.parts.append(data)
|
||||
|
||||
collector = Collector()
|
||||
try:
|
||||
collector.feed(value)
|
||||
collector.close()
|
||||
except Exception:
|
||||
return ""
|
||||
text = "".join(collector.parts)
|
||||
text = text.replace("\r", "")
|
||||
text = re.sub(r"[ \t]+\n", "\n", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def sanitize_html(value: str, images: dict[str, str] | None = None) -> str:
|
||||
"""Keep a formatted reading. Scripts, active content and remote loads stay out."""
|
||||
images = images or {}
|
||||
|
||||
class Cleaner(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts: list[str] = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
||||
self.skip += 1
|
||||
return
|
||||
if self.skip or name not in _ALLOWED_TAGS:
|
||||
return
|
||||
rendered = _clean_attrs(name, attrs, images)
|
||||
if rendered is None:
|
||||
return
|
||||
self.parts.append(f"<{name}{rendered}>")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
name = tag.lower()
|
||||
if name in {"script", "style", "iframe", "object", "embed", "form", "link", "meta"}:
|
||||
if self.skip:
|
||||
self.skip -= 1
|
||||
return
|
||||
if self.skip or name not in _ALLOWED_TAGS or name in _VOID_TAGS:
|
||||
return
|
||||
self.parts.append(f"</{name}>")
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.skip:
|
||||
self.parts.append(html.escape(data))
|
||||
|
||||
def handle_entityref(self, name):
|
||||
if not self.skip:
|
||||
self.parts.append(f"&{name};")
|
||||
|
||||
def handle_charref(self, name):
|
||||
if not self.skip:
|
||||
self.parts.append(f"&#{name};")
|
||||
|
||||
cleaner = Cleaner()
|
||||
try:
|
||||
cleaner.feed(value)
|
||||
cleaner.close()
|
||||
except Exception:
|
||||
return ""
|
||||
return "".join(cleaner.parts).strip()
|
||||
|
||||
|
||||
def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None:
|
||||
kept: list[str] = []
|
||||
for key, value in attrs:
|
||||
name = (key or "").lower()
|
||||
if name.startswith("on") or name in {"style", "srcset"}:
|
||||
continue
|
||||
if tag == "a" and name == "href":
|
||||
target = (value or "").strip()
|
||||
if not _SAFE_LINK.match(target):
|
||||
continue
|
||||
kept.append(f' href="{html.escape(target, quote=True)}"')
|
||||
elif tag == "img" and name == "src":
|
||||
source = (value or "").strip()
|
||||
if source.lower().startswith("cid:"):
|
||||
token = source[4:].strip().strip("<>")
|
||||
source = images.get(token.casefold(), "")
|
||||
if not _DATA_IMAGE.match(source):
|
||||
continue
|
||||
kept.append(f' src="{source}"')
|
||||
elif tag == "img" and name == "alt":
|
||||
kept.append(f' alt="{html.escape(value or "", quote=True)}"')
|
||||
elif name in {"colspan", "rowspan"} and str(value or "").isdigit():
|
||||
kept.append(f" {name}=\"{int(value)}\"")
|
||||
if tag == "img" and not any(item.startswith(" src=") for item in kept):
|
||||
return None
|
||||
if tag == "a":
|
||||
kept.append(' rel="noreferrer"')
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def _walk(message: EmailMessage):
|
||||
if message.get_content_maintype() == "message" and message.get_content_subtype() == "rfc822":
|
||||
yield "message", message
|
||||
return
|
||||
if message.is_multipart():
|
||||
for child in message.iter_parts():
|
||||
yield from _walk(child)
|
||||
return
|
||||
yield "leaf", message
|
||||
|
||||
|
||||
def _decoded_text(part: EmailMessage) -> str | None:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(payload, (bytes, bytearray)):
|
||||
raw = part.get_payload()
|
||||
return raw.strip() if isinstance(raw, str) else None
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
return decode_transfer(bytes(payload), charset).strip()
|
||||
except LookupError:
|
||||
return decode_transfer(bytes(payload), "utf-8").strip()
|
||||
|
||||
|
||||
def _leaf_bytes(part: EmailMessage) -> bytes | None:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(payload, (bytes, bytearray)):
|
||||
return bytes(payload)
|
||||
return None
|
||||
|
||||
|
||||
def _message_bytes(part: EmailMessage) -> bytes | None:
|
||||
payload = part.get_payload()
|
||||
if isinstance(payload, list) and payload:
|
||||
inner = payload[0]
|
||||
if isinstance(inner, EmailMessage):
|
||||
return inner.as_bytes()
|
||||
data = _leaf_bytes(part)
|
||||
return data or None
|
||||
|
||||
|
||||
def _content_id(part: EmailMessage) -> str:
|
||||
token = str(part.get("Content-ID") or "").strip().strip("<>").strip()
|
||||
return token
|
||||
|
||||
|
||||
def _attachment(part: EmailMessage, filename: str, label: str, data: bytes, inline: bool) -> AttachmentBytes:
|
||||
return AttachmentBytes(
|
||||
filename=filename,
|
||||
media_type=(part.get_content_type() or "application/octet-stream")[:120],
|
||||
data=data,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
display_name=label or filename,
|
||||
content_id=_content_id(part),
|
||||
inline=inline,
|
||||
)
|
||||
|
||||
|
||||
def _inline_images(attachments: list[AttachmentBytes]) -> dict[str, str]:
|
||||
import base64
|
||||
|
||||
images: dict[str, str] = {}
|
||||
for item in attachments:
|
||||
if not item.content_id or not item.media_type.startswith("image/"):
|
||||
continue
|
||||
if len(item.data) > 1_500_000:
|
||||
continue
|
||||
subtype = item.media_type.split("/", 1)[1].split(";", 1)[0].lower()
|
||||
if subtype == "jpg":
|
||||
subtype = "jpeg"
|
||||
if subtype not in {"png", "jpeg", "gif", "webp"}:
|
||||
continue
|
||||
encoded = base64.b64encode(item.data).decode("ascii")
|
||||
images[item.content_id.casefold()] = f"data:image/{subtype};base64,{encoded}"
|
||||
return images
|
||||
|
||||
|
||||
def _message_date(value) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return parsedate_to_datetime(str(value)).date().isoformat()
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _bare(value: str) -> str:
|
||||
return value.strip().strip("<>").strip().casefold()
|
||||
|
||||
|
||||
def _header_text(value) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
text = str(value).replace("\r", " ").replace("\n", " ")
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _unique_filename(name: str, used: set[str]) -> str:
|
||||
cleaned = name.replace("\\", "/").split("/")[-1].strip() or "anlage.bin"
|
||||
cleaned = cleaned[:180]
|
||||
cleaned = re.sub(r"[\r\n\x00]", "", cleaned)[:180] or "anlage.bin"
|
||||
if cleaned not in used:
|
||||
used.add(cleaned)
|
||||
return cleaned
|
||||
stem, dot, ext = cleaned.rpartition(".")
|
||||
if not dot:
|
||||
if not dot or not stem:
|
||||
stem, ext = cleaned, ""
|
||||
else:
|
||||
ext = f".{ext}"
|
||||
@@ -140,36 +418,3 @@ def _unique_filename(name: str, used: set[str]) -> str:
|
||||
used.add(candidate)
|
||||
return candidate
|
||||
number += 1
|
||||
|
||||
|
||||
def _body_text(message) -> str:
|
||||
plain: list[str] = []
|
||||
html: list[str] = []
|
||||
parts = message.walk() if message.is_multipart() else [message]
|
||||
for part in parts:
|
||||
if part.is_multipart() or part.get_content_disposition() == "attachment" or part.get_filename():
|
||||
continue
|
||||
kind = part.get_content_type()
|
||||
payload = part.get_payload(decode=True)
|
||||
if isinstance(payload, (bytes, bytearray)):
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
text = bytes(payload).decode(charset, errors="replace").strip()
|
||||
else:
|
||||
raw = part.get_payload()
|
||||
text = raw.strip() if isinstance(raw, str) else ""
|
||||
if kind == "text/plain":
|
||||
plain.append(text)
|
||||
elif kind == "text/html":
|
||||
html.append(_html_text(text))
|
||||
body = "\n".join(chunk for chunk in plain if chunk).strip()
|
||||
if not body:
|
||||
body = "\n".join(chunk for chunk in html if chunk).strip()
|
||||
return body
|
||||
|
||||
|
||||
def _html_text(value: str) -> str:
|
||||
text = re.sub(r"(?is)<(script|style)\b.*?>.*?</\1>", " ", value)
|
||||
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
|
||||
text = re.sub(r"(?i)</p>", "\n", text)
|
||||
text = re.sub(r"<[^>]+>", " ", text)
|
||||
return re.sub(r"[ \t]{2,}", " ", text).strip()
|
||||
Reference in new issue
Block a user