diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 05ecba0..6ca79ae 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -13,7 +13,8 @@ from app.api.deps import Principal, get_conn, get_principal from app.api.routes import _created, _ctx_model from app.api.schemas import ConnectorIn, ExternalIn, SourceAssignmentIn from app.core.errors import DOWNLOAD_REFUSED, forbidden, not_found, unprocessable, unsupported -from app.mailpack import parse_mail +from app.mailpack import _inline_images, parse_mail, sanitize_html +from app.remoteimages import RemoteImages from app.media.storage import content_disposition from app.ops import create_external from app.retrieval import RetrievalFailure @@ -133,6 +134,7 @@ def view_mail( "message_cc": parsed.copy, "text": parsed.text, "html": parsed.html, + "remote_images": len(parsed.remote_images), "text_truncated": parsed.text_truncated, "complete": not parsed.missing, "notices": parsed.notices, @@ -190,6 +192,25 @@ def _complete_mail(conn, principal: Principal, element, context, ref): return parsed, raw +@router.get("/api/v1/external-refs/{element_id}/mail/html-with-images") +def view_mail_with_images( + element_id: UUID, + context_kind: str | None = None, + context_id: str | None = None, + principal: Principal = Depends(get_principal), + conn: Connection = Depends(get_conn), +): + """The formatted mail with its pictures from other servers, loaded by the server on request.""" + element, context, ref = _opened_ref(conn, principal, element_id, context_kind, context_id) + parsed, _raw = _complete_mail(conn, principal, element, context, ref) + loader = RemoteImages() + rendered = sanitize_html(parsed.html_source, _inline_images(parsed.attachments), remote=loader) if parsed.html_source else "" + return JSONResponse( + content={"html": rendered, "loaded": loader.loaded, "failed": loader.failed}, + headers={"Cache-Control": "private, no-store", "Referrer-Policy": "no-referrer"}, + ) + + @router.get("/api/v1/external-refs/{element_id}/mail/original") def download_mail( element_id: UUID, diff --git a/backend/app/mailpack.py b/backend/app/mailpack.py index 3416e94..0dda38f 100644 --- a/backend/app/mailpack.py +++ b/backend/app/mailpack.py @@ -70,6 +70,8 @@ class ParsedMail: attachments: list[AttachmentBytes] = field(default_factory=list) missing: list[str] = field(default_factory=list) html: str = "" + html_source: str = "" + remote_images: list[str] = field(default_factory=list) notices: list[str] = field(default_factory=list) text_truncated: bool = False document_on: str | None = None @@ -168,7 +170,9 @@ def parse_mail(raw: bytes) -> ParsedMail: simplified = "\n".join(chunk for chunk in plain if chunk).strip() rendered = "" if html_parts: - rendered = sanitize_html("\n".join(html_parts), _inline_images(attachments)) + source = "\n".join(html_parts) + remote_found: list[str] = [] + rendered = sanitize_html(source, _inline_images(attachments), found=remote_found) if not simplified: simplified = simplified_text("\n".join(html_parts)) if missing: @@ -183,6 +187,8 @@ def parse_mail(raw: bytes) -> ParsedMail: attachments=attachments, missing=missing, html=rendered, + html_source=source if html_parts else "", + remote_images=remote_found if html_parts else [], notices=notices, document_on=_message_date(message.get("Date")), ) @@ -235,8 +241,11 @@ def simplified_text(value: str) -> str: return text.strip() -def sanitize_html(value: str, images: dict[str, str] | None = None) -> str: - """Keep a formatted reading. Scripts, active content and remote loads stay out.""" +def sanitize_html(value: str, images: dict[str, str] | None = None, *, remote=None, found: list[str] | None = None) -> str: + """Keep a formatted reading. Scripts and active content stay out. + + Pictures from other servers stay out too, unless `remote` turns an address into an inline picture. + `found` collects the addresses that were met.""" images = images or {} class Cleaner(HTMLParser): @@ -252,7 +261,7 @@ def sanitize_html(value: str, images: dict[str, str] | None = None) -> str: return if self.skip or name not in _ALLOWED_TAGS: return - rendered = _clean_attrs(name, attrs, images) + rendered = _clean_attrs(name, attrs, images, remote, found) if rendered is None: return self.parts.append(f"<{name}{rendered}>") @@ -299,7 +308,7 @@ def _safe_style(value: str) -> str: return "; ".join(kept) -def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None: +def _clean_attrs(tag: str, attrs, images: dict[str, str], remote=None, found: list[str] | None = None) -> str | None: kept: list[str] = [] for key, value in attrs: name = (key or "").lower() @@ -332,6 +341,10 @@ def _clean_attrs(tag: str, attrs, images: dict[str, str]) -> str | None: if source.lower().startswith("cid:"): token = source[4:].strip().strip("<>") source = images.get(token.casefold(), "") + elif re.match(r"https://", source, re.IGNORECASE): + if found is not None and source not in found: + found.append(source) + source = remote(source) if remote is not None else "" if not _DATA_IMAGE.match(source): continue kept.append(f' src="{source}"') diff --git a/backend/app/remoteimages.py b/backend/app/remoteimages.py new file mode 100644 index 0000000..8a19173 --- /dev/null +++ b/backend/app/remoteimages.py @@ -0,0 +1,133 @@ +"""Fetching the pictures a mail refers to, on the server and only on request. + +The browser never contacts the sender: the server loads each picture, checks that it really is +a PNG, JPEG, GIF or WebP, and hands it over inline. This hides the reader's address and device. +It cannot hide that the mail was opened, because the sender sees the request. + +Only public https addresses are fetched. The address is resolved once, every result must be a +public one, and the connection goes to exactly that address, so a changing DNS answer cannot +lead to an internal host. Redirects are followed a few times with the same checks. +""" + +from __future__ import annotations + +import base64 +import http.client +import ipaddress +import socket +import ssl +from urllib.parse import urljoin, urlsplit + +MAX_IMAGE_BYTES = 1_500_000 +MAX_REDIRECTS = 3 +TIMEOUT_SECONDS = 5 +MAX_IMAGES = 12 +MAX_TOTAL_BYTES = 6_000_000 + +_SIGNATURES = ( + (b"\x89PNG\r\n\x1a\n", "png"), + (b"\xff\xd8\xff", "jpeg"), + (b"GIF87a", "gif"), + (b"GIF89a", "gif"), +) + + +class _Pinned(http.client.HTTPSConnection): + """An https connection that goes to a fixed, already checked address.""" + + def __init__(self, host: str, address: str, port: int) -> None: + super().__init__(host, port, timeout=TIMEOUT_SECONDS, context=ssl.create_default_context()) + self._address = address + + def connect(self) -> None: + sock = socket.create_connection((self._address, self.port), self.timeout) + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + + +def is_public(address: str) -> bool: + try: + ip = ipaddress.ip_address(address) + except ValueError: + return False + return ip.is_global and not ip.is_multicast + + +def _public_address(host: str, port: int) -> str | None: + try: + found = {item[4][0] for item in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)} + except OSError: + return None + if not found or not all(is_public(address) for address in found): + return None + return sorted(found)[0] + + +def _kind(data: bytes) -> str | None: + for signature, kind in _SIGNATURES: + if data.startswith(signature): + return kind + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "webp" + return None + + +def fetch_image(url: str) -> bytes | None: + """The bytes of a picture, or None when anything about it is not safe.""" + for _ in range(MAX_REDIRECTS + 1): + parts = urlsplit(url) + if parts.scheme != "https" or not parts.hostname or parts.username or parts.password: + return None + port = parts.port or 443 + if port != 443: + return None + address = _public_address(parts.hostname, port) + if address is None: + return None + target = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") + connection = _Pinned(parts.hostname, address, port) + try: + connection.request("GET", target, headers={"User-Agent": "Mozilla/5.0", "Accept": "image/png,image/jpeg,image/gif,image/webp", "Connection": "close"}) + response = connection.getresponse() + if response.status in {301, 302, 303, 307, 308}: + location = response.getheader("Location") + if not location: + return None + url = urljoin(url, location) + continue + if response.status != 200: + return None + data = response.read(MAX_IMAGE_BYTES + 1) + except (OSError, http.client.HTTPException, ssl.SSLError): + return None + finally: + connection.close() + if len(data) > MAX_IMAGE_BYTES or _kind(data) is None: + return None + return data + return None + + +class RemoteImages: + """Loads the pictures of one mail within a budget and remembers each address once.""" + + def __init__(self) -> None: + self.cache: dict[str, str] = {} + self.total = 0 + self.loaded = 0 + self.failed = 0 + + def __call__(self, url: str) -> str: + if url in self.cache: + return self.cache[url] + result = "" + if len(self.cache) < MAX_IMAGES and self.total < MAX_TOTAL_BYTES: + data = fetch_image(url) + if data is not None: + self.total += len(data) + result = f"data:image/{_kind(data)};base64,{base64.b64encode(data).decode('ascii')}" + self.cache[url] = result + if result: + self.loaded += 1 + else: + self.failed += 1 + return result diff --git a/backend/tests/test_mail_html.py b/backend/tests/test_mail_html.py index be1c00c..95c7efe 100644 --- a/backend/tests/test_mail_html.py +++ b/backend/tests/test_mail_html.py @@ -36,3 +36,46 @@ def test_links_and_images_are_limited(): assert 'href="https://ok.example/x"' in cleaned assert 'rel="noreferrer noopener"' in cleaned and 'target="_blank"' in cleaned assert 'src="data:image/png;base64,AAAA"' in cleaned + + +def test_remote_pictures_stay_out_unless_the_server_loads_them(): + source = 'a

Text

' + found: list[str] = [] + assert "") is None + assert module._kind(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "webp" diff --git a/docs/architecture/BENUTZERVERWALTUNG.md b/docs/architecture/BENUTZERVERWALTUNG.md index e4aa7cd..72d387e 100644 --- a/docs/architecture/BENUTZERVERWALTUNG.md +++ b/docs/architecture/BENUTZERVERWALTUNG.md @@ -108,3 +108,7 @@ Der Baum „Meine Ablage“ steht links auf den Listenseiten und auf den Seiten ### E-Mails als HTML Die Mail wird formatiert (Standard) oder als Text gezeigt. Der Server bereinigt das HTML mit einer Positivliste (keine Skripte, Formulare, Rahmen, `