From 7a1206b18cb4b614705a1001dfc7b2b1885e153b Mon Sep 17 00:00:00 2001 From: Lars Date: Fri, 23 Oct 2026 13:00:25 +0200 Subject: [PATCH] =?UTF-8?q?Speichere=20die=20neue=20Position=20einer=20ver?= =?UTF-8?q?schobenen=20IMAP-Nachricht,=20damit=20der=20n=C3=A4chste=20Aufr?= =?UTF-8?q?uf=20sie=20direkt=20=C3=B6ffnet.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- backend/app/retrieval.py | 2 +- backend/app/sourcebind.py | 39 +++++++ backend/app/sources.py | 85 +++++++++++--- backend/tests/test_mailboxes.py | 192 ++++++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 19 deletions(-) diff --git a/backend/app/retrieval.py b/backend/app/retrieval.py index dc041e3..050a2db 100644 --- a/backend/app/retrieval.py +++ b/backend/app/retrieval.py @@ -44,7 +44,7 @@ def load_external(conn, actor_id: UUID, element_id: UUID, assigned: UUID | None) ref = conn.execute( """ - SELECT source, external_id, detail, connector_id, resolution + SELECT element_id, source, external_id, detail, connector_id, resolution FROM external_references WHERE element_id = %s """, (element_id,), diff --git a/backend/app/sourcebind.py b/backend/app/sourcebind.py index f61ee52..a5a0ae8 100644 --- a/backend/app/sourcebind.py +++ b/backend/app/sourcebind.py @@ -217,3 +217,42 @@ def assign_reference(conn, actor_id: UUID, element_id: UUID, connector_id: UUID) detail = dict(ref["detail"] or {}) detail["connector_id"] = str(connector_id) _set_binding(conn, element_id, connector_id, "bound", detail) + + +def remember_imap_location( + conn, + element_id: UUID, + external_id: str, + previous_mailbox: str, + previous_uid: int, + mailbox: str, + uid: int, +) -> None: + """Store the folder and UID of a message found at a new place. + + Only the previous place is replaced. The message identity, the connector + and an adopted direction stay as they were. + """ + if ( + not mailbox + or any(char in mailbox for char in "\r\n\x00") + or uid < 1 + or (mailbox == previous_mailbox and uid == previous_uid) + ): + return + conn.execute( + """ + UPDATE external_references + SET detail = jsonb_set( + jsonb_set(detail, '{mailbox}', to_jsonb(%s::text)), + '{uid}', + to_jsonb(%s::bigint) + ) + WHERE element_id = %s + AND source = 'imap' + AND external_id = %s + AND detail->>'mailbox' = %s + AND detail->>'uid' = %s + """, + (mailbox, uid, element_id, external_id, previous_mailbox, str(previous_uid)), + ) diff --git a/backend/app/sources.py b/backend/app/sources.py index 050cf83..b302840 100644 --- a/backend/app/sources.py +++ b/backend/app/sources.py @@ -7,6 +7,7 @@ import imaplib import json import re import ssl +from dataclasses import dataclass from email.header import decode_header from email.parser import BytesParser from email.utils import parsedate_to_datetime @@ -33,7 +34,7 @@ from app.mailpack import ( simplified_text, ) from app.media.storage import sniff -from app.sourcebind import resolve_for_read +from app.sourcebind import remember_imap_location, resolve_for_read from app.sourceaccess import ( ORIGINAL_UNAVAILABLE, assert_still_owned, @@ -509,7 +510,8 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r def fetch(row): return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT) - return complete_borrow(conn, borrowed, fetch) + found = complete_borrow(conn, borrowed, fetch) + return _keep_moved_mail(conn, element["id"], external_id, mailbox, uid, found) def _uuid(value) -> UUID | None: @@ -1470,11 +1472,33 @@ def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, u row = _owned(conn, owner_id, connector_id) if row["kind"] != "imap": raise unprocessable("Das ist kein E-Mail-Konto.") - return pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT) + found = pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT) + return found.raw if isinstance(found, LocatedMail) else found -def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> bytes: - """Fetch the referenced message. A different message at the old UID is not returned.""" +@dataclass(frozen=True) +class LocatedMail: + """Bytes plus the folder they were read from. `relocated` is a complete, unique hit.""" + + raw: bytes + mailbox: str + uid: int + relocated: bool = False + + +@dataclass(frozen=True) +class _MailHit: + raw: bytes + mailbox: str + uid: int + + +def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> LocatedMail: + """Fetch the referenced message. A different message at the old UID is not returned. + + The stored folder is tried first. A single Message-ID hit elsewhere comes back + marked relocated; the caller stores that place after the access check. + """ client = _imap_login(settings, secret) try: _select_mailbox(client, mailbox) @@ -1484,23 +1508,45 @@ def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) - alternates: list[bytes] = [] - if candidate is None or not identities_match(header_message_id(candidate), expected_id): - alternates = _search_rfc822(client, expected_id, max_bytes) + if candidate is not None and identities_match(header_message_id(candidate), expected_id): + return LocatedMail(candidate, mailbox, uid, relocated=False) + hits, capped = _search_rfc822(client, expected_id, max_bytes) try: - chosen = choose_message(candidate, alternates, expected_id) + chosen = choose_message(None, [hit.raw for hit in hits], expected_id) except MailIdentityError as exc: raise unprocessable(exc.message) from exc - if len(chosen) > max_bytes: + match = next( + hit for hit in hits + if hit.raw == chosen and identities_match(header_message_id(hit.raw), expected_id) + ) + if len(match.raw) > max_bytes: raise unprocessable( "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) - return chosen + moved = not capped and (match.mailbox != mailbox or match.uid != uid) + return LocatedMail(match.raw, match.mailbox, match.uid, relocated=moved) finally: _imap_logout(client) +def _keep_moved_mail(conn, element_id, external_id: str, mailbox: str, uid: int | None, found) -> bytes: + """Return the bytes. A unique new place is written only after the fetch has finished.""" + if not isinstance(found, LocatedMail): + return found + if found.relocated and element_id is not None and uid is not None: + remember_imap_location( + conn, + element_id, + external_id, + mailbox, + uid, + found.mailbox, + found.uid, + ) + return found.raw + + def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None: try: status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])") @@ -1511,11 +1557,12 @@ def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None: return _literal(fetched) -def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> list[bytes]: +def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> tuple[list[_MailHit], bool]: + """Search every selectable folder. The second value is true when the scan stopped early.""" token = (expected_id or "").strip() if not token or token.casefold().startswith("uid:"): - return [] - found: list[bytes] = [] + return [], False + found: list[_MailHit] = [] for folder in _imap_mailboxes(client): if not folder["selectable"]: continue @@ -1539,10 +1586,10 @@ def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> l "Die vollständige Nachricht überschreitet die Exportgrenze. " "Eine gekürzte Vorschau wird nicht als Original übernommen." ) - found.append(blob) + found.append(_MailHit(blob, folder["name"], uid)) if len(found) > 5: - return found - return found + return found, True + return found, False PAPERLESS_FIELDS = ( @@ -1667,13 +1714,15 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned: owner_id = resolved.row["owner_id"] release_for_external(conn) try: - raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT) + found = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT) except Problem as exc: return failure_from_problem(exc) try: assert_still_owned(conn, connector_id, owner_id) except Problem as exc: return failure_from_problem(exc) + element_id = ref["element_id"] if ref is not None and "element_id" in ref.keys() else None + raw = _keep_moved_mail(conn, element_id, str(external_id), mailbox, uid, found) if len(raw) > EXPORT_BYTE_LIMIT: return RetrievalFailure( "too_large", diff --git a/backend/tests/test_mailboxes.py b/backend/tests/test_mailboxes.py index c1756af..51205be 100644 --- a/backend/tests/test_mailboxes.py +++ b/backend/tests/test_mailboxes.py @@ -285,3 +285,195 @@ def test_an_adopted_message_loads_in_full_with_its_attachment(client, db, monkey ada.request("GET", f"/api/v1/external-refs/{wrong['element']['id']}/mail", status=422) ben.login() ben.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=404) + + +def _letter(message_id: str, body: str) -> bytes: + return ( + f"From: Ada Beispiel \r\n" + f"To: Gegenseite \r\n" + f"Subject: Verschoben\r\n" + f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n" + f"Message-ID: {message_id}\r\n" + f"MIME-Version: 1.0\r\n" + f"Content-Type: text/plain; charset=utf-8\r\n" + f"\r\n" + f"{body}\r\n" + ).encode() + + +class RelocatingIMAP: + """The saved UID holds another message. The Message-ID lives in Archiv.""" + + log: list[tuple] = [] + copies: dict[tuple[str, int], bytes] = {} + header_uids: dict[str, bytes] | None = None + + def __init__(self, *args, **kwargs): + self.selected = "" + + def login(self, username, secret): + assert secret == "Postfach-Geheim" + return "OK", [b"logged"] + + def list(self): + return "OK", [ + b'(\\HasNoChildren) "/" "INBOX"', + b'(\\HasNoChildren) "/" "Archiv"', + ] + + def select(self, mailbox, readonly=False): + assert readonly is True + self.selected = mailbox + RelocatingIMAP.log.append(("select", mailbox)) + return "OK", [b"2"] + + def uid(self, command, *args): + if command == "SEARCH": + if args and args[0] == "HEADER": + RelocatingIMAP.log.append(("header", self.selected)) + override = (RelocatingIMAP.header_uids or {}).get(self.selected) + if override is not None: + return "OK", [override] + found = [ + str(uid) + for (folder, uid), raw in RelocatingIMAP.copies.items() + if folder == self.selected and b"Message-ID: " in raw + ] + return "OK", [" ".join(found).encode()] + return "OK", [b""] + uid = int(args[0]) + spec = " ".join(str(part) for part in args[1:]) + if "BODY.PEEK[]" in spec: + RelocatingIMAP.log.append(("fetch", self.selected, uid)) + raw = RelocatingIMAP.copies.get((self.selected, uid)) + if raw is None: + return "OK", [None] + return "OK", [(b"rfc822", raw)] + return "OK", [None] + + def logout(self): + return "BYE", [b""] + + +def _stored_mail(db, element_id: str) -> dict: + row = db.execute( + "SELECT detail FROM external_references WHERE element_id = %s", + (element_id,), + ).fetchone() + return dict(row["detail"]) + + +def _linked_mail(actor: Actor, account: str, matter_id: str) -> dict: + return actor.request( + "POST", + "/api/v1/external-refs", + json={ + "title": "Verschoben", + "source": "imap", + "external_id": "", + "context": {"kind": "matter", "id": matter_id}, + "detail": { + "connector_id": account, + "mailbox": "INBOX", + "uid": 2, + "direction": "in", + }, + }, + status=201, + ) + + +def test_a_moved_message_remembers_its_new_folder(client, db, monkeypatch): + from uuid import UUID + + from app.retrieval import MailPayload, load_external + + ada = Actor(client, "ada-move@example.test", "Ada Umzug") + account_row = register(db, ada.email, ada.name) + ada.login() + RelocatingIMAP.log = [] + RelocatingIMAP.header_uids = None + RelocatingIMAP.copies = { + ("INBOX", 2): _letter("", "Eine andere Nachricht."), + ("Archiv", 9): _letter("", "Der Brief liegt jetzt im Archiv."), + } + account = _account(monkeypatch, ada) + monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP) + matter = matter_of(ada, "Ablage") + created = _linked_mail(ada, account, matter["id"]) + + opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") + assert "Archiv" in opened["text"] + stored = _stored_mail(db, created["id"]) + assert stored["mailbox"] == "Archiv" + assert stored["uid"] == 9 + assert stored["direction"] == "in" + assert stored["connector_id"] == account + assert any(item[0] == "header" for item in RelocatingIMAP.log) + + RelocatingIMAP.log = [] + again = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") + assert again["text"] == opened["text"] + assert ("fetch", "Archiv", 9) in RelocatingIMAP.log + assert not any(item[0] == "header" for item in RelocatingIMAP.log) + assert _stored_mail(db, created["id"])["uid"] == 9 + + db.execute( + """ + UPDATE external_references + SET detail = jsonb_set( + jsonb_set(detail, '{mailbox}', to_jsonb('INBOX'::text)), + '{uid}', + to_jsonb(2) + ) + WHERE element_id = %s + """, + (created["id"],), + ) + db.commit() + RelocatingIMAP.log = [] + loaded = load_external(db, account_row["id"], UUID(created["id"]), None) + assert isinstance(loaded, MailPayload) + assert "Archiv" in loaded.parsed.text + exported = _stored_mail(db, created["id"]) + assert exported["mailbox"] == "Archiv" + assert exported["uid"] == 9 + assert exported["direction"] == "in" + assert any(item[0] == "header" for item in RelocatingIMAP.log) + + +def test_an_ambiguous_or_truncated_move_does_not_replace_the_stored_place(client, db, monkeypatch): + ada = Actor(client, "ada-ambiguous@example.test", "Ada Zweifach") + register(db, ada.email, ada.name) + ada.login() + RelocatingIMAP.header_uids = None + RelocatingIMAP.copies = { + ("INBOX", 2): _letter("", "Altbestand."), + ("INBOX", 4): _letter("", "Erste Kopie."), + ("Archiv", 9): _letter("", "Zweite Kopie."), + } + account = _account(monkeypatch, ada) + monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP) + matter = matter_of(ada, "Zweifach") + created = _linked_mail(ada, account, matter["id"]) + ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=422) + assert _stored_mail(db, created["id"])["mailbox"] == "INBOX" + assert _stored_mail(db, created["id"])["uid"] == 2 + + RelocatingIMAP.copies = { + ("INBOX", 2): _letter("", "Altbestand."), + ("INBOX", 11): _letter("", "Treffer in der abgebrochenen Suche."), + ("INBOX", 12): _letter("", "a"), + ("INBOX", 13): _letter("", "b"), + ("INBOX", 14): _letter("", "c"), + ("INBOX", 15): _letter("", "d"), + ("INBOX", 16): _letter("", "e"), + ("Archiv", 9): _letter("", "Eigentlich auch hier."), + } + RelocatingIMAP.header_uids = {"INBOX": b"11 12 13 14 15 16"} + RelocatingIMAP.log = [] + opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") + assert "abgebrochenen Suche" in opened["text"] + assert not any(item == ("header", "Archiv") for item in RelocatingIMAP.log) + assert _stored_mail(db, created["id"])["mailbox"] == "INBOX" + assert _stored_mail(db, created["id"])["uid"] == 2