"""IMAP folders are browsed; search stays inside the open folder.""" from __future__ import annotations import app.sources as sources from tests.support import Actor, matter_of, register class FakeIMAP: selects: list[str] = [] def __init__(self, *args, **kwargs): pass def login(self, username, secret): assert secret == "Postfach-Geheim" return "OK", [b"logged"] def list(self): return "OK", [ b'(\\HasNoChildren) "/" "INBOX"', b'(\\HasNoChildren) "/" "&AMQ-nderungen"', b'(\\Noselect \\HasChildren) "/" "[Gmail]"', b'(\\HasNoChildren) "/" "[Gmail]/Gesendet"', ] def select(self, mailbox, readonly=False): assert readonly is True FakeIMAP.selects.append(mailbox) return "OK", [b"5"] def uid(self, command, *args): if command == "SEARCH": if any(isinstance(arg, str) and "Einwendung" in arg for arg in args): return "OK", [b"2"] return "OK", [b"1 2 3 4 5"] uid = int(args[0]) spec = " ".join(str(part) for part in args[1:]) if "BODY.PEEK[]" in spec: raw = ( f"From: Ada Beispiel \r\n" f"To: Gegenseite \r\n" f"Subject: Nachricht {uid}\r\n" f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n" f"Message-ID: <{uid}@example.test>\r\n" f"MIME-Version: 1.0\r\n" f'Content-Type: multipart/mixed; boundary="grenze"\r\n' f"\r\n" f"--grenze\r\n" f"Content-Type: text/plain; charset=utf-8\r\n" f"\r\n" f"Bitte die Einwendung pruefen.\r\n" f"--grenze\r\n" f"Content-Type: application/pdf\r\n" f"Content-Disposition: attachment; filename=\"einwendung.pdf\"\r\n" f"\r\n" f"%PDF\r\n" f"--grenze--\r\n" ).encode() return "OK", [(b"rfc822", raw)] if "BODYSTRUCTURE" in spec: structure = b'("ATTACHMENT" ("FILENAME" "einwendung.pdf"))' return "OK", [structure] header = ( f"From: Ada Beispiel \r\n" f"To: Gegenseite \r\n" f"Subject: Nachricht {uid}\r\n" f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n" f"Message-ID: <{uid}@example.test>\r\n" ).encode() return "OK", [(b"body", header), b")"] def logout(self): return "BYE", [b""] def _account(monkeypatch, actor: Actor) -> str: monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", FakeIMAP) created = actor.request( "POST", "/api/v1/connectors", json={ "kind": "imap", "name": "Kanzlei", "host": "imap.example.test", "port": 993, "username": "ada@example.test", "password": "Postfach-Geheim", "security": "ssl", }, status=201, ) return created["id"] def test_mailboxes_are_named_and_inbox_comes_first(client, db, monkeypatch): ada = Actor(client, "ada-mail@example.test", "Ada Post") register(db, ada.email, ada.name) ada.login() account = _account(monkeypatch, ada) listed = ada.request("GET", "/api/v1/sources/imap/mailboxes", params={"connector_id": account}) assert [item["label"] for item in listed["items"]] == ["Posteingang", "[Gmail]", "[Gmail]/Gesendet", "Änderungen"] gmail = next(item for item in listed["items"] if item["name"] == "[Gmail]") sent = next(item for item in listed["items"] if item["name"] == "[Gmail]/Gesendet") assert gmail["selectable"] is False assert sent["selectable"] is True assert sent["depth"] == 1 def test_folder_pages_newest_first_and_search_stays_inside(client, db, monkeypatch): ada = Actor(client, "ada-page@example.test", "Ada Seite") register(db, ada.email, ada.name) ada.login() account = _account(monkeypatch, ada) monkeypatch.setattr(sources, "PAGE_SIZE", 2) FakeIMAP.selects = [] first = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "[Gmail]/Gesendet"}, ) assert first["mailbox"] == "[Gmail]/Gesendet" assert [item["title"] for item in first["items"]] == ["Nachricht 5", "Nachricht 4"] assert first["items"][0]["document_on"] == "2024-05-02" assert first["items"][0]["message_from"] == "Ada Beispiel " assert first["items"][0]["message_to"] == "Gegenseite " assert first["items"][0]["direction"] == "out" assert first["items"][0]["attachments"] == [{"filename": "einwendung.pdf"}] assert first["next_before"] == 4 older = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "[Gmail]/Gesendet", "before": first["next_before"]}, ) assert [item["title"] for item in older["items"]] == ["Nachricht 3", "Nachricht 2"] found = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX", "q": "Einwendung"}, ) assert [item["title"] for item in found["items"]] == ["Nachricht 2"] assert found["items"][0]["direction"] == "in" assert found["next_before"] is None preview = ada.request( "GET", "/api/v1/sources/imap/message", params={"connector_id": account, "mailbox": "INBOX", "uid": 2}, ) assert preview["direction"] == "in" assert preview["message_to"] == "Gegenseite " assert "Einwendung" in preview["text"] other = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "&AMQ-nderungen"}, ) assert other["items"][0]["direction"] is None assert preview["attachments"][0]["filename"] == "einwendung.pdf" ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "Geheim"}, status=422, ) assert "Geheim" not in FakeIMAP.selects def test_foreign_mailbox_stays_hidden(client, db, monkeypatch): ada = Actor(client, "ada-own@example.test", "Ada Eigen") ben = Actor(client, "ben-own@example.test", "Ben Fremd") register(db, ada.email, ada.name) register(db, ben.email, ben.name) ada.login() account = _account(monkeypatch, ada) ben.login() ben.request("GET", "/api/v1/sources/imap/mailboxes", params={"connector_id": account}, status=404) ben.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account}, status=404) class HugeIMAP(FakeIMAP): def uid(self, command, *args): spec = " ".join(str(part) for part in args[1:]) if len(args) > 1 else "" if command == "FETCH" and "BODY.PEEK[]" in spec and "<" not in spec: return "OK", [(b"rfc822", b"x" * 8_000_001)] if "BODYSTRUCTURE" in spec: structure = b'("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 40 1 NIL NIL NIL NIL)' return "OK", [(b"body", structure)] if "BODY.PEEK[1]" in spec: return "OK", [(b"text", "Kurzer Auszug der grossen Nachricht.".encode())] return super().uid(command, *args) def test_oversized_message_still_returns_text(client, db, monkeypatch): ada = Actor(client, "ada-huge@example.test", "Ada Gross") register(db, ada.email, ada.name) ada.login() monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", HugeIMAP) account = ada.request( "POST", "/api/v1/connectors", json={ "kind": "imap", "name": "Gross", "host": "imap.example.test", "port": 993, "username": "ada@example.test", "password": "Postfach-Geheim", "security": "ssl", }, status=201, )["id"] preview = ada.request( "GET", "/api/v1/sources/imap/message", params={"connector_id": account, "mailbox": "INBOX", "uid": 2}, ) assert "Kurzer Auszug" in preview["text"] assert preview["text_truncated"] is True assert preview["direction"] == "in" assert preview["message_from"] def _raw(actor: Actor, path: str): return actor.client.get( path, headers={ "Origin": "http://testserver", "X-CSRF-Token": actor.csrf, "X-Scope-Kind": actor.scope["kind"], "X-Scope-Id": actor.scope["id"], }, ) def test_an_adopted_message_loads_in_full_with_its_attachment(client, db, monkeypatch): ada = Actor(client, "ada-voll@example.test", "Ada Voll") ben = Actor(client, "ben-voll@example.test", "Ben Voll") register(db, ada.email, ada.name) register(db, ben.email, ben.name) ada.login() account = _account(monkeypatch, ada) matter = matter_of(ada, "Nebenkosten") created = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Nachricht 2", "source": "imap", "external_id": "<2@example.test>", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": account, "mailbox": "INBOX", "uid": 2, "direction": "in"}, }, status=201, ) letter = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") assert letter["complete"] is True assert letter["text_truncated"] is False assert "Einwendung" in letter["text"] assert letter["attachments"][0]["filename"] == "einwendung.pdf" assert letter["attachments"][0]["content_type"] == "application/pdf" assert letter["attachments"][0]["byte_size"] >= 4 original = _raw(ada, f"/api/v1/external-refs/{created['id']}/mail/original") assert original.status_code == 200 assert b"Message-ID: <2@example.test>" in original.content assert b"%PDF" in original.content attachment = _raw(ada, f"/api/v1/external-refs/{created['id']}/mail/attachments/0") assert attachment.status_code == 200 assert attachment.content.startswith(b"%PDF") assert attachment.headers["content-type"].startswith("application/pdf") assert attachment.headers["content-disposition"].startswith("inline;") ada.request( "POST", "/api/v1/external-refs", json={ "title": "Fremde Nachricht", "source": "imap", "external_id": "", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": account, "mailbox": "INBOX", "uid": 2}, }, status=201, ) shown = ada.request("GET", f"/api/v1/matters/{matter['id']}") wrong = next(item for item in shown["bindings"] if item["element"]["title"] == "Fremde Nachricht") ada.request("GET", f"/api/v1/external-refs/{wrong['element']['id']}/mail", status=422) ben.login() ben.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=404) def _letter(message_id: str, body: str) -> bytes: return ( f"From: Ada Beispiel \r\n" f"To: Gegenseite \r\n" f"Subject: Verschoben\r\n" f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n" f"Message-ID: {message_id}\r\n" f"MIME-Version: 1.0\r\n" f"Content-Type: text/plain; charset=utf-8\r\n" f"\r\n" f"{body}\r\n" ).encode() class RelocatingIMAP: """The saved UID holds another message. The Message-ID lives in Archiv.""" log: list[tuple] = [] copies: dict[tuple[str, int], bytes] = {} header_uids: dict[str, bytes] | None = None def __init__(self, *args, **kwargs): self.selected = "" def login(self, username, secret): assert secret == "Postfach-Geheim" return "OK", [b"logged"] def list(self): return "OK", [ b'(\\HasNoChildren) "/" "INBOX"', b'(\\HasNoChildren) "/" "Archiv"', ] def select(self, mailbox, readonly=False): assert readonly is True self.selected = mailbox RelocatingIMAP.log.append(("select", mailbox)) return "OK", [b"2"] def uid(self, command, *args): if command == "SEARCH": if args and args[0] == "HEADER": RelocatingIMAP.log.append(("header", self.selected)) override = (RelocatingIMAP.header_uids or {}).get(self.selected) if override is not None: return "OK", [override] found = [ str(uid) for (folder, uid), raw in RelocatingIMAP.copies.items() if folder == self.selected and b"Message-ID: " in raw ] return "OK", [" ".join(found).encode()] return "OK", [b""] uid = int(args[0]) spec = " ".join(str(part) for part in args[1:]) if "BODY.PEEK[]" in spec: RelocatingIMAP.log.append(("fetch", self.selected, uid)) raw = RelocatingIMAP.copies.get((self.selected, uid)) if raw is None: return "OK", [None] return "OK", [(b"rfc822", raw)] return "OK", [None] def logout(self): return "BYE", [b""] def _stored_mail(db, element_id: str) -> dict: row = db.execute( "SELECT detail FROM external_references WHERE element_id = %s", (element_id,), ).fetchone() return dict(row["detail"]) def _linked_mail(actor: Actor, account: str, matter_id: str) -> dict: return actor.request( "POST", "/api/v1/external-refs", json={ "title": "Verschoben", "source": "imap", "external_id": "", "context": {"kind": "matter", "id": matter_id}, "detail": { "connector_id": account, "mailbox": "INBOX", "uid": 2, "direction": "in", }, }, status=201, ) def test_a_moved_message_remembers_its_new_folder(client, db, monkeypatch): from uuid import UUID from app.retrieval import MailPayload, load_external ada = Actor(client, "ada-move@example.test", "Ada Umzug") account_row = register(db, ada.email, ada.name) ada.login() RelocatingIMAP.log = [] RelocatingIMAP.header_uids = None RelocatingIMAP.copies = { ("INBOX", 2): _letter("", "Eine andere Nachricht."), ("Archiv", 9): _letter("", "Der Brief liegt jetzt im Archiv."), } account = _account(monkeypatch, ada) monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP) matter = matter_of(ada, "Ablage") created = _linked_mail(ada, account, matter["id"]) opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") assert "Archiv" in opened["text"] stored = _stored_mail(db, created["id"]) assert stored["mailbox"] == "Archiv" assert stored["uid"] == 9 assert stored["direction"] == "in" assert stored["connector_id"] == account assert any(item[0] == "header" for item in RelocatingIMAP.log) RelocatingIMAP.log = [] again = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") assert again["text"] == opened["text"] assert ("fetch", "Archiv", 9) in RelocatingIMAP.log assert not any(item[0] == "header" for item in RelocatingIMAP.log) assert _stored_mail(db, created["id"])["uid"] == 9 db.execute( """ UPDATE external_references SET detail = jsonb_set( jsonb_set(detail, '{mailbox}', to_jsonb('INBOX'::text)), '{uid}', to_jsonb(2) ) WHERE element_id = %s """, (created["id"],), ) db.commit() RelocatingIMAP.log = [] loaded = load_external(db, account_row["id"], UUID(created["id"]), None) assert isinstance(loaded, MailPayload) assert "Archiv" in loaded.parsed.text exported = _stored_mail(db, created["id"]) assert exported["mailbox"] == "Archiv" assert exported["uid"] == 9 assert exported["direction"] == "in" assert any(item[0] == "header" for item in RelocatingIMAP.log) def test_an_ambiguous_or_truncated_move_does_not_replace_the_stored_place(client, db, monkeypatch): ada = Actor(client, "ada-ambiguous@example.test", "Ada Zweifach") register(db, ada.email, ada.name) ada.login() RelocatingIMAP.header_uids = None RelocatingIMAP.copies = { ("INBOX", 2): _letter("", "Altbestand."), ("INBOX", 4): _letter("", "Erste Kopie."), ("Archiv", 9): _letter("", "Zweite Kopie."), } account = _account(monkeypatch, ada) monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP) matter = matter_of(ada, "Zweifach") created = _linked_mail(ada, account, matter["id"]) ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=422) assert _stored_mail(db, created["id"])["mailbox"] == "INBOX" assert _stored_mail(db, created["id"])["uid"] == 2 RelocatingIMAP.copies = { ("INBOX", 2): _letter("", "Altbestand."), ("INBOX", 11): _letter("", "Treffer in der abgebrochenen Suche."), ("INBOX", 12): _letter("", "a"), ("INBOX", 13): _letter("", "b"), ("INBOX", 14): _letter("", "c"), ("INBOX", 15): _letter("", "d"), ("INBOX", 16): _letter("", "e"), ("Archiv", 9): _letter("", "Eigentlich auch hier."), } RelocatingIMAP.header_uids = {"INBOX": b"11 12 13 14 15 16"} RelocatingIMAP.log = [] opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail") assert "abgebrochenen Suche" in opened["text"] assert not any(item == ("header", "Archiv") for item in RelocatingIMAP.log) assert _stored_mail(db, created["id"])["mailbox"] == "INBOX" assert _stored_mail(db, created["id"])["uid"] == 2