Speichere die neue Position einer verschobenen IMAP-Nachricht, damit der nächste Aufruf sie direkt öffnet.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
cd239a3ab8
commit
7a1206b18c
4 files changed
+299
-19
No files matched your search
@@ -44,7 +44,7 @@ def load_external(conn, actor_id: UUID, element_id: UUID, assigned: UUID | None)
|
||||
|
||||
ref = conn.execute(
|
||||
"""
|
||||
SELECT source, external_id, detail, connector_id, resolution
|
||||
SELECT element_id, source, external_id, detail, connector_id, resolution
|
||||
FROM external_references WHERE element_id = %s
|
||||
""",
|
||||
(element_id,),
|
||||
|
||||
@@ -217,3 +217,42 @@ def assign_reference(conn, actor_id: UUID, element_id: UUID, connector_id: UUID)
|
||||
detail = dict(ref["detail"] or {})
|
||||
detail["connector_id"] = str(connector_id)
|
||||
_set_binding(conn, element_id, connector_id, "bound", detail)
|
||||
|
||||
|
||||
def remember_imap_location(
|
||||
conn,
|
||||
element_id: UUID,
|
||||
external_id: str,
|
||||
previous_mailbox: str,
|
||||
previous_uid: int,
|
||||
mailbox: str,
|
||||
uid: int,
|
||||
) -> None:
|
||||
"""Store the folder and UID of a message found at a new place.
|
||||
|
||||
Only the previous place is replaced. The message identity, the connector
|
||||
and an adopted direction stay as they were.
|
||||
"""
|
||||
if (
|
||||
not mailbox
|
||||
or any(char in mailbox for char in "\r\n\x00")
|
||||
or uid < 1
|
||||
or (mailbox == previous_mailbox and uid == previous_uid)
|
||||
):
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET detail = jsonb_set(
|
||||
jsonb_set(detail, '{mailbox}', to_jsonb(%s::text)),
|
||||
'{uid}',
|
||||
to_jsonb(%s::bigint)
|
||||
)
|
||||
WHERE element_id = %s
|
||||
AND source = 'imap'
|
||||
AND external_id = %s
|
||||
AND detail->>'mailbox' = %s
|
||||
AND detail->>'uid' = %s
|
||||
""",
|
||||
(mailbox, uid, element_id, external_id, previous_mailbox, str(previous_uid)),
|
||||
)
|
||||
+67
-18
@@ -7,6 +7,7 @@ import imaplib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
from email.header import decode_header
|
||||
from email.parser import BytesParser
|
||||
from email.utils import parsedate_to_datetime
|
||||
@@ -33,7 +34,7 @@ from app.mailpack import (
|
||||
simplified_text,
|
||||
)
|
||||
from app.media.storage import sniff
|
||||
from app.sourcebind import resolve_for_read
|
||||
from app.sourcebind import remember_imap_location, resolve_for_read
|
||||
from app.sourceaccess import (
|
||||
ORIGINAL_UNAVAILABLE,
|
||||
assert_still_owned,
|
||||
@@ -509,7 +510,8 @@ def read_linked_mail(conn, actor_id: UUID, element: dict, context: Ctx | None, r
|
||||
def fetch(row):
|
||||
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, external_id, COMPLETE_MAIL_LIMIT)
|
||||
|
||||
return complete_borrow(conn, borrowed, fetch)
|
||||
found = complete_borrow(conn, borrowed, fetch)
|
||||
return _keep_moved_mail(conn, element["id"], external_id, mailbox, uid, found)
|
||||
|
||||
|
||||
def _uuid(value) -> UUID | None:
|
||||
@@ -1470,11 +1472,33 @@ def load_imap_original(conn, owner_id: UUID, connector_id: UUID, mailbox: str, u
|
||||
row = _owned(conn, owner_id, connector_id)
|
||||
if row["kind"] != "imap":
|
||||
raise unprocessable("Das ist kein E-Mail-Konto.")
|
||||
return pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT)
|
||||
found = pull_rfc822(row["settings"], row["secret"], mailbox, uid, expected_id, COMPLETE_MAIL_LIMIT)
|
||||
return found.raw if isinstance(found, LocatedMail) else found
|
||||
|
||||
|
||||
def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> bytes:
|
||||
"""Fetch the referenced message. A different message at the old UID is not returned."""
|
||||
@dataclass(frozen=True)
|
||||
class LocatedMail:
|
||||
"""Bytes plus the folder they were read from. `relocated` is a complete, unique hit."""
|
||||
|
||||
raw: bytes
|
||||
mailbox: str
|
||||
uid: int
|
||||
relocated: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _MailHit:
|
||||
raw: bytes
|
||||
mailbox: str
|
||||
uid: int
|
||||
|
||||
|
||||
def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id: str, max_bytes: int) -> LocatedMail:
|
||||
"""Fetch the referenced message. A different message at the old UID is not returned.
|
||||
|
||||
The stored folder is tried first. A single Message-ID hit elsewhere comes back
|
||||
marked relocated; the caller stores that place after the access check.
|
||||
"""
|
||||
client = _imap_login(settings, secret)
|
||||
try:
|
||||
_select_mailbox(client, mailbox)
|
||||
@@ -1484,23 +1508,45 @@ def pull_rfc822(settings: dict, secret: str, mailbox: str, uid: int, expected_id
|
||||
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
||||
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
||||
)
|
||||
alternates: list[bytes] = []
|
||||
if candidate is None or not identities_match(header_message_id(candidate), expected_id):
|
||||
alternates = _search_rfc822(client, expected_id, max_bytes)
|
||||
if candidate is not None and identities_match(header_message_id(candidate), expected_id):
|
||||
return LocatedMail(candidate, mailbox, uid, relocated=False)
|
||||
hits, capped = _search_rfc822(client, expected_id, max_bytes)
|
||||
try:
|
||||
chosen = choose_message(candidate, alternates, expected_id)
|
||||
chosen = choose_message(None, [hit.raw for hit in hits], expected_id)
|
||||
except MailIdentityError as exc:
|
||||
raise unprocessable(exc.message) from exc
|
||||
if len(chosen) > max_bytes:
|
||||
match = next(
|
||||
hit for hit in hits
|
||||
if hit.raw == chosen and identities_match(header_message_id(hit.raw), expected_id)
|
||||
)
|
||||
if len(match.raw) > max_bytes:
|
||||
raise unprocessable(
|
||||
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
||||
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
||||
)
|
||||
return chosen
|
||||
moved = not capped and (match.mailbox != mailbox or match.uid != uid)
|
||||
return LocatedMail(match.raw, match.mailbox, match.uid, relocated=moved)
|
||||
finally:
|
||||
_imap_logout(client)
|
||||
|
||||
|
||||
def _keep_moved_mail(conn, element_id, external_id: str, mailbox: str, uid: int | None, found) -> bytes:
|
||||
"""Return the bytes. A unique new place is written only after the fetch has finished."""
|
||||
if not isinstance(found, LocatedMail):
|
||||
return found
|
||||
if found.relocated and element_id is not None and uid is not None:
|
||||
remember_imap_location(
|
||||
conn,
|
||||
element_id,
|
||||
external_id,
|
||||
mailbox,
|
||||
uid,
|
||||
found.mailbox,
|
||||
found.uid,
|
||||
)
|
||||
return found.raw
|
||||
|
||||
|
||||
def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None:
|
||||
try:
|
||||
status, fetched = client.uid("FETCH", str(uid), "(BODY.PEEK[])")
|
||||
@@ -1511,11 +1557,12 @@ def _fetch_rfc822(client: imaplib.IMAP4, uid: int) -> bytes | None:
|
||||
return _literal(fetched)
|
||||
|
||||
|
||||
def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> list[bytes]:
|
||||
def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> tuple[list[_MailHit], bool]:
|
||||
"""Search every selectable folder. The second value is true when the scan stopped early."""
|
||||
token = (expected_id or "").strip()
|
||||
if not token or token.casefold().startswith("uid:"):
|
||||
return []
|
||||
found: list[bytes] = []
|
||||
return [], False
|
||||
found: list[_MailHit] = []
|
||||
for folder in _imap_mailboxes(client):
|
||||
if not folder["selectable"]:
|
||||
continue
|
||||
@@ -1539,10 +1586,10 @@ def _search_rfc822(client: imaplib.IMAP4, expected_id: str, max_bytes: int) -> l
|
||||
"Die vollständige Nachricht überschreitet die Exportgrenze. "
|
||||
"Eine gekürzte Vorschau wird nicht als Original übernommen."
|
||||
)
|
||||
found.append(blob)
|
||||
found.append(_MailHit(blob, folder["name"], uid))
|
||||
if len(found) > 5:
|
||||
return found
|
||||
return found
|
||||
return found, True
|
||||
return found, False
|
||||
|
||||
|
||||
PAPERLESS_FIELDS = (
|
||||
@@ -1667,13 +1714,15 @@ def _export_imap(conn, actor_id: UUID, external_id: str, detail: dict, assigned:
|
||||
owner_id = resolved.row["owner_id"]
|
||||
release_for_external(conn)
|
||||
try:
|
||||
raw = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
found = pull_rfc822(settings, secret, mailbox, uid, str(external_id), EXPORT_BYTE_LIMIT)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
try:
|
||||
assert_still_owned(conn, connector_id, owner_id)
|
||||
except Problem as exc:
|
||||
return failure_from_problem(exc)
|
||||
element_id = ref["element_id"] if ref is not None and "element_id" in ref.keys() else None
|
||||
raw = _keep_moved_mail(conn, element_id, str(external_id), mailbox, uid, found)
|
||||
if len(raw) > EXPORT_BYTE_LIMIT:
|
||||
return RetrievalFailure(
|
||||
"too_large",
|
||||
|
||||
@@ -285,3 +285,195 @@ def test_an_adopted_message_loads_in_full_with_its_attachment(client, db, monkey
|
||||
ada.request("GET", f"/api/v1/external-refs/{wrong['element']['id']}/mail", status=422)
|
||||
ben.login()
|
||||
ben.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=404)
|
||||
|
||||
|
||||
def _letter(message_id: str, body: str) -> bytes:
|
||||
return (
|
||||
f"From: Ada Beispiel <ada@example.test>\r\n"
|
||||
f"To: Gegenseite <gegen@example.test>\r\n"
|
||||
f"Subject: Verschoben\r\n"
|
||||
f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||||
f"Message-ID: {message_id}\r\n"
|
||||
f"MIME-Version: 1.0\r\n"
|
||||
f"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
f"\r\n"
|
||||
f"{body}\r\n"
|
||||
).encode()
|
||||
|
||||
|
||||
class RelocatingIMAP:
|
||||
"""The saved UID holds another message. The Message-ID lives in Archiv."""
|
||||
|
||||
log: list[tuple] = []
|
||||
copies: dict[tuple[str, int], bytes] = {}
|
||||
header_uids: dict[str, bytes] | None = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.selected = ""
|
||||
|
||||
def login(self, username, secret):
|
||||
assert secret == "Postfach-Geheim"
|
||||
return "OK", [b"logged"]
|
||||
|
||||
def list(self):
|
||||
return "OK", [
|
||||
b'(\\HasNoChildren) "/" "INBOX"',
|
||||
b'(\\HasNoChildren) "/" "Archiv"',
|
||||
]
|
||||
|
||||
def select(self, mailbox, readonly=False):
|
||||
assert readonly is True
|
||||
self.selected = mailbox
|
||||
RelocatingIMAP.log.append(("select", mailbox))
|
||||
return "OK", [b"2"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
if command == "SEARCH":
|
||||
if args and args[0] == "HEADER":
|
||||
RelocatingIMAP.log.append(("header", self.selected))
|
||||
override = (RelocatingIMAP.header_uids or {}).get(self.selected)
|
||||
if override is not None:
|
||||
return "OK", [override]
|
||||
found = [
|
||||
str(uid)
|
||||
for (folder, uid), raw in RelocatingIMAP.copies.items()
|
||||
if folder == self.selected and b"Message-ID: <moved@example.test>" in raw
|
||||
]
|
||||
return "OK", [" ".join(found).encode()]
|
||||
return "OK", [b""]
|
||||
uid = int(args[0])
|
||||
spec = " ".join(str(part) for part in args[1:])
|
||||
if "BODY.PEEK[]" in spec:
|
||||
RelocatingIMAP.log.append(("fetch", self.selected, uid))
|
||||
raw = RelocatingIMAP.copies.get((self.selected, uid))
|
||||
if raw is None:
|
||||
return "OK", [None]
|
||||
return "OK", [(b"rfc822", raw)]
|
||||
return "OK", [None]
|
||||
|
||||
def logout(self):
|
||||
return "BYE", [b""]
|
||||
|
||||
|
||||
def _stored_mail(db, element_id: str) -> dict:
|
||||
row = db.execute(
|
||||
"SELECT detail FROM external_references WHERE element_id = %s",
|
||||
(element_id,),
|
||||
).fetchone()
|
||||
return dict(row["detail"])
|
||||
|
||||
|
||||
def _linked_mail(actor: Actor, account: str, matter_id: str) -> dict:
|
||||
return actor.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Verschoben",
|
||||
"source": "imap",
|
||||
"external_id": "<moved@example.test>",
|
||||
"context": {"kind": "matter", "id": matter_id},
|
||||
"detail": {
|
||||
"connector_id": account,
|
||||
"mailbox": "INBOX",
|
||||
"uid": 2,
|
||||
"direction": "in",
|
||||
},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
def test_a_moved_message_remembers_its_new_folder(client, db, monkeypatch):
|
||||
from uuid import UUID
|
||||
|
||||
from app.retrieval import MailPayload, load_external
|
||||
|
||||
ada = Actor(client, "ada-move@example.test", "Ada Umzug")
|
||||
account_row = register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
RelocatingIMAP.log = []
|
||||
RelocatingIMAP.header_uids = None
|
||||
RelocatingIMAP.copies = {
|
||||
("INBOX", 2): _letter("<other@example.test>", "Eine andere Nachricht."),
|
||||
("Archiv", 9): _letter("<moved@example.test>", "Der Brief liegt jetzt im Archiv."),
|
||||
}
|
||||
account = _account(monkeypatch, ada)
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP)
|
||||
matter = matter_of(ada, "Ablage")
|
||||
created = _linked_mail(ada, account, matter["id"])
|
||||
|
||||
opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail")
|
||||
assert "Archiv" in opened["text"]
|
||||
stored = _stored_mail(db, created["id"])
|
||||
assert stored["mailbox"] == "Archiv"
|
||||
assert stored["uid"] == 9
|
||||
assert stored["direction"] == "in"
|
||||
assert stored["connector_id"] == account
|
||||
assert any(item[0] == "header" for item in RelocatingIMAP.log)
|
||||
|
||||
RelocatingIMAP.log = []
|
||||
again = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail")
|
||||
assert again["text"] == opened["text"]
|
||||
assert ("fetch", "Archiv", 9) in RelocatingIMAP.log
|
||||
assert not any(item[0] == "header" for item in RelocatingIMAP.log)
|
||||
assert _stored_mail(db, created["id"])["uid"] == 9
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE external_references
|
||||
SET detail = jsonb_set(
|
||||
jsonb_set(detail, '{mailbox}', to_jsonb('INBOX'::text)),
|
||||
'{uid}',
|
||||
to_jsonb(2)
|
||||
)
|
||||
WHERE element_id = %s
|
||||
""",
|
||||
(created["id"],),
|
||||
)
|
||||
db.commit()
|
||||
RelocatingIMAP.log = []
|
||||
loaded = load_external(db, account_row["id"], UUID(created["id"]), None)
|
||||
assert isinstance(loaded, MailPayload)
|
||||
assert "Archiv" in loaded.parsed.text
|
||||
exported = _stored_mail(db, created["id"])
|
||||
assert exported["mailbox"] == "Archiv"
|
||||
assert exported["uid"] == 9
|
||||
assert exported["direction"] == "in"
|
||||
assert any(item[0] == "header" for item in RelocatingIMAP.log)
|
||||
|
||||
|
||||
def test_an_ambiguous_or_truncated_move_does_not_replace_the_stored_place(client, db, monkeypatch):
|
||||
ada = Actor(client, "ada-ambiguous@example.test", "Ada Zweifach")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
RelocatingIMAP.header_uids = None
|
||||
RelocatingIMAP.copies = {
|
||||
("INBOX", 2): _letter("<other@example.test>", "Altbestand."),
|
||||
("INBOX", 4): _letter("<moved@example.test>", "Erste Kopie."),
|
||||
("Archiv", 9): _letter("<moved@example.test>", "Zweite Kopie."),
|
||||
}
|
||||
account = _account(monkeypatch, ada)
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", RelocatingIMAP)
|
||||
matter = matter_of(ada, "Zweifach")
|
||||
created = _linked_mail(ada, account, matter["id"])
|
||||
ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail", status=422)
|
||||
assert _stored_mail(db, created["id"])["mailbox"] == "INBOX"
|
||||
assert _stored_mail(db, created["id"])["uid"] == 2
|
||||
|
||||
RelocatingIMAP.copies = {
|
||||
("INBOX", 2): _letter("<other@example.test>", "Altbestand."),
|
||||
("INBOX", 11): _letter("<moved@example.test>", "Treffer in der abgebrochenen Suche."),
|
||||
("INBOX", 12): _letter("<a@example.test>", "a"),
|
||||
("INBOX", 13): _letter("<b@example.test>", "b"),
|
||||
("INBOX", 14): _letter("<c@example.test>", "c"),
|
||||
("INBOX", 15): _letter("<d@example.test>", "d"),
|
||||
("INBOX", 16): _letter("<e@example.test>", "e"),
|
||||
("Archiv", 9): _letter("<moved@example.test>", "Eigentlich auch hier."),
|
||||
}
|
||||
RelocatingIMAP.header_uids = {"INBOX": b"11 12 13 14 15 16"}
|
||||
RelocatingIMAP.log = []
|
||||
opened = ada.request("GET", f"/api/v1/external-refs/{created['id']}/mail")
|
||||
assert "abgebrochenen Suche" in opened["text"]
|
||||
assert not any(item == ("header", "Archiv") for item in RelocatingIMAP.log)
|
||||
assert _stored_mail(db, created["id"])["mailbox"] == "INBOX"
|
||||
assert _stored_mail(db, created["id"])["uid"] == 2
|
||||
Reference in new issue
Block a user