Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
34c4084a7c
commit
c29bd29f34
29 files changed
+2562
-410
No files matched your search
@@ -0,0 +1,469 @@
|
||||
"""Corrections K1–K3 and K7. Synthetic only; no live mailbox or Paperless."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import imaplib
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from email.message import EmailMessage
|
||||
|
||||
import app.sources as sources
|
||||
from app.core.errors import unprocessable
|
||||
from app.mailpack import parse_mail
|
||||
from tests.support import Actor, register
|
||||
|
||||
|
||||
def test_mailbox_names_are_quoted_protocol_strings():
|
||||
client = imaplib.IMAP4.__new__(imaplib.IMAP4)
|
||||
client.sent = []
|
||||
client.state = "AUTH"
|
||||
client._encoding = "ascii"
|
||||
client.tagpre = b"T"
|
||||
client.tagnum = 1
|
||||
client.tagged_commands = {}
|
||||
client.untagged_responses = {}
|
||||
client.is_readonly = True
|
||||
client.literal = None
|
||||
client.debug = 0
|
||||
client._mesg = lambda *args: None
|
||||
client._log = lambda *args: None
|
||||
client.send = lambda data: client.sent.append(data)
|
||||
|
||||
value = "Sent Items"
|
||||
client._command("EXAMINE", sources._mailbox_argument(value))
|
||||
assert client.sent[-1] == b'T1 EXAMINE "Sent Items"\r\n'
|
||||
value = 'Ordner "A"\\B'
|
||||
client._command("EXAMINE", sources._mailbox_argument(value))
|
||||
assert client.sent[-1] == b'T2 EXAMINE "Ordner \\"A\\"\\\\B"\r\n'
|
||||
assert sources._mailbox_argument("INBOX") == "INBOX"
|
||||
with pytest.raises(Exception):
|
||||
sources._mailbox_argument("Sent\nItems")
|
||||
|
||||
|
||||
def test_parse_mail_keeps_names_html_and_attached_messages():
|
||||
inner = EmailMessage()
|
||||
inner["Subject"] = "Innen"
|
||||
inner.set_content("innen bleibt drin")
|
||||
inner.add_attachment(b"innen-pdf", maintype="application", subtype="pdf", filename="innen.pdf")
|
||||
outer = EmailMessage()
|
||||
outer["From"] = "Ada <ada@example.test>"
|
||||
outer["To"] = "Ben <ben@example.test>"
|
||||
outer["Subject"] = "=?utf-8?q?Aussen?="
|
||||
outer.set_content("aussen bleibt aussen")
|
||||
outer.add_attachment(b"%PDF-same", maintype="application", subtype="pdf", filename="lage.pdf")
|
||||
outer.add_attachment(b"%PDF-other", maintype="application", subtype="pdf", filename="lage.pdf")
|
||||
outer.add_attachment(inner, subtype="rfc822", filename="weitergeleitet.eml")
|
||||
html = (
|
||||
"<p>A & B</p><table><tr><td>Zelle</td><td>Zwei</td></tr></table>"
|
||||
"<script>alert(1)</script><img src=\"https://evil.test/a.png\">"
|
||||
"<a href=\"javascript:alert(1)\">x</a><img src=\"cid:logo@x\">"
|
||||
)
|
||||
mixed = EmailMessage()
|
||||
mixed["Subject"] = "HTML"
|
||||
mixed.set_content("nur text")
|
||||
mixed.add_alternative(html, subtype="html")
|
||||
logo = b"\x89PNG\r\n"
|
||||
mixed.add_attachment(logo, maintype="image", subtype="png", filename="logo.png", cid="<logo@x>")
|
||||
encoded = EmailMessage()
|
||||
encoded["Subject"] = "Namen"
|
||||
encoded.set_content("datei")
|
||||
encoded.add_attachment(
|
||||
b"pdf",
|
||||
maintype="application",
|
||||
subtype="pdf",
|
||||
filename="=?utf-8?q?Buero_Uebergabe=2Epdf?=",
|
||||
)
|
||||
|
||||
parsed = parse_mail(outer.as_bytes())
|
||||
assert parsed.subject == "Aussen"
|
||||
assert parsed.text == "aussen bleibt aussen"
|
||||
assert "innen" not in parsed.text
|
||||
names = [item.display_name for item in parsed.attachments]
|
||||
assert "weitergeleitet.eml" in names
|
||||
assert names.count("lage.pdf") == 2
|
||||
archives = [item.filename for item in parsed.attachments]
|
||||
assert len(set(archives)) == len(archives)
|
||||
attached = next(item for item in parsed.attachments if item.display_name == "weitergeleitet.eml")
|
||||
assert b"innen bleibt drin" in attached.data or b"Innen" in attached.data
|
||||
assert attached.sha256 == hashlib.sha256(attached.data).hexdigest()
|
||||
|
||||
view = parse_mail(mixed.as_bytes())
|
||||
assert "<table>" in view.html
|
||||
assert "<script" not in view.html
|
||||
assert "https://evil.test" not in view.html
|
||||
assert "javascript:" not in view.html
|
||||
assert "data:image/png;base64," in view.html
|
||||
assert "A & B" in view.html or "A & B" in view.html
|
||||
assert "Zelle" in view.html
|
||||
|
||||
named = parse_mail(encoded.as_bytes())
|
||||
assert named.attachments[0].display_name == "Buero Uebergabe.pdf"
|
||||
assert "=?utf-8?" not in named.attachments[0].display_name
|
||||
assert named.attachments[0].data == b"pdf"
|
||||
|
||||
|
||||
class QuotedIMAP:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.untagged_responses = {}
|
||||
|
||||
def login(self, username, secret):
|
||||
return "OK", [b"logged"]
|
||||
|
||||
def list(self):
|
||||
return "OK", [b'(\\HasNoChildren) "/" "Sent Items"']
|
||||
|
||||
def select(self, mailbox, readonly=False):
|
||||
assert readonly is True
|
||||
assert mailbox == '"Sent Items"'
|
||||
self.untagged_responses = {
|
||||
"EXISTS": [b"4200"],
|
||||
"UIDVALIDITY": [b"8"],
|
||||
"UIDNEXT": [b"2"],
|
||||
}
|
||||
return "OK", [b"4200"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
if command == "SEARCH":
|
||||
return "OK", [b""]
|
||||
header = b"From: Ada <ada@example.test>\r\nSubject: Leer\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||||
return "OK", [(b"body", header), b")"]
|
||||
|
||||
def logout(self):
|
||||
return "BYE", [b""]
|
||||
|
||||
|
||||
def test_a_filled_folder_with_spaces_is_not_reported_empty(client, db, monkeypatch):
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
|
||||
ada = Actor(client, "ada-sent@example.test", "Ada Gesendet")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
account = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={
|
||||
"kind": "imap",
|
||||
"name": "Post",
|
||||
"host": "imap.example.test",
|
||||
"port": 993,
|
||||
"username": "ada@example.test",
|
||||
"password": "Postfach-Geheim",
|
||||
"security": "ssl",
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
page = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/messages",
|
||||
params={"connector_id": account["id"], "mailbox": "Sent Items"},
|
||||
)
|
||||
assert page["items"] == []
|
||||
assert page["contradictory"] is True
|
||||
assert page["exists"] == 4200
|
||||
assert "leer" not in str(page).casefold()
|
||||
|
||||
|
||||
def _multipart(text: str, name: str, payload: bytes) -> bytes:
|
||||
message = EmailMessage()
|
||||
message["From"] = "Ada <ada@example.test>"
|
||||
message["To"] = "Ben <ben@example.test>"
|
||||
message["Subject"] = "Grenze"
|
||||
message["Message-ID"] = "<grenze@example.test>"
|
||||
message.set_content(text)
|
||||
message.add_attachment(payload, maintype="application", subtype="pdf", filename=name)
|
||||
return message.as_bytes()
|
||||
|
||||
|
||||
class LimitIMAP:
|
||||
raw = b""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.untagged_responses = {}
|
||||
|
||||
def login(self, username, secret):
|
||||
return "OK", [b"ok"]
|
||||
|
||||
def list(self):
|
||||
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
|
||||
|
||||
def select(self, mailbox, readonly=False):
|
||||
self.untagged_responses = {"UIDVALIDITY": [b"1"], "UIDNEXT": [b"3"], "EXISTS": [b"1"]}
|
||||
return "OK", [b"1"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
spec = " ".join(str(part) for part in args[1:])
|
||||
if "BODY.PEEK[]" in spec and "<" not in spec:
|
||||
return "OK", [(b"rfc822", type(self).raw)]
|
||||
if "BODYSTRUCTURE" in spec:
|
||||
structure = (
|
||||
b'(("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 12 1 NIL NIL NIL NIL)'
|
||||
b'("APPLICATION" "PDF" NIL NIL NIL "BASE64" 20 NIL '
|
||||
b'("ATTACHMENT" ("FILENAME" "=?utf-8?q?Buero_Uebergabe=2Epdf?=")) NIL NIL)'
|
||||
b' "MIXED" ("BOUNDARY" "b") NIL NIL NIL)'
|
||||
)
|
||||
return "OK", [(b"body", structure)]
|
||||
if "BODY.PEEK[1]" in spec:
|
||||
return "OK", [(b"part", b"gleicher text")]
|
||||
if "BODY.PEEK[2]" in spec:
|
||||
return "OK", [(b"part", b"nicht text")]
|
||||
header = (
|
||||
b"From: Ada <ada@example.test>\r\nTo: Ben <ben@example.test>\r\n"
|
||||
b"Subject: Grenze\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||||
)
|
||||
return "OK", [(b"body", header), b")"]
|
||||
|
||||
def logout(self):
|
||||
return "BYE", [b""]
|
||||
|
||||
|
||||
def test_preview_below_and_above_the_limit_uses_the_same_parts(client, db, monkeypatch):
|
||||
payload = b"%PDF-plan"
|
||||
small = _multipart("gleicher text", "Buero Uebergabe.pdf", payload)
|
||||
LimitIMAP.raw = small + b"X" * 50
|
||||
monkeypatch.setattr(sources, "PREVIEW_BYTE_LIMIT", len(small))
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", LimitIMAP)
|
||||
ada = Actor(client, "ada-limit@example.test", "Ada Grenze")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
account = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={
|
||||
"kind": "imap",
|
||||
"name": "Post",
|
||||
"host": "imap.example.test",
|
||||
"port": 993,
|
||||
"username": "ada@example.test",
|
||||
"password": "Postfach-Geheim",
|
||||
"security": "ssl",
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
under = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/message",
|
||||
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
|
||||
)
|
||||
LimitIMAP.raw = small + b"Y"
|
||||
over = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/imap/message",
|
||||
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
|
||||
)
|
||||
assert "gleicher text" in under["text"]
|
||||
assert "gleicher text" in over["text"]
|
||||
assert under["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
|
||||
assert over["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
|
||||
assert "Content-Transfer-Encoding" not in over["text"]
|
||||
assert "BASE64" not in over["text"].upper()
|
||||
assert over["preview_limited"] is True
|
||||
|
||||
|
||||
def test_upload_and_mail_fetch_leave_the_domain_transaction(client, db, monkeypatch):
|
||||
seen = []
|
||||
|
||||
import app.api.routes as routes
|
||||
import app.ops as ops
|
||||
from app.media.port import publish as real_publish
|
||||
|
||||
def watching(adapter, key, data):
|
||||
seen.append(("publish", ops_conn[0].info.transaction_status))
|
||||
return real_publish(adapter, key, data)
|
||||
|
||||
ops_conn = [None]
|
||||
real_create = ops.create_file
|
||||
|
||||
def wrapped_create(conn, *args, **kwargs):
|
||||
ops_conn[0] = conn
|
||||
return real_create(conn, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(ops, "publish", watching)
|
||||
monkeypatch.setattr(routes, "create_file", wrapped_create)
|
||||
ada = Actor(client, "ada-idle@example.test", "Ada Leerlauf")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
|
||||
ada.request(
|
||||
"POST",
|
||||
"/api/v1/files",
|
||||
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
|
||||
files={"upload": ("scan.pdf", b"%PDF-1.4\n", "application/pdf")},
|
||||
status=201,
|
||||
)
|
||||
assert seen[0][1] == psycopg.pq.TransactionStatus.IDLE
|
||||
|
||||
import app.sourceaccess as sourceaccess
|
||||
|
||||
phase = {"external": False, "status": None}
|
||||
real_release = sourceaccess.release_for_external
|
||||
|
||||
def spy_release(conn):
|
||||
real_release(conn)
|
||||
phase["external"] = True
|
||||
phase["status"] = conn.info.transaction_status
|
||||
|
||||
def spy_owned(*args, **kwargs):
|
||||
if phase["external"]:
|
||||
raise AssertionError("Datenbankzugriff im externen Abschnitt")
|
||||
return real_owned(*args, **kwargs)
|
||||
|
||||
def spy_pull(*args, **kwargs):
|
||||
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
|
||||
return _multipart("brief", "a.pdf", b"%PDF")
|
||||
|
||||
real_owned = sources._owned
|
||||
monkeypatch.setattr(sourceaccess, "release_for_external", spy_release)
|
||||
monkeypatch.setattr(sources, "_owned", spy_owned)
|
||||
monkeypatch.setattr(sources, "pull_rfc822", spy_pull)
|
||||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
|
||||
account = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={
|
||||
"kind": "imap",
|
||||
"name": "Post",
|
||||
"host": "imap.example.test",
|
||||
"port": 993,
|
||||
"username": "ada@example.test",
|
||||
"password": "Postfach-Geheim",
|
||||
"security": "ssl",
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
|
||||
linked = ada.request(
|
||||
"POST",
|
||||
"/api/v1/external-refs",
|
||||
json={
|
||||
"title": "Brief",
|
||||
"source": "imap",
|
||||
"external_id": "<grenze@example.test>",
|
||||
"context": {"kind": "matter", "id": matter["id"]},
|
||||
"detail": {"connector_id": account["id"], "mailbox": "Sent Items", "uid": 4},
|
||||
},
|
||||
status=201,
|
||||
)
|
||||
opened = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
|
||||
assert opened["text"]
|
||||
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
|
||||
|
||||
|
||||
def test_failed_conclusion_keeps_no_partial_file(client, db, monkeypatch):
|
||||
import app.ops as ops
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise unprocessable("Abschluss fehlgeschlagen")
|
||||
|
||||
monkeypatch.setattr(ops, "_new_element", fail)
|
||||
ada = Actor(client, "ada-fail@example.test", "Ada Abschluss")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
|
||||
ada.request(
|
||||
"POST",
|
||||
"/api/v1/files",
|
||||
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
|
||||
files={"upload": ("scan.pdf", b"%PDF-FAIL-CONCLUSION\n", "application/pdf")},
|
||||
status=422,
|
||||
)
|
||||
stored = db.execute("SELECT count(*) AS n FROM elements WHERE title = 'Scan'").fetchone()
|
||||
assert stored["n"] == 0
|
||||
from pathlib import Path
|
||||
from app.core.settings import get_settings
|
||||
|
||||
root = Path(get_settings().media_root)
|
||||
left = [path for path in root.rglob("*") if path.is_file() and b"PDF-FAIL-CONCLUSION" in path.read_bytes()]
|
||||
assert left == []
|
||||
|
||||
|
||||
def test_paperless_filters_change_the_hits(client, db, monkeypatch):
|
||||
catalog = []
|
||||
for number in range(1, 46):
|
||||
catalog.append({
|
||||
"id": number,
|
||||
"title": "Vertrag" if number % 5 == 0 else f"Schrift {number}",
|
||||
"content": "nurinhalt" if number == 41 else f"text {number}",
|
||||
"correspondent": 4 if number % 2 == 0 else 9,
|
||||
"document_type": 8 if number < 30 else 1,
|
||||
"tags": [2, 5] if number % 3 == 0 else [2],
|
||||
"created": "2024-06-01" if number > 20 else "2023-01-01",
|
||||
})
|
||||
|
||||
def fake_get(base, token, path, params):
|
||||
if path != "/api/documents/":
|
||||
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
|
||||
matched = []
|
||||
for item in catalog:
|
||||
query = params.get("query") or ""
|
||||
title = params.get("title__icontains") or ""
|
||||
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
|
||||
continue
|
||||
if title and title.casefold() not in item["title"].casefold():
|
||||
continue
|
||||
if params.get("correspondent__id") and str(item["correspondent"]) != params["correspondent__id"]:
|
||||
continue
|
||||
if params.get("document_type__id") and str(item["document_type"]) != params["document_type__id"]:
|
||||
continue
|
||||
if params.get("tags__id__all"):
|
||||
needed = {int(part) for part in params["tags__id__all"].split(",") if part}
|
||||
if not needed.issubset(set(item["tags"])):
|
||||
continue
|
||||
start = params.get("created__date__gte")
|
||||
end = params.get("created__date__lte")
|
||||
if start and item["created"] < start:
|
||||
continue
|
||||
if end and item["created"] > end:
|
||||
continue
|
||||
matched.append(item)
|
||||
page = int(params.get("page") or 1)
|
||||
start_at = (page - 1) * 20
|
||||
rows = matched[start_at:start_at + 20]
|
||||
return {
|
||||
"count": len(matched),
|
||||
"results": [{"id": item["id"], "title": item["title"], "created": item["created"]} for item in rows],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(sources, "_paperless_get", fake_get)
|
||||
ada = Actor(client, "ada-sense@example.test", "Ada Sinn")
|
||||
register(db, ada.email, ada.name)
|
||||
ada.login()
|
||||
account = ada.request(
|
||||
"POST",
|
||||
"/api/v1/connectors",
|
||||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||||
status=201,
|
||||
)
|
||||
content = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": account["id"], "q": "nurinhalt"},
|
||||
)
|
||||
assert [item["id"] for item in content["items"]] == ["41"]
|
||||
title = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": account["id"], "title": "Vertrag"},
|
||||
)
|
||||
assert title["items"]
|
||||
assert all("Vertrag" in item["title"] for item in title["items"])
|
||||
assert all(item["id"] != "41" for item in title["items"])
|
||||
combined = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={
|
||||
"connector_id": account["id"],
|
||||
"correspondent": "4",
|
||||
"document_type": "8",
|
||||
"tag": "2,5",
|
||||
"document_from": "2023-01-01",
|
||||
"document_to": "2024-12-31",
|
||||
},
|
||||
)
|
||||
plain = ada.request(
|
||||
"GET",
|
||||
"/api/v1/sources/paperless/documents",
|
||||
params={"connector_id": account["id"], "cursor": "2"},
|
||||
)
|
||||
assert [item["id"] for item in combined["items"]] == ["6", "12", "18", "24"]
|
||||
assert [item["id"] for item in plain["items"]][0] == "21"
|
||||
@@ -180,9 +180,12 @@ def test_foreign_mailbox_stays_hidden(client, db, monkeypatch):
|
||||
class HugeIMAP(FakeIMAP):
|
||||
def uid(self, command, *args):
|
||||
spec = " ".join(str(part) for part in args[1:]) if len(args) > 1 else ""
|
||||
if command == "FETCH" and "BODY.PEEK[]" in spec:
|
||||
if command == "FETCH" and "BODY.PEEK[]" in spec and "<" not in spec:
|
||||
return "OK", [(b"rfc822", b"x" * 8_000_001)]
|
||||
if "BODY.PEEK[TEXT]" in spec:
|
||||
if "BODYSTRUCTURE" in spec:
|
||||
structure = b'("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 40 1 NIL NIL NIL NIL)'
|
||||
return "OK", [(b"body", structure)]
|
||||
if "BODY.PEEK[1]" in spec:
|
||||
return "OK", [(b"text", "Kurzer Auszug der grossen Nachricht.".encode())]
|
||||
return super().uid(command, *args)
|
||||
|
||||
|
||||
Reference in new issue
Block a user