Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Deploy Development / deploy (push) Successful in 39s
Test Suite / pytest-backend (push) Successful in 2m29s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 20s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-20 08:47:24 +02:00
1 parent 34c4084a7c
commit c29bd29f34
29 files changed
+2562 -410

No files matched your search

+469
View File
@@ -0,0 +1,469 @@
"""Corrections K1–K3 and K7. Synthetic only; no live mailbox or Paperless."""
from __future__ import annotations
import hashlib
import imaplib
import psycopg
import pytest
from email.message import EmailMessage
import app.sources as sources
from app.core.errors import unprocessable
from app.mailpack import parse_mail
from tests.support import Actor, register
def test_mailbox_names_are_quoted_protocol_strings():
client = imaplib.IMAP4.__new__(imaplib.IMAP4)
client.sent = []
client.state = "AUTH"
client._encoding = "ascii"
client.tagpre = b"T"
client.tagnum = 1
client.tagged_commands = {}
client.untagged_responses = {}
client.is_readonly = True
client.literal = None
client.debug = 0
client._mesg = lambda *args: None
client._log = lambda *args: None
client.send = lambda data: client.sent.append(data)
value = "Sent Items"
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T1 EXAMINE "Sent Items"\r\n'
value = 'Ordner "A"\\B'
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T2 EXAMINE "Ordner \\"A\\"\\\\B"\r\n'
assert sources._mailbox_argument("INBOX") == "INBOX"
with pytest.raises(Exception):
sources._mailbox_argument("Sent\nItems")
def test_parse_mail_keeps_names_html_and_attached_messages():
inner = EmailMessage()
inner["Subject"] = "Innen"
inner.set_content("innen bleibt drin")
inner.add_attachment(b"innen-pdf", maintype="application", subtype="pdf", filename="innen.pdf")
outer = EmailMessage()
outer["From"] = "Ada <ada@example.test>"
outer["To"] = "Ben <ben@example.test>"
outer["Subject"] = "=?utf-8?q?Aussen?="
outer.set_content("aussen bleibt aussen")
outer.add_attachment(b"%PDF-same", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(b"%PDF-other", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(inner, subtype="rfc822", filename="weitergeleitet.eml")
html = (
"<p>A &amp; B</p><table><tr><td>Zelle</td><td>Zwei</td></tr></table>"
"<script>alert(1)</script><img src=\"https://evil.test/a.png\">"
"<a href=\"javascript:alert(1)\">x</a><img src=\"cid:logo@x\">"
)
mixed = EmailMessage()
mixed["Subject"] = "HTML"
mixed.set_content("nur text")
mixed.add_alternative(html, subtype="html")
logo = b"\x89PNG\r\n"
mixed.add_attachment(logo, maintype="image", subtype="png", filename="logo.png", cid="<logo@x>")
encoded = EmailMessage()
encoded["Subject"] = "Namen"
encoded.set_content("datei")
encoded.add_attachment(
b"pdf",
maintype="application",
subtype="pdf",
filename="=?utf-8?q?Buero_Uebergabe=2Epdf?=",
)
parsed = parse_mail(outer.as_bytes())
assert parsed.subject == "Aussen"
assert parsed.text == "aussen bleibt aussen"
assert "innen" not in parsed.text
names = [item.display_name for item in parsed.attachments]
assert "weitergeleitet.eml" in names
assert names.count("lage.pdf") == 2
archives = [item.filename for item in parsed.attachments]
assert len(set(archives)) == len(archives)
attached = next(item for item in parsed.attachments if item.display_name == "weitergeleitet.eml")
assert b"innen bleibt drin" in attached.data or b"Innen" in attached.data
assert attached.sha256 == hashlib.sha256(attached.data).hexdigest()
view = parse_mail(mixed.as_bytes())
assert "<table>" in view.html
assert "<script" not in view.html
assert "https://evil.test" not in view.html
assert "javascript:" not in view.html
assert "data:image/png;base64," in view.html
assert "A & B" in view.html or "A &amp; B" in view.html
assert "Zelle" in view.html
named = parse_mail(encoded.as_bytes())
assert named.attachments[0].display_name == "Buero Uebergabe.pdf"
assert "=?utf-8?" not in named.attachments[0].display_name
assert named.attachments[0].data == b"pdf"
class QuotedIMAP:
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"logged"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "Sent Items"']
def select(self, mailbox, readonly=False):
assert readonly is True
assert mailbox == '"Sent Items"'
self.untagged_responses = {
"EXISTS": [b"4200"],
"UIDVALIDITY": [b"8"],
"UIDNEXT": [b"2"],
}
return "OK", [b"4200"]
def uid(self, command, *args):
if command == "SEARCH":
return "OK", [b""]
header = b"From: Ada <ada@example.test>\r\nSubject: Leer\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_a_filled_folder_with_spaces_is_not_reported_empty(client, db, monkeypatch):
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
ada = Actor(client, "ada-sent@example.test", "Ada Gesendet")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
page = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account["id"], "mailbox": "Sent Items"},
)
assert page["items"] == []
assert page["contradictory"] is True
assert page["exists"] == 4200
assert "leer" not in str(page).casefold()
def _multipart(text: str, name: str, payload: bytes) -> bytes:
message = EmailMessage()
message["From"] = "Ada <ada@example.test>"
message["To"] = "Ben <ben@example.test>"
message["Subject"] = "Grenze"
message["Message-ID"] = "<grenze@example.test>"
message.set_content(text)
message.add_attachment(payload, maintype="application", subtype="pdf", filename=name)
return message.as_bytes()
class LimitIMAP:
raw = b""
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"ok"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
def select(self, mailbox, readonly=False):
self.untagged_responses = {"UIDVALIDITY": [b"1"], "UIDNEXT": [b"3"], "EXISTS": [b"1"]}
return "OK", [b"1"]
def uid(self, command, *args):
spec = " ".join(str(part) for part in args[1:])
if "BODY.PEEK[]" in spec and "<" not in spec:
return "OK", [(b"rfc822", type(self).raw)]
if "BODYSTRUCTURE" in spec:
structure = (
b'(("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 12 1 NIL NIL NIL NIL)'
b'("APPLICATION" "PDF" NIL NIL NIL "BASE64" 20 NIL '
b'("ATTACHMENT" ("FILENAME" "=?utf-8?q?Buero_Uebergabe=2Epdf?=")) NIL NIL)'
b' "MIXED" ("BOUNDARY" "b") NIL NIL NIL)'
)
return "OK", [(b"body", structure)]
if "BODY.PEEK[1]" in spec:
return "OK", [(b"part", b"gleicher text")]
if "BODY.PEEK[2]" in spec:
return "OK", [(b"part", b"nicht text")]
header = (
b"From: Ada <ada@example.test>\r\nTo: Ben <ben@example.test>\r\n"
b"Subject: Grenze\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
)
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_preview_below_and_above_the_limit_uses_the_same_parts(client, db, monkeypatch):
payload = b"%PDF-plan"
small = _multipart("gleicher text", "Buero Uebergabe.pdf", payload)
LimitIMAP.raw = small + b"X" * 50
monkeypatch.setattr(sources, "PREVIEW_BYTE_LIMIT", len(small))
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", LimitIMAP)
ada = Actor(client, "ada-limit@example.test", "Ada Grenze")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
under = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
LimitIMAP.raw = small + b"Y"
over = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
assert "gleicher text" in under["text"]
assert "gleicher text" in over["text"]
assert under["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert over["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert "Content-Transfer-Encoding" not in over["text"]
assert "BASE64" not in over["text"].upper()
assert over["preview_limited"] is True
def test_upload_and_mail_fetch_leave_the_domain_transaction(client, db, monkeypatch):
seen = []
import app.api.routes as routes
import app.ops as ops
from app.media.port import publish as real_publish
def watching(adapter, key, data):
seen.append(("publish", ops_conn[0].info.transaction_status))
return real_publish(adapter, key, data)
ops_conn = [None]
real_create = ops.create_file
def wrapped_create(conn, *args, **kwargs):
ops_conn[0] = conn
return real_create(conn, *args, **kwargs)
monkeypatch.setattr(ops, "publish", watching)
monkeypatch.setattr(routes, "create_file", wrapped_create)
ada = Actor(client, "ada-idle@example.test", "Ada Leerlauf")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-1.4\n", "application/pdf")},
status=201,
)
assert seen[0][1] == psycopg.pq.TransactionStatus.IDLE
import app.sourceaccess as sourceaccess
phase = {"external": False, "status": None}
real_release = sourceaccess.release_for_external
def spy_release(conn):
real_release(conn)
phase["external"] = True
phase["status"] = conn.info.transaction_status
def spy_owned(*args, **kwargs):
if phase["external"]:
raise AssertionError("Datenbankzugriff im externen Abschnitt")
return real_owned(*args, **kwargs)
def spy_pull(*args, **kwargs):
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
return _multipart("brief", "a.pdf", b"%PDF")
real_owned = sources._owned
monkeypatch.setattr(sourceaccess, "release_for_external", spy_release)
monkeypatch.setattr(sources, "_owned", spy_owned)
monkeypatch.setattr(sources, "pull_rfc822", spy_pull)
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Brief",
"source": "imap",
"external_id": "<grenze@example.test>",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": account["id"], "mailbox": "Sent Items", "uid": 4},
},
status=201,
)
opened = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
assert opened["text"]
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
def test_failed_conclusion_keeps_no_partial_file(client, db, monkeypatch):
import app.ops as ops
def fail(*args, **kwargs):
raise unprocessable("Abschluss fehlgeschlagen")
monkeypatch.setattr(ops, "_new_element", fail)
ada = Actor(client, "ada-fail@example.test", "Ada Abschluss")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-FAIL-CONCLUSION\n", "application/pdf")},
status=422,
)
stored = db.execute("SELECT count(*) AS n FROM elements WHERE title = 'Scan'").fetchone()
assert stored["n"] == 0
from pathlib import Path
from app.core.settings import get_settings
root = Path(get_settings().media_root)
left = [path for path in root.rglob("*") if path.is_file() and b"PDF-FAIL-CONCLUSION" in path.read_bytes()]
assert left == []
def test_paperless_filters_change_the_hits(client, db, monkeypatch):
catalog = []
for number in range(1, 46):
catalog.append({
"id": number,
"title": "Vertrag" if number % 5 == 0 else f"Schrift {number}",
"content": "nurinhalt" if number == 41 else f"text {number}",
"correspondent": 4 if number % 2 == 0 else 9,
"document_type": 8 if number < 30 else 1,
"tags": [2, 5] if number % 3 == 0 else [2],
"created": "2024-06-01" if number > 20 else "2023-01-01",
})
def fake_get(base, token, path, params):
if path != "/api/documents/":
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
matched = []
for item in catalog:
query = params.get("query") or ""
title = params.get("title__icontains") or ""
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
continue
if title and title.casefold() not in item["title"].casefold():
continue
if params.get("correspondent__id") and str(item["correspondent"]) != params["correspondent__id"]:
continue
if params.get("document_type__id") and str(item["document_type"]) != params["document_type__id"]:
continue
if params.get("tags__id__all"):
needed = {int(part) for part in params["tags__id__all"].split(",") if part}
if not needed.issubset(set(item["tags"])):
continue
start = params.get("created__date__gte")
end = params.get("created__date__lte")
if start and item["created"] < start:
continue
if end and item["created"] > end:
continue
matched.append(item)
page = int(params.get("page") or 1)
start_at = (page - 1) * 20
rows = matched[start_at:start_at + 20]
return {
"count": len(matched),
"results": [{"id": item["id"], "title": item["title"], "created": item["created"]} for item in rows],
}
monkeypatch.setattr(sources, "_paperless_get", fake_get)
ada = Actor(client, "ada-sense@example.test", "Ada Sinn")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
content = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "q": "nurinhalt"},
)
assert [item["id"] for item in content["items"]] == ["41"]
title = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "title": "Vertrag"},
)
assert title["items"]
assert all("Vertrag" in item["title"] for item in title["items"])
assert all(item["id"] != "41" for item in title["items"])
combined = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={
"connector_id": account["id"],
"correspondent": "4",
"document_type": "8",
"tag": "2,5",
"document_from": "2023-01-01",
"document_to": "2024-12-31",
},
)
plain = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "cursor": "2"},
)
assert [item["id"] for item in combined["items"]] == ["6", "12", "18", "24"]
assert [item["id"] for item in plain["items"]][0] == "21"