Files
tsumugu/backend/tests/test_correction_k1_k7.py
T
LarsandCursor c29bd29f34
Deploy Development / deploy (push) Successful in 39s
Test Suite / pytest-backend (push) Successful in 2m29s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 20s
Korrigiere IMAP-Ordner, MIME-Ansicht und die gemeinsamen Leser, damit große Postfächer, Notizen und verspätete Quellenantworten zuverlässig bleiben.
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-20 08:47:24 +02:00

470 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Corrections K1–K3 and K7. Synthetic only; no live mailbox or Paperless."""
from __future__ import annotations
import hashlib
import imaplib
import psycopg
import pytest
from email.message import EmailMessage
import app.sources as sources
from app.core.errors import unprocessable
from app.mailpack import parse_mail
from tests.support import Actor, register
def test_mailbox_names_are_quoted_protocol_strings():
client = imaplib.IMAP4.__new__(imaplib.IMAP4)
client.sent = []
client.state = "AUTH"
client._encoding = "ascii"
client.tagpre = b"T"
client.tagnum = 1
client.tagged_commands = {}
client.untagged_responses = {}
client.is_readonly = True
client.literal = None
client.debug = 0
client._mesg = lambda *args: None
client._log = lambda *args: None
client.send = lambda data: client.sent.append(data)
value = "Sent Items"
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T1 EXAMINE "Sent Items"\r\n'
value = 'Ordner "A"\\B'
client._command("EXAMINE", sources._mailbox_argument(value))
assert client.sent[-1] == b'T2 EXAMINE "Ordner \\"A\\"\\\\B"\r\n'
assert sources._mailbox_argument("INBOX") == "INBOX"
with pytest.raises(Exception):
sources._mailbox_argument("Sent\nItems")
def test_parse_mail_keeps_names_html_and_attached_messages():
inner = EmailMessage()
inner["Subject"] = "Innen"
inner.set_content("innen bleibt drin")
inner.add_attachment(b"innen-pdf", maintype="application", subtype="pdf", filename="innen.pdf")
outer = EmailMessage()
outer["From"] = "Ada <ada@example.test>"
outer["To"] = "Ben <ben@example.test>"
outer["Subject"] = "=?utf-8?q?Aussen?="
outer.set_content("aussen bleibt aussen")
outer.add_attachment(b"%PDF-same", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(b"%PDF-other", maintype="application", subtype="pdf", filename="lage.pdf")
outer.add_attachment(inner, subtype="rfc822", filename="weitergeleitet.eml")
html = (
"<p>A &amp; B</p><table><tr><td>Zelle</td><td>Zwei</td></tr></table>"
"<script>alert(1)</script><img src=\"https://evil.test/a.png\">"
"<a href=\"javascript:alert(1)\">x</a><img src=\"cid:logo@x\">"
)
mixed = EmailMessage()
mixed["Subject"] = "HTML"
mixed.set_content("nur text")
mixed.add_alternative(html, subtype="html")
logo = b"\x89PNG\r\n"
mixed.add_attachment(logo, maintype="image", subtype="png", filename="logo.png", cid="<logo@x>")
encoded = EmailMessage()
encoded["Subject"] = "Namen"
encoded.set_content("datei")
encoded.add_attachment(
b"pdf",
maintype="application",
subtype="pdf",
filename="=?utf-8?q?Buero_Uebergabe=2Epdf?=",
)
parsed = parse_mail(outer.as_bytes())
assert parsed.subject == "Aussen"
assert parsed.text == "aussen bleibt aussen"
assert "innen" not in parsed.text
names = [item.display_name for item in parsed.attachments]
assert "weitergeleitet.eml" in names
assert names.count("lage.pdf") == 2
archives = [item.filename for item in parsed.attachments]
assert len(set(archives)) == len(archives)
attached = next(item for item in parsed.attachments if item.display_name == "weitergeleitet.eml")
assert b"innen bleibt drin" in attached.data or b"Innen" in attached.data
assert attached.sha256 == hashlib.sha256(attached.data).hexdigest()
view = parse_mail(mixed.as_bytes())
assert "<table>" in view.html
assert "<script" not in view.html
assert "https://evil.test" not in view.html
assert "javascript:" not in view.html
assert "data:image/png;base64," in view.html
assert "A & B" in view.html or "A &amp; B" in view.html
assert "Zelle" in view.html
named = parse_mail(encoded.as_bytes())
assert named.attachments[0].display_name == "Buero Uebergabe.pdf"
assert "=?utf-8?" not in named.attachments[0].display_name
assert named.attachments[0].data == b"pdf"
class QuotedIMAP:
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"logged"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "Sent Items"']
def select(self, mailbox, readonly=False):
assert readonly is True
assert mailbox == '"Sent Items"'
self.untagged_responses = {
"EXISTS": [b"4200"],
"UIDVALIDITY": [b"8"],
"UIDNEXT": [b"2"],
}
return "OK", [b"4200"]
def uid(self, command, *args):
if command == "SEARCH":
return "OK", [b""]
header = b"From: Ada <ada@example.test>\r\nSubject: Leer\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_a_filled_folder_with_spaces_is_not_reported_empty(client, db, monkeypatch):
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
ada = Actor(client, "ada-sent@example.test", "Ada Gesendet")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
page = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account["id"], "mailbox": "Sent Items"},
)
assert page["items"] == []
assert page["contradictory"] is True
assert page["exists"] == 4200
assert "leer" not in str(page).casefold()
def _multipart(text: str, name: str, payload: bytes) -> bytes:
message = EmailMessage()
message["From"] = "Ada <ada@example.test>"
message["To"] = "Ben <ben@example.test>"
message["Subject"] = "Grenze"
message["Message-ID"] = "<grenze@example.test>"
message.set_content(text)
message.add_attachment(payload, maintype="application", subtype="pdf", filename=name)
return message.as_bytes()
class LimitIMAP:
raw = b""
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
return "OK", [b"ok"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
def select(self, mailbox, readonly=False):
self.untagged_responses = {"UIDVALIDITY": [b"1"], "UIDNEXT": [b"3"], "EXISTS": [b"1"]}
return "OK", [b"1"]
def uid(self, command, *args):
spec = " ".join(str(part) for part in args[1:])
if "BODY.PEEK[]" in spec and "<" not in spec:
return "OK", [(b"rfc822", type(self).raw)]
if "BODYSTRUCTURE" in spec:
structure = (
b'(("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 12 1 NIL NIL NIL NIL)'
b'("APPLICATION" "PDF" NIL NIL NIL "BASE64" 20 NIL '
b'("ATTACHMENT" ("FILENAME" "=?utf-8?q?Buero_Uebergabe=2Epdf?=")) NIL NIL)'
b' "MIXED" ("BOUNDARY" "b") NIL NIL NIL)'
)
return "OK", [(b"body", structure)]
if "BODY.PEEK[1]" in spec:
return "OK", [(b"part", b"gleicher text")]
if "BODY.PEEK[2]" in spec:
return "OK", [(b"part", b"nicht text")]
header = (
b"From: Ada <ada@example.test>\r\nTo: Ben <ben@example.test>\r\n"
b"Subject: Grenze\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
)
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def test_preview_below_and_above_the_limit_uses_the_same_parts(client, db, monkeypatch):
payload = b"%PDF-plan"
small = _multipart("gleicher text", "Buero Uebergabe.pdf", payload)
LimitIMAP.raw = small + b"X" * 50
monkeypatch.setattr(sources, "PREVIEW_BYTE_LIMIT", len(small))
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", LimitIMAP)
ada = Actor(client, "ada-limit@example.test", "Ada Grenze")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
under = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
LimitIMAP.raw = small + b"Y"
over = ada.request(
"GET",
"/api/v1/sources/imap/message",
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
)
assert "gleicher text" in under["text"]
assert "gleicher text" in over["text"]
assert under["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert over["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
assert "Content-Transfer-Encoding" not in over["text"]
assert "BASE64" not in over["text"].upper()
assert over["preview_limited"] is True
def test_upload_and_mail_fetch_leave_the_domain_transaction(client, db, monkeypatch):
seen = []
import app.api.routes as routes
import app.ops as ops
from app.media.port import publish as real_publish
def watching(adapter, key, data):
seen.append(("publish", ops_conn[0].info.transaction_status))
return real_publish(adapter, key, data)
ops_conn = [None]
real_create = ops.create_file
def wrapped_create(conn, *args, **kwargs):
ops_conn[0] = conn
return real_create(conn, *args, **kwargs)
monkeypatch.setattr(ops, "publish", watching)
monkeypatch.setattr(routes, "create_file", wrapped_create)
ada = Actor(client, "ada-idle@example.test", "Ada Leerlauf")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-1.4\n", "application/pdf")},
status=201,
)
assert seen[0][1] == psycopg.pq.TransactionStatus.IDLE
import app.sourceaccess as sourceaccess
phase = {"external": False, "status": None}
real_release = sourceaccess.release_for_external
def spy_release(conn):
real_release(conn)
phase["external"] = True
phase["status"] = conn.info.transaction_status
def spy_owned(*args, **kwargs):
if phase["external"]:
raise AssertionError("Datenbankzugriff im externen Abschnitt")
return real_owned(*args, **kwargs)
def spy_pull(*args, **kwargs):
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
return _multipart("brief", "a.pdf", b"%PDF")
real_owned = sources._owned
monkeypatch.setattr(sourceaccess, "release_for_external", spy_release)
monkeypatch.setattr(sources, "_owned", spy_owned)
monkeypatch.setattr(sources, "pull_rfc822", spy_pull)
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
account = ada.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Post",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Brief",
"source": "imap",
"external_id": "<grenze@example.test>",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": account["id"], "mailbox": "Sent Items", "uid": 4},
},
status=201,
)
opened = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
assert opened["text"]
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
def test_failed_conclusion_keeps_no_partial_file(client, db, monkeypatch):
import app.ops as ops
def fail(*args, **kwargs):
raise unprocessable("Abschluss fehlgeschlagen")
monkeypatch.setattr(ops, "_new_element", fail)
ada = Actor(client, "ada-fail@example.test", "Ada Abschluss")
register(db, ada.email, ada.name)
ada.login()
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
ada.request(
"POST",
"/api/v1/files",
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
files={"upload": ("scan.pdf", b"%PDF-FAIL-CONCLUSION\n", "application/pdf")},
status=422,
)
stored = db.execute("SELECT count(*) AS n FROM elements WHERE title = 'Scan'").fetchone()
assert stored["n"] == 0
from pathlib import Path
from app.core.settings import get_settings
root = Path(get_settings().media_root)
left = [path for path in root.rglob("*") if path.is_file() and b"PDF-FAIL-CONCLUSION" in path.read_bytes()]
assert left == []
def test_paperless_filters_change_the_hits(client, db, monkeypatch):
catalog = []
for number in range(1, 46):
catalog.append({
"id": number,
"title": "Vertrag" if number % 5 == 0 else f"Schrift {number}",
"content": "nurinhalt" if number == 41 else f"text {number}",
"correspondent": 4 if number % 2 == 0 else 9,
"document_type": 8 if number < 30 else 1,
"tags": [2, 5] if number % 3 == 0 else [2],
"created": "2024-06-01" if number > 20 else "2023-01-01",
})
def fake_get(base, token, path, params):
if path != "/api/documents/":
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
matched = []
for item in catalog:
query = params.get("query") or ""
title = params.get("title__icontains") or ""
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
continue
if title and title.casefold() not in item["title"].casefold():
continue
if params.get("correspondent__id") and str(item["correspondent"]) != params["correspondent__id"]:
continue
if params.get("document_type__id") and str(item["document_type"]) != params["document_type__id"]:
continue
if params.get("tags__id__all"):
needed = {int(part) for part in params["tags__id__all"].split(",") if part}
if not needed.issubset(set(item["tags"])):
continue
start = params.get("created__date__gte")
end = params.get("created__date__lte")
if start and item["created"] < start:
continue
if end and item["created"] > end:
continue
matched.append(item)
page = int(params.get("page") or 1)
start_at = (page - 1) * 20
rows = matched[start_at:start_at + 20]
return {
"count": len(matched),
"results": [{"id": item["id"], "title": item["title"], "created": item["created"]} for item in rows],
}
monkeypatch.setattr(sources, "_paperless_get", fake_get)
ada = Actor(client, "ada-sense@example.test", "Ada Sinn")
register(db, ada.email, ada.name)
ada.login()
account = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
content = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "q": "nurinhalt"},
)
assert [item["id"] for item in content["items"]] == ["41"]
title = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "title": "Vertrag"},
)
assert title["items"]
assert all("Vertrag" in item["title"] for item in title["items"])
assert all(item["id"] != "41" for item in title["items"])
combined = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={
"connector_id": account["id"],
"correspondent": "4",
"document_type": "8",
"tag": "2,5",
"document_from": "2023-01-01",
"document_to": "2024-12-31",
},
)
plain = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": account["id"], "cursor": "2"},
)
assert [item["id"] for item in combined["items"]] == ["6", "12", "18", "24"]
assert [item["id"] for item in plain["items"]][0] == "21"