Standard ist jetzt Titel und Inhalt ohne Index; Titel, Inhalt und Volltextindex sind wählbar. Neue Filter: Speicherpfad, Tag-Verknüpfung (alle, eines, keines), "ohne" bei Korrespondent, Typ und Pfad, Hinzugefügt-Datum und Archivnummer. Treffer zeigen Vorschaubild, Textauszug, Tags, Typ und Seitenzahl; die Vorschaubilder laufen über den Server. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
504 lines
18 KiB
Python
504 lines
18 KiB
Python
"""Corrections K1–K3 and K7. Synthetic only; no live mailbox or Paperless."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import imaplib
|
||
|
||
import psycopg
|
||
import pytest
|
||
from email.message import EmailMessage
|
||
|
||
import app.sources as sources
|
||
from app.core.errors import unprocessable
|
||
from app.mailpack import parse_mail
|
||
from tests.support import Actor, register
|
||
|
||
|
||
def test_mailbox_names_are_quoted_protocol_strings():
|
||
client = imaplib.IMAP4.__new__(imaplib.IMAP4)
|
||
client.sent = []
|
||
client.state = "AUTH"
|
||
client._encoding = "ascii"
|
||
client.tagpre = b"T"
|
||
client.tagnum = 1
|
||
client.tagged_commands = {}
|
||
client.untagged_responses = {}
|
||
client.is_readonly = True
|
||
client.literal = None
|
||
client.debug = 0
|
||
client._mesg = lambda *args: None
|
||
client._log = lambda *args: None
|
||
client.send = lambda data: client.sent.append(data)
|
||
|
||
value = "Sent Items"
|
||
client._command("EXAMINE", sources._mailbox_argument(value))
|
||
assert client.sent[-1] == b'T1 EXAMINE "Sent Items"\r\n'
|
||
value = 'Ordner "A"\\B'
|
||
client._command("EXAMINE", sources._mailbox_argument(value))
|
||
assert client.sent[-1] == b'T2 EXAMINE "Ordner \\"A\\"\\\\B"\r\n'
|
||
assert sources._mailbox_argument("INBOX") == "INBOX"
|
||
with pytest.raises(Exception):
|
||
sources._mailbox_argument("Sent\nItems")
|
||
|
||
|
||
def test_parse_mail_keeps_names_html_and_attached_messages():
|
||
inner = EmailMessage()
|
||
inner["Subject"] = "Innen"
|
||
inner.set_content("innen bleibt drin")
|
||
inner.add_attachment(b"innen-pdf", maintype="application", subtype="pdf", filename="innen.pdf")
|
||
outer = EmailMessage()
|
||
outer["From"] = "Ada <ada@example.test>"
|
||
outer["To"] = "Ben <ben@example.test>"
|
||
outer["Subject"] = "=?utf-8?q?Aussen?="
|
||
outer.set_content("aussen bleibt aussen")
|
||
outer.add_attachment(b"%PDF-same", maintype="application", subtype="pdf", filename="lage.pdf")
|
||
outer.add_attachment(b"%PDF-other", maintype="application", subtype="pdf", filename="lage.pdf")
|
||
outer.add_attachment(inner, subtype="rfc822", filename="weitergeleitet.eml")
|
||
html = (
|
||
"<p>A & B</p><table><tr><td>Zelle</td><td>Zwei</td></tr></table>"
|
||
"<script>alert(1)</script><img src=\"https://evil.test/a.png\">"
|
||
"<a href=\"javascript:alert(1)\">x</a><img src=\"cid:logo@x\">"
|
||
)
|
||
mixed = EmailMessage()
|
||
mixed["Subject"] = "HTML"
|
||
mixed.set_content("nur text")
|
||
mixed.add_alternative(html, subtype="html")
|
||
logo = b"\x89PNG\r\n"
|
||
mixed.add_attachment(logo, maintype="image", subtype="png", filename="logo.png", cid="<logo@x>")
|
||
encoded = EmailMessage()
|
||
encoded["Subject"] = "Namen"
|
||
encoded.set_content("datei")
|
||
encoded.add_attachment(
|
||
b"pdf",
|
||
maintype="application",
|
||
subtype="pdf",
|
||
filename="=?utf-8?q?Buero_Uebergabe=2Epdf?=",
|
||
)
|
||
|
||
parsed = parse_mail(outer.as_bytes())
|
||
assert parsed.subject == "Aussen"
|
||
assert parsed.text == "aussen bleibt aussen"
|
||
assert "innen" not in parsed.text
|
||
names = [item.display_name for item in parsed.attachments]
|
||
assert "weitergeleitet.eml" in names
|
||
assert names.count("lage.pdf") == 2
|
||
archives = [item.filename for item in parsed.attachments]
|
||
assert len(set(archives)) == len(archives)
|
||
attached = next(item for item in parsed.attachments if item.display_name == "weitergeleitet.eml")
|
||
assert b"innen bleibt drin" in attached.data or b"Innen" in attached.data
|
||
assert attached.sha256 == hashlib.sha256(attached.data).hexdigest()
|
||
|
||
view = parse_mail(mixed.as_bytes())
|
||
assert "<table>" in view.html
|
||
assert "<script" not in view.html
|
||
assert "https://evil.test" not in view.html
|
||
assert "javascript:" not in view.html
|
||
assert "data:image/png;base64," in view.html
|
||
assert "A & B" in view.html or "A & B" in view.html
|
||
assert "Zelle" in view.html
|
||
|
||
named = parse_mail(encoded.as_bytes())
|
||
assert named.attachments[0].display_name == "Buero Uebergabe.pdf"
|
||
assert "=?utf-8?" not in named.attachments[0].display_name
|
||
assert named.attachments[0].data == b"pdf"
|
||
|
||
|
||
def test_an_undecoded_pdf_stays_a_named_pdf():
|
||
import base64
|
||
from app.media.storage import content_disposition
|
||
|
||
pdf = b"%PDF-1.4\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\n%%EOF\n"
|
||
raw = (
|
||
b"Subject: Scan\r\n"
|
||
b"MIME-Version: 1.0\r\n"
|
||
b"Content-Type: multipart/mixed; boundary=bound\r\n"
|
||
b"\r\n"
|
||
b"--bound\r\n"
|
||
b"Content-Type: text/plain; charset=utf-8\r\n"
|
||
b"\r\n"
|
||
b"text\r\n"
|
||
b"--bound\r\n"
|
||
b"Content-Type: application/octet-stream\r\n"
|
||
b"Content-Transfer-Encoding: 8bit\r\n"
|
||
b"Content-Disposition: attachment; filename=\"scan-7f3a9c\"\r\n"
|
||
b"\r\n"
|
||
+ base64.b64encode(pdf) + b"\r\n"
|
||
b"--bound--\r\n"
|
||
)
|
||
item = parse_mail(raw).attachments[0]
|
||
assert item.data.startswith(b"%PDF")
|
||
assert item.media_type == "application/pdf"
|
||
assert item.display_name == "scan-7f3a9c.pdf"
|
||
header = content_disposition("\u00dcbergabe.pdf", inline=True)
|
||
assert header.startswith("inline;")
|
||
header.encode("ascii")
|
||
assert ".pdf" in header
|
||
assert "filename*=UTF-8''" in header
|
||
assert "%C3%9Cbergabe.pdf" in header
|
||
|
||
|
||
class QuotedIMAP:
|
||
def __init__(self, *args, **kwargs):
|
||
self.untagged_responses = {}
|
||
|
||
def login(self, username, secret):
|
||
return "OK", [b"logged"]
|
||
|
||
def list(self):
|
||
return "OK", [b'(\\HasNoChildren) "/" "Sent Items"']
|
||
|
||
def select(self, mailbox, readonly=False):
|
||
assert readonly is True
|
||
assert mailbox == '"Sent Items"'
|
||
self.untagged_responses = {
|
||
"EXISTS": [b"4200"],
|
||
"UIDVALIDITY": [b"8"],
|
||
"UIDNEXT": [b"2"],
|
||
}
|
||
return "OK", [b"4200"]
|
||
|
||
def uid(self, command, *args):
|
||
if command == "SEARCH":
|
||
return "OK", [b""]
|
||
header = b"From: Ada <ada@example.test>\r\nSubject: Leer\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||
return "OK", [(b"body", header), b")"]
|
||
|
||
def logout(self):
|
||
return "BYE", [b""]
|
||
|
||
|
||
def test_a_filled_folder_with_spaces_is_not_reported_empty(client, db, monkeypatch):
|
||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
|
||
ada = Actor(client, "ada-sent@example.test", "Ada Gesendet")
|
||
register(db, ada.email, ada.name)
|
||
ada.login()
|
||
account = ada.request(
|
||
"POST",
|
||
"/api/v1/connectors",
|
||
json={
|
||
"kind": "imap",
|
||
"name": "Post",
|
||
"host": "imap.example.test",
|
||
"port": 993,
|
||
"username": "ada@example.test",
|
||
"password": "Postfach-Geheim",
|
||
"security": "ssl",
|
||
},
|
||
status=201,
|
||
)
|
||
page = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/imap/messages",
|
||
params={"connector_id": account["id"], "mailbox": "Sent Items"},
|
||
)
|
||
assert page["items"] == []
|
||
assert page["contradictory"] is True
|
||
assert page["exists"] == 4200
|
||
assert "leer" not in str(page).casefold()
|
||
|
||
|
||
def _multipart(text: str, name: str, payload: bytes) -> bytes:
|
||
message = EmailMessage()
|
||
message["From"] = "Ada <ada@example.test>"
|
||
message["To"] = "Ben <ben@example.test>"
|
||
message["Subject"] = "Grenze"
|
||
message["Message-ID"] = "<grenze@example.test>"
|
||
message.set_content(text)
|
||
message.add_attachment(payload, maintype="application", subtype="pdf", filename=name)
|
||
return message.as_bytes()
|
||
|
||
|
||
class LimitIMAP:
|
||
raw = b""
|
||
|
||
def __init__(self, *args, **kwargs):
|
||
self.untagged_responses = {}
|
||
|
||
def login(self, username, secret):
|
||
return "OK", [b"ok"]
|
||
|
||
def list(self):
|
||
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
|
||
|
||
def select(self, mailbox, readonly=False):
|
||
self.untagged_responses = {"UIDVALIDITY": [b"1"], "UIDNEXT": [b"3"], "EXISTS": [b"1"]}
|
||
return "OK", [b"1"]
|
||
|
||
def uid(self, command, *args):
|
||
spec = " ".join(str(part) for part in args[1:])
|
||
if "BODY.PEEK[]" in spec and "<" not in spec:
|
||
return "OK", [(b"rfc822", type(self).raw)]
|
||
if "BODYSTRUCTURE" in spec:
|
||
structure = (
|
||
b'(("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "7BIT" 12 1 NIL NIL NIL NIL)'
|
||
b'("APPLICATION" "PDF" NIL NIL NIL "BASE64" 20 NIL '
|
||
b'("ATTACHMENT" ("FILENAME" "=?utf-8?q?Buero_Uebergabe=2Epdf?=")) NIL NIL)'
|
||
b' "MIXED" ("BOUNDARY" "b") NIL NIL NIL)'
|
||
)
|
||
return "OK", [(b"body", structure)]
|
||
if "BODY.PEEK[1]" in spec:
|
||
return "OK", [(b"part", b"gleicher text")]
|
||
if "BODY.PEEK[2]" in spec:
|
||
return "OK", [(b"part", b"nicht text")]
|
||
header = (
|
||
b"From: Ada <ada@example.test>\r\nTo: Ben <ben@example.test>\r\n"
|
||
b"Subject: Grenze\r\nDate: Thu, 02 May 2024 08:00:00 +0000\r\n"
|
||
)
|
||
return "OK", [(b"body", header), b")"]
|
||
|
||
def logout(self):
|
||
return "BYE", [b""]
|
||
|
||
|
||
def test_preview_below_and_above_the_limit_uses_the_same_parts(client, db, monkeypatch):
|
||
payload = b"%PDF-plan"
|
||
small = _multipart("gleicher text", "Buero Uebergabe.pdf", payload)
|
||
LimitIMAP.raw = small + b"X" * 50
|
||
monkeypatch.setattr(sources, "PREVIEW_BYTE_LIMIT", len(small))
|
||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", LimitIMAP)
|
||
ada = Actor(client, "ada-limit@example.test", "Ada Grenze")
|
||
register(db, ada.email, ada.name)
|
||
ada.login()
|
||
account = ada.request(
|
||
"POST",
|
||
"/api/v1/connectors",
|
||
json={
|
||
"kind": "imap",
|
||
"name": "Post",
|
||
"host": "imap.example.test",
|
||
"port": 993,
|
||
"username": "ada@example.test",
|
||
"password": "Postfach-Geheim",
|
||
"security": "ssl",
|
||
},
|
||
status=201,
|
||
)
|
||
under = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/imap/message",
|
||
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
|
||
)
|
||
LimitIMAP.raw = small + b"Y"
|
||
over = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/imap/message",
|
||
params={"connector_id": account["id"], "mailbox": "INBOX", "uid": 1},
|
||
)
|
||
assert "gleicher text" in under["text"]
|
||
assert "gleicher text" in over["text"]
|
||
assert under["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
|
||
assert over["attachments"][0]["filename"] == "Buero Uebergabe.pdf"
|
||
assert "Content-Transfer-Encoding" not in over["text"]
|
||
assert "BASE64" not in over["text"].upper()
|
||
assert over["preview_limited"] is True
|
||
|
||
|
||
def test_upload_and_mail_fetch_leave_the_domain_transaction(client, db, monkeypatch):
|
||
seen = []
|
||
|
||
import app.api.routes as routes
|
||
import app.ops as ops
|
||
from app.media.port import publish as real_publish
|
||
|
||
def watching(adapter, key, data):
|
||
seen.append(("publish", ops_conn[0].info.transaction_status))
|
||
return real_publish(adapter, key, data)
|
||
|
||
ops_conn = [None]
|
||
real_create = ops.create_file
|
||
|
||
def wrapped_create(conn, *args, **kwargs):
|
||
ops_conn[0] = conn
|
||
return real_create(conn, *args, **kwargs)
|
||
|
||
monkeypatch.setattr(ops, "publish", watching)
|
||
monkeypatch.setattr(routes, "create_file", wrapped_create)
|
||
ada = Actor(client, "ada-idle@example.test", "Ada Leerlauf")
|
||
register(db, ada.email, ada.name)
|
||
ada.login()
|
||
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
|
||
ada.request(
|
||
"POST",
|
||
"/api/v1/files",
|
||
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
|
||
files={"upload": ("scan.pdf", b"%PDF-1.4\n", "application/pdf")},
|
||
status=201,
|
||
)
|
||
assert seen[0][1] == psycopg.pq.TransactionStatus.IDLE
|
||
|
||
import app.sourceaccess as sourceaccess
|
||
|
||
phase = {"external": False, "status": None}
|
||
real_release = sourceaccess.release_for_external
|
||
|
||
def spy_release(conn):
|
||
real_release(conn)
|
||
phase["external"] = True
|
||
phase["status"] = conn.info.transaction_status
|
||
|
||
def spy_owned(*args, **kwargs):
|
||
if phase["external"]:
|
||
raise AssertionError("Datenbankzugriff im externen Abschnitt")
|
||
return real_owned(*args, **kwargs)
|
||
|
||
def spy_pull(*args, **kwargs):
|
||
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
|
||
return _multipart("brief", "a.pdf", b"%PDF")
|
||
|
||
real_owned = sources._owned
|
||
monkeypatch.setattr(sourceaccess, "release_for_external", spy_release)
|
||
monkeypatch.setattr(sources, "_owned", spy_owned)
|
||
monkeypatch.setattr(sources, "pull_rfc822", spy_pull)
|
||
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", QuotedIMAP)
|
||
account = ada.request(
|
||
"POST",
|
||
"/api/v1/connectors",
|
||
json={
|
||
"kind": "imap",
|
||
"name": "Post",
|
||
"host": "imap.example.test",
|
||
"port": 993,
|
||
"username": "ada@example.test",
|
||
"password": "Postfach-Geheim",
|
||
"security": "ssl",
|
||
},
|
||
status=201,
|
||
)
|
||
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
|
||
linked = ada.request(
|
||
"POST",
|
||
"/api/v1/external-refs",
|
||
json={
|
||
"title": "Brief",
|
||
"source": "imap",
|
||
"external_id": "<grenze@example.test>",
|
||
"context": {"kind": "matter", "id": matter["id"]},
|
||
"detail": {"connector_id": account["id"], "mailbox": "Sent Items", "uid": 4},
|
||
},
|
||
status=201,
|
||
)
|
||
opened = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
|
||
assert opened["text"]
|
||
assert phase["status"] == psycopg.pq.TransactionStatus.IDLE
|
||
|
||
|
||
def test_failed_conclusion_keeps_no_partial_file(client, db, monkeypatch):
|
||
import app.ops as ops
|
||
|
||
def fail(*args, **kwargs):
|
||
raise unprocessable("Abschluss fehlgeschlagen")
|
||
|
||
monkeypatch.setattr(ops, "_new_element", fail)
|
||
ada = Actor(client, "ada-fail@example.test", "Ada Abschluss")
|
||
register(db, ada.email, ada.name)
|
||
ada.login()
|
||
case = ada.request("POST", "/api/v1/cases", json={"title": "Ablage"}, status=201)
|
||
ada.request(
|
||
"POST",
|
||
"/api/v1/files",
|
||
data={"title": "Scan", "context_kind": "case", "context_id": case["id"]},
|
||
files={"upload": ("scan.pdf", b"%PDF-FAIL-CONCLUSION\n", "application/pdf")},
|
||
status=422,
|
||
)
|
||
stored = db.execute("SELECT count(*) AS n FROM elements WHERE title = 'Scan'").fetchone()
|
||
assert stored["n"] == 0
|
||
from pathlib import Path
|
||
from app.core.settings import get_settings
|
||
|
||
root = Path(get_settings().media_root)
|
||
left = [path for path in root.rglob("*") if path.is_file() and b"PDF-FAIL-CONCLUSION" in path.read_bytes()]
|
||
assert left == []
|
||
|
||
|
||
def test_paperless_filters_change_the_hits(client, db, monkeypatch):
|
||
catalog = []
|
||
for number in range(1, 46):
|
||
catalog.append({
|
||
"id": number,
|
||
"title": "Vertrag" if number % 5 == 0 else f"Schrift {number}",
|
||
"content": "nurinhalt" if number == 41 else f"text {number}",
|
||
"correspondent": 4 if number % 2 == 0 else 9,
|
||
"document_type": 8 if number < 30 else 1,
|
||
"tags": [2, 5] if number % 3 == 0 else [2],
|
||
"created": "2024-06-01" if number > 20 else "2023-01-01",
|
||
})
|
||
|
||
def fake_get(base, token, path, params):
|
||
if path != "/api/documents/":
|
||
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
|
||
matched = []
|
||
for item in catalog:
|
||
query = params.get("title_content") or params.get("query") or ""
|
||
title = params.get("title__icontains") or ""
|
||
if query and query.casefold() not in f"{item['title']} {item['content']}".casefold():
|
||
continue
|
||
if title and title.casefold() not in item["title"].casefold():
|
||
continue
|
||
if params.get("correspondent__id") and str(item["correspondent"]) != params["correspondent__id"]:
|
||
continue
|
||
if params.get("document_type__id") and str(item["document_type"]) != params["document_type__id"]:
|
||
continue
|
||
if params.get("tags__id__all"):
|
||
needed = {int(part) for part in params["tags__id__all"].split(",") if part}
|
||
if not needed.issubset(set(item["tags"])):
|
||
continue
|
||
start = params.get("created__date__gte")
|
||
end = params.get("created__date__lte")
|
||
if start and item["created"] < start:
|
||
continue
|
||
if end and item["created"] > end:
|
||
continue
|
||
matched.append(item)
|
||
page = int(params.get("page") or 1)
|
||
start_at = (page - 1) * 20
|
||
rows = matched[start_at:start_at + 20]
|
||
return {
|
||
"count": len(matched),
|
||
"results": [{"id": item["id"], "title": item["title"], "created": item["created"]} for item in rows],
|
||
}
|
||
|
||
monkeypatch.setattr(sources, "_paperless_get", fake_get)
|
||
ada = Actor(client, "ada-sense@example.test", "Ada Sinn")
|
||
register(db, ada.email, ada.name)
|
||
ada.login()
|
||
account = ada.request(
|
||
"POST",
|
||
"/api/v1/connectors",
|
||
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
|
||
status=201,
|
||
)
|
||
content = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/paperless/documents",
|
||
params={"connector_id": account["id"], "q": "nurinhalt"},
|
||
)
|
||
assert [item["id"] for item in content["items"]] == ["41"]
|
||
title = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/paperless/documents",
|
||
params={"connector_id": account["id"], "title": "Vertrag"},
|
||
)
|
||
assert title["items"]
|
||
assert all("Vertrag" in item["title"] for item in title["items"])
|
||
assert all(item["id"] != "41" for item in title["items"])
|
||
combined = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/paperless/documents",
|
||
params={
|
||
"connector_id": account["id"],
|
||
"correspondent": "4",
|
||
"document_type": "8",
|
||
"tag": "2,5",
|
||
"document_from": "2023-01-01",
|
||
"document_to": "2024-12-31",
|
||
},
|
||
)
|
||
plain = ada.request(
|
||
"GET",
|
||
"/api/v1/sources/paperless/documents",
|
||
params={"connector_id": account["id"], "cursor": "2"},
|
||
)
|
||
assert [item["id"] for item in combined["items"]] == ["6", "12", "18", "24"]
|
||
assert [item["id"] for item in plain["items"]][0] == "21"
|