Files
tsumugu/backend/tests/test_connector_followup.py
T
StommerandClaude Sonnet 5.5 bee48bf4a5
Deploy Development / deploy (push) Successful in 52s
Behebe die fehlschlagenden Paperless-Tests und lasse test.yml nur noch nach dem Dev-Deploy laufen.
Der Hinweis zu Tags und Zusatzfeld sitzt wieder in paperless_documents, damit die Signatur von _paperless_get bleibt. Der Push-Trigger entfällt, weil er einen zweiten Lauf auf derselben Testdatenbank startete.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 07:29:45 +01:00

1094 lines
41 KiB
Python

"""Follow-up checks for paging, filters, storage resolution and notes.
IMAP and Paperless doubles stand in for the external servers. They do not
prove a live mailbox or a live Paperless instance.
"""
from __future__ import annotations
import io
import zipfile
from email.parser import BytesParser
import pytest
from app.core.errors import unprocessable
from app.media.port import MemoryAdapter
from app.sourceaccess import borrow_original, complete_borrow
from app.sourceregistry import SourceModule, unregister_source, register_source
from tests.support import ORIGIN, Actor, document, register
PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\nM\n"
LETTER = (
b"From: Ada <ada@example.test>\r\n"
b"To: Ben <ben@example.test>\r\n"
b"Subject: Brief\r\n"
b"Message-ID: <brief@example.test>\r\n"
b"MIME-Version: 1.0\r\n"
b'Content-Type: multipart/mixed; boundary="grenze"\r\n'
b"\r\n"
b"--grenze\r\n"
b"Content-Type: text/plain; charset=utf-8\r\n"
b"\r\n"
b"Hallo aus der zweiten Quelle.\r\n"
b"--grenze\r\n"
b"Content-Type: application/pdf\r\n"
b'Content-Disposition: attachment; filename="plan.pdf"\r\n'
b"\r\n"
b"%PDF-plan\r\n"
b"--grenze--\r\n"
)
def _headers(actor: Actor) -> dict:
return {
"Origin": ORIGIN,
"X-CSRF-Token": actor.csrf,
"X-Scope-Kind": actor.scope["kind"],
"X-Scope-Id": actor.scope["id"],
}
def _install_letterbox() -> None:
def parse_config(payload: dict):
name = str(payload.get("name") or "Briefkasten").strip() or "Briefkasten"
return name, {"catalog": "letters"}, "letter-secret"
def probe(row):
return {"ok": True, "detail": "Der Briefkasten antwortet."}
def find(conn, owner_id, connector_id, query, options):
del conn, owner_id, query, options
return {"items": [{"id": "<brief@example.test>", "title": "Brief"}], "connector_id": str(connector_id)}
def open_original(conn, actor_id, element, context, ref):
borrowed = borrow_original(conn, actor_id, element, context, ref, "letterbox")
return complete_borrow(conn, borrowed, lambda row: (LETTER, "message/rfc822", "nachricht.eml"))
def export_original(conn, actor_id, external_id, detail, assigned, ref):
del conn, actor_id, external_id, detail, assigned, ref
from app.retrieval import FilePayload
return FilePayload("nachricht.eml", "message/rfc822", LETTER, "")
register_source(SourceModule(
kind="letterbox",
label="Briefkasten",
representation="email",
capabilities={
"kind": "letterbox",
"navigate": False,
"read_original": True,
"read_preview": False,
"read_attachment": True,
"edit_content": False,
"edit_metadata": False,
"search": {"query": True, "continuation": False},
},
parse_config=parse_config,
probe=probe,
find=find,
open_original=open_original,
export_original=export_original,
))
class RangeIMAP:
searches: list = []
validity = b"3"
uidnext = b"50001"
fail_headers = False
search_status = "OK"
parts = False
def __init__(self, *args, **kwargs):
self.untagged_responses = {}
def login(self, username, secret):
assert secret == "Postfach-Geheim"
return "OK", [b"logged"]
def list(self):
return "OK", [b'(\\HasNoChildren) "/" "INBOX"']
def select(self, mailbox, readonly=False):
assert readonly is True
self.untagged_responses = {"UIDVALIDITY": [type(self).validity], "UIDNEXT": [type(self).uidnext]}
return "OK", [b"1"]
def uid(self, command, *args):
if command == "SEARCH":
type(self).searches.append(args)
if type(self).search_status != "OK":
return type(self).search_status, [None]
if "*" in args and not any(isinstance(arg, str) and ":" in arg for arg in args):
return "OK", [b"50000"]
low, high = _range_of(args)
if type(self).parts and low <= 5 <= high:
return "OK", [b"4 5", b"3"]
needle = _needle(args)
if needle:
hit = 80 if low <= 80 <= high else None
if needle.startswith("=?") or "nderung" in needle or "Änderung" in needle:
hit = 80 if low <= 80 <= high else None
return ("OK", [str(hit).encode()]) if hit else ("OK", [b""])
found = [str(uid).encode() for uid in range(low, high + 1)]
return "OK", [b" ".join(found)]
uid = int(args[0])
if type(self).fail_headers:
return "NO", [None]
subject = "Seltener Treffer" if uid == 80 else f"Nachricht {uid}"
header = (
f"From: Ada <ada@example.test>\r\nSubject: {subject}\r\n"
f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n"
).encode()
return "OK", [(b"body", header), b")"]
def logout(self):
return "BYE", [b""]
def _range_of(args) -> tuple[int, int]:
for arg in args:
text = arg.decode() if isinstance(arg, bytes) else str(arg)
if ":" in text and text.replace(":", "").isdigit():
low, high = text.split(":", 1)
return int(low), int(high)
raise AssertionError(args)
def _needle(args) -> str:
for arg in args:
text = arg.decode() if isinstance(arg, bytes) else str(arg)
if text.startswith('"') or "nderung" in text or "Änderung" in text:
return text
return ""
def _account(monkeypatch, actor: Actor, fake) -> str:
import app.sources as sources
monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", fake)
created = actor.request(
"POST",
"/api/v1/connectors",
json={
"kind": "imap",
"name": "Kanzlei",
"host": "imap.example.test",
"port": 993,
"username": "ada@example.test",
"password": "Postfach-Geheim",
"security": "ssl",
},
status=201,
)
return created["id"]
def _spans(searches) -> list[int]:
spans = []
for args in searches:
if any(arg == "*" for arg in args):
continue
low, high = _range_of(args)
spans.append(high - low + 1)
return spans
def test_large_folder_is_bounded_and_a_late_hit_is_not_empty(client, db, monkeypatch):
import app.sources as sources
ada = Actor(client, "ada-range@example.test", "Ada Bereich")
register(db, ada.email, ada.name)
ada.login()
RangeIMAP.searches = []
RangeIMAP.validity = b"3"
RangeIMAP.uidnext = b"50001"
RangeIMAP.fail_headers = False
RangeIMAP.search_status = "OK"
RangeIMAP.parts = False
account = _account(monkeypatch, ada, RangeIMAP)
first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
assert first["items"][0]["title"] == "Nachricht 50000"
assert len(first["items"]) == sources.PAGE_SIZE
assert first["incomplete"] is False
assert first["next_before"] is not None
assert _spans(RangeIMAP.searches)
assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW
assert all("ALL" not in args for args in RangeIMAP.searches)
RangeIMAP.searches = []
page = first
seen = {item["uid"] for item in page["items"]}
hops = 0
while 80 not in seen:
hops += 1
assert hops < 40
assert page["next_before"] or page["incomplete"]
page = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account, "mailbox": "INBOX", "q": "Seltener", "before": page["next_before"]},
)
if not page["items"] and page["incomplete"]:
assert page["next_before"]
seen.update(item["uid"] for item in page["items"])
assert any(item["title"] == "Seltener Treffer" for item in page["items"])
assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW
def test_imap_errors_are_not_reported_as_an_empty_folder(client, db, monkeypatch):
ada = Actor(client, "ada-imap-state@example.test", "Ada Zustand")
register(db, ada.email, ada.name)
ada.login()
RangeIMAP.searches = []
RangeIMAP.uidnext = b"1"
RangeIMAP.fail_headers = False
RangeIMAP.search_status = "OK"
RangeIMAP.parts = False
account = _account(monkeypatch, ada, RangeIMAP)
empty = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
assert empty["items"] == []
assert empty["incomplete"] is False
RangeIMAP.uidnext = b"31"
RangeIMAP.search_status = "NO"
refused = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account, "mailbox": "INBOX"},
status=422,
)
assert "abgelehnt" in refused["detail"]
assert "leer" not in refused["detail"].casefold()
RangeIMAP.search_status = "OK"
RangeIMAP.fail_headers = True
failed = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": account, "mailbox": "INBOX"},
status=422,
)
assert "Nachrichtenköpfe" in failed["detail"]
assert "leer" not in failed["detail"].casefold()
def test_uidvalidity_change_restarts_and_split_responses_join(client, db, monkeypatch):
ada = Actor(client, "ada-validity@example.test", "Ada Geltung")
register(db, ada.email, ada.name)
ada.login()
RangeIMAP.searches = []
RangeIMAP.validity = b"4"
RangeIMAP.uidnext = b"11"
RangeIMAP.fail_headers = False
RangeIMAP.search_status = "OK"
RangeIMAP.parts = False
account = _account(monkeypatch, ada, RangeIMAP)
first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
RangeIMAP.validity = b"9"
restarted = ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={
"connector_id": account,
"mailbox": "INBOX",
**({"before": first["next_before"]} if first["next_before"] else {}),
"uidvalidity": first["uidvalidity"],
},
)
assert restarted["restarted"] is True
assert restarted["items"][0]["title"] == "Nachricht 10"
RangeIMAP.validity = b"4"
RangeIMAP.parts = True
RangeIMAP.uidnext = b"6"
joined = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"})
assert [item["uid"] for item in joined["items"]] == [5, 4, 3]
def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch):
import app.sources as sources
calls = []
def fake_get(base, token, path, params):
calls.append({"path": path, "params": dict(params), "token": token})
if path != "/api/documents/":
return {"count": 1, "results": [{"id": 4, "name": "Haus"}]}
page = int(params.get("page") or 1)
start = (page - 1) * 20
return {
"count": 45,
"results": [
{"id": number + 1, "title": f"Titel {number + 1}", "created": "2024-03-01"}
for number in range(start, min(start + 20, 45))
],
}
def fake_bytes(row, document_id):
return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf"
monkeypatch.setattr(sources, "_paperless_get", fake_get)
monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes)
ada = Actor(client, "ada-filter@example.test", "Ada Filter")
register(db, ada.email, ada.name)
ada.login()
first = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
second = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Zweitarchiv", "base_url": "https://paper.example.test", "token": "token-bravo"},
status=201,
)
facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]})
assert facets["tag_mode"] == "all"
assert facets["correspondents"] == [{"id": "4", "name": "Haus"}]
ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "document_from": "2024-05-02", "document_to": "2024-01-01"},
status=422,
)
found = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={
"connector_id": first["id"],
"q": "nurinhalt",
"mode": "index",
"title": "Vertrag",
"correspondent": "4",
"document_type": "8",
"storage_path": "6",
"tag": "2,5",
"tag_mode": "any",
"document_from": "2024-01-01",
"document_to": "2024-12-31",
"added_from": "2024-02-01",
"added_to": "2024-02-28",
"asn": "1042",
"sort": "-created",
"cursor": "3",
},
)
sent = calls[-1]["params"]
assert sent["query"] == "nurinhalt"
assert "title_content" not in sent
assert sent["title__icontains"] == "Vertrag"
assert sent["correspondent__id"] == "4"
assert sent["document_type__id"] == "8"
assert sent["storage_path__id"] == "6"
assert sent["tags__id__in"] == "2,5"
assert "tags__id__all" not in sent
assert sent["added__date__gte"] == "2024-02-01"
assert sent["added__date__lte"] == "2024-02-28"
assert sent["archive_serial_number"] == "1042"
assert sent["created__date__gte"] == "2024-01-01"
assert sent["created__date__lte"] == "2024-12-31"
assert sent["ordering"] == "-created"
assert sent["page"] == "3"
ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "q": "nurinhalt", "mode": "index", "sort": "-score"},
)
assert "ordering" not in calls[-1]["params"]
ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"},
status=422,
)
assert found["items"][0]["id"] == "41"
assert found["connector_id"] == first["id"]
preview = ada.client.get(
"/api/v1/sources/paperless/documents/41/preview",
params={"connector_id": first["id"]},
headers=_headers(ada),
)
assert preview.status_code == 200
assert preview.content.startswith(b"%PDF-token-alpha-41")
matter = ada.request("POST", "/api/v1/matters", json={"title": "Suche"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Seite drei",
"source": "paperless",
"external_id": "41",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": first["id"]},
},
status=201,
)
other = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Dieselbe Nummer",
"source": "paperless",
"external_id": "41",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": second["id"]},
},
status=201,
)
opened = ada.client.get(f"/api/v1/external-refs/{linked['id']}/content", headers=_headers(ada))
twin = ada.client.get(f"/api/v1/external-refs/{other['id']}/content", headers=_headers(ada))
assert opened.content.startswith(b"%PDF-token-alpha-41")
assert twin.content.startswith(b"%PDF-token-bravo-41")
ben = Actor(client, "ben-filter@example.test", "Ben Filter")
register(db, ben.email, ben.name)
ben.login()
ben.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]}, status=404)
ben.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "q": "nurinhalt"},
status=404,
)
def test_paperless_default_search_reads_title_and_content_and_shows_a_preview(client, db, monkeypatch):
import app.sources as sources
calls = []
body = "Vorspann " * 30 + "Die Kündigung des Mietvertrags erfolgt zum Jahresende. " + "Nachspann " * 30
def fake_get(base, token, path, params):
calls.append({"path": path, "params": dict(params)})
if path == "/api/documents/":
return {
"count": 1,
"results": [
{
"id": 7,
"title": "Schreiben",
"created": "2024-03-01T00:00:00+01:00",
"added": "2024-03-05T08:00:00Z",
"correspondent": 4,
"document_type": 8,
"storage_path": None,
"tags": [2, 5, 99],
"archive_serial_number": 1042,
"page_count": 3,
"custom_fields": [
{"field": 1, "value": "EUR100.50"},
{"field": 2, "value": "2024-03-09"},
{"field": 3, "value": "b"},
{"field": 4, "value": True},
{"field": 5, "value": None},
{"field": 99, "value": "fremd"},
],
"content": body,
}
],
}
if path == "/api/storage_paths/":
raise sources.unprocessable("Paperless hat die Anfrage nicht beantwortet.")
if path == "/api/tags/":
return {"results": [{"id": 2, "name": "Miete"}, {"id": 5, "name": "Wohnung"}]}
if path == "/api/correspondents/":
return {"results": [{"id": 4, "name": "Hausverwaltung"}]}
if path == "/api/custom_fields/":
return {
"results": [
{"id": 1, "name": "Rechnungsbetrag", "data_type": "monetary"},
{"id": 2, "name": "bezahlt_am", "data_type": "date"},
{"id": 3, "name": "Status", "data_type": "select", "extra_data": {"select_options": [{"id": "b", "label": "bezahlt"}]}},
{"id": 4, "name": "Geprüft", "data_type": "boolean"},
{"id": 5, "name": "Notiz", "data_type": "string"},
]
}
return {"results": [{"id": 8, "name": "Brief"}]}
monkeypatch.setattr(sources, "_paperless_get", fake_get)
ada = Actor(client, "ada-preview@example.test", "Ada Preview")
register(db, ada.email, ada.name)
ada.login()
first = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
found = ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "q": "kündigung", "correspondent": "none", "tag": "2", "tag_mode": "none"},
)
sent = [call for call in calls if call["path"] == "/api/documents/"][-1]["params"]
assert sent["title_content"] == "kündigung"
assert "query" not in sent
assert sent["correspondent__isnull"] == "true"
assert sent["tags__id__none"] == "2"
item = found["items"][0]
assert item["correspondent"] == "Hausverwaltung"
assert item["document_type"] == "Brief"
assert item["tags"] == ["Miete", "Wohnung"]
assert item["pages"] == 3
assert item["asn"] == 1042
assert item["document_on"] == "2024-03-01"
assert item["added_on"] == "2024-03-05"
assert "Kündigung des Mietvertrags" in item["snippet"]
assert len(item["snippet"]) <= 262
assert item["snippet"].startswith("…")
assert item["custom_fields"] == [
{"name": "Rechnungsbetrag", "type": "monetary", "value": "EUR100.50"},
{"name": "bezahlt_am", "type": "date", "value": "2024-03-09"},
{"name": "Status", "type": "select", "value": "bezahlt"},
{"name": "Geprüft", "type": "boolean", "value": "Ja"},
]
assert found["total"] == 1
assert found["previous_page"] is None
facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]})
assert facets["storage_paths"] == []
assert facets["tag_modes"] == ["all", "any", "none"]
# A filter of another source is refused, not ignored.
ada.request(
"GET",
"/api/v1/sources/imap/messages",
params={"connector_id": first["id"], "mailbox": "INBOX", "mode": "title"},
status=422,
)
for params in (
{"mode": "regex"},
{"tag_mode": "x"},
{"asn": "x1"},
{"storage_path": "a"},
{"q": "miete", "mode": "title", "title": "vertrag"},
{"bogus": "1"},
):
ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], **params},
status=422,
)
def test_paperless_custom_field_filters_use_the_instance_fields(client, db, monkeypatch):
import json
import app.sources as sources
calls = []
def fake_get(base, token, path, params):
calls.append({"path": path, "params": dict(params)})
if path == "/api/custom_fields/":
return {
"results": [
{"id": 1, "name": "Rechnungsbetrag", "data_type": "monetary"},
{"id": 2, "name": "bezahlt_am", "data_type": "date"},
{
"id": 3,
"name": "Status",
"data_type": "select",
"extra_data": {"select_options": [{"id": "a", "label": "offen"}, {"id": "b", "label": "bezahlt"}]},
},
{"id": 4, "name": "Verweis", "data_type": "documentlink"},
]
}
if path == "/api/documents/":
if failing:
raise unprocessable(failing[0])
return {"count": 0, "results": []}
return {"results": []}
failing = []
monkeypatch.setattr(sources, "_paperless_get", fake_get)
ada = Actor(client, "ada-custom@example.test", "Ada Custom")
register(db, ada.email, ada.name)
ada.login()
first = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]})
assert [item["name"] for item in facets["custom_fields"]] == ["Rechnungsbetrag", "bezahlt_am", "Status"]
assert facets["custom_fields"][2]["options"] == [{"id": "a", "name": "offen"}, {"id": "b", "name": "bezahlt"}]
def ask(rows, status=200):
return ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "custom": json.dumps(rows)},
status=status,
)
ask([{"field": "2", "op": "range", "value": ["2024-01-01", "2024-06-30"]}])
sent = [call for call in calls if call["path"] == "/api/documents/"][-1]["params"]
assert json.loads(sent["custom_field_query"]) == ["bezahlt_am", "range", ["2024-01-01", "2024-06-30"]]
ask(
[
{"field": "1", "op": "gte", "value": "100,5"},
{"field": "3", "op": "exact", "value": "a"},
{"field": "2", "op": "exists", "value": False},
]
)
sent = [call for call in calls if call["path"] == "/api/documents/"][-1]["params"]
assert json.loads(sent["custom_field_query"]) == [
"AND",
[["Rechnungsbetrag", "gte", 100.5], ["Status", "exact", "a"], ["bezahlt_am", "exists", False]],
]
ask([], 200)
assert "custom_field_query" not in [call for call in calls if call["path"] == "/api/documents/"][-1]["params"]
for rows in (
[{"field": "9", "op": "exact", "value": "x"}],
[{"field": "4", "op": "contains", "value": "1"}],
[{"field": "2", "op": "icontains", "value": "2024"}],
[{"field": "3", "op": "exact", "value": "z"}],
[{"field": "1", "op": "gt", "value": "viel"}],
[{"field": "2", "op": "range", "value": ["2024-06-30", "2024-01-01"]}],
[{"field": "2", "op": "exists", "value": "ja"}],
[{"field": "1", "op": "exact", "value": "1"}] * 6,
):
ask(rows, 422)
ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "custom": "kein json"},
status=422,
)
# Paperless answers 500 for all-tags together with a custom field: the message names that pairing.
failing.append("Paperless hat die Anfrage nicht beantwortet (HTTP 500 auf /api/documents/).")
row = json.dumps([{"field": "2", "op": "exists", "value": True}])
def refused(extra):
return ada.request(
"GET",
"/api/v1/sources/paperless/documents",
params={"connector_id": first["id"], "custom": row, **extra},
status=422,
)
assert "Alle gewählten Tags" in str(refused({"tag": "1,2"}))
assert "Alle gewählten Tags" not in str(refused({"tag": "1,2", "tag_mode": "any"}))
assert "Alle gewählten Tags" not in str(refused({"tag": "1"}))
def test_paperless_refusal_names_the_status_and_reason(monkeypatch):
import io
from urllib.error import HTTPError
import app.sources as sources
from app.core.errors import Problem
def refuse(code, body):
def fake(request, timeout=0):
raise HTTPError(request.full_url, code, "x", {}, io.BytesIO(body))
return fake
monkeypatch.setattr(sources, "urlopen", refuse(400, '{"custom_field_query": ["Ungültig"]}'.encode()))
with pytest.raises(Problem) as caught:
sources._paperless_get("https://paper.example.test", "token-alpha", "/api/documents/", {"page_size": "20"})
assert "HTTP 400 auf /api/documents/" in caught.value.detail
assert "custom_field_query" in caught.value.detail
assert "token-alpha" not in caught.value.detail
monkeypatch.setattr(sources, "urlopen", refuse(500, b"<html>Fehler</html>"))
with pytest.raises(Problem) as caught:
sources._paperless_get("https://paper.example.test", "token-alpha", "/api/documents/", {})
assert caught.value.detail.endswith("(HTTP 500 auf /api/documents/).")
monkeypatch.setattr(sources, "urlopen", refuse(403, b"{}"))
with pytest.raises(Problem) as caught:
sources._paperless_get("https://paper.example.test", "token-alpha", "/api/documents/", {})
assert "Schlüssel abgelehnt" in caught.value.detail
def test_paperless_thumbnail_is_bound_to_the_owners_instance(client, db, monkeypatch):
import app.sources as sources
seen = []
def fake_file(base, token, path, max_bytes=0):
seen.append((token, path))
if path.endswith("/9/thumb/"):
return b"RIFFxxxxWEBP", "image/webp", ""
if path.endswith("/8/thumb/"):
return b"<p>", "text/html", ""
if path.endswith("/6/thumb/"):
return b"\x89PNG\r\n\x1a\nxxxx", "application/octet-stream", ""
if path.endswith("/7/thumb/"):
return b"<svg xmlns='http://www.w3.org/2000/svg'/>", "image/svg+xml", ""
return None
monkeypatch.setattr(sources, "_paperless_file", fake_file)
ada = Actor(client, "ada-thumb@example.test", "Ada Thumb")
register(db, ada.email, ada.name)
ada.login()
first = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
url = "/api/v1/sources/paperless/thumbnail"
def get(actor, item, kind_url=url):
return actor.client.get(kind_url, params={"connector_id": first["id"], "item": item}, headers=_headers(actor))
ok = get(ada, 9)
assert ok.status_code == 200
assert ok.headers["content-type"] == "image/webp"
assert ok.headers["cache-control"] == "private, no-store"
assert seen[-1] == ("token-alpha", "/api/documents/9/thumb/")
typed = get(ada, 6)
assert typed.status_code == 200 and typed.headers["content-type"] == "image/png"
assert get(ada, 8).status_code == 422
assert get(ada, 7).status_code == 422
assert get(ada, 5).status_code == 404
assert get(ada, "x1").status_code == 422
# IMAP declares no thumbnails, and the module must declare the capability.
assert get(ada, 9, "/api/v1/sources/imap/thumbnail").status_code == 422
ben = Actor(client, "ben-thumb@example.test", "Ben Thumb")
register(db, ben.email, ben.name)
ben.login()
assert get(ben, 9).status_code == 404
assert len(seen) == 5
def test_revoked_grant_hides_bytes_fetched_during_the_call(client, db, monkeypatch):
import app.sources as sources
def fake_bytes(row, document_id):
db.execute("UPDATE access_grants SET can_read = false")
db.commit()
return b"%PDF-1.4\nsecret\n", "application/pdf", "geheim.pdf"
monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes)
ada = Actor(client, "ada-revoke@example.test", "Ada Widerruf")
ben = Actor(client, "ben-revoke@example.test", "Ben Widerruf")
register(db, ada.email, ada.name)
register(db, ben.email, ben.name)
ada.login()
paper = ada.request(
"POST",
"/api/v1/connectors",
json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"},
status=201,
)
matter = ada.request("POST", "/api/v1/matters", json={"title": "Freigabe"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Vertrag",
"source": "paperless",
"external_id": "15",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": paper["id"]},
},
status=201,
)
ada.request(
"POST",
f"/api/v1/matters/{matter['id']}/grants",
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True},
status=201,
)
ben.login()
hidden = ben.client.get(
f"/api/v1/external-refs/{linked['id']}/content",
params={"context_kind": "matter", "context_id": matter["id"]},
headers=_headers(ben),
)
assert hidden.status_code == 422
assert b"secret" not in hidden.content
def test_email_representation_uses_the_mail_reader_without_imap(client, db):
ada = Actor(client, "ada-letter@example.test", "Ada Brief")
register(db, ada.email, ada.name)
ada.login()
_install_letterbox()
try:
box = ada.request("POST", "/api/v1/connectors", json={"kind": "letterbox", "name": "Briefe"}, status=201)
matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201)
linked = ada.request(
"POST",
"/api/v1/external-refs",
json={
"title": "Brief",
"source": "letterbox",
"external_id": "<brief@example.test>",
"context": {"kind": "matter", "id": matter["id"]},
"detail": {"connector_id": box["id"]},
},
status=201,
)
mail = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail")
assert mail["attachments"][0]["filename"] == "plan.pdf"
attachment = ada.client.get(
f"/api/v1/external-refs/{linked['id']}/mail/attachments/0",
headers=_headers(ada),
)
assert attachment.status_code == 200
assert attachment.content.startswith(b"%PDF-plan")
raw = ada.client.get(f"/api/v1/external-refs/{linked['id']}/mail/original", headers=_headers(ada))
parsed = BytesParser().parsebytes(raw.content)
assert parsed.get("subject") == "Brief"
finally:
unregister_source("letterbox")
def test_domain_storage_uses_the_resolved_adapter(client, db, monkeypatch):
import app.media.port as port
memory = MemoryAdapter()
monkeypatch.setitem(port._FACTORIES, "local", lambda row: memory)
ada = Actor(client, "ada-store@example.test", "Ada Speicher")
register(db, ada.email, ada.name)
ada.login()
matter = ada.request("POST", "/api/v1/matters", json={"title": "Ablage"}, status=201)
ctx = {"kind": "matter", "id": matter["id"]}
note = ada.request(
"POST",
"/api/v1/notes",
json={"title": "Schreiben", "document": document("Text"), "context": ctx},
status=201,
)
uploaded = ada.request(
"POST",
"/api/v1/files",
data={"title": "Plan", "context_kind": "matter", "context_id": matter["id"], "document_on": "2024-01-02"},
files={"upload": ("plan.pdf", PDF, "application/pdf")},
status=201,
)
detail = ada.request("GET", f"/api/v1/matters/{matter['id']}")
host = next(item for item in detail["bindings"] if item["element"]["id"] == note["id"])
ada.request(
"POST",
f"/api/v1/elements/{note['id']}/attachments",
json={
"expected_version": host["element"]["version"],
"context": ctx,
"attachment_element_ids": [uploaded["id"]],
"fold": "attachment",
},
status=201,
)
folded = ada.request("GET", f"/api/v1/matters/{matter['id']}")
host = next(item for item in folded["bindings"] if item["element"]["id"] == note["id"])
dispatch = ada.request(
"POST",
f"/api/v1/elements/{note['id']}/dispatches",
json={
"expected_version": host["element"]["version"],
"context": ctx,
"channel": "post",
"sent_on": "2024-02-02",
"attachment_element_ids": [uploaded["id"]],
"capture_editions": True,
},
status=201,
)
content = ada.client.get(f"/api/v1/files/{uploaded['id']}/content", headers=_headers(ada))
assert content.content == PDF
item = next(entry for entry in dispatch["items"] if entry["element_id"] == uploaded["id"])
snapshot = ada.client.get(
f"/api/v1/dispatches/{dispatch['id']}/items/{item['id']}/content",
params={"context_kind": "matter", "context_id": matter["id"]},
headers=_headers(ada),
)
assert snapshot.content == PDF
asset = db.execute(
"SELECT object_key, storage_generation, sha256 FROM media_assets WHERE element_id = %s",
(uploaded["id"],),
).fetchone()
assert memory.open(asset["object_key"]) == PDF
tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"]
selected = [
node["id"]
for node in _walk(tree)
if node.get("exportable") and node.get("kind") in {"binding", "attachment", "dispatch_item"}
]
package = ada.client.post(
f"/api/v1/matters/{matter['id']}/export",
json={"context": ctx, "selected": selected},
headers={**_headers(ada), "Content-Type": "application/json"},
)
assert package.status_code == 200, package.text
archive = zipfile.ZipFile(io.BytesIO(package.content))
assert any(archive.read(name) == PDF for name in archive.namelist())
version = next(
item["element"]["version"]
for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["bindings"]
if item["element"]["id"] == uploaded["id"]
)
ada.request(
"POST",
f"/api/v1/elements/{uploaded['id']}/delete",
json={"expected_version": version},
status=204,
)
try:
memory.open(asset["object_key"])
except Exception:
removed = True
else:
removed = False
assert removed is True
assert asset["storage_generation"]
def test_note_markdown_is_shared_exported_and_kept_on_conflict(client, db):
ada = Actor(client, "ada-note-md@example.test", "Ada Notiz")
ben = Actor(client, "ben-note-md@example.test", "Ben Notiz")
register(db, ada.email, ada.name)
register(db, ben.email, ben.name)
ada.login()
left = ada.request("POST", "/api/v1/matters", json={"title": "Links"}, status=201)
right = ada.request("POST", "/api/v1/matters", json={"title": "Rechts"}, status=201)
body = "\n".join([
"# Überschrift",
"",
"Ein **fetter** und *kursiver* Satz.",
"",
"- erster",
"- zweiter",
"",
"> Zitat",
"",
"Siehe [Quelle](https://example.test/hinweis).",
"",
"`code`",
"",
"```",
"block",
"```",
"",
"<script>alert(1)</script>",
"",
"![bild](https://example.test/bild.png)",
])
created = ada.request(
"POST",
"/api/v1/notes",
json={
"title": "Merkzettel",
"document": {"version": 1, "blocks": [{"type": "markdown", "text": body}]},
"context": {"kind": "matter", "id": left["id"]},
},
status=201,
)
ada.request(
"POST",
"/api/v1/bindings",
json={
"element_id": created["id"],
"source_context": {"kind": "matter", "id": left["id"]},
"target": {"kind": "matter", "id": right["id"]},
},
status=201,
)
detail = ada.request("GET", f"/api/v1/matters/{left['id']}")
element = next(item["element"] for item in detail["bindings"] if item["element"]["id"] == created["id"])
changed = "Geändert in beiden Vorgängen.\n\n" + body
ada.request(
"PATCH",
f"/api/v1/notes/{created['id']}",
json={
"expected_version": element["version"],
"document": {"version": 1, "blocks": [{"type": "markdown", "text": changed}]},
"context": {"kind": "matter", "id": right["id"]},
},
)
again = ada.request("GET", f"/api/v1/matters/{left['id']}")
shared = next(item["element"] for item in again["bindings"] if item["element"]["id"] == created["id"])
assert changed in _text(shared)
ada.request(
"POST",
f"/api/v1/bindings/{next(item['id'] for item in again['bindings'] if item['element']['id'] == created['id'])}/annotations",
json={"body": "Nur links"},
params={"context_kind": "matter", "context_id": left["id"]},
status=201,
)
left_detail = ada.request("GET", f"/api/v1/matters/{left['id']}")
right_detail = ada.request("GET", f"/api/v1/matters/{right['id']}")
left_binding = next(item for item in left_detail["bindings"] if item["element"]["id"] == created["id"])
right_binding = next(item for item in right_detail["bindings"] if item["element"]["id"] == created["id"])
assert any(note["body"] == "Nur links" for note in left_binding["annotations"])
assert all(note["body"] != "Nur links" for note in right_binding["annotations"])
ada.request(
"POST",
f"/api/v1/matters/{left['id']}/grants",
json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False},
status=201,
)
ben.login()
ben.request(
"PATCH",
f"/api/v1/notes/{created['id']}",
json={
"expected_version": shared["version"],
"document": {"version": 1, "blocks": [{"type": "markdown", "text": "fremd"}]},
"context": {"kind": "matter", "id": left["id"]},
},
status=403,
)
ada.login()
current = next(
item["element"]["version"]
for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"]
if item["element"]["id"] == created["id"]
)
conflict = ada.request(
"PATCH",
f"/api/v1/notes/{created['id']}",
json={
"expected_version": current - 1,
"document": {"version": 1, "blocks": [{"type": "markdown", "text": "darf nicht überschreiben"}]},
"context": {"kind": "matter", "id": left["id"]},
},
status=409,
)
assert conflict["code"] == "conflict"
stored = next(
item["element"]
for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"]
if item["element"]["id"] == created["id"]
)
assert "darf nicht überschreiben" not in _text(stored)
assert "<script>alert(1)</script>" in _text(stored)
tree = ada.request("GET", f"/api/v1/matters/{left['id']}/export-tree")["tree"]
selected = [node["id"] for node in _walk(tree) if node.get("kind") == "binding" and node.get("exportable")]
package = ada.client.post(
f"/api/v1/matters/{left['id']}/export",
json={"context": {"kind": "matter", "id": left["id"]}, "selected": selected},
headers={**_headers(ada), "Content-Type": "application/json"},
)
assert package.status_code == 200, package.text
archive = zipfile.ZipFile(io.BytesIO(package.content))
exported = "\n".join(
archive.read(name).decode("utf-8", errors="replace")
for name in archive.namelist()
if name.endswith((".md", ".txt"))
)
assert "```" in exported
assert "<script>alert(1)</script>" in exported
assert "https://example.test/bild.png" in exported
def _text(element: dict) -> str:
return "\n".join(block.get("text") or "" for block in (element.get("document") or {}).get("blocks") or [])
def _walk(node):
if isinstance(node, dict) and "kind" in node:
yield node
for child in node.get("children") or []:
yield from _walk(child)
elif isinstance(node, dict):
for value in node.values():
yield from _walk(value)
elif isinstance(node, list):
for item in node:
yield from _walk(item)