"""Section A: source binding and the storage port, with today's behaviour kept.""" from __future__ import annotations import hashlib import json from pathlib import Path from uuid import uuid4 import pytest from app.core.errors import Problem from app.media.port import ( LEGACY_GENERATION, LocalAdapter, MemoryAdapter, check_object_key, instance_for_write, publish, read_clear_bytes, ) from app.sourcebind import backfill_source_bindings from tests.support import ORIGIN, Actor, register PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" def _raw(actor: Actor, path: str): return actor.client.get( path, headers={ "Origin": ORIGIN, "X-CSRF-Token": actor.csrf, "X-Scope-Kind": actor.scope["kind"], "X-Scope-Id": actor.scope["id"], }, ) def _case(actor: Actor) -> dict: return actor.request("POST", "/api/v1/cases", json={"title": "Bestand"}, status=201) def _paperless(actor: Actor, name: str, token: str) -> str: row = actor.request( "POST", "/api/v1/connectors", json={"kind": "paperless", "name": name, "base_url": "https://paper.example.test", "token": token}, status=201, ) return row["id"] def _link(actor: Actor, case_id: str, connector_id: str | None, external_id: str, title: str) -> str: body = { "title": title, "source": "paperless", "external_id": external_id, "context": {"kind": "case", "id": case_id}, } if connector_id: body["detail"] = {"connector_id": connector_id} created = actor.request("POST", "/api/v1/external-refs", json=body, status=201) return created["id"] def test_storage_adapters_roundtrip_and_reject_escape(tmp_path): payload = b"opaque-bytes" key = str(uuid4()) revision = "rev-1" for adapter in (LocalAdapter(tmp_path), MemoryAdapter()): publish(adapter, key, payload) assert adapter.open(key) == payload assert adapter.describe(key)["byte_size"] == len(payload) assert adapter.open(key) == payload assert revision == "rev-1" memory = MemoryAdapter() memory.fail_put = True with pytest.raises(Problem) as failed: publish(memory, str(uuid4()), payload) assert failed.value.code == "storage_unavailable" assert memory._items == {} with pytest.raises(Problem): check_object_key("../secret") with pytest.raises(Problem): check_object_key("/etc/passwd") with pytest.raises(Problem): LocalAdapter(tmp_path).open("dispatch/../../" + str(uuid4())) def test_changed_root_does_not_retarget_legacy(db, database): legacy = db.execute( "SELECT id, root_path FROM storage_instances WHERE generation = %s", (LEGACY_GENERATION,), ).fetchone() key = str(uuid4()) original = b"%PDF-legacy-root" publish(LocalAdapter(legacy["root_path"]), key, original) other = Path(database.media_root).parent / f"other-root-{uuid4().hex}" other.mkdir() moved = instance_for_write(db, str(other)) assert moved["id"] != legacy["id"] assert moved["generation"] != LEGACY_GENERATION publish(LocalAdapter(moved["root_path"]), key, b"%PDF-other-root") seen = read_clear_bytes( db, instance_id=legacy["id"], generation=LEGACY_GENERATION, object_key=key, media_root=str(other), protection_profile="legacy-plaintext", expected_sha256=None, ) assert seen == original db.rollback() def test_existing_file_keeps_id_bytes_and_revision(client, db): register(db, "ada-file@example.test", "Ada File") ada = Actor(client, "ada-file@example.test", "Ada File") ada.login() case = _case(ada) upload = ada.request( "POST", "/api/v1/files", data={"title": "Scan", "context_kind": "case", "context_id": case["id"]}, files={"upload": ("scan.pdf", PDF, "application/pdf")}, status=201, ) stored = db.execute( """ SELECT a.id, a.object_key, a.content_revision, a.storage_generation, a.protection_profile, s.root_path FROM media_assets a JOIN storage_instances s ON s.id = a.storage_instance_id WHERE a.element_id = %s """, (upload["id"],), ).fetchone() assert stored["protection_profile"] == "legacy-plaintext" assert stored["object_key"] == str(stored["id"]) assert Path(stored["root_path"]) == Path(client.app.state.settings.media_root).resolve() assert stored["storage_generation"] first = stored["content_revision"] opened = _raw(ada, f"/api/v1/files/{upload['id']}/content") assert opened.status_code == 200 assert opened.content == PDF again = db.execute( "SELECT content_revision FROM media_assets WHERE element_id = %s", (upload["id"],), ).fetchone() assert again["content_revision"] == first refused = ada.client.patch( f"/api/v1/notes/{upload['id']}", json={"expected_version": 1, "document": {"version": 1, "blocks": [{"type": "paragraph", "text": "nein"}]}}, headers={ "Origin": ORIGIN, "X-CSRF-Token": ada.csrf, "X-Scope-Kind": ada.scope["kind"], "X-Scope-Id": ada.scope["id"], }, ) assert refused.status_code == 422 assert refused.json()["code"] == "unsupported" def test_two_paperless_instances_stay_bound_when_order_changes(client, db, monkeypatch): register(db, "ada-two@example.test", "Ada Two") ada = Actor(client, "ada-two@example.test", "Ada Two") ada.login() case = _case(ada) older = _paperless(ada, "Zweitens", "token-older") newer = _paperless(ada, "Erstens", "token-newer") assert older != newer def fake_bytes(row, document_id): return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf" monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes) left = _link(ada, case["id"], older, "15", "Altbestand") right = _link(ada, case["id"], newer, "15", "Zweiter Bestand") db.execute("UPDATE connectors SET created_at = now() - interval '2 days' WHERE id = %s", (newer,)) db.execute("UPDATE connectors SET created_at = now() WHERE id = %s", (older,)) db.commit() assert _raw(ada, f"/api/v1/external-refs/{left}/content").content.startswith(b"%PDF-token-older-15") assert _raw(ada, f"/api/v1/external-refs/{right}/content").content.startswith(b"%PDF-token-newer-15") calls = [] def fake_get(*args, **kwargs): calls.append(args) return {"results": [{"id": 15, "title": "Treffer", "created": "2024-01-02"}]} monkeypatch.setattr("app.sources._paperless_get", fake_get) headers = { "Origin": ORIGIN, "X-CSRF-Token": ada.csrf, "X-Scope-Kind": ada.scope["kind"], "X-Scope-Id": ada.scope["id"], } refused = ada.client.get("/api/v1/sources/paperless/documents", params={"q": "Treffer"}, headers=headers) assert refused.status_code == 422 assert calls == [] found = ada.request( "GET", "/api/v1/sources/paperless/documents", params={"q": "Treffer", "connector_id": older}, ) assert found["items"][0]["title"] == "Treffer" assert found["connector_id"] == older assert found["capabilities"]["search"]["correspondent"] is True assert "correspondent" not in found["unsupported"] filtered = ada.request( "GET", "/api/v1/sources/paperless/documents", params={"q": "Treffer", "connector_id": older, "correspondent": "4"}, ) assert filtered["items"][0]["title"] == "Treffer" assert calls[-1][3]["correspondent__id"] == "4" assert calls[-1][3]["title_content"] == "Treffer" def test_ambiguous_legacy_stays_until_assignment(client, db, monkeypatch): register(db, "ada-old@example.test", "Ada Old") ada = Actor(client, "ada-old@example.test", "Ada Old") ada.login() case = _case(ada) loose = _link(ada, case["id"], None, "15", "Ohne Instanz") _paperless(ada, "Alpha", "token-alpha") second = _paperless(ada, "Beta", "token-beta") db.execute( """ UPDATE external_references SET connector_id = NULL, resolution = 'unresolved', detail = '{}'::jsonb WHERE element_id = %s """, (loose,), ) backfill_source_bindings(db) state = db.execute( "SELECT connector_id, resolution FROM external_references WHERE element_id = %s", (loose,), ).fetchone() db.commit() assert state["resolution"] == "ambiguous" assert state["connector_id"] is None opened = ada.request("GET", f"/api/v1/cases/{case['id']}") assert opened["bindings"][0]["element"]["id"] == loose assert opened["bindings"][0]["element"]["preview"]["media"] == "unassigned" blocked = _raw(ada, f"/api/v1/external-refs/{loose}/content") assert blocked.status_code == 422 assert "eindeutiger Zugang" in blocked.json()["detail"] def fake_bytes(row, document_id): del document_id return f"%PDF-{row['secret']}\n".encode(), "application/pdf", "dokument.pdf" monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes) ada.request( "POST", f"/api/v1/external-refs/{loose}/assignment", json={"connector_id": second}, ) _paperless(ada, "Gamma", "token-gamma") assert _raw(ada, f"/api/v1/external-refs/{loose}/content").content.startswith(b"%PDF-token-beta") def test_legacy_stock_rebinds_and_case_and_matter_open(client, db, monkeypatch): from app.sourceregistry import get_source, unregister_source register(db, "ada-keep@example.test", "Ada Keep") ada = Actor(client, "ada-keep@example.test", "Ada Keep") ada.login() case = _case(ada) matter = ada.request("POST", "/api/v1/matters", json={"title": "Bestand"}, status=201) kept = _paperless(ada, "Archiv", "token-kept") case_ref = _link(ada, case["id"], None, "15", "Vertragsakte") matter_ref = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Vertragsvorgang", "source": "paperless", "external_id": "16", "context": {"kind": "matter", "id": matter["id"]}, }, status=201, )["id"] db.execute( """ UPDATE external_references SET connector_id = NULL, resolution = 'unresolved', detail = '{}'::jsonb WHERE element_id IN (%s, %s) """, (case_ref, matter_ref), ) db.commit() unregister_source("paperless") unregister_source("imap") assert get_source("paperless") is None try: backfill_source_bindings(db) db.commit() for element_id in (case_ref, matter_ref): state = db.execute( "SELECT connector_id::text AS connector_id, resolution FROM external_references WHERE element_id = %s", (element_id,), ).fetchone() assert state["resolution"] == "bound" assert state["connector_id"] == kept assert get_source("paperless") is not None case_page = ada.request("GET", f"/api/v1/cases/{case['id']}") matter_page = ada.request("GET", f"/api/v1/matters/{matter['id']}") assert case_page["bindings"][0]["element"]["preview"]["available"] is True assert case_page["bindings"][0]["element"]["preview"]["media"] == "pdf" assert matter_page["bindings"][0]["element"]["preview"]["available"] is True monkeypatch.setattr( "app.sources._paperless_bytes", lambda row, document_id: (f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf"), ) assert _raw(ada, f"/api/v1/external-refs/{case_ref}/content").content.startswith(b"%PDF-token-kept-15") assert _raw(ada, f"/api/v1/external-refs/{matter_ref}/content").content.startswith(b"%PDF-token-kept-16") finally: from app.sources import _install_sources _install_sources() def test_connectors_remain_with_their_account(client, db, monkeypatch): calls = [] monkeypatch.setattr( "app.sources._paperless_get", lambda *args, **kwargs: calls.append(args) or {"results": []}, ) register(db, "ada-own@example.test", "Ada Own") register(db, "ben-own@example.test", "Ben Own") ada = Actor(client, "ada-own@example.test", "Ada Own") ben = Actor(client, "ben-own@example.test", "Ben Own") ada.login() ada_paper = _paperless(ada, "Ada Archiv", "token-ada") ben.login() ben_paper = _paperless(ben, "Ben Archiv", "token-ben") listed = ben.request("GET", "/api/v1/connectors") ids = {item["id"] for item in listed["items"]} assert ben_paper in ids assert ada_paper not in ids assert all(item["name"] != "Ada Archiv" for item in listed["items"]) ben.request("POST", f"/api/v1/connectors/{ada_paper}/probe", status=404) ben.request( "GET", "/api/v1/sources/paperless/documents", params={"q": "fremd", "connector_id": ada_paper}, status=404, ) assert calls == [] ada.login() own = {item["id"] for item in ada.request("GET", "/api/v1/connectors")["items"]} assert ada_paper in own assert ben_paper not in own def test_locked_profile_serves_metadata_not_cleartext(client, db): register(db, "ada-lock@example.test", "Ada Lock") ada = Actor(client, "ada-lock@example.test", "Ada Lock") ada.login() case = _case(ada) upload = ada.request( "POST", "/api/v1/files", data={"title": "Geschuetzt", "context_kind": "case", "context_id": case["id"]}, files={"upload": ("geheim.pdf", PDF, "application/pdf")}, status=201, ) dummy = b"cipher-dummy-not-clear" asset = db.execute( "SELECT id, object_key, storage_instance_id FROM media_assets WHERE element_id = %s", (upload["id"],), ).fetchone() root = db.execute( "SELECT root_path FROM storage_instances WHERE id = %s", (asset["storage_instance_id"],), ).fetchone()["root_path"] path = Path(root) / asset["object_key"] path.write_bytes(dummy) db.execute( """ UPDATE media_assets SET protection_profile = 'client-encrypted', key_responsibility = 'client', sha256 = %s WHERE element_id = %s """, (hashlib.sha256(dummy).hexdigest(), upload["id"]), ) db.commit() detail = ada.request("GET", f"/api/v1/cases/{case['id']}") rendered = json.dumps(detail) assert "client-encrypted" in rendered assert "Geschuetzt" in rendered assert dummy.decode() not in rendered opened = _raw(ada, f"/api/v1/files/{upload['id']}/content") assert opened.status_code == 422 assert opened.json()["code"] == "unsupported" assert path.read_bytes() == dummy def test_credential_update_keeps_the_instance(client, db): register(db, "ada-key@example.test", "Ada Key") ada = Actor(client, "ada-key@example.test", "Ada Key") ada.login() first = _paperless(ada, "Archiv", "token-first-1") second = _paperless(ada, "Anderes Archiv", "token-second-2") renewed = ada.request( "PATCH", f"/api/v1/connectors/{first}", json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-renewed"}, ) assert renewed["id"] == first secrets = { str(row["id"]): row["secret"] for row in db.execute("SELECT id, secret FROM connectors").fetchall() } assert secrets[first] == "token-renewed" assert secrets[second] == "token-second-2" caps = ada.request("GET", "/api/v1/sources/capabilities") assert caps["editors"]["note"] is True assert caps["editors"]["markdown_file"] is False assert caps["protection_profiles"]["client-encrypted"] == "locked" paperless = next(item for item in caps["sources"] if item["kind"] == "paperless") assert paperless["search"]["query"] is True assert paperless["edit_content"] is False def test_legacy_reference_stays_bound_and_foreign_connectors_stay_hidden(client, db, database, monkeypatch): """A startup without loaded modules must still keep the one existing connection.""" from app.db.migrate import _prepare_source_storage from app.sourceregistry import unregister_source from app.sources import _install_sources def fake_bytes(row, document_id): del document_id return f"%PDF-{row['secret']}\n".encode(), "application/pdf", "dokument.pdf" monkeypatch.setattr("app.sources._paperless_bytes", fake_bytes) register(db, "ada-legacy@example.test", "Ada Legacy") register(db, "ben-legacy@example.test", "Ben Legacy") ada = Actor(client, "ada-legacy@example.test", "Ada Legacy") ben = Actor(client, "ben-legacy@example.test", "Ben Legacy") ada.login() case = _case(ada) matter = ada.request("POST", "/api/v1/matters", json={"title": "Altbestand"}, status=201) connector = _paperless(ada, "Archiv", "token-legacy") case_ref = _link(ada, case["id"], None, "15", "Akte") matter_ref = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Vorgang", "source": "paperless", "external_id": "16", "context": {"kind": "matter", "id": matter["id"]}, }, status=201, )["id"] db.execute( """ UPDATE external_references SET connector_id = NULL, resolution = 'unresolved', detail = '{}'::jsonb WHERE element_id IN (%s, %s) """, (case_ref, matter_ref), ) db.commit() opened = ada.request("GET", f"/api/v1/cases/{case['id']}") shown = next(item["element"] for item in opened["bindings"] if item["element"]["id"] == case_ref) assert shown["preview"]["media"] == "pdf" assert shown["preview"]["available"] is True assert _raw(ada, f"/api/v1/external-refs/{case_ref}/content?context_kind=case&context_id={case['id']}").content.startswith( b"%PDF-token-legacy" ) matter_view = ada.request("GET", f"/api/v1/matters/{matter['id']}") matter_element = next(item["element"] for item in matter_view["bindings"] if item["element"]["id"] == matter_ref) assert matter_element["preview"]["media"] == "pdf" unregister_source("paperless") unregister_source("imap") try: _prepare_source_storage(database) finally: _install_sources() state = db.execute( "SELECT connector_id::text AS connector_id, resolution FROM external_references WHERE element_id = %s", (case_ref,), ).fetchone() assert state["connector_id"] == connector assert state["resolution"] == "bound" ben.login() own = _paperless(ben, "Bens Archiv", "token-ben") listed = {item["id"] for item in ben.request("GET", "/api/v1/connectors")["items"]} assert listed == {own} ben.request("POST", f"/api/v1/connectors/{connector}/probe", status=404) ben.request( "GET", "/api/v1/sources/paperless/documents", params={"q": "Alt", "connector_id": connector}, status=404, ) ada.login() ada.request( "POST", f"/api/v1/matters/{matter['id']}/grants", json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": False}, status=201, ) ben.login() shared = ben.request("GET", f"/api/v1/matters/{matter['id']}") shared_element = next(item["element"] for item in shared["bindings"] if item["element"]["id"] == matter_ref) assert "connector_id" not in shared_element["external"]["detail"] assert "external_id" not in shared_element["external"] blocked = _raw( ben, f"/api/v1/external-refs/{matter_ref}/content?context_kind=matter&context_id={matter['id']}", ) assert blocked.status_code == 422 ada.login() ada.request( "POST", f"/api/v1/matters/{matter['id']}/grants", json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True}, status=201, ) ben.login() allowed = _raw( ben, f"/api/v1/external-refs/{matter_ref}/content?context_kind=matter&context_id={matter['id']}", ) assert allowed.status_code == 200 assert allowed.content.startswith(b"%PDF-token-legacy")