"""Follow-up checks for paging, filters, storage resolution and notes. IMAP and Paperless doubles stand in for the external servers. They do not prove a live mailbox or a live Paperless instance. """ from __future__ import annotations import io import zipfile from email.parser import BytesParser from app.media.port import MemoryAdapter from app.sourceaccess import borrow_original, complete_borrow from app.sourceregistry import SourceModule, unregister_source, register_source from tests.support import ORIGIN, Actor, document, register PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\nM\n" LETTER = ( b"From: Ada \r\n" b"To: Ben \r\n" b"Subject: Brief\r\n" b"Message-ID: \r\n" b"MIME-Version: 1.0\r\n" b'Content-Type: multipart/mixed; boundary="grenze"\r\n' b"\r\n" b"--grenze\r\n" b"Content-Type: text/plain; charset=utf-8\r\n" b"\r\n" b"Hallo aus der zweiten Quelle.\r\n" b"--grenze\r\n" b"Content-Type: application/pdf\r\n" b'Content-Disposition: attachment; filename="plan.pdf"\r\n' b"\r\n" b"%PDF-plan\r\n" b"--grenze--\r\n" ) def _headers(actor: Actor) -> dict: return { "Origin": ORIGIN, "X-CSRF-Token": actor.csrf, "X-Scope-Kind": actor.scope["kind"], "X-Scope-Id": actor.scope["id"], } def _install_letterbox() -> None: def parse_config(payload: dict): name = str(payload.get("name") or "Briefkasten").strip() or "Briefkasten" return name, {"catalog": "letters"}, "letter-secret" def probe(row): return {"ok": True, "detail": "Der Briefkasten antwortet."} def find(conn, owner_id, connector_id, query, options): del conn, owner_id, query, options return {"items": [{"id": "", "title": "Brief"}], "connector_id": str(connector_id)} def open_original(conn, actor_id, element, context, ref): borrowed = borrow_original(conn, actor_id, element, context, ref, "letterbox") return complete_borrow(conn, borrowed, lambda row: (LETTER, "message/rfc822", "nachricht.eml")) def export_original(conn, actor_id, external_id, detail, assigned, ref): del conn, actor_id, external_id, detail, assigned, ref from app.retrieval import FilePayload return FilePayload("nachricht.eml", "message/rfc822", LETTER, "") register_source(SourceModule( kind="letterbox", label="Briefkasten", representation="email", capabilities={ "kind": "letterbox", "navigate": False, "read_original": True, "read_preview": False, "read_attachment": True, "edit_content": False, "edit_metadata": False, "search": {"query": True, "continuation": False}, }, parse_config=parse_config, probe=probe, find=find, open_original=open_original, export_original=export_original, )) class RangeIMAP: searches: list = [] validity = b"3" uidnext = b"50001" fail_headers = False search_status = "OK" parts = False def __init__(self, *args, **kwargs): self.untagged_responses = {} def login(self, username, secret): assert secret == "Postfach-Geheim" return "OK", [b"logged"] def list(self): return "OK", [b'(\\HasNoChildren) "/" "INBOX"'] def select(self, mailbox, readonly=False): assert readonly is True self.untagged_responses = {"UIDVALIDITY": [type(self).validity], "UIDNEXT": [type(self).uidnext]} return "OK", [b"1"] def uid(self, command, *args): if command == "SEARCH": type(self).searches.append(args) if type(self).search_status != "OK": return type(self).search_status, [None] if "*" in args and not any(isinstance(arg, str) and ":" in arg for arg in args): return "OK", [b"50000"] low, high = _range_of(args) if type(self).parts and low <= 5 <= high: return "OK", [b"4 5", b"3"] needle = _needle(args) if needle: hit = 80 if low <= 80 <= high else None if needle.startswith("=?") or "nderung" in needle or "Änderung" in needle: hit = 80 if low <= 80 <= high else None return ("OK", [str(hit).encode()]) if hit else ("OK", [b""]) found = [str(uid).encode() for uid in range(low, high + 1)] return "OK", [b" ".join(found)] uid = int(args[0]) if type(self).fail_headers: return "NO", [None] subject = "Seltener Treffer" if uid == 80 else f"Nachricht {uid}" header = ( f"From: Ada \r\nSubject: {subject}\r\n" f"Date: Thu, 02 May 2024 08:00:00 +0000\r\n" ).encode() return "OK", [(b"body", header), b")"] def logout(self): return "BYE", [b""] def _range_of(args) -> tuple[int, int]: for arg in args: text = arg.decode() if isinstance(arg, bytes) else str(arg) if ":" in text and text.replace(":", "").isdigit(): low, high = text.split(":", 1) return int(low), int(high) raise AssertionError(args) def _needle(args) -> str: for arg in args: text = arg.decode() if isinstance(arg, bytes) else str(arg) if text.startswith('"') or "nderung" in text or "Änderung" in text: return text return "" def _account(monkeypatch, actor: Actor, fake) -> str: import app.sources as sources monkeypatch.setattr(sources.imaplib, "IMAP4_SSL", fake) created = actor.request( "POST", "/api/v1/connectors", json={ "kind": "imap", "name": "Kanzlei", "host": "imap.example.test", "port": 993, "username": "ada@example.test", "password": "Postfach-Geheim", "security": "ssl", }, status=201, ) return created["id"] def _spans(searches) -> list[int]: spans = [] for args in searches: if any(arg == "*" for arg in args): continue low, high = _range_of(args) spans.append(high - low + 1) return spans def test_large_folder_is_bounded_and_a_late_hit_is_not_empty(client, db, monkeypatch): import app.sources as sources ada = Actor(client, "ada-range@example.test", "Ada Bereich") register(db, ada.email, ada.name) ada.login() RangeIMAP.searches = [] RangeIMAP.validity = b"3" RangeIMAP.uidnext = b"50001" RangeIMAP.fail_headers = False RangeIMAP.search_status = "OK" RangeIMAP.parts = False account = _account(monkeypatch, ada, RangeIMAP) first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}) assert first["items"][0]["title"] == "Nachricht 50000" assert len(first["items"]) == sources.PAGE_SIZE assert first["incomplete"] is False assert first["next_before"] is not None assert _spans(RangeIMAP.searches) assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW assert all("ALL" not in args for args in RangeIMAP.searches) RangeIMAP.searches = [] page = first seen = {item["uid"] for item in page["items"]} hops = 0 while 80 not in seen: hops += 1 assert hops < 40 assert page["next_before"] or page["incomplete"] page = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX", "q": "Seltener", "before": page["next_before"]}, ) if not page["items"] and page["incomplete"]: assert page["next_before"] seen.update(item["uid"] for item in page["items"]) assert any(item["title"] == "Seltener Treffer" for item in page["items"]) assert max(_spans(RangeIMAP.searches)) <= sources.UID_WINDOW def test_imap_errors_are_not_reported_as_an_empty_folder(client, db, monkeypatch): ada = Actor(client, "ada-imap-state@example.test", "Ada Zustand") register(db, ada.email, ada.name) ada.login() RangeIMAP.searches = [] RangeIMAP.uidnext = b"1" RangeIMAP.fail_headers = False RangeIMAP.search_status = "OK" RangeIMAP.parts = False account = _account(monkeypatch, ada, RangeIMAP) empty = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}) assert empty["items"] == [] assert empty["incomplete"] is False RangeIMAP.uidnext = b"31" RangeIMAP.search_status = "NO" refused = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}, status=422, ) assert "abgelehnt" in refused["detail"] assert "leer" not in refused["detail"].casefold() RangeIMAP.search_status = "OK" RangeIMAP.fail_headers = True failed = ada.request( "GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}, status=422, ) assert "Nachrichtenköpfe" in failed["detail"] assert "leer" not in failed["detail"].casefold() def test_uidvalidity_change_restarts_and_split_responses_join(client, db, monkeypatch): ada = Actor(client, "ada-validity@example.test", "Ada Geltung") register(db, ada.email, ada.name) ada.login() RangeIMAP.searches = [] RangeIMAP.validity = b"4" RangeIMAP.uidnext = b"11" RangeIMAP.fail_headers = False RangeIMAP.search_status = "OK" RangeIMAP.parts = False account = _account(monkeypatch, ada, RangeIMAP) first = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}) RangeIMAP.validity = b"9" restarted = ada.request( "GET", "/api/v1/sources/imap/messages", params={ "connector_id": account, "mailbox": "INBOX", **({"before": first["next_before"]} if first["next_before"] else {}), "uidvalidity": first["uidvalidity"], }, ) assert restarted["restarted"] is True assert restarted["items"][0]["title"] == "Nachricht 10" RangeIMAP.validity = b"4" RangeIMAP.parts = True RangeIMAP.uidnext = b"6" joined = ada.request("GET", "/api/v1/sources/imap/messages", params={"connector_id": account, "mailbox": "INBOX"}) assert [item["uid"] for item in joined["items"]] == [5, 4, 3] def test_paperless_filters_reach_the_api_and_cross_pages(client, db, monkeypatch): import app.sources as sources calls = [] def fake_get(base, token, path, params): calls.append({"path": path, "params": dict(params), "token": token}) if path != "/api/documents/": return {"count": 1, "results": [{"id": 4, "name": "Haus"}]} page = int(params.get("page") or 1) start = (page - 1) * 20 return { "count": 45, "results": [ {"id": number + 1, "title": f"Titel {number + 1}", "created": "2024-03-01"} for number in range(start, min(start + 20, 45)) ], } def fake_bytes(row, document_id): return f"%PDF-{row['secret']}-{document_id}\n".encode(), "application/pdf", "dokument.pdf" monkeypatch.setattr(sources, "_paperless_get", fake_get) monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes) ada = Actor(client, "ada-filter@example.test", "Ada Filter") register(db, ada.email, ada.name) ada.login() first = ada.request( "POST", "/api/v1/connectors", json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"}, status=201, ) second = ada.request( "POST", "/api/v1/connectors", json={"kind": "paperless", "name": "Zweitarchiv", "base_url": "https://paper.example.test", "token": "token-bravo"}, status=201, ) facets = ada.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]}) assert facets["tag_mode"] == "all" assert facets["correspondents"] == [{"id": "4", "name": "Haus"}] ada.request( "GET", "/api/v1/sources/paperless/documents", params={"connector_id": first["id"], "document_from": "2024-05-02", "document_to": "2024-01-01"}, status=422, ) found = ada.request( "GET", "/api/v1/sources/paperless/documents", params={ "connector_id": first["id"], "q": "nurinhalt", "title": "Vertrag", "correspondent": "4", "document_type": "8", "tag": "2,5", "document_from": "2024-01-01", "document_to": "2024-12-31", "sort": "-created", "cursor": "3", }, ) sent = calls[-1]["params"] assert sent["query"] == "nurinhalt" assert sent["title__icontains"] == "Vertrag" assert sent["correspondent__id"] == "4" assert sent["document_type__id"] == "8" assert sent["tags__id__all"] == "2,5" assert sent["created__date__gte"] == "2024-01-01" assert sent["created__date__lte"] == "2024-12-31" assert sent["ordering"] == "-created" assert sent["page"] == "3" ada.request( "GET", "/api/v1/sources/paperless/documents", params={"connector_id": first["id"], "q": "nurinhalt", "sort": "-score"}, ) assert "ordering" not in calls[-1]["params"] assert found["items"][0]["id"] == "41" assert found["connector_id"] == first["id"] preview = ada.client.get( "/api/v1/sources/paperless/documents/41/preview", params={"connector_id": first["id"]}, headers=_headers(ada), ) assert preview.status_code == 200 assert preview.content.startswith(b"%PDF-token-alpha-41") matter = ada.request("POST", "/api/v1/matters", json={"title": "Suche"}, status=201) linked = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Seite drei", "source": "paperless", "external_id": "41", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": first["id"]}, }, status=201, ) other = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Dieselbe Nummer", "source": "paperless", "external_id": "41", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": second["id"]}, }, status=201, ) opened = ada.client.get(f"/api/v1/external-refs/{linked['id']}/content", headers=_headers(ada)) twin = ada.client.get(f"/api/v1/external-refs/{other['id']}/content", headers=_headers(ada)) assert opened.content.startswith(b"%PDF-token-alpha-41") assert twin.content.startswith(b"%PDF-token-bravo-41") ben = Actor(client, "ben-filter@example.test", "Ben Filter") register(db, ben.email, ben.name) ben.login() ben.request("GET", "/api/v1/sources/paperless/facets", params={"connector_id": first["id"]}, status=404) ben.request( "GET", "/api/v1/sources/paperless/documents", params={"connector_id": first["id"], "q": "nurinhalt"}, status=404, ) def test_revoked_grant_hides_bytes_fetched_during_the_call(client, db, monkeypatch): import app.sources as sources def fake_bytes(row, document_id): db.execute("UPDATE access_grants SET can_read = false") db.commit() return b"%PDF-1.4\nsecret\n", "application/pdf", "geheim.pdf" monkeypatch.setattr(sources, "_paperless_bytes", fake_bytes) ada = Actor(client, "ada-revoke@example.test", "Ada Widerruf") ben = Actor(client, "ben-revoke@example.test", "Ben Widerruf") register(db, ada.email, ada.name) register(db, ben.email, ben.name) ada.login() paper = ada.request( "POST", "/api/v1/connectors", json={"kind": "paperless", "name": "Archiv", "base_url": "https://paper.example.test", "token": "token-alpha"}, status=201, ) matter = ada.request("POST", "/api/v1/matters", json={"title": "Freigabe"}, status=201) linked = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Vertrag", "source": "paperless", "external_id": "15", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": paper["id"]}, }, status=201, ) ada.request( "POST", f"/api/v1/matters/{matter['id']}/grants", json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False, "use_owner_sources": True}, status=201, ) ben.login() hidden = ben.client.get( f"/api/v1/external-refs/{linked['id']}/content", params={"context_kind": "matter", "context_id": matter["id"]}, headers=_headers(ben), ) assert hidden.status_code == 422 assert b"secret" not in hidden.content def test_email_representation_uses_the_mail_reader_without_imap(client, db): ada = Actor(client, "ada-letter@example.test", "Ada Brief") register(db, ada.email, ada.name) ada.login() _install_letterbox() try: box = ada.request("POST", "/api/v1/connectors", json={"kind": "letterbox", "name": "Briefe"}, status=201) matter = ada.request("POST", "/api/v1/matters", json={"title": "Post"}, status=201) linked = ada.request( "POST", "/api/v1/external-refs", json={ "title": "Brief", "source": "letterbox", "external_id": "", "context": {"kind": "matter", "id": matter["id"]}, "detail": {"connector_id": box["id"]}, }, status=201, ) mail = ada.request("GET", f"/api/v1/external-refs/{linked['id']}/mail") assert mail["attachments"][0]["filename"] == "plan.pdf" attachment = ada.client.get( f"/api/v1/external-refs/{linked['id']}/mail/attachments/0", headers=_headers(ada), ) assert attachment.status_code == 200 assert attachment.content.startswith(b"%PDF-plan") raw = ada.client.get(f"/api/v1/external-refs/{linked['id']}/mail/original", headers=_headers(ada)) parsed = BytesParser().parsebytes(raw.content) assert parsed.get("subject") == "Brief" finally: unregister_source("letterbox") def test_domain_storage_uses_the_resolved_adapter(client, db, monkeypatch): import app.media.port as port memory = MemoryAdapter() monkeypatch.setitem(port._FACTORIES, "local", lambda row: memory) ada = Actor(client, "ada-store@example.test", "Ada Speicher") register(db, ada.email, ada.name) ada.login() matter = ada.request("POST", "/api/v1/matters", json={"title": "Ablage"}, status=201) ctx = {"kind": "matter", "id": matter["id"]} note = ada.request( "POST", "/api/v1/notes", json={"title": "Schreiben", "document": document("Text"), "context": ctx}, status=201, ) uploaded = ada.request( "POST", "/api/v1/files", data={"title": "Plan", "context_kind": "matter", "context_id": matter["id"], "document_on": "2024-01-02"}, files={"upload": ("plan.pdf", PDF, "application/pdf")}, status=201, ) detail = ada.request("GET", f"/api/v1/matters/{matter['id']}") host = next(item for item in detail["bindings"] if item["element"]["id"] == note["id"]) ada.request( "POST", f"/api/v1/elements/{note['id']}/attachments", json={ "expected_version": host["element"]["version"], "context": ctx, "attachment_element_ids": [uploaded["id"]], "fold": "attachment", }, status=201, ) folded = ada.request("GET", f"/api/v1/matters/{matter['id']}") host = next(item for item in folded["bindings"] if item["element"]["id"] == note["id"]) dispatch = ada.request( "POST", f"/api/v1/elements/{note['id']}/dispatches", json={ "expected_version": host["element"]["version"], "context": ctx, "channel": "post", "sent_on": "2024-02-02", "attachment_element_ids": [uploaded["id"]], "capture_editions": True, }, status=201, ) content = ada.client.get(f"/api/v1/files/{uploaded['id']}/content", headers=_headers(ada)) assert content.content == PDF item = next(entry for entry in dispatch["items"] if entry["element_id"] == uploaded["id"]) snapshot = ada.client.get( f"/api/v1/dispatches/{dispatch['id']}/items/{item['id']}/content", params={"context_kind": "matter", "context_id": matter["id"]}, headers=_headers(ada), ) assert snapshot.content == PDF asset = db.execute( "SELECT object_key, storage_generation, sha256 FROM media_assets WHERE element_id = %s", (uploaded["id"],), ).fetchone() assert memory.open(asset["object_key"]) == PDF tree = ada.request("GET", f"/api/v1/matters/{matter['id']}/export-tree")["tree"] selected = [ node["id"] for node in _walk(tree) if node.get("exportable") and node.get("kind") in {"binding", "attachment", "dispatch_item"} ] package = ada.client.post( f"/api/v1/matters/{matter['id']}/export", json={"context": ctx, "selected": selected}, headers={**_headers(ada), "Content-Type": "application/json"}, ) assert package.status_code == 200, package.text archive = zipfile.ZipFile(io.BytesIO(package.content)) assert any(archive.read(name) == PDF for name in archive.namelist()) version = next( item["element"]["version"] for item in ada.request("GET", f"/api/v1/matters/{matter['id']}")["bindings"] if item["element"]["id"] == uploaded["id"] ) ada.request( "POST", f"/api/v1/elements/{uploaded['id']}/delete", json={"expected_version": version}, status=204, ) try: memory.open(asset["object_key"]) except Exception: removed = True else: removed = False assert removed is True assert asset["storage_generation"] def test_note_markdown_is_shared_exported_and_kept_on_conflict(client, db): ada = Actor(client, "ada-note-md@example.test", "Ada Notiz") ben = Actor(client, "ben-note-md@example.test", "Ben Notiz") register(db, ada.email, ada.name) register(db, ben.email, ben.name) ada.login() left = ada.request("POST", "/api/v1/matters", json={"title": "Links"}, status=201) right = ada.request("POST", "/api/v1/matters", json={"title": "Rechts"}, status=201) body = "\n".join([ "# Überschrift", "", "Ein **fetter** und *kursiver* Satz.", "", "- erster", "- zweiter", "", "> Zitat", "", "Siehe [Quelle](https://example.test/hinweis).", "", "`code`", "", "```", "block", "```", "", "", "", "![bild](https://example.test/bild.png)", ]) created = ada.request( "POST", "/api/v1/notes", json={ "title": "Merkzettel", "document": {"version": 1, "blocks": [{"type": "markdown", "text": body}]}, "context": {"kind": "matter", "id": left["id"]}, }, status=201, ) ada.request( "POST", "/api/v1/bindings", json={ "element_id": created["id"], "source_context": {"kind": "matter", "id": left["id"]}, "target": {"kind": "matter", "id": right["id"]}, }, status=201, ) detail = ada.request("GET", f"/api/v1/matters/{left['id']}") element = next(item["element"] for item in detail["bindings"] if item["element"]["id"] == created["id"]) changed = "Geändert in beiden Vorgängen.\n\n" + body ada.request( "PATCH", f"/api/v1/notes/{created['id']}", json={ "expected_version": element["version"], "document": {"version": 1, "blocks": [{"type": "markdown", "text": changed}]}, "context": {"kind": "matter", "id": right["id"]}, }, ) again = ada.request("GET", f"/api/v1/matters/{left['id']}") shared = next(item["element"] for item in again["bindings"] if item["element"]["id"] == created["id"]) assert changed in _text(shared) ada.request( "POST", f"/api/v1/bindings/{next(item['id'] for item in again['bindings'] if item['element']['id'] == created['id'])}/annotations", json={"body": "Nur links"}, params={"context_kind": "matter", "context_id": left["id"]}, status=201, ) left_detail = ada.request("GET", f"/api/v1/matters/{left['id']}") right_detail = ada.request("GET", f"/api/v1/matters/{right['id']}") left_binding = next(item for item in left_detail["bindings"] if item["element"]["id"] == created["id"]) right_binding = next(item for item in right_detail["bindings"] if item["element"]["id"] == created["id"]) assert any(note["body"] == "Nur links" for note in left_binding["annotations"]) assert all(note["body"] != "Nur links" for note in right_binding["annotations"]) ada.request( "POST", f"/api/v1/matters/{left['id']}/grants", json={"email": ben.email, "can_read": True, "can_edit": False, "can_share": False}, status=201, ) ben.login() ben.request( "PATCH", f"/api/v1/notes/{created['id']}", json={ "expected_version": shared["version"], "document": {"version": 1, "blocks": [{"type": "markdown", "text": "fremd"}]}, "context": {"kind": "matter", "id": left["id"]}, }, status=403, ) ada.login() current = next( item["element"]["version"] for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"] if item["element"]["id"] == created["id"] ) conflict = ada.request( "PATCH", f"/api/v1/notes/{created['id']}", json={ "expected_version": current - 1, "document": {"version": 1, "blocks": [{"type": "markdown", "text": "darf nicht überschreiben"}]}, "context": {"kind": "matter", "id": left["id"]}, }, status=409, ) assert conflict["code"] == "conflict" stored = next( item["element"] for item in ada.request("GET", f"/api/v1/matters/{left['id']}")["bindings"] if item["element"]["id"] == created["id"] ) assert "darf nicht überschreiben" not in _text(stored) assert "" in _text(stored) tree = ada.request("GET", f"/api/v1/matters/{left['id']}/export-tree")["tree"] selected = [node["id"] for node in _walk(tree) if node.get("kind") == "binding" and node.get("exportable")] package = ada.client.post( f"/api/v1/matters/{left['id']}/export", json={"context": {"kind": "matter", "id": left["id"]}, "selected": selected}, headers={**_headers(ada), "Content-Type": "application/json"}, ) assert package.status_code == 200, package.text archive = zipfile.ZipFile(io.BytesIO(package.content)) exported = "\n".join( archive.read(name).decode("utf-8", errors="replace") for name in archive.namelist() if name.endswith((".md", ".txt")) ) assert "```" in exported assert "" in exported assert "https://example.test/bild.png" in exported def _text(element: dict) -> str: return "\n".join(block.get("text") or "" for block in (element.get("document") or {}).get("blocks") or []) def _walk(node): if isinstance(node, dict) and "kind" in node: yield node for child in node.get("children") or []: yield from _walk(child) elif isinstance(node, dict): for value in node.values(): yield from _walk(value) elif isinstance(node, list): for item in node: yield from _walk(item)