code header
This commit is contained in:
1 parent
ec759dd1dc
commit
83bb18b6a7
42 files changed
+469
-615
No files matched your search
@@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/chunk_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||
- neighbors_prev / neighbors_next sind Listen ([], [id]).
|
||||
- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
|
||||
- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
|
||||
- WP-11 Update: Injects 'title' into chunk payload for Discovery Service.
|
||||
FILE: app/core/chunk_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'. Inkludiert Nachbarschafts-IDs (prev/next) und Titel.
|
||||
VERSION: 2.0.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, os
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
"""
|
||||
FILE: app/core/chunker.py
|
||||
DESCRIPTION: Zerlegt Texte in Chunks (Sliding Window oder nach Headings). Orchestriert die Smart-Edge-Allocation via SemanticAnalyzer.
|
||||
VERSION: 2.5.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.services.semantic_analyzer, app.core.derive_edges, markdown_it, yaml, asyncio
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Dict, Optional, Tuple, Any, Set
|
||||
|
||||
@@ -1,26 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/derive_edges.py
|
||||
Zweck:
|
||||
- Bewahrt bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||
- Ergänzt typenbasierte Default-Kanten (edge_defaults aus config/types.yaml)
|
||||
- Unterstützt "typed inline relations":
|
||||
* [[rel:KIND | Target]]
|
||||
* [[rel:KIND Target]]
|
||||
* rel: KIND [[Target]]
|
||||
- Unterstützt Obsidian-Callouts:
|
||||
* > [!edge] KIND: [[Target]] [[Target2]] ...
|
||||
Kompatibilität:
|
||||
- build_edges_for_note(...) Signatur unverändert
|
||||
- rule_id Werte:
|
||||
* structure:belongs_to
|
||||
* structure:order
|
||||
* explicit:wikilink
|
||||
* inline:rel
|
||||
* callout:edge
|
||||
* edge_defaults:<type>:<relation>
|
||||
* derived:backlink
|
||||
FILE: app/core/derive_edges.py
|
||||
DESCRIPTION: Extrahiert Graph-Kanten aus Text. Unterstützt Wikilinks, Inline-Relations ([[rel:type|target]]) und Obsidian Callouts.
|
||||
VERSION: 2.0.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: re, os, yaml, typing
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion
|
||||
|
||||
Zweck:
|
||||
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph.
|
||||
|
||||
Kompatibilität:
|
||||
- WP-04a: Liefert Scores (edge_bonus, centrality).
|
||||
- WP-04b: Liefert jetzt auch Struktur-Daten für Erklärungen (Reverse-Lookup).
|
||||
|
||||
Version:
|
||||
0.4.0 (Update für WP-04b: Reverse Adjacency für Explainability)
|
||||
FILE: app/core/graph_adapter.py
|
||||
DESCRIPTION: Lädt Kanten aus Qdrant und baut einen In-Memory Subgraphen für Scoring (Centrality) und Explanation.
|
||||
VERSION: 0.4.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, app.core.qdrant
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
"""
|
||||
app/core/ingestion.py
|
||||
|
||||
Zentraler Service für die Transformation von Markdown-Dateien in Qdrant-Objekte.
|
||||
Version: 2.5.2 (Full Feature: Change Detection + Robust IO + Clean Config)
|
||||
FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Haupt-Ingestion-Logik. Liest Markdown, prüft Hashes (Change Detection), zerlegt in Chunks und schreibt in Qdrant.
|
||||
VERSION: 2.5.2
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
|
||||
@@ -1,17 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/note_payload.py
|
||||
Version: 2.1.0 (WP-11 Update: Aliases support)
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Erzeugt ein robustes Note-Payload. Werte wie `retriever_weight`, `chunk_profile`
|
||||
und `edge_defaults` werden in folgender Priorität bestimmt:
|
||||
1) Frontmatter (Note)
|
||||
2) Typ-Registry (config/types.yaml: types.<type>.*)
|
||||
3) Registry-Defaults (config/types.yaml: defaults.*)
|
||||
4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE)
|
||||
FILE: app/core/note_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_notes'. Wendet Vererbung für Configs an (Frontmatter > Type > Default).
|
||||
VERSION: 2.1.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, os, json, pathlib
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
+6
-39
@@ -1,43 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/parser.py
|
||||
Version: 1.7.1 (fault-tolerant, API-kompatibel)
|
||||
Datum: 2025-10-01
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Fehlertolerantes Einlesen von Markdown-Dateien mit YAML-Frontmatter.
|
||||
Kompatibel zur bisherigen Parser-API, aber robust gegenüber Nicht-UTF-8-Dateien:
|
||||
- Versucht nacheinander: utf-8 → utf-8-sig → cp1252 → latin-1.
|
||||
- Bei Fallback wird ein JSON-Warnhinweis auf stdout ausgegeben; der Import bricht NICHT ab.
|
||||
- YAML-Frontmatter wird mit '---' am Anfang und '---' als Abschluss erkannt.
|
||||
- extract_wikilinks() normalisiert [[id#anchor|label]] → 'id'.
|
||||
|
||||
Öffentliche API (kompatibel):
|
||||
- class ParsedNote(frontmatter: dict, body: str, path: str)
|
||||
- read_markdown(path) -> ParsedNote | None
|
||||
- normalize_frontmatter(fm) -> dict
|
||||
- validate_required_frontmatter(fm, required: tuple[str,...]=("id","title")) -> None
|
||||
- extract_wikilinks(text) -> list[str]
|
||||
- FRONTMATTER_RE (Kompatibilitäts-Konstante; Regex für '---'-Zeilen)
|
||||
|
||||
Beispiele
|
||||
---------
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
parsed = read_markdown("./vault/30_projects/project-demo.md")
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
body = parsed.body
|
||||
|
||||
from app.core.parser import extract_wikilinks
|
||||
links = extract_wikilinks(body)
|
||||
|
||||
Abhängigkeiten
|
||||
--------------
|
||||
- PyYAML (yaml)
|
||||
|
||||
Lizenz: MIT (projektintern)
|
||||
FILE: app/core/parser.py
|
||||
DESCRIPTION: Liest Markdown-Dateien fehlertolerant (Encoding-Fallback). Trennt Frontmatter (YAML) vom Body.
|
||||
VERSION: 1.7.1
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, re, dataclasses, json, io, os
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
+6
-24
@@ -1,28 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/qdrant.py
|
||||
Version: 2.2.0 (2025-11-11)
|
||||
|
||||
Aufgabe
|
||||
-------
|
||||
- Zentraler Qdrant-Zugriff (Client, Config)
|
||||
- Collection-Anlage (notes/chunks/edges)
|
||||
- **Payload-Indizes sicherstellen** (idempotent)
|
||||
|
||||
Hinweis
|
||||
-------
|
||||
Diese Datei ist als Drop-in-Ersatz gedacht, falls in deinem Projekt noch keine
|
||||
robuste ensure_payload_indexes()-Implementierung vorliegt. Die Signaturen
|
||||
bleiben kompatibel zu scripts.import_markdown und scripts.reset_qdrant.
|
||||
|
||||
API-Notizen
|
||||
-----------
|
||||
- Payload-Indizes werden mit `create_payload_index` angelegt.
|
||||
- Typen stammen aus `qdrant_client.http.models.PayloadSchemaType`:
|
||||
KEYWORD | TEXT | INTEGER | FLOAT | BOOL | GEO | DATETIME
|
||||
- Für häufige Filterfelder (note_id, kind, scope, type, tags, ...) legen wir
|
||||
Indizes an. Das ist laut Qdrant-Doku Best Practice für performante Filter.
|
||||
FILE: app/core/qdrant.py
|
||||
DESCRIPTION: Qdrant-Client Factory und Schema-Management. Erstellt Collections und Payload-Indizes.
|
||||
VERSION: 2.2.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, dataclasses, os
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/qdrant_points.py - robust points helpers for Qdrant
|
||||
|
||||
- Single source of truth for building PointStruct for notes/chunks/edges
|
||||
- Backward-compatible payloads for edges
|
||||
- Handles both Single-Vector and Named-Vector collections
|
||||
- Deterministic overrides via ENV to avoid auto-detection traps:
|
||||
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
|
||||
* MINDNET_VECTOR_NAME (fallback)
|
||||
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
|
||||
> Set to "__single__" (or "single") to force Single-Vector
|
||||
|
||||
Version: 1.5.0 (2025-11-08)
|
||||
FILE: app/core/qdrant_points.py
|
||||
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges) in PointStructs und generiert deterministische UUIDs.
|
||||
VERSION: 1.5.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, uuid, os
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""
|
||||
app/core/retriever.py — Hybrider Such-Algorithmus
|
||||
|
||||
Version:
|
||||
0.5.3 (WP-06 Fix: Populate 'payload' in QueryHit for meta-data access)
|
||||
FILE: app/core/retriever.py
|
||||
DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability).
|
||||
VERSION: 0.5.3
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,30 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/type_registry.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-11-08
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
|
||||
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
|
||||
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
|
||||
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
|
||||
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
|
||||
|
||||
Öffentliche API
|
||||
---------------
|
||||
- load_type_registry(path: str = "config/types.yaml") -> dict
|
||||
- get_type_config(note_type: str, reg: dict) -> dict
|
||||
- resolve_note_type(fm_type: str | None, reg: dict) -> str
|
||||
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
|
||||
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
|
||||
|
||||
Hinweis
|
||||
-------
|
||||
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
|
||||
Verhalten exakt wie im Release-Stand 20251105.
|
||||
FILE: app/core/type_registry.py
|
||||
DESCRIPTION: Loader für types.yaml. Achtung: Wird in der aktuellen Pipeline meist durch lokale Loader in 'ingestion.py' oder 'note_payload.py' umgangen.
|
||||
VERSION: 1.0.0
|
||||
STATUS: Deprecated (Redundant)
|
||||
DEPENDENCIES: yaml, os, functools
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
Reference in new issue
Block a user