Update ingestion_db.py, ingestion_processor.py, and import_markdown.py: Enhance documentation and logging clarity, improve artifact purging and symmetry injection logic, and implement stricter authority checks. Update versioning to 2.6.0 and 3.3.7 to reflect changes in functionality and maintain compatibility with the ingestion service.

This commit is contained in:
Lars committed 2026-01-10 08:06:07 +01:00
1 parent 57656bbaaf
commit ec89d83916
3 files changed
+117 -91

No files matched your search

+12 -22
View File
@@ -2,19 +2,11 @@
# -*- coding: utf-8 -*-
"""
FILE: scripts/import_markdown.py
VERSION: 2.5.0 (2026-01-10)
VERSION: 2.6.0 (2026-01-10)
STATUS: Active (Core)
COMPATIBILITY: IngestionProcessor v3.3.5+
Zweck:
-------
Hauptwerkzeug zum Importieren von Markdown-Dateien aus einem Vault in Qdrant.
Implementiert die globale 2-Phasen-Schreibstrategie.
Änderungen v2.5.0:
------------------
- Globale Phasentrennung: commit_vault_symmetries() wird erst am Ende aufgerufen.
- Erweiterter Ordner-Filter: Schließt .trash und andere Systemordner aus.
COMPATIBILITY: IngestionProcessor v3.3.7+
Zweck: Hauptwerkzeug zum Importieren von Markdown-Dateien.
Implementiert die globale 2-Phasen-Schreibstrategie.
"""
import asyncio
import os
@@ -24,10 +16,8 @@ import sys
from pathlib import Path
from dotenv import load_dotenv
# Setzt das Level global auf INFO
# Root Logger Setup
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
# Stelle sicher, dass das Root-Verzeichnis im Python-Pfad ist
sys.path.append(os.getcwd())
from app.core.ingestion import IngestionService
@@ -41,14 +31,13 @@ async def main_async(args):
logger.error(f"Vault path does not exist: {vault_path}")
return
# 1. Service initialisieren
logger.info(f"Initializing IngestionService (Prefix: {args.prefix})")
service = IngestionService(collection_prefix=args.prefix)
logger.info(f"Scanning {vault_path}...")
all_files = list(vault_path.rglob("*.md"))
# --- ORDNER-FILTER ---
# --- GLOBALER ORDNER-FILTER ---
files = []
ignore_folders = [".trash", ".obsidian", ".sync", "templates", "_system"]
for f in all_files:
@@ -74,7 +63,7 @@ async def main_async(args):
except Exception: pass
# =========================================================================
# PHASE 1: Batch-Import (Explicit Edges only)
# PHASE 1: Batch-Import (Notes & Explicit Edges)
# =========================================================================
stats = {"processed": 0, "skipped": 0, "errors": 0}
sem = asyncio.Semaphore(5)
@@ -82,6 +71,7 @@ async def main_async(args):
async def process_with_limit(f_path):
async with sem:
try:
# Nutzt process_file (v3.3.7)
return await service.process_file(
file_path=str(f_path), vault_root=str(vault_path),
force_replace=args.force, apply=args.apply, purge_before=True
@@ -101,15 +91,15 @@ async def main_async(args):
else: stats["skipped"] += 1
# =========================================================================
# PHASE 2: Global Symmetry Injection
# PHASE 2: Global Symmetry Injection (Nach Abschluss aller Batches)
# =========================================================================
if args.apply:
logger.info(f"🔄 [Phase 2] Starting global symmetry injection...")
logger.info(f"🔄 [Phase 2] Starting global symmetry injection for the entire vault...")
sym_res = await service.commit_vault_symmetries()
if sym_res.get("status") == "success":
logger.info(f"✅ Added {sym_res.get('added', 0)} protected symmetry edges.")
logger.info(f"✅ Finished global symmetry injection. Added: {sym_res.get('added', 0)}")
logger.info(f"Done. Final Stats: {stats}")
logger.info(f"Final Stats: {stats}")
def main():
load_dotenv()