Update ingestion_db.py, ingestion_processor.py, and import_markdown.py: Enhance documentation and logging clarity, improve artifact purging and symmetry injection logic, and implement stricter authority checks. Update versioning to 2.6.0 and 3.3.7 to reflect changes in functionality and maintain compatibility with the ingestion service.
This commit is contained in:
1 parent
57656bbaaf
commit
ec89d83916
3 files changed
+117
-91
No files matched your search
+12
-22
@@ -2,19 +2,11 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
FILE: scripts/import_markdown.py
|
||||
VERSION: 2.5.0 (2026-01-10)
|
||||
VERSION: 2.6.0 (2026-01-10)
|
||||
STATUS: Active (Core)
|
||||
COMPATIBILITY: IngestionProcessor v3.3.5+
|
||||
|
||||
Zweck:
|
||||
-------
|
||||
Hauptwerkzeug zum Importieren von Markdown-Dateien aus einem Vault in Qdrant.
|
||||
Implementiert die globale 2-Phasen-Schreibstrategie.
|
||||
|
||||
Änderungen v2.5.0:
|
||||
------------------
|
||||
- Globale Phasentrennung: commit_vault_symmetries() wird erst am Ende aufgerufen.
|
||||
- Erweiterter Ordner-Filter: Schließt .trash und andere Systemordner aus.
|
||||
COMPATIBILITY: IngestionProcessor v3.3.7+
|
||||
Zweck: Hauptwerkzeug zum Importieren von Markdown-Dateien.
|
||||
Implementiert die globale 2-Phasen-Schreibstrategie.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
@@ -24,10 +16,8 @@ import sys
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Setzt das Level global auf INFO
|
||||
# Root Logger Setup
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
|
||||
# Stelle sicher, dass das Root-Verzeichnis im Python-Pfad ist
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from app.core.ingestion import IngestionService
|
||||
@@ -41,14 +31,13 @@ async def main_async(args):
|
||||
logger.error(f"Vault path does not exist: {vault_path}")
|
||||
return
|
||||
|
||||
# 1. Service initialisieren
|
||||
logger.info(f"Initializing IngestionService (Prefix: {args.prefix})")
|
||||
service = IngestionService(collection_prefix=args.prefix)
|
||||
|
||||
logger.info(f"Scanning {vault_path}...")
|
||||
all_files = list(vault_path.rglob("*.md"))
|
||||
|
||||
# --- ORDNER-FILTER ---
|
||||
# --- GLOBALER ORDNER-FILTER ---
|
||||
files = []
|
||||
ignore_folders = [".trash", ".obsidian", ".sync", "templates", "_system"]
|
||||
for f in all_files:
|
||||
@@ -74,7 +63,7 @@ async def main_async(args):
|
||||
except Exception: pass
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 1: Batch-Import (Explicit Edges only)
|
||||
# PHASE 1: Batch-Import (Notes & Explicit Edges)
|
||||
# =========================================================================
|
||||
stats = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
sem = asyncio.Semaphore(5)
|
||||
@@ -82,6 +71,7 @@ async def main_async(args):
|
||||
async def process_with_limit(f_path):
|
||||
async with sem:
|
||||
try:
|
||||
# Nutzt process_file (v3.3.7)
|
||||
return await service.process_file(
|
||||
file_path=str(f_path), vault_root=str(vault_path),
|
||||
force_replace=args.force, apply=args.apply, purge_before=True
|
||||
@@ -101,15 +91,15 @@ async def main_async(args):
|
||||
else: stats["skipped"] += 1
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 2: Global Symmetry Injection
|
||||
# PHASE 2: Global Symmetry Injection (Nach Abschluss aller Batches)
|
||||
# =========================================================================
|
||||
if args.apply:
|
||||
logger.info(f"🔄 [Phase 2] Starting global symmetry injection...")
|
||||
logger.info(f"🔄 [Phase 2] Starting global symmetry injection for the entire vault...")
|
||||
sym_res = await service.commit_vault_symmetries()
|
||||
if sym_res.get("status") == "success":
|
||||
logger.info(f"✅ Added {sym_res.get('added', 0)} protected symmetry edges.")
|
||||
logger.info(f"✅ Finished global symmetry injection. Added: {sym_res.get('added', 0)}")
|
||||
|
||||
logger.info(f"Done. Final Stats: {stats}")
|
||||
logger.info(f"Final Stats: {stats}")
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
|
||||
Reference in new issue
Block a user