"""Persistence for the local frame. SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine. Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL. """ from __future__ import annotations import json import os import sqlite3 from contextlib import contextmanager from pathlib import Path from typing import Any from sql_compat import ( adapt_sql, postgres_connect_kwargs, rewrite_catalog_sql, split_sql, sqlite_schema_to_postgres, use_postgres, ) _env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip() DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data" SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql" SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json" PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json" _env_db = os.environ.get("KANSHO_DB_PATH") DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite" SQLITE_HISTORY_MIGRATIONS = ( "001_frame", "002_platform", "003_dialogue_memory", "004_mvp_journal", "005_provider_settings", "006_writing_profile_dialogue_style", "007_journal_day_scratch", "008_journal_source_refs", "009_conversation_signals", "010_profile_governance", "011_profile_review", "012_profile_shell", "013_journal_generate_narration", "014_identity_registry", "015_journal_generation_settings", "016_generation_instruction_fragments", "017_generation_guidelines", "018_debug_runs", "019_debug_run_placement", "020_voice_style_context", "021_voice_legacy_immutable", ) _PROFILE_COLUMNS = { "status": "TEXT NOT NULL DEFAULT 'active'", "tier_id": "TEXT NOT NULL DEFAULT 'local'", } _PROMPT_COLUMNS = { "description": "TEXT NOT NULL DEFAULT ''", "category": "TEXT NOT NULL DEFAULT 'uncategorized'", "stages_json": "TEXT", "graph_json": "TEXT", "output_format": "TEXT NOT NULL DEFAULT 'text'", "output_schema_json": "TEXT", "required_feature": "TEXT NOT NULL DEFAULT 'ai_calls'", "default_template": "TEXT NOT NULL DEFAULT ''", "seed_revision": "TEXT NOT NULL DEFAULT ''", "sort_order": "INTEGER NOT NULL DEFAULT 0", "updated": "TEXT NOT NULL DEFAULT (datetime('now'))", } _CONVERSATION_COLUMNS = { "space_id": "TEXT", "journal_day_id": "TEXT", "narrative_mode": "TEXT NOT NULL DEFAULT ''", "reflection_depth": "TEXT NOT NULL DEFAULT ''", "current_focus": "TEXT NOT NULL DEFAULT ''", "emotional_intensity": "TEXT NOT NULL DEFAULT ''", "long_story": "INTEGER NOT NULL DEFAULT 0", } _WRITING_PROFILE_COLUMNS = { "governance": "TEXT NOT NULL DEFAULT 'learning'", "version": "INTEGER NOT NULL DEFAULT 0", "review_ready": "INTEGER NOT NULL DEFAULT 0", "last_reviewed": "TEXT", "lifecycle": "TEXT NOT NULL DEFAULT 'uninitialized'", } _WRITING_SOURCE_COLUMNS = { "occurred_at": "TEXT", "context_hint": "TEXT NOT NULL DEFAULT ''", } _WRITING_SUGGESTION_COLUMNS = { "trait_slug": "TEXT NOT NULL DEFAULT ''", "action": "TEXT NOT NULL DEFAULT 'update'", "payload_json": "TEXT NOT NULL DEFAULT '{}'", } _WRITING_VERSION_COLUMNS = { "lifecycle": "TEXT NOT NULL DEFAULT ''", "traits_json": "TEXT NOT NULL DEFAULT '[]'", } _JOURNAL_DAY_COLUMNS = { "scratch_json": "TEXT NOT NULL DEFAULT '[]'", } _JOURNAL_DRAFT_COLUMNS = { "generation_snapshot": "TEXT NOT NULL DEFAULT '{}'", } _GUIDELINE_COLUMNS = { "style_context_json": "TEXT NOT NULL DEFAULT '{}'", } _DEBUG_RUN_COLUMNS = { "space_id": "TEXT", "journal_day_id": "TEXT", "space_title": "TEXT NOT NULL DEFAULT ''", "calendar_date": "TEXT NOT NULL DEFAULT ''", "conversation_title": "TEXT NOT NULL DEFAULT ''", } _IDENTITY_MAPPING_COLUMNS = { "canonical_label": "TEXT NOT NULL DEFAULT ''", "entity_type": "TEXT NOT NULL DEFAULT 'PERSON'", "status": "TEXT NOT NULL DEFAULT 'legacy_review_required'", "origin": "TEXT NOT NULL DEFAULT 'legacy_auto'", "aliases_json": "TEXT NOT NULL DEFAULT '[]'", # SQLite ALTER TABLE cannot use datetime('now'); backfill in migrate_legacy_identity_rows. "updated": "TEXT NOT NULL DEFAULT ''", "confirmed_at": "TEXT", } class PostgresCompat: """sqlite3-like execute/commit surface over psycopg, with SQL translation.""" def __init__(self, raw: Any): self._raw = raw def execute(self, sql: str, params: Any = None): special = rewrite_catalog_sql(sql) if special: adapted, extra = special args = extra if extra is not None else params else: adapted = adapt_sql(sql) args = params if args is None: return self._raw.execute(adapted) return self._raw.execute(adapted, args) def executemany(self, sql: str, seq_of_params: Any): adapted = adapt_sql(sql) cursor = None for params in seq_of_params: cursor = self._raw.execute(adapted, params) return cursor def executescript(self, script: str): for statement in split_sql(sqlite_schema_to_postgres(script)): self._raw.execute(statement) return self def commit(self): self._raw.commit() def rollback(self): self._raw.rollback() def close(self): self._raw.close() def _connect_postgres() -> PostgresCompat: import psycopg from psycopg.rows import dict_row kwargs = postgres_connect_kwargs() if "conninfo" in kwargs: raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row) else: raw = psycopg.connect(row_factory=dict_row, **kwargs) return PostgresCompat(raw) def _connect(): if use_postgres(): return _connect_postgres() DB_PATH.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(DB_PATH, timeout=30) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") return conn @contextmanager def get_db(): conn = _connect() try: yield conn conn.commit() except Exception: conn.rollback() raise finally: conn.close() def row_to_dict(row: Any) -> dict | None: if row is None: return None return dict(row) def _column_names(conn: Any, table: str) -> set[str]: rows = conn.execute(f"PRAGMA table_info({table})").fetchall() return {row["name"] for row in rows} def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None: existing = _column_names(conn, table) for name, ddl in columns.items(): if name not in existing: conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}") def _mark(conn: Any, migration_id: str) -> None: conn.execute( "INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)", (migration_id,), ) def _seed_platform(conn: Any) -> None: seed = json.loads(SEED_PATH.read_text(encoding="utf-8")) for tier in seed.get("tiers", []): conn.execute( """ INSERT OR IGNORE INTO tiers (id, name, description, sort_order) VALUES (?, ?, ?, ?) """, (tier["id"], tier["name"], tier.get("description") or "", tier.get("sort_order") or 0), ) for feature in seed.get("features", []): conn.execute( """ INSERT OR IGNORE INTO features (id, name, description, category, limit_type, default_limit, reset_period) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( feature["id"], feature["name"], feature.get("description") or "", feature.get("category") or "platform", feature["limit_type"], feature.get("default_limit"), feature.get("reset_period") or "none", ), ) for limit in seed.get("tier_limits", []): conn.execute( """ INSERT OR IGNORE INTO tier_limits (tier_id, feature_id, limit_value) VALUES (?, ?, ?) """, (limit["tier_id"], limit["feature_id"], limit.get("limit_value")), ) def _seed_prompts(conn: Any) -> None: """Prompts come from JSON/DB, never from Python string literals. Untouched system prompts (template == default_template) receive the seed. Independently edited prompts keep their template; only default_template and seed_revision are refreshed so Reset-to-default stays possible. """ items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8")) for item in items: template = item.get("template") or "" revision = item.get("seed_revision") or "" existing = row_to_dict( conn.execute("SELECT * FROM ai_prompts WHERE slug = ?", (item["slug"],)).fetchone() ) if existing is None: conn.execute( """ INSERT INTO ai_prompts (id, slug, name, description, category, prompt_type, template, required_feature, is_system_default, default_template, seed_revision) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) """, ( item["id"], item["slug"], item["name"], item.get("description") or "", item.get("category") or "uncategorized", item.get("prompt_type") or "base", template, item.get("required_feature") or "ai_calls", template, revision, ), ) continue untouched = (existing.get("template") or "") == (existing.get("default_template") or "") conn.execute( """ UPDATE ai_prompts SET name = ?, description = ?, category = ?, prompt_type = ?, required_feature = ?, is_system_default = 1, default_template = ?, seed_revision = ?, template = CASE WHEN ? = 1 THEN ? ELSE template END, updated = datetime('now') WHERE slug = ? """, ( item["name"], item.get("description") or "", item.get("category") or "uncategorized", item.get("prompt_type") or "base", item.get("required_feature") or "ai_calls", template, revision, 1 if untouched else 0, template, item["slug"], ), ) def _parse_legacy_ids(raw: str | None) -> list[str]: if not raw: return [] try: data = json.loads(raw) except json.JSONDecodeError: return [] if not isinstance(data, list): return [] return [str(item) for item in data if item] def _insert_source_refs( conn: Any, table: str, owner_col: str, owner_id: str, profile_id: str, conversation_ids: list[str], message_ids: list[str], ) -> None: conn.execute(f"DELETE FROM {table} WHERE {owner_col} = ?", (owner_id,)) for index, source_id in enumerate(conversation_ids): conn.execute( f""" INSERT OR IGNORE INTO {table} ({owner_col}, profile_id, source_kind, source_id, sort_order) VALUES (?, ?, 'conversation', ?, ?) """, (owner_id, profile_id, source_id, index), ) for index, source_id in enumerate(message_ids): conn.execute( f""" INSERT OR IGNORE INTO {table} ({owner_col}, profile_id, source_kind, source_id, sort_order) VALUES (?, ?, 'message', ?, ?) """, (owner_id, profile_id, source_id, index), ) def migrate_journal_source_refs(conn: Any) -> None: """Copy JSON id lists into relational source-ref tables without touching Source.""" existing = { row["name"] for row in conn.execute( "SELECT name FROM sqlite_master WHERE type = 'table'" ).fetchall() } if "journal_draft_source_refs" not in existing or "journal_entry_version_source_refs" not in existing: return for row in conn.execute( "SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_drafts" ).fetchall(): already = conn.execute( "SELECT 1 FROM journal_draft_source_refs WHERE draft_id = ? LIMIT 1", (row["id"],), ).fetchone() if already: continue _insert_source_refs( conn, "journal_draft_source_refs", "draft_id", row["id"], row["profile_id"], _parse_legacy_ids(row["source_conversation_ids"]), _parse_legacy_ids(row["source_message_ids"]), ) for row in conn.execute( "SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_entry_versions" ).fetchall(): already = conn.execute( "SELECT 1 FROM journal_entry_version_source_refs WHERE version_id = ? LIMIT 1", (row["id"],), ).fetchone() if already: continue _insert_source_refs( conn, "journal_entry_version_source_refs", "version_id", row["id"], row["profile_id"], _parse_legacy_ids(row["source_conversation_ids"]), _parse_legacy_ids(row["source_message_ids"]), ) def _migrate_writing_profile_dialogue_style(conn: Any) -> None: """Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind.""" row = row_to_dict( conn.execute( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_sources'" ).fetchone() ) current = (row or {}).get("sql") or "" if "dialogue_style" in current: return conn.executescript( """ CREATE TABLE writing_profile_sources_new ( id TEXT PRIMARY KEY, profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, kind TEXT NOT NULL CHECK (kind IN ('imported_text', 'journal_entry', 'dialogue_style')), body TEXT NOT NULL DEFAULT '', entry_id TEXT REFERENCES journal_entries(id), weight REAL NOT NULL DEFAULT 1, created TEXT NOT NULL DEFAULT (datetime('now')) ); INSERT INTO writing_profile_sources_new (id, profile_id, kind, body, entry_id, weight, created) SELECT id, profile_id, kind, body, entry_id, weight, created FROM writing_profile_sources; DROP TABLE writing_profile_sources; ALTER TABLE writing_profile_sources_new RENAME TO writing_profile_sources; """ ) def _migrate_writing_profile_shell(conn: Any) -> None: """Layers stay; inferred count-facets are not the profile. Manual style keys become traits.""" from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key facet_sql = row_to_dict( conn.execute( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_facets'" ).fetchone() ) current = (facet_sql or {}).get("sql") or "" if current and ("inferred" in current or "layer" not in current): conn.executescript( """ CREATE TABLE writing_profile_facets_new ( id TEXT PRIMARY KEY, profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, facet_key TEXT NOT NULL, layer TEXT NOT NULL DEFAULT 'context', value TEXT NOT NULL DEFAULT '', evidence TEXT NOT NULL DEFAULT '', origin TEXT NOT NULL DEFAULT 'manual', locked INTEGER NOT NULL DEFAULT 0, updated TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE (profile_id, facet_key) ); INSERT INTO writing_profile_facets_new (id, profile_id, facet_key, layer, value, evidence, origin, locked, updated) SELECT id, profile_id, facet_key, 'context', value, evidence, CASE WHEN origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build') THEN origin ELSE 'manual' END, locked, updated FROM writing_profile_facets WHERE origin IS NOT NULL AND origin != 'inferred'; DROP TABLE writing_profile_facets; ALTER TABLE writing_profile_facets_new RENAME TO writing_profile_facets; """ ) names = _column_names(conn, "writing_profile_facets") if "layer" not in names: conn.execute("ALTER TABLE writing_profile_facets ADD COLUMN layer TEXT NOT NULL DEFAULT 'context'") rows = [ dict(item) for item in conn.execute("SELECT * FROM writing_profile_facets").fetchall() ] for item in rows: key = normalize_facet_key(item.get("facet_key") or "") origin = item.get("origin") or "manual" if (item.get("facet_key") or "") in LEGACY_STYLE_KEYS: slug = coerce_slug(item.get("facet_key") or "trait") conn.execute( """ INSERT OR IGNORE INTO writing_profile_traits (id, profile_id, facet_key, slug, label, statement, origin, locked, status) VALUES (?, ?, 'core', ?, ?, ?, ?, ?, 'active') """, ( item["id"], item["profile_id"], slug, item.get("facet_key") or slug, item.get("value") or "", origin if origin in {"manual", "imported", "accepted_suggestion", "initial_build"} else "manual", int(item.get("locked") or 0), ), ) conn.execute("DELETE FROM writing_profile_facets WHERE id = ?", (item["id"],)) continue if key != (item.get("facet_key") or ""): conn.execute( "UPDATE writing_profile_facets SET facet_key = ? WHERE id = ?", (key, item["id"]), ) layer = facet_layer(key) conn.execute("UPDATE writing_profile_facets SET layer = ? WHERE id = ?", (layer, item["id"])) for row in conn.execute("SELECT profile_id FROM writing_profiles").fetchall(): profile_id = row["profile_id"] remaining = conn.execute( "SELECT 1 FROM writing_profile_facets WHERE profile_id = ? LIMIT 1", (profile_id,), ).fetchone() traits = conn.execute( "SELECT 1 FROM writing_profile_traits WHERE profile_id = ? AND status = 'active' LIMIT 1", (profile_id,), ).fetchone() if remaining or traits: conn.execute( """ UPDATE writing_profiles SET lifecycle = 'confirmed' WHERE profile_id = ? AND lifecycle = 'uninitialized' """, (profile_id,), ) def _seed_runtime(conn: Any) -> None: _seed_platform(conn) _seed_prompts(conn) from provider_settings import seed_provider_settings from journal_generation_policy import backfill_missing_settings, seed_generation_instructions seed_provider_settings(conn) seed_generation_instructions(conn) backfill_missing_settings(conn) seed_generation_instructions(conn) seed_generation_instructions(conn) def refresh_runtime_seed() -> None: """Re-seed catalog rows after a data truncate. Schema must already exist.""" with get_db() as conn: _seed_runtime(conn) from writing_profile_store import bootstrap_from_existing bootstrap_from_existing() def init_db() -> None: if use_postgres(): from db_init import ensure_postgres_ready ensure_postgres_ready() refresh_runtime_seed() return schema = SCHEMA_PATH.read_text(encoding="utf-8") with get_db() as conn: conn.executescript(schema) _seed_platform(conn) _ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS) _seed_prompts(conn) from provider_settings import seed_provider_settings seed_provider_settings(conn) _ensure_columns(conn, "profiles", _PROFILE_COLUMNS) _ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS) _ensure_columns(conn, "journal_days", _JOURNAL_DAY_COLUMNS) _ensure_columns(conn, "journal_drafts", _JOURNAL_DRAFT_COLUMNS) _ensure_columns(conn, "writing_profiles", _WRITING_PROFILE_COLUMNS) _migrate_writing_profile_dialogue_style(conn) _ensure_columns(conn, "writing_profile_sources", _WRITING_SOURCE_COLUMNS) _ensure_columns(conn, "writing_profile_suggestions", _WRITING_SUGGESTION_COLUMNS) _ensure_columns(conn, "writing_profile_versions", _WRITING_VERSION_COLUMNS) migrate_journal_source_refs(conn) _migrate_writing_profile_shell(conn) _ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS) from identity_store import migrate_legacy_identity_rows migrate_legacy_identity_rows(conn) for migration_id in SQLITE_HISTORY_MIGRATIONS: _mark(conn, migration_id) from journal_generation_policy import backfill_missing_settings, seed_generation_instructions _ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS) seed_generation_instructions(conn) backfill_missing_settings(conn) _ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS) seed_generation_instructions(conn) seed_generation_instructions(conn) from writing_profile_store import bootstrap_from_existing bootstrap_from_existing()