Kansho/backend/db.py
Lars 79e4fd0d4b
All checks were successful
Deploy Development / deploy (push) Successful in 56s
Test Suite / pytest-backend (push) Successful in 2m43s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 15s
Load the test schema once and truncate data between files.
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 15:07:33 +02:00

612 lines
22 KiB
Python

"""Persistence for the local frame.
SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine.
Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL.
"""
from __future__ import annotations
import json
import os
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from typing import Any
from sql_compat import (
adapt_sql,
postgres_connect_kwargs,
rewrite_catalog_sql,
split_sql,
sqlite_schema_to_postgres,
use_postgres,
)
_env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data"
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
_env_db = os.environ.get("KANSHO_DB_PATH")
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
SQLITE_HISTORY_MIGRATIONS = (
"001_frame",
"002_platform",
"003_dialogue_memory",
"004_mvp_journal",
"005_provider_settings",
"006_writing_profile_dialogue_style",
"007_journal_day_scratch",
"008_journal_source_refs",
"009_conversation_signals",
"010_profile_governance",
"011_profile_review",
"012_profile_shell",
"013_journal_generate_narration",
"014_identity_registry",
"015_journal_generation_settings",
"016_generation_instruction_fragments",
"017_generation_guidelines",
"018_debug_runs",
"019_debug_run_placement",
"020_voice_style_context",
"021_voice_legacy_immutable",
)
_PROFILE_COLUMNS = {
"status": "TEXT NOT NULL DEFAULT 'active'",
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
}
_PROMPT_COLUMNS = {
"description": "TEXT NOT NULL DEFAULT ''",
"category": "TEXT NOT NULL DEFAULT 'uncategorized'",
"stages_json": "TEXT",
"graph_json": "TEXT",
"output_format": "TEXT NOT NULL DEFAULT 'text'",
"output_schema_json": "TEXT",
"required_feature": "TEXT NOT NULL DEFAULT 'ai_calls'",
"default_template": "TEXT NOT NULL DEFAULT ''",
"seed_revision": "TEXT NOT NULL DEFAULT ''",
"sort_order": "INTEGER NOT NULL DEFAULT 0",
"updated": "TEXT NOT NULL DEFAULT (datetime('now'))",
}
_CONVERSATION_COLUMNS = {
"space_id": "TEXT",
"journal_day_id": "TEXT",
"narrative_mode": "TEXT NOT NULL DEFAULT ''",
"reflection_depth": "TEXT NOT NULL DEFAULT ''",
"current_focus": "TEXT NOT NULL DEFAULT ''",
"emotional_intensity": "TEXT NOT NULL DEFAULT ''",
"long_story": "INTEGER NOT NULL DEFAULT 0",
}
_WRITING_PROFILE_COLUMNS = {
"governance": "TEXT NOT NULL DEFAULT 'learning'",
"version": "INTEGER NOT NULL DEFAULT 0",
"review_ready": "INTEGER NOT NULL DEFAULT 0",
"last_reviewed": "TEXT",
"lifecycle": "TEXT NOT NULL DEFAULT 'uninitialized'",
}
_WRITING_SOURCE_COLUMNS = {
"occurred_at": "TEXT",
"context_hint": "TEXT NOT NULL DEFAULT ''",
}
_WRITING_SUGGESTION_COLUMNS = {
"trait_slug": "TEXT NOT NULL DEFAULT ''",
"action": "TEXT NOT NULL DEFAULT 'update'",
"payload_json": "TEXT NOT NULL DEFAULT '{}'",
}
_WRITING_VERSION_COLUMNS = {
"lifecycle": "TEXT NOT NULL DEFAULT ''",
"traits_json": "TEXT NOT NULL DEFAULT '[]'",
}
_JOURNAL_DAY_COLUMNS = {
"scratch_json": "TEXT NOT NULL DEFAULT '[]'",
}
_JOURNAL_DRAFT_COLUMNS = {
"generation_snapshot": "TEXT NOT NULL DEFAULT '{}'",
}
_GUIDELINE_COLUMNS = {
"style_context_json": "TEXT NOT NULL DEFAULT '{}'",
}
_DEBUG_RUN_COLUMNS = {
"space_id": "TEXT",
"journal_day_id": "TEXT",
"space_title": "TEXT NOT NULL DEFAULT ''",
"calendar_date": "TEXT NOT NULL DEFAULT ''",
"conversation_title": "TEXT NOT NULL DEFAULT ''",
}
_IDENTITY_MAPPING_COLUMNS = {
"canonical_label": "TEXT NOT NULL DEFAULT ''",
"entity_type": "TEXT NOT NULL DEFAULT 'PERSON'",
"status": "TEXT NOT NULL DEFAULT 'legacy_review_required'",
"origin": "TEXT NOT NULL DEFAULT 'legacy_auto'",
"aliases_json": "TEXT NOT NULL DEFAULT '[]'",
# SQLite ALTER TABLE cannot use datetime('now'); backfill in migrate_legacy_identity_rows.
"updated": "TEXT NOT NULL DEFAULT ''",
"confirmed_at": "TEXT",
}
class PostgresCompat:
"""sqlite3-like execute/commit surface over psycopg, with SQL translation."""
def __init__(self, raw: Any):
self._raw = raw
def execute(self, sql: str, params: Any = None):
special = rewrite_catalog_sql(sql)
if special:
adapted, extra = special
args = extra if extra is not None else params
else:
adapted = adapt_sql(sql)
args = params
if args is None:
return self._raw.execute(adapted)
return self._raw.execute(adapted, args)
def executemany(self, sql: str, seq_of_params: Any):
adapted = adapt_sql(sql)
cursor = None
for params in seq_of_params:
cursor = self._raw.execute(adapted, params)
return cursor
def executescript(self, script: str):
for statement in split_sql(sqlite_schema_to_postgres(script)):
self._raw.execute(statement)
return self
def commit(self):
self._raw.commit()
def rollback(self):
self._raw.rollback()
def close(self):
self._raw.close()
def _connect_postgres() -> PostgresCompat:
import psycopg
from psycopg.rows import dict_row
kwargs = postgres_connect_kwargs()
if "conninfo" in kwargs:
raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row)
else:
raw = psycopg.connect(row_factory=dict_row, **kwargs)
return PostgresCompat(raw)
def _connect():
if use_postgres():
return _connect_postgres()
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH, timeout=30)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = ON")
return conn
@contextmanager
def get_db():
conn = _connect()
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def row_to_dict(row: Any) -> dict | None:
if row is None:
return None
return dict(row)
def _column_names(conn: Any, table: str) -> set[str]:
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
return {row["name"] for row in rows}
def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None:
existing = _column_names(conn, table)
for name, ddl in columns.items():
if name not in existing:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
def _mark(conn: Any, migration_id: str) -> None:
conn.execute(
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
(migration_id,),
)
def _seed_platform(conn: Any) -> None:
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
for tier in seed.get("tiers", []):
conn.execute(
"""
INSERT OR IGNORE INTO tiers (id, name, description, sort_order)
VALUES (?, ?, ?, ?)
""",
(tier["id"], tier["name"], tier.get("description") or "", tier.get("sort_order") or 0),
)
for feature in seed.get("features", []):
conn.execute(
"""
INSERT OR IGNORE INTO features
(id, name, description, category, limit_type, default_limit, reset_period)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(
feature["id"],
feature["name"],
feature.get("description") or "",
feature.get("category") or "platform",
feature["limit_type"],
feature.get("default_limit"),
feature.get("reset_period") or "none",
),
)
for limit in seed.get("tier_limits", []):
conn.execute(
"""
INSERT OR IGNORE INTO tier_limits (tier_id, feature_id, limit_value)
VALUES (?, ?, ?)
""",
(limit["tier_id"], limit["feature_id"], limit.get("limit_value")),
)
def _seed_prompts(conn: Any) -> None:
"""Prompts come from JSON/DB, never from Python string literals.
Untouched system prompts (template == default_template) receive the seed.
Independently edited prompts keep their template; only default_template and
seed_revision are refreshed so Reset-to-default stays possible.
"""
items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8"))
for item in items:
template = item.get("template") or ""
revision = item.get("seed_revision") or ""
existing = row_to_dict(
conn.execute("SELECT * FROM ai_prompts WHERE slug = ?", (item["slug"],)).fetchone()
)
if existing is None:
conn.execute(
"""
INSERT INTO ai_prompts
(id, slug, name, description, category, prompt_type, template,
required_feature, is_system_default, default_template, seed_revision)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
""",
(
item["id"],
item["slug"],
item["name"],
item.get("description") or "",
item.get("category") or "uncategorized",
item.get("prompt_type") or "base",
template,
item.get("required_feature") or "ai_calls",
template,
revision,
),
)
continue
untouched = (existing.get("template") or "") == (existing.get("default_template") or "")
conn.execute(
"""
UPDATE ai_prompts
SET name = ?, description = ?, category = ?, prompt_type = ?,
required_feature = ?, is_system_default = 1, default_template = ?,
seed_revision = ?,
template = CASE WHEN ? = 1 THEN ? ELSE template END,
updated = datetime('now')
WHERE slug = ?
""",
(
item["name"],
item.get("description") or "",
item.get("category") or "uncategorized",
item.get("prompt_type") or "base",
item.get("required_feature") or "ai_calls",
template,
revision,
1 if untouched else 0,
template,
item["slug"],
),
)
def _parse_legacy_ids(raw: str | None) -> list[str]:
if not raw:
return []
try:
data = json.loads(raw)
except json.JSONDecodeError:
return []
if not isinstance(data, list):
return []
return [str(item) for item in data if item]
def _insert_source_refs(
conn: Any,
table: str,
owner_col: str,
owner_id: str,
profile_id: str,
conversation_ids: list[str],
message_ids: list[str],
) -> None:
conn.execute(f"DELETE FROM {table} WHERE {owner_col} = ?", (owner_id,))
for index, source_id in enumerate(conversation_ids):
conn.execute(
f"""
INSERT OR IGNORE INTO {table}
({owner_col}, profile_id, source_kind, source_id, sort_order)
VALUES (?, ?, 'conversation', ?, ?)
""",
(owner_id, profile_id, source_id, index),
)
for index, source_id in enumerate(message_ids):
conn.execute(
f"""
INSERT OR IGNORE INTO {table}
({owner_col}, profile_id, source_kind, source_id, sort_order)
VALUES (?, ?, 'message', ?, ?)
""",
(owner_id, profile_id, source_id, index),
)
def migrate_journal_source_refs(conn: Any) -> None:
"""Copy JSON id lists into relational source-ref tables without touching Source."""
existing = {
row["name"]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
).fetchall()
}
if "journal_draft_source_refs" not in existing or "journal_entry_version_source_refs" not in existing:
return
for row in conn.execute(
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_drafts"
).fetchall():
already = conn.execute(
"SELECT 1 FROM journal_draft_source_refs WHERE draft_id = ? LIMIT 1",
(row["id"],),
).fetchone()
if already:
continue
_insert_source_refs(
conn,
"journal_draft_source_refs",
"draft_id",
row["id"],
row["profile_id"],
_parse_legacy_ids(row["source_conversation_ids"]),
_parse_legacy_ids(row["source_message_ids"]),
)
for row in conn.execute(
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_entry_versions"
).fetchall():
already = conn.execute(
"SELECT 1 FROM journal_entry_version_source_refs WHERE version_id = ? LIMIT 1",
(row["id"],),
).fetchone()
if already:
continue
_insert_source_refs(
conn,
"journal_entry_version_source_refs",
"version_id",
row["id"],
row["profile_id"],
_parse_legacy_ids(row["source_conversation_ids"]),
_parse_legacy_ids(row["source_message_ids"]),
)
def _migrate_writing_profile_dialogue_style(conn: Any) -> None:
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
row = row_to_dict(
conn.execute(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_sources'"
).fetchone()
)
current = (row or {}).get("sql") or ""
if "dialogue_style" in current:
return
conn.executescript(
"""
CREATE TABLE writing_profile_sources_new (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN ('imported_text', 'journal_entry', 'dialogue_style')),
body TEXT NOT NULL DEFAULT '',
entry_id TEXT REFERENCES journal_entries(id),
weight REAL NOT NULL DEFAULT 1,
created TEXT NOT NULL DEFAULT (datetime('now'))
);
INSERT INTO writing_profile_sources_new
(id, profile_id, kind, body, entry_id, weight, created)
SELECT id, profile_id, kind, body, entry_id, weight, created
FROM writing_profile_sources;
DROP TABLE writing_profile_sources;
ALTER TABLE writing_profile_sources_new RENAME TO writing_profile_sources;
"""
)
def _migrate_writing_profile_shell(conn: Any) -> None:
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
facet_sql = row_to_dict(
conn.execute(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_facets'"
).fetchone()
)
current = (facet_sql or {}).get("sql") or ""
if current and ("inferred" in current or "layer" not in current):
conn.executescript(
"""
CREATE TABLE writing_profile_facets_new (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
facet_key TEXT NOT NULL,
layer TEXT NOT NULL DEFAULT 'context',
value TEXT NOT NULL DEFAULT '',
evidence TEXT NOT NULL DEFAULT '',
origin TEXT NOT NULL DEFAULT 'manual',
locked INTEGER NOT NULL DEFAULT 0,
updated TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE (profile_id, facet_key)
);
INSERT INTO writing_profile_facets_new
(id, profile_id, facet_key, layer, value, evidence, origin, locked, updated)
SELECT id, profile_id, facet_key, 'context', value, evidence,
CASE WHEN origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build')
THEN origin ELSE 'manual' END,
locked, updated
FROM writing_profile_facets
WHERE origin IS NOT NULL AND origin != 'inferred';
DROP TABLE writing_profile_facets;
ALTER TABLE writing_profile_facets_new RENAME TO writing_profile_facets;
"""
)
names = _column_names(conn, "writing_profile_facets")
if "layer" not in names:
conn.execute("ALTER TABLE writing_profile_facets ADD COLUMN layer TEXT NOT NULL DEFAULT 'context'")
rows = [
dict(item)
for item in conn.execute("SELECT * FROM writing_profile_facets").fetchall()
]
for item in rows:
key = normalize_facet_key(item.get("facet_key") or "")
origin = item.get("origin") or "manual"
if (item.get("facet_key") or "") in LEGACY_STYLE_KEYS:
slug = coerce_slug(item.get("facet_key") or "trait")
conn.execute(
"""
INSERT OR IGNORE INTO writing_profile_traits
(id, profile_id, facet_key, slug, label, statement, origin, locked, status)
VALUES (?, ?, 'core', ?, ?, ?, ?, ?, 'active')
""",
(
item["id"],
item["profile_id"],
slug,
item.get("facet_key") or slug,
item.get("value") or "",
origin if origin in {"manual", "imported", "accepted_suggestion", "initial_build"} else "manual",
int(item.get("locked") or 0),
),
)
conn.execute("DELETE FROM writing_profile_facets WHERE id = ?", (item["id"],))
continue
if key != (item.get("facet_key") or ""):
conn.execute(
"UPDATE writing_profile_facets SET facet_key = ? WHERE id = ?",
(key, item["id"]),
)
layer = facet_layer(key)
conn.execute("UPDATE writing_profile_facets SET layer = ? WHERE id = ?", (layer, item["id"]))
for row in conn.execute("SELECT profile_id FROM writing_profiles").fetchall():
profile_id = row["profile_id"]
remaining = conn.execute(
"SELECT 1 FROM writing_profile_facets WHERE profile_id = ? LIMIT 1",
(profile_id,),
).fetchone()
traits = conn.execute(
"SELECT 1 FROM writing_profile_traits WHERE profile_id = ? AND status = 'active' LIMIT 1",
(profile_id,),
).fetchone()
if remaining or traits:
conn.execute(
"""
UPDATE writing_profiles
SET lifecycle = 'confirmed'
WHERE profile_id = ? AND lifecycle = 'uninitialized'
""",
(profile_id,),
)
def _seed_runtime(conn: Any) -> None:
_seed_platform(conn)
_seed_prompts(conn)
from provider_settings import seed_provider_settings
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
seed_provider_settings(conn)
seed_generation_instructions(conn)
backfill_missing_settings(conn)
seed_generation_instructions(conn)
seed_generation_instructions(conn)
def refresh_runtime_seed() -> None:
"""Re-seed catalog rows after a data truncate. Schema must already exist."""
with get_db() as conn:
_seed_runtime(conn)
from writing_profile_store import bootstrap_from_existing
bootstrap_from_existing()
def init_db() -> None:
if use_postgres():
from db_init import ensure_postgres_ready
ensure_postgres_ready()
refresh_runtime_seed()
return
schema = SCHEMA_PATH.read_text(encoding="utf-8")
with get_db() as conn:
conn.executescript(schema)
_seed_platform(conn)
_ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS)
_seed_prompts(conn)
from provider_settings import seed_provider_settings
seed_provider_settings(conn)
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
_ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS)
_ensure_columns(conn, "journal_days", _JOURNAL_DAY_COLUMNS)
_ensure_columns(conn, "journal_drafts", _JOURNAL_DRAFT_COLUMNS)
_ensure_columns(conn, "writing_profiles", _WRITING_PROFILE_COLUMNS)
_migrate_writing_profile_dialogue_style(conn)
_ensure_columns(conn, "writing_profile_sources", _WRITING_SOURCE_COLUMNS)
_ensure_columns(conn, "writing_profile_suggestions", _WRITING_SUGGESTION_COLUMNS)
_ensure_columns(conn, "writing_profile_versions", _WRITING_VERSION_COLUMNS)
migrate_journal_source_refs(conn)
_migrate_writing_profile_shell(conn)
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
from identity_store import migrate_legacy_identity_rows
migrate_legacy_identity_rows(conn)
for migration_id in SQLITE_HISTORY_MIGRATIONS:
_mark(conn, migration_id)
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
seed_generation_instructions(conn)
backfill_missing_settings(conn)
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
seed_generation_instructions(conn)
seed_generation_instructions(conn)
from writing_profile_store import bootstrap_from_existing
bootstrap_from_existing()