480 lines
18 KiB
Python
480 lines
18 KiB
Python
"""SQLite persistence for the local frame. PostgreSQL remains the later target."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
|
|
DATA_DIR = Path(__file__).resolve().parent / "data"
|
|
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
|
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
|
|
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
|
|
_env_db = os.environ.get("KANSHO_DB_PATH")
|
|
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
|
|
|
|
_PROFILE_COLUMNS = {
|
|
"status": "TEXT NOT NULL DEFAULT 'active'",
|
|
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
|
|
}
|
|
_PROMPT_COLUMNS = {
|
|
"description": "TEXT NOT NULL DEFAULT ''",
|
|
"category": "TEXT NOT NULL DEFAULT 'uncategorized'",
|
|
"stages_json": "TEXT",
|
|
"graph_json": "TEXT",
|
|
"output_format": "TEXT NOT NULL DEFAULT 'text'",
|
|
"output_schema_json": "TEXT",
|
|
"required_feature": "TEXT NOT NULL DEFAULT 'ai_calls'",
|
|
"default_template": "TEXT NOT NULL DEFAULT ''",
|
|
"seed_revision": "TEXT NOT NULL DEFAULT ''",
|
|
"sort_order": "INTEGER NOT NULL DEFAULT 0",
|
|
"updated": "TEXT NOT NULL DEFAULT (datetime('now'))",
|
|
}
|
|
_CONVERSATION_COLUMNS = {
|
|
"space_id": "TEXT",
|
|
"journal_day_id": "TEXT",
|
|
"narrative_mode": "TEXT NOT NULL DEFAULT ''",
|
|
"reflection_depth": "TEXT NOT NULL DEFAULT ''",
|
|
"current_focus": "TEXT NOT NULL DEFAULT ''",
|
|
"emotional_intensity": "TEXT NOT NULL DEFAULT ''",
|
|
"long_story": "INTEGER NOT NULL DEFAULT 0",
|
|
}
|
|
_WRITING_PROFILE_COLUMNS = {
|
|
"governance": "TEXT NOT NULL DEFAULT 'learning'",
|
|
"version": "INTEGER NOT NULL DEFAULT 0",
|
|
"review_ready": "INTEGER NOT NULL DEFAULT 0",
|
|
"last_reviewed": "TEXT",
|
|
"lifecycle": "TEXT NOT NULL DEFAULT 'uninitialized'",
|
|
}
|
|
_WRITING_SOURCE_COLUMNS = {
|
|
"occurred_at": "TEXT",
|
|
"context_hint": "TEXT NOT NULL DEFAULT ''",
|
|
}
|
|
_WRITING_SUGGESTION_COLUMNS = {
|
|
"trait_slug": "TEXT NOT NULL DEFAULT ''",
|
|
"action": "TEXT NOT NULL DEFAULT 'update'",
|
|
"payload_json": "TEXT NOT NULL DEFAULT '{}'",
|
|
}
|
|
_WRITING_VERSION_COLUMNS = {
|
|
"lifecycle": "TEXT NOT NULL DEFAULT ''",
|
|
"traits_json": "TEXT NOT NULL DEFAULT '[]'",
|
|
}
|
|
_JOURNAL_DAY_COLUMNS = {
|
|
"scratch_json": "TEXT NOT NULL DEFAULT '[]'",
|
|
}
|
|
_IDENTITY_MAPPING_COLUMNS = {
|
|
"canonical_label": "TEXT NOT NULL DEFAULT ''",
|
|
"entity_type": "TEXT NOT NULL DEFAULT 'PERSON'",
|
|
"status": "TEXT NOT NULL DEFAULT 'legacy_review_required'",
|
|
"origin": "TEXT NOT NULL DEFAULT 'legacy_auto'",
|
|
"aliases_json": "TEXT NOT NULL DEFAULT '[]'",
|
|
# SQLite ALTER TABLE cannot use datetime('now'); backfill in migrate_legacy_identity_rows.
|
|
"updated": "TEXT NOT NULL DEFAULT ''",
|
|
"confirmed_at": "TEXT",
|
|
}
|
|
|
|
|
|
def _connect() -> sqlite3.Connection:
|
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(DB_PATH, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
return conn
|
|
|
|
|
|
@contextmanager
|
|
def get_db():
|
|
conn = _connect()
|
|
try:
|
|
yield conn
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
|
|
if row is None:
|
|
return None
|
|
return dict(row)
|
|
|
|
|
|
def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
|
return {row["name"] for row in rows}
|
|
|
|
|
|
def _ensure_columns(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
|
|
existing = _column_names(conn, table)
|
|
for name, ddl in columns.items():
|
|
if name not in existing:
|
|
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
|
|
|
|
|
def _mark(conn: sqlite3.Connection, migration_id: str) -> None:
|
|
conn.execute(
|
|
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
|
|
(migration_id,),
|
|
)
|
|
|
|
|
|
def _seed_platform(conn: sqlite3.Connection) -> None:
|
|
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
|
for tier in seed.get("tiers", []):
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO tiers (id, name, description, sort_order)
|
|
VALUES (?, ?, ?, ?)
|
|
""",
|
|
(tier["id"], tier["name"], tier.get("description") or "", tier.get("sort_order") or 0),
|
|
)
|
|
for feature in seed.get("features", []):
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO features
|
|
(id, name, description, category, limit_type, default_limit, reset_period)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
feature["id"],
|
|
feature["name"],
|
|
feature.get("description") or "",
|
|
feature.get("category") or "platform",
|
|
feature["limit_type"],
|
|
feature.get("default_limit"),
|
|
feature.get("reset_period") or "none",
|
|
),
|
|
)
|
|
for limit in seed.get("tier_limits", []):
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO tier_limits (tier_id, feature_id, limit_value)
|
|
VALUES (?, ?, ?)
|
|
""",
|
|
(limit["tier_id"], limit["feature_id"], limit.get("limit_value")),
|
|
)
|
|
|
|
|
|
def _seed_prompts(conn: sqlite3.Connection) -> None:
|
|
"""Prompts come from JSON/DB, never from Python string literals.
|
|
|
|
Untouched system prompts (template == default_template) receive the seed.
|
|
Independently edited prompts keep their template; only default_template and
|
|
seed_revision are refreshed so Reset-to-default stays possible.
|
|
"""
|
|
items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8"))
|
|
for item in items:
|
|
template = item.get("template") or ""
|
|
revision = item.get("seed_revision") or ""
|
|
existing = row_to_dict(
|
|
conn.execute("SELECT * FROM ai_prompts WHERE slug = ?", (item["slug"],)).fetchone()
|
|
)
|
|
if existing is None:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO ai_prompts
|
|
(id, slug, name, description, category, prompt_type, template,
|
|
required_feature, is_system_default, default_template, seed_revision)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
""",
|
|
(
|
|
item["id"],
|
|
item["slug"],
|
|
item["name"],
|
|
item.get("description") or "",
|
|
item.get("category") or "uncategorized",
|
|
item.get("prompt_type") or "base",
|
|
template,
|
|
item.get("required_feature") or "ai_calls",
|
|
template,
|
|
revision,
|
|
),
|
|
)
|
|
continue
|
|
untouched = (existing.get("template") or "") == (existing.get("default_template") or "")
|
|
conn.execute(
|
|
"""
|
|
UPDATE ai_prompts
|
|
SET name = ?, description = ?, category = ?, prompt_type = ?,
|
|
required_feature = ?, is_system_default = 1, default_template = ?,
|
|
seed_revision = ?,
|
|
template = CASE WHEN ? THEN ? ELSE template END,
|
|
updated = datetime('now')
|
|
WHERE slug = ?
|
|
""",
|
|
(
|
|
item["name"],
|
|
item.get("description") or "",
|
|
item.get("category") or "uncategorized",
|
|
item.get("prompt_type") or "base",
|
|
item.get("required_feature") or "ai_calls",
|
|
template,
|
|
revision,
|
|
1 if untouched else 0,
|
|
template,
|
|
item["slug"],
|
|
),
|
|
)
|
|
|
|
|
|
def _parse_legacy_ids(raw: str | None) -> list[str]:
|
|
if not raw:
|
|
return []
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return []
|
|
if not isinstance(data, list):
|
|
return []
|
|
return [str(item) for item in data if item]
|
|
|
|
|
|
def _insert_source_refs(
|
|
conn: sqlite3.Connection,
|
|
table: str,
|
|
owner_col: str,
|
|
owner_id: str,
|
|
profile_id: str,
|
|
conversation_ids: list[str],
|
|
message_ids: list[str],
|
|
) -> None:
|
|
conn.execute(f"DELETE FROM {table} WHERE {owner_col} = ?", (owner_id,))
|
|
for index, source_id in enumerate(conversation_ids):
|
|
conn.execute(
|
|
f"""
|
|
INSERT OR IGNORE INTO {table}
|
|
({owner_col}, profile_id, source_kind, source_id, sort_order)
|
|
VALUES (?, ?, 'conversation', ?, ?)
|
|
""",
|
|
(owner_id, profile_id, source_id, index),
|
|
)
|
|
for index, source_id in enumerate(message_ids):
|
|
conn.execute(
|
|
f"""
|
|
INSERT OR IGNORE INTO {table}
|
|
({owner_col}, profile_id, source_kind, source_id, sort_order)
|
|
VALUES (?, ?, 'message', ?, ?)
|
|
""",
|
|
(owner_id, profile_id, source_id, index),
|
|
)
|
|
|
|
|
|
def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
|
|
"""Copy JSON id lists into relational source-ref tables without touching Source."""
|
|
existing = {
|
|
row["name"]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
|
).fetchall()
|
|
}
|
|
if "journal_draft_source_refs" not in existing or "journal_entry_version_source_refs" not in existing:
|
|
return
|
|
for row in conn.execute(
|
|
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_drafts"
|
|
).fetchall():
|
|
already = conn.execute(
|
|
"SELECT 1 FROM journal_draft_source_refs WHERE draft_id = ? LIMIT 1",
|
|
(row["id"],),
|
|
).fetchone()
|
|
if already:
|
|
continue
|
|
_insert_source_refs(
|
|
conn,
|
|
"journal_draft_source_refs",
|
|
"draft_id",
|
|
row["id"],
|
|
row["profile_id"],
|
|
_parse_legacy_ids(row["source_conversation_ids"]),
|
|
_parse_legacy_ids(row["source_message_ids"]),
|
|
)
|
|
for row in conn.execute(
|
|
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_entry_versions"
|
|
).fetchall():
|
|
already = conn.execute(
|
|
"SELECT 1 FROM journal_entry_version_source_refs WHERE version_id = ? LIMIT 1",
|
|
(row["id"],),
|
|
).fetchone()
|
|
if already:
|
|
continue
|
|
_insert_source_refs(
|
|
conn,
|
|
"journal_entry_version_source_refs",
|
|
"version_id",
|
|
row["id"],
|
|
row["profile_id"],
|
|
_parse_legacy_ids(row["source_conversation_ids"]),
|
|
_parse_legacy_ids(row["source_message_ids"]),
|
|
)
|
|
|
|
|
|
def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
|
|
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
|
|
row = row_to_dict(
|
|
conn.execute(
|
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_sources'"
|
|
).fetchone()
|
|
)
|
|
current = (row or {}).get("sql") or ""
|
|
if "dialogue_style" in current:
|
|
return
|
|
conn.executescript(
|
|
"""
|
|
CREATE TABLE writing_profile_sources_new (
|
|
id TEXT PRIMARY KEY,
|
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
|
kind TEXT NOT NULL CHECK (kind IN ('imported_text', 'journal_entry', 'dialogue_style')),
|
|
body TEXT NOT NULL DEFAULT '',
|
|
entry_id TEXT REFERENCES journal_entries(id),
|
|
weight REAL NOT NULL DEFAULT 1,
|
|
created TEXT NOT NULL DEFAULT (datetime('now'))
|
|
);
|
|
INSERT INTO writing_profile_sources_new
|
|
(id, profile_id, kind, body, entry_id, weight, created)
|
|
SELECT id, profile_id, kind, body, entry_id, weight, created
|
|
FROM writing_profile_sources;
|
|
DROP TABLE writing_profile_sources;
|
|
ALTER TABLE writing_profile_sources_new RENAME TO writing_profile_sources;
|
|
"""
|
|
)
|
|
|
|
|
|
def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
|
|
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
|
|
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
|
|
|
|
facet_sql = row_to_dict(
|
|
conn.execute(
|
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_facets'"
|
|
).fetchone()
|
|
)
|
|
current = (facet_sql or {}).get("sql") or ""
|
|
if current and ("inferred" in current or "layer" not in current):
|
|
conn.executescript(
|
|
"""
|
|
CREATE TABLE writing_profile_facets_new (
|
|
id TEXT PRIMARY KEY,
|
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
|
facet_key TEXT NOT NULL,
|
|
layer TEXT NOT NULL DEFAULT 'context',
|
|
value TEXT NOT NULL DEFAULT '',
|
|
evidence TEXT NOT NULL DEFAULT '',
|
|
origin TEXT NOT NULL DEFAULT 'manual',
|
|
locked INTEGER NOT NULL DEFAULT 0,
|
|
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
|
UNIQUE (profile_id, facet_key)
|
|
);
|
|
INSERT INTO writing_profile_facets_new
|
|
(id, profile_id, facet_key, layer, value, evidence, origin, locked, updated)
|
|
SELECT id, profile_id, facet_key, 'context', value, evidence,
|
|
CASE WHEN origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build')
|
|
THEN origin ELSE 'manual' END,
|
|
locked, updated
|
|
FROM writing_profile_facets
|
|
WHERE origin IS NOT NULL AND origin != 'inferred';
|
|
DROP TABLE writing_profile_facets;
|
|
ALTER TABLE writing_profile_facets_new RENAME TO writing_profile_facets;
|
|
"""
|
|
)
|
|
names = _column_names(conn, "writing_profile_facets")
|
|
if "layer" not in names:
|
|
conn.execute("ALTER TABLE writing_profile_facets ADD COLUMN layer TEXT NOT NULL DEFAULT 'context'")
|
|
rows = [
|
|
dict(item)
|
|
for item in conn.execute("SELECT * FROM writing_profile_facets").fetchall()
|
|
]
|
|
for item in rows:
|
|
key = normalize_facet_key(item.get("facet_key") or "")
|
|
origin = item.get("origin") or "manual"
|
|
if (item.get("facet_key") or "") in LEGACY_STYLE_KEYS:
|
|
slug = coerce_slug(item.get("facet_key") or "trait")
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO writing_profile_traits
|
|
(id, profile_id, facet_key, slug, label, statement, origin, locked, status)
|
|
VALUES (?, ?, 'core', ?, ?, ?, ?, ?, 'active')
|
|
""",
|
|
(
|
|
item["id"],
|
|
item["profile_id"],
|
|
slug,
|
|
item.get("facet_key") or slug,
|
|
item.get("value") or "",
|
|
origin if origin in {"manual", "imported", "accepted_suggestion", "initial_build"} else "manual",
|
|
int(item.get("locked") or 0),
|
|
),
|
|
)
|
|
conn.execute("DELETE FROM writing_profile_facets WHERE id = ?", (item["id"],))
|
|
continue
|
|
if key != (item.get("facet_key") or ""):
|
|
conn.execute(
|
|
"UPDATE writing_profile_facets SET facet_key = ? WHERE id = ?",
|
|
(key, item["id"]),
|
|
)
|
|
layer = facet_layer(key)
|
|
conn.execute("UPDATE writing_profile_facets SET layer = ? WHERE id = ?", (layer, item["id"]))
|
|
for row in conn.execute("SELECT profile_id FROM writing_profiles").fetchall():
|
|
profile_id = row["profile_id"]
|
|
remaining = conn.execute(
|
|
"SELECT 1 FROM writing_profile_facets WHERE profile_id = ? LIMIT 1",
|
|
(profile_id,),
|
|
).fetchone()
|
|
traits = conn.execute(
|
|
"SELECT 1 FROM writing_profile_traits WHERE profile_id = ? AND status = 'active' LIMIT 1",
|
|
(profile_id,),
|
|
).fetchone()
|
|
if remaining or traits:
|
|
conn.execute(
|
|
"""
|
|
UPDATE writing_profiles
|
|
SET lifecycle = 'confirmed'
|
|
WHERE profile_id = ? AND lifecycle = 'uninitialized'
|
|
""",
|
|
(profile_id,),
|
|
)
|
|
|
|
|
|
def init_db() -> None:
|
|
schema = SCHEMA_PATH.read_text(encoding="utf-8")
|
|
with get_db() as conn:
|
|
conn.executescript(schema)
|
|
_seed_platform(conn)
|
|
_ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS)
|
|
_seed_prompts(conn)
|
|
from provider_settings import seed_provider_settings
|
|
|
|
seed_provider_settings(conn)
|
|
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
|
|
_ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS)
|
|
_ensure_columns(conn, "journal_days", _JOURNAL_DAY_COLUMNS)
|
|
_ensure_columns(conn, "writing_profiles", _WRITING_PROFILE_COLUMNS)
|
|
_migrate_writing_profile_dialogue_style(conn)
|
|
_ensure_columns(conn, "writing_profile_sources", _WRITING_SOURCE_COLUMNS)
|
|
_ensure_columns(conn, "writing_profile_suggestions", _WRITING_SUGGESTION_COLUMNS)
|
|
_ensure_columns(conn, "writing_profile_versions", _WRITING_VERSION_COLUMNS)
|
|
migrate_journal_source_refs(conn)
|
|
_migrate_writing_profile_shell(conn)
|
|
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
|
from identity_store import migrate_legacy_identity_rows
|
|
|
|
migrate_legacy_identity_rows(conn)
|
|
_mark(conn, "001_frame")
|
|
_mark(conn, "002_platform")
|
|
_mark(conn, "003_dialogue_memory")
|
|
_mark(conn, "004_mvp_journal")
|
|
_mark(conn, "005_provider_settings")
|
|
_mark(conn, "006_writing_profile_dialogue_style")
|
|
_mark(conn, "007_journal_day_scratch")
|
|
_mark(conn, "008_journal_source_refs")
|
|
_mark(conn, "009_conversation_signals")
|
|
_mark(conn, "010_profile_governance")
|
|
_mark(conn, "011_profile_review")
|
|
_mark(conn, "012_profile_shell")
|
|
_mark(conn, "013_journal_generate_narration")
|
|
_mark(conn, "014_identity_registry")
|
|
from writing_profile_store import bootstrap_from_existing
|
|
|
|
bootstrap_from_existing()
|