Add Compose, Gitea deploy, and Postgres dual-backend on develop.
Use host ports 3006/8005 and 3096/8096 so Kansho does not collide with Bookstack on 3005 or the sister products. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
32acdc220b
commit
07479f6a9f
37 files changed
+1613
-102
No files matched your search
@@ -22,3 +22,11 @@ KANSHO_DETECT_PROVIDER_KEY=
|
||||
# Nur Tests
|
||||
# KANSHO_FAKE_PROVIDER=1
|
||||
# KANSHO_FAKE_DETECT=1
|
||||
|
||||
# Docker/Server only. Local Windows stays SQLite (default).
|
||||
# KANSHO_DB_BACKEND=postgres
|
||||
# DB_HOST=postgres
|
||||
# DB_NAME=kansho
|
||||
# DB_USER=kansho
|
||||
# DB_PASSWORD=
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
ENV PIP_DEFAULT_TIMEOUT=120
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
RUN chmod +x /app/startup.sh \
|
||||
&& mkdir -p /app/data/media
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["/app/startup.sh"]
|
||||
+132
-34
@@ -1,4 +1,8 @@
|
||||
"""SQLite persistence for the local frame. PostgreSQL remains the later target."""
|
||||
"""Persistence for the local frame.
|
||||
|
||||
SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine.
|
||||
Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -6,14 +10,49 @@ import os
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent / "data"
|
||||
from sql_compat import (
|
||||
adapt_sql,
|
||||
postgres_connect_kwargs,
|
||||
rewrite_catalog_sql,
|
||||
split_sql,
|
||||
sqlite_schema_to_postgres,
|
||||
use_postgres,
|
||||
)
|
||||
|
||||
_env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data"
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
|
||||
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
|
||||
_env_db = os.environ.get("KANSHO_DB_PATH")
|
||||
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
|
||||
|
||||
SQLITE_HISTORY_MIGRATIONS = (
|
||||
"001_frame",
|
||||
"002_platform",
|
||||
"003_dialogue_memory",
|
||||
"004_mvp_journal",
|
||||
"005_provider_settings",
|
||||
"006_writing_profile_dialogue_style",
|
||||
"007_journal_day_scratch",
|
||||
"008_journal_source_refs",
|
||||
"009_conversation_signals",
|
||||
"010_profile_governance",
|
||||
"011_profile_review",
|
||||
"012_profile_shell",
|
||||
"013_journal_generate_narration",
|
||||
"014_identity_registry",
|
||||
"015_journal_generation_settings",
|
||||
"016_generation_instruction_fragments",
|
||||
"017_generation_guidelines",
|
||||
"018_debug_runs",
|
||||
"019_debug_run_placement",
|
||||
"020_voice_style_context",
|
||||
"021_voice_legacy_immutable",
|
||||
)
|
||||
|
||||
_PROFILE_COLUMNS = {
|
||||
"status": "TEXT NOT NULL DEFAULT 'active'",
|
||||
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
|
||||
@@ -88,7 +127,61 @@ _IDENTITY_MAPPING_COLUMNS = {
|
||||
}
|
||||
|
||||
|
||||
def _connect() -> sqlite3.Connection:
|
||||
class PostgresCompat:
|
||||
"""sqlite3-like execute/commit surface over psycopg, with SQL translation."""
|
||||
|
||||
def __init__(self, raw: Any):
|
||||
self._raw = raw
|
||||
|
||||
def execute(self, sql: str, params: Any = None):
|
||||
special = rewrite_catalog_sql(sql)
|
||||
if special:
|
||||
adapted, extra = special
|
||||
args = extra if extra is not None else params
|
||||
else:
|
||||
adapted = adapt_sql(sql)
|
||||
args = params
|
||||
if args is None:
|
||||
return self._raw.execute(adapted)
|
||||
return self._raw.execute(adapted, args)
|
||||
|
||||
def executemany(self, sql: str, seq_of_params: Any):
|
||||
adapted = adapt_sql(sql)
|
||||
cursor = None
|
||||
for params in seq_of_params:
|
||||
cursor = self._raw.execute(adapted, params)
|
||||
return cursor
|
||||
|
||||
def executescript(self, script: str):
|
||||
for statement in split_sql(sqlite_schema_to_postgres(script)):
|
||||
self._raw.execute(statement)
|
||||
return self
|
||||
|
||||
def commit(self):
|
||||
self._raw.commit()
|
||||
|
||||
def rollback(self):
|
||||
self._raw.rollback()
|
||||
|
||||
def close(self):
|
||||
self._raw.close()
|
||||
|
||||
|
||||
def _connect_postgres() -> PostgresCompat:
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row)
|
||||
else:
|
||||
raw = psycopg.connect(row_factory=dict_row, **kwargs)
|
||||
return PostgresCompat(raw)
|
||||
|
||||
|
||||
def _connect():
|
||||
if use_postgres():
|
||||
return _connect_postgres()
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
@@ -109,32 +202,32 @@ def get_db():
|
||||
conn.close()
|
||||
|
||||
|
||||
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
|
||||
def row_to_dict(row: Any) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
|
||||
def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
|
||||
def _column_names(conn: Any, table: str) -> set[str]:
|
||||
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
||||
return {row["name"] for row in rows}
|
||||
|
||||
|
||||
def _ensure_columns(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
|
||||
def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None:
|
||||
existing = _column_names(conn, table)
|
||||
for name, ddl in columns.items():
|
||||
if name not in existing:
|
||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
||||
|
||||
|
||||
def _mark(conn: sqlite3.Connection, migration_id: str) -> None:
|
||||
def _mark(conn: Any, migration_id: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
|
||||
(migration_id,),
|
||||
)
|
||||
|
||||
|
||||
def _seed_platform(conn: sqlite3.Connection) -> None:
|
||||
def _seed_platform(conn: Any) -> None:
|
||||
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
||||
for tier in seed.get("tiers", []):
|
||||
conn.execute(
|
||||
@@ -171,7 +264,7 @@ def _seed_platform(conn: sqlite3.Connection) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _seed_prompts(conn: sqlite3.Connection) -> None:
|
||||
def _seed_prompts(conn: Any) -> None:
|
||||
"""Prompts come from JSON/DB, never from Python string literals.
|
||||
|
||||
Untouched system prompts (template == default_template) receive the seed.
|
||||
@@ -246,7 +339,7 @@ def _parse_legacy_ids(raw: str | None) -> list[str]:
|
||||
|
||||
|
||||
def _insert_source_refs(
|
||||
conn: sqlite3.Connection,
|
||||
conn: Any,
|
||||
table: str,
|
||||
owner_col: str,
|
||||
owner_id: str,
|
||||
@@ -275,7 +368,7 @@ def _insert_source_refs(
|
||||
)
|
||||
|
||||
|
||||
def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
|
||||
def migrate_journal_source_refs(conn: Any) -> None:
|
||||
"""Copy JSON id lists into relational source-ref tables without touching Source."""
|
||||
existing = {
|
||||
row["name"]
|
||||
@@ -323,7 +416,7 @@ def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
|
||||
def _migrate_writing_profile_dialogue_style(conn: Any) -> None:
|
||||
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
@@ -354,7 +447,7 @@ def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
|
||||
def _migrate_writing_profile_shell(conn: Any) -> None:
|
||||
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
|
||||
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
|
||||
|
||||
@@ -449,7 +542,31 @@ def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _seed_runtime(conn: Any) -> None:
|
||||
_seed_platform(conn)
|
||||
_seed_prompts(conn)
|
||||
from provider_settings import seed_provider_settings
|
||||
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||
|
||||
seed_provider_settings(conn)
|
||||
seed_generation_instructions(conn)
|
||||
backfill_missing_settings(conn)
|
||||
seed_generation_instructions(conn)
|
||||
seed_generation_instructions(conn)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if use_postgres():
|
||||
from db_init import ensure_postgres_ready
|
||||
|
||||
ensure_postgres_ready()
|
||||
with get_db() as conn:
|
||||
_seed_runtime(conn)
|
||||
from writing_profile_store import bootstrap_from_existing
|
||||
|
||||
bootstrap_from_existing()
|
||||
return
|
||||
|
||||
schema = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
with get_db() as conn:
|
||||
conn.executescript(schema)
|
||||
@@ -474,35 +591,16 @@ def init_db() -> None:
|
||||
from identity_store import migrate_legacy_identity_rows
|
||||
|
||||
migrate_legacy_identity_rows(conn)
|
||||
_mark(conn, "001_frame")
|
||||
_mark(conn, "002_platform")
|
||||
_mark(conn, "003_dialogue_memory")
|
||||
_mark(conn, "004_mvp_journal")
|
||||
_mark(conn, "005_provider_settings")
|
||||
_mark(conn, "006_writing_profile_dialogue_style")
|
||||
_mark(conn, "007_journal_day_scratch")
|
||||
_mark(conn, "008_journal_source_refs")
|
||||
_mark(conn, "009_conversation_signals")
|
||||
_mark(conn, "010_profile_governance")
|
||||
_mark(conn, "011_profile_review")
|
||||
_mark(conn, "012_profile_shell")
|
||||
_mark(conn, "013_journal_generate_narration")
|
||||
_mark(conn, "014_identity_registry")
|
||||
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||
_mark(conn, migration_id)
|
||||
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||
|
||||
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
|
||||
seed_generation_instructions(conn)
|
||||
backfill_missing_settings(conn)
|
||||
_mark(conn, "015_journal_generation_settings")
|
||||
_mark(conn, "016_generation_instruction_fragments")
|
||||
_mark(conn, "017_generation_guidelines")
|
||||
_mark(conn, "018_debug_runs")
|
||||
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
|
||||
_mark(conn, "019_debug_run_placement")
|
||||
seed_generation_instructions(conn)
|
||||
_mark(conn, "020_voice_style_context")
|
||||
seed_generation_instructions(conn)
|
||||
_mark(conn, "021_voice_legacy_immutable")
|
||||
from writing_profile_store import bootstrap_from_existing
|
||||
|
||||
bootstrap_from_existing()
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for PostgreSQL, load schema.sql, apply numbered SQL migrations. Fail-fast."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from sql_compat import postgres_connect_kwargs, split_sql, sqlite_schema_to_postgres, use_postgres
|
||||
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations"
|
||||
SQLITE_HISTORY_MIGRATIONS = (
|
||||
"001_frame",
|
||||
"002_platform",
|
||||
"003_dialogue_memory",
|
||||
"004_mvp_journal",
|
||||
"005_provider_settings",
|
||||
"006_writing_profile_dialogue_style",
|
||||
"007_journal_day_scratch",
|
||||
"008_journal_source_refs",
|
||||
"009_conversation_signals",
|
||||
"010_profile_governance",
|
||||
"011_profile_review",
|
||||
"012_profile_shell",
|
||||
"013_journal_generate_narration",
|
||||
"014_identity_registry",
|
||||
"015_journal_generation_settings",
|
||||
"016_generation_instruction_fragments",
|
||||
"017_generation_guidelines",
|
||||
"018_debug_runs",
|
||||
"019_debug_run_placement",
|
||||
"020_voice_style_context",
|
||||
"021_voice_legacy_immutable",
|
||||
)
|
||||
_LEADING_DIGITS = re.compile(r"^(\d{3})_.*\.sql$")
|
||||
|
||||
|
||||
def _connect_raw():
|
||||
import psycopg
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
return psycopg.connect(kwargs["conninfo"], autocommit=False)
|
||||
return psycopg.connect(autocommit=False, **kwargs)
|
||||
|
||||
|
||||
def wait_for_postgres(max_retries: int = 30) -> None:
|
||||
print("Checking PostgreSQL connection...")
|
||||
last_error = None
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
conn = _connect_raw()
|
||||
conn.close()
|
||||
print("PostgreSQL ready")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — fail-fast after retries
|
||||
last_error = exc
|
||||
print(f" waiting ({attempt}/{max_retries})")
|
||||
time.sleep(2)
|
||||
print(f"PostgreSQL not ready: {last_error}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _table_exists(conn, name: str) -> bool:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
""",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _load_schema(conn) -> None:
|
||||
sql = sqlite_schema_to_postgres(SCHEMA_PATH.read_text(encoding="utf-8"))
|
||||
for statement in split_sql(sql):
|
||||
conn.execute(statement)
|
||||
print("Schema loaded from schema.sql (Postgres dialect)")
|
||||
|
||||
|
||||
def _applied_ids(conn) -> set[str]:
|
||||
rows = conn.execute("SELECT id FROM schema_migrations").fetchall()
|
||||
return {row[0] for row in rows}
|
||||
|
||||
|
||||
def _record(conn, migration_id: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
|
||||
(migration_id,),
|
||||
)
|
||||
|
||||
|
||||
def _migration_files() -> list[tuple[str, Path]]:
|
||||
if not MIGRATIONS_DIR.is_dir():
|
||||
return []
|
||||
rows: list[tuple[str, Path]] = []
|
||||
for path in sorted(MIGRATIONS_DIR.iterdir()):
|
||||
match = _LEADING_DIGITS.match(path.name)
|
||||
if not match or path.suffix != ".sql":
|
||||
continue
|
||||
rows.append((path.stem, path))
|
||||
return rows
|
||||
|
||||
|
||||
def apply_schema_and_migrations() -> None:
|
||||
conn = _connect_raw()
|
||||
try:
|
||||
if not _table_exists(conn, "schema_migrations"):
|
||||
_load_schema(conn)
|
||||
conn.commit()
|
||||
applied = _applied_ids(conn)
|
||||
if "001_frame" not in applied:
|
||||
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||
_record(conn, migration_id)
|
||||
conn.commit()
|
||||
print("Recorded SQLite history 001-021 as applied (greenfield Postgres schema)")
|
||||
applied = _applied_ids(conn)
|
||||
for stem, path in _migration_files():
|
||||
if stem in applied:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
for statement in split_sql(sql):
|
||||
conn.execute(statement)
|
||||
_record(conn, stem)
|
||||
conn.commit()
|
||||
print(f"Applied migration {stem}")
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
print(f"Database initialization failed: {exc}")
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def ensure_postgres_ready() -> None:
|
||||
if not use_postgres():
|
||||
return
|
||||
wait_for_postgres()
|
||||
try:
|
||||
apply_schema_and_migrations()
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("KANSHO_DB_BACKEND", "postgres")
|
||||
ensure_postgres_ready()
|
||||
print("db_init complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+24
-1
@@ -2,6 +2,8 @@ from env_loader import load_env_file
|
||||
|
||||
load_env_file()
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
@@ -10,11 +12,32 @@ import data_layer_dialogue # noqa: F401
|
||||
from routers import admin, auth, dialogue, generation_instructions, journal, placeholders, prompts, subscription, users
|
||||
from version import APP_VERSION
|
||||
|
||||
|
||||
def allowed_origins() -> list[str]:
|
||||
origins = [
|
||||
"http://localhost:5188",
|
||||
"http://127.0.0.1:5188",
|
||||
"https://kansho.jinkendo.de",
|
||||
"https://dev.kansho.jinkendo.de",
|
||||
]
|
||||
extra = (os.environ.get("ALLOWED_ORIGINS") or os.environ.get("KANSHO_ALLOWED_ORIGINS") or "").strip()
|
||||
if extra:
|
||||
origins.extend(item.strip() for item in extra.split(",") if item.strip())
|
||||
app_url = (os.environ.get("APP_URL") or "").strip().rstrip("/")
|
||||
if app_url:
|
||||
origins.append(app_url)
|
||||
seen: list[str] = []
|
||||
for item in origins:
|
||||
if item not in seen:
|
||||
seen.append(item)
|
||||
return seen
|
||||
|
||||
|
||||
app = FastAPI(title="Kanshō", version=APP_VERSION)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://localhost:5188", "http://127.0.0.1:5188"],
|
||||
allow_origins=allowed_origins(),
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Postgres runtime baseline for Kanshō.
|
||||
-- Greenfield schema is backend/schema.sql translated at startup
|
||||
-- (datetime('now') → CURRENT_TIMESTAMP::text). Historical SQLite
|
||||
-- inline migrations 001-021 are recorded as applied, not replayed.
|
||||
-- Future incremental DDL belongs in 023_*.sql and later.
|
||||
|
||||
SELECT 1;
|
||||
@@ -3,3 +3,4 @@ uvicorn[standard]==0.34.0
|
||||
bcrypt==4.2.1
|
||||
httpx==0.28.1
|
||||
python-multipart==0.0.20
|
||||
psycopg[binary]==3.2.6
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Translate SQLite-oriented SQL so the same stores can run against PostgreSQL.
|
||||
|
||||
Local Windows development and tests keep SQLite. Docker/Server set
|
||||
KANSHO_DB_BACKEND=postgres. Identity, keys and journal bodies are not logged here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
_INSERT_OR_IGNORE = re.compile(r"INSERT\s+OR\s+IGNORE\s+INTO", re.IGNORECASE)
|
||||
_ON_CONFLICT_PAREN = re.compile(r"ON\s+CONFLICT\s*\(", re.IGNORECASE)
|
||||
_PRAGMA_TABLE = re.compile(
|
||||
r"^\s*PRAGMA\s+table_info\(\s*['\"]?(\w+)['\"]?\s*\)\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SQLITE_MASTER_TABLES = re.compile(
|
||||
r"^\s*SELECT\s+name\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SQLITE_MASTER_SQL = re.compile(
|
||||
r"^\s*SELECT\s+sql\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s+AND\s+name\s*=\s*\?\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def use_postgres() -> bool:
|
||||
"""Decide the engine at connect time, not at import time.
|
||||
|
||||
Explicit KANSHO_DB_BACKEND wins. KANSHO_DB_PATH without an explicit postgres
|
||||
backend keeps tests on isolated SQLite even inside a Compose container.
|
||||
"""
|
||||
raw = (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
|
||||
if raw in {"sqlite", "sqlite3"}:
|
||||
return False
|
||||
if raw in {"postgres", "postgresql", "pg"}:
|
||||
return True
|
||||
if (os.environ.get("KANSHO_DB_PATH") or "").strip():
|
||||
return False
|
||||
url = (os.environ.get("DATABASE_URL") or "").strip().lower()
|
||||
if url.startswith("postgres"):
|
||||
return True
|
||||
if (os.environ.get("DB_HOST") or "").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def postgres_connect_kwargs() -> dict:
|
||||
url = (os.environ.get("DATABASE_URL") or "").strip()
|
||||
if url:
|
||||
return {"conninfo": url}
|
||||
host = (os.environ.get("DB_HOST") or "postgres").strip() or "postgres"
|
||||
port = int((os.environ.get("DB_PORT") or "5432").strip() or "5432")
|
||||
dbname = (os.environ.get("DB_NAME") or "kansho").strip() or "kansho"
|
||||
user = (os.environ.get("DB_USER") or "kansho").strip() or "kansho"
|
||||
password = os.environ.get("DB_PASSWORD") or ""
|
||||
return {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"dbname": dbname,
|
||||
"user": user,
|
||||
"password": password,
|
||||
}
|
||||
|
||||
|
||||
def sqlite_schema_to_postgres(sql: str) -> str:
|
||||
return sql.replace("datetime('now')", "CURRENT_TIMESTAMP::text")
|
||||
|
||||
|
||||
def replace_placeholders(sql: str) -> str:
|
||||
"""Replace SQLite `?` placeholders with psycopg `%s`, ignoring quoted text."""
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
in_single = False
|
||||
in_double = False
|
||||
while i < len(sql):
|
||||
ch = sql[i]
|
||||
if ch == "'" and not in_double:
|
||||
if in_single and i + 1 < len(sql) and sql[i + 1] == "'":
|
||||
out.append("''")
|
||||
i += 2
|
||||
continue
|
||||
in_single = not in_single
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"' and not in_single:
|
||||
in_double = not in_double
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == "?" and not in_single and not in_double:
|
||||
out.append("%s")
|
||||
i += 1
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def adapt_sql(sql: str) -> str:
|
||||
"""Make a SQLite-shaped statement runnable on PostgreSQL."""
|
||||
sql = sqlite_schema_to_postgres(sql)
|
||||
sql = _ON_CONFLICT_PAREN.sub("ON CONFLICT (", sql)
|
||||
used_ignore = bool(_INSERT_OR_IGNORE.search(sql))
|
||||
sql = _INSERT_OR_IGNORE.sub("INSERT INTO", sql)
|
||||
if used_ignore and re.search(r"ON\s+CONFLICT", sql, re.IGNORECASE) is None:
|
||||
stripped = sql.rstrip()
|
||||
ended = stripped.endswith(";")
|
||||
if ended:
|
||||
stripped = stripped[:-1].rstrip()
|
||||
stripped = f"{stripped} ON CONFLICT DO NOTHING"
|
||||
sql = stripped + (";" if ended else "")
|
||||
return replace_placeholders(sql)
|
||||
|
||||
|
||||
def rewrite_catalog_sql(sql: str) -> tuple[str, tuple | None] | None:
|
||||
"""Rewrite SQLite catalog queries. Returns (sql, extra_params) or None."""
|
||||
match = _PRAGMA_TABLE.match(sql)
|
||||
if match:
|
||||
return (
|
||||
"""
|
||||
SELECT column_name AS name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(match.group(1).lower(),),
|
||||
)
|
||||
if _SQLITE_MASTER_TABLES.match(sql):
|
||||
return (
|
||||
"""
|
||||
SELECT tablename AS name
|
||||
FROM pg_catalog.pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
""",
|
||||
None,
|
||||
)
|
||||
if _SQLITE_MASTER_SQL.match(sql):
|
||||
return ("SELECT NULL AS sql WHERE FALSE", None)
|
||||
return None
|
||||
|
||||
|
||||
def split_sql(script: str) -> list[str]:
|
||||
"""Split a SQL script into statements, respecting quotes."""
|
||||
statements: list[str] = []
|
||||
buf: list[str] = []
|
||||
in_single = False
|
||||
in_double = False
|
||||
i = 0
|
||||
while i < len(script):
|
||||
ch = script[i]
|
||||
if ch == "'" and not in_double:
|
||||
if in_single and i + 1 < len(script) and script[i + 1] == "'":
|
||||
buf.append("''")
|
||||
i += 2
|
||||
continue
|
||||
in_single = not in_single
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"' and not in_single:
|
||||
in_double = not in_double
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == ";" and not in_single and not in_double:
|
||||
stmt = "".join(buf).strip()
|
||||
if stmt:
|
||||
statements.append(stmt)
|
||||
buf = []
|
||||
i += 1
|
||||
continue
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
tail = "".join(buf).strip()
|
||||
if tail:
|
||||
statements.append(tail)
|
||||
return statements
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot SQLite → PostgreSQL copy for the laptop/home journal instance.
|
||||
|
||||
Probe on a copy, never on the only backup. Does not send data to providers.
|
||||
Media files are copied separately into the destination media root.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sql_compat import postgres_connect_kwargs
|
||||
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
DEFAULT_SQLITE = Path(__file__).resolve().parent / "data" / "kansho.sqlite"
|
||||
DEFAULT_MEDIA = Path(__file__).resolve().parent / "data" / "media"
|
||||
|
||||
# Parent tables first — matches schema.sql CREATE order.
|
||||
TABLES = [
|
||||
"schema_migrations",
|
||||
"tiers",
|
||||
"features",
|
||||
"tier_limits",
|
||||
"profiles",
|
||||
"sessions",
|
||||
"user_feature_restrictions",
|
||||
"user_feature_usage",
|
||||
"ai_prompts",
|
||||
"usage_sessions",
|
||||
"conversations",
|
||||
"messages",
|
||||
"threads",
|
||||
"conversation_threads",
|
||||
"spaces",
|
||||
"thread_spaces",
|
||||
"derived_records",
|
||||
"handoffs",
|
||||
"external_refs",
|
||||
"re_grounding_events",
|
||||
"identity_mappings",
|
||||
"identity_review_proposals",
|
||||
"journal_days",
|
||||
"journal_drafts",
|
||||
"journal_entries",
|
||||
"journal_entry_versions",
|
||||
"journal_draft_source_refs",
|
||||
"journal_entry_version_source_refs",
|
||||
"media_assets",
|
||||
"writing_profiles",
|
||||
"writing_profile_sources",
|
||||
"writing_profile_facets",
|
||||
"writing_profile_traits",
|
||||
"writing_profile_trait_refs",
|
||||
"writing_profile_suggestions",
|
||||
"interaction_profiles",
|
||||
"interaction_preferences",
|
||||
"interaction_suggestions",
|
||||
"writing_profile_evidence",
|
||||
"writing_profile_reviews",
|
||||
"writing_profile_versions",
|
||||
"provider_settings",
|
||||
"journal_generation_selection",
|
||||
"generation_guidelines",
|
||||
"app_settings",
|
||||
"debug_runs",
|
||||
]
|
||||
|
||||
|
||||
def _sqlite_path() -> Path:
|
||||
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "kansho.sqlite"
|
||||
return DEFAULT_SQLITE
|
||||
|
||||
|
||||
def _media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "media"
|
||||
return DEFAULT_MEDIA
|
||||
|
||||
|
||||
def _connect_pg():
|
||||
import psycopg
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
return psycopg.connect(kwargs["conninfo"])
|
||||
return psycopg.connect(**kwargs)
|
||||
|
||||
|
||||
def _pg_columns(conn, table: str) -> list[str]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(table,),
|
||||
).fetchall()
|
||||
return [row[0] for row in rows]
|
||||
|
||||
|
||||
def _sqlite_rows(sqlite_path: Path, table: str) -> list[dict]:
|
||||
conn = sqlite3.connect(str(sqlite_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = conn.execute(f"SELECT * FROM {table}").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
except sqlite3.OperationalError:
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _copy_media(src: Path, dest: Path) -> int:
|
||||
if not src.is_dir():
|
||||
print(f"No media directory at {src}")
|
||||
return 0
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
count = 0
|
||||
for path in src.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(src)
|
||||
target = dest / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def migrate(sqlite_path: Path, media_src: Path, media_dest: Path, replace: bool) -> dict[str, tuple[int, int]]:
|
||||
if not sqlite_path.is_file():
|
||||
raise SystemExit(f"SQLite not found: {sqlite_path}")
|
||||
|
||||
pg = _connect_pg()
|
||||
stats: dict[str, tuple[int, int]] = {}
|
||||
try:
|
||||
existing = pg.execute("SELECT COUNT(*) FROM profiles").fetchone()[0]
|
||||
if existing and not replace:
|
||||
raise SystemExit(
|
||||
f"PostgreSQL already has {existing} profiles. Pass --replace after a probe copy, not on the only backup."
|
||||
)
|
||||
if existing and replace:
|
||||
print(f"Replacing {existing} existing Postgres profiles")
|
||||
for table in reversed(TABLES):
|
||||
pg.execute(f"TRUNCATE TABLE {table} CASCADE")
|
||||
|
||||
for table in TABLES:
|
||||
rows = _sqlite_rows(sqlite_path, table)
|
||||
columns = _pg_columns(pg, table)
|
||||
if not columns:
|
||||
print(f" skip {table} (missing in Postgres schema)")
|
||||
stats[table] = (len(rows), 0)
|
||||
continue
|
||||
if not rows:
|
||||
print(f" {table}: empty")
|
||||
stats[table] = (0, 0)
|
||||
continue
|
||||
usable = [col for col in columns if col in rows[0]]
|
||||
placeholders = ", ".join(["%s"] * len(usable))
|
||||
col_sql = ", ".join(usable)
|
||||
sql = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
|
||||
values = [tuple(row.get(col) for col in usable) for row in rows]
|
||||
with pg.cursor() as cur:
|
||||
cur.executemany(sql, values)
|
||||
count = pg.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
print(f" {table}: {len(rows)} → {count}")
|
||||
stats[table] = (len(rows), int(count))
|
||||
pg.commit()
|
||||
except Exception:
|
||||
pg.rollback()
|
||||
raise
|
||||
finally:
|
||||
pg.close()
|
||||
|
||||
copied = _copy_media(media_src, media_dest)
|
||||
print(f"Media files copied: {copied} → {media_dest}")
|
||||
return stats
|
||||
|
||||
|
||||
def verify(stats: dict[str, tuple[int, int]]) -> None:
|
||||
failed = False
|
||||
print("\nVerification")
|
||||
for table, (src, dest) in stats.items():
|
||||
mark = "ok" if src == dest else "MISMATCH"
|
||||
if src != dest:
|
||||
failed = True
|
||||
print(f" {mark:8} {table:36} sqlite={src:5} postgres={dest:5}")
|
||||
if failed:
|
||||
raise SystemExit("Row counts do not match")
|
||||
print("Row counts match")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Copy Kanshō SQLite data into PostgreSQL")
|
||||
parser.add_argument("--sqlite", type=Path, default=_sqlite_path())
|
||||
parser.add_argument("--media-src", type=Path, default=_media_root())
|
||||
parser.add_argument("--media-dest", type=Path, default=Path(os.environ.get("KANSHO_MEDIA_DEST") or "/app/data/media"))
|
||||
parser.add_argument("--confirm", action="store_true", help="Required. Refuse a silent copy.")
|
||||
parser.add_argument("--replace", action="store_true", help="Truncate Postgres tables first")
|
||||
args = parser.parse_args()
|
||||
if not args.confirm:
|
||||
raise SystemExit("Refusing to copy personal data without --confirm")
|
||||
print(f"SQLite source: {args.sqlite}")
|
||||
print(f"Media source: {args.media_src}")
|
||||
print(f"Media dest: {args.media_dest}")
|
||||
stats = migrate(args.sqlite, args.media_src, args.media_dest, args.replace)
|
||||
verify(stats)
|
||||
print("Import complete. Check entry/message counts and one media GET before Prod.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "=== Kanshō backend startup ==="
|
||||
python /app/db_init.py
|
||||
echo "=== starting uvicorn ==="
|
||||
exec uvicorn main:app --host 0.0.0.0 --port 8000
|
||||
@@ -0,0 +1,75 @@
|
||||
"""SQL dialect adapter for the SQLite/Postgres dual backend."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from sql_compat import adapt_sql, split_sql, sqlite_schema_to_postgres, use_postgres
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.pop("KANSHO_DB_BACKEND", None)
|
||||
os.environ.pop("KANSHO_DB_PATH", None)
|
||||
os.environ.pop("DB_HOST", None)
|
||||
os.environ.pop("DATABASE_URL", None)
|
||||
expect(use_postgres() is False, "default engine is sqlite")
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = "/tmp/isolated.sqlite"
|
||||
os.environ["DB_HOST"] = "postgres"
|
||||
expect(use_postgres() is False, "KANSHO_DB_PATH keeps tests on sqlite")
|
||||
os.environ["KANSHO_DB_BACKEND"] = "postgres"
|
||||
expect(use_postgres() is True, "explicit postgres backend wins")
|
||||
os.environ["KANSHO_DB_BACKEND"] = "sqlite"
|
||||
expect(use_postgres() is False, "explicit sqlite backend wins")
|
||||
os.environ.pop("KANSHO_DB_BACKEND", None)
|
||||
os.environ.pop("KANSHO_DB_PATH", None)
|
||||
expect(use_postgres() is True, "DB_HOST alone selects postgres")
|
||||
os.environ.pop("DB_HOST", None)
|
||||
|
||||
translated = sqlite_schema_to_postgres("created TEXT NOT NULL DEFAULT (datetime('now'))")
|
||||
expect("CURRENT_TIMESTAMP::text" in translated, "schema datetime default")
|
||||
expect("datetime('now')" not in translated, "no sqlite datetime left in schema")
|
||||
|
||||
ignore = adapt_sql("INSERT OR IGNORE INTO tiers (id) VALUES (?)")
|
||||
expect("INSERT INTO tiers" in ignore, "insert or ignore becomes insert")
|
||||
expect("ON CONFLICT DO NOTHING" in ignore, "conflict do nothing appended")
|
||||
expect("%s" in ignore and "?" not in ignore, "placeholders converted")
|
||||
|
||||
upsert = adapt_sql(
|
||||
"INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
||||
)
|
||||
expect("ON CONFLICT (key)" in upsert, "on conflict parenthesis spacing")
|
||||
expect(upsert.count("ON CONFLICT") == 1, "existing on conflict not duplicated")
|
||||
expect("%s" in upsert, "upsert placeholders")
|
||||
|
||||
quoted = adapt_sql("SELECT '?' AS q, name FROM profiles WHERE id = ?")
|
||||
expect("'?'" in quoted, "question mark inside quotes kept")
|
||||
expect(quoted.endswith("%s") or quoted.rstrip().endswith("%s"), "trailing placeholder converted")
|
||||
|
||||
parts = split_sql("CREATE TABLE a (id TEXT); CREATE TABLE b (id TEXT);")
|
||||
expect(parts == ["CREATE TABLE a (id TEXT)", "CREATE TABLE b (id TEXT)"], "split two statements")
|
||||
schema = (ROOT / "schema.sql").read_text(encoding="utf-8")
|
||||
pg_schema = sqlite_schema_to_postgres(schema)
|
||||
expect("datetime('now')" not in pg_schema, "translated schema has no sqlite datetime")
|
||||
expect("CURRENT_TIMESTAMP::text" in pg_schema, "translated schema uses timestamp text default")
|
||||
expect(len(split_sql(pg_schema)) > 40, "schema splits into many statements")
|
||||
from sqlite_to_postgres import TABLES
|
||||
|
||||
tables = re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", schema)
|
||||
expect(tables == list(TABLES), "import table order matches schema.sql")
|
||||
print("sql compat: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user