Add Compose, Gitea deploy, and Postgres dual-backend on develop.
Deploy Development / deploy (push) Successful in 44s
Test Suite / backend-sqlite (push) Failing after 2s
Test Suite / frontend-build (push) Successful in 23s

Use host ports 3006/8005 and 3096/8096 so Kansho does not collide with Bookstack on 3005 or the sister products.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-09-07 12:15:08 +02:00
1 parent 32acdc220b
commit 07479f6a9f
37 files changed
+1613 -102

No files matched your search

+8
View File
@@ -22,3 +22,11 @@ KANSHO_DETECT_PROVIDER_KEY=
# Nur Tests
# KANSHO_FAKE_PROVIDER=1
# KANSHO_FAKE_DETECT=1
# Docker/Server only. Local Windows stays SQLite (default).
# KANSHO_DB_BACKEND=postgres
# DB_HOST=postgres
# DB_NAME=kansho
# DB_USER=kansho
# DB_PASSWORD=
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.12-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
ENV PIP_DEFAULT_TIMEOUT=120
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
RUN chmod +x /app/startup.sh \
&& mkdir -p /app/data/media
EXPOSE 8000
CMD ["/app/startup.sh"]
+132 -34
View File
@@ -1,4 +1,8 @@
"""SQLite persistence for the local frame. PostgreSQL remains the later target."""
"""Persistence for the local frame.
SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine.
Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL.
"""
from __future__ import annotations
import json
@@ -6,14 +10,49 @@ import os
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from typing import Any
DATA_DIR = Path(__file__).resolve().parent / "data"
from sql_compat import (
adapt_sql,
postgres_connect_kwargs,
rewrite_catalog_sql,
split_sql,
sqlite_schema_to_postgres,
use_postgres,
)
_env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data"
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
_env_db = os.environ.get("KANSHO_DB_PATH")
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
SQLITE_HISTORY_MIGRATIONS = (
"001_frame",
"002_platform",
"003_dialogue_memory",
"004_mvp_journal",
"005_provider_settings",
"006_writing_profile_dialogue_style",
"007_journal_day_scratch",
"008_journal_source_refs",
"009_conversation_signals",
"010_profile_governance",
"011_profile_review",
"012_profile_shell",
"013_journal_generate_narration",
"014_identity_registry",
"015_journal_generation_settings",
"016_generation_instruction_fragments",
"017_generation_guidelines",
"018_debug_runs",
"019_debug_run_placement",
"020_voice_style_context",
"021_voice_legacy_immutable",
)
_PROFILE_COLUMNS = {
"status": "TEXT NOT NULL DEFAULT 'active'",
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
@@ -88,7 +127,61 @@ _IDENTITY_MAPPING_COLUMNS = {
}
def _connect() -> sqlite3.Connection:
class PostgresCompat:
"""sqlite3-like execute/commit surface over psycopg, with SQL translation."""
def __init__(self, raw: Any):
self._raw = raw
def execute(self, sql: str, params: Any = None):
special = rewrite_catalog_sql(sql)
if special:
adapted, extra = special
args = extra if extra is not None else params
else:
adapted = adapt_sql(sql)
args = params
if args is None:
return self._raw.execute(adapted)
return self._raw.execute(adapted, args)
def executemany(self, sql: str, seq_of_params: Any):
adapted = adapt_sql(sql)
cursor = None
for params in seq_of_params:
cursor = self._raw.execute(adapted, params)
return cursor
def executescript(self, script: str):
for statement in split_sql(sqlite_schema_to_postgres(script)):
self._raw.execute(statement)
return self
def commit(self):
self._raw.commit()
def rollback(self):
self._raw.rollback()
def close(self):
self._raw.close()
def _connect_postgres() -> PostgresCompat:
import psycopg
from psycopg.rows import dict_row
kwargs = postgres_connect_kwargs()
if "conninfo" in kwargs:
raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row)
else:
raw = psycopg.connect(row_factory=dict_row, **kwargs)
return PostgresCompat(raw)
def _connect():
if use_postgres():
return _connect_postgres()
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH, timeout=30)
conn.row_factory = sqlite3.Row
@@ -109,32 +202,32 @@ def get_db():
conn.close()
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
def row_to_dict(row: Any) -> dict | None:
if row is None:
return None
return dict(row)
def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
def _column_names(conn: Any, table: str) -> set[str]:
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
return {row["name"] for row in rows}
def _ensure_columns(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None:
existing = _column_names(conn, table)
for name, ddl in columns.items():
if name not in existing:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
def _mark(conn: sqlite3.Connection, migration_id: str) -> None:
def _mark(conn: Any, migration_id: str) -> None:
conn.execute(
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
(migration_id,),
)
def _seed_platform(conn: sqlite3.Connection) -> None:
def _seed_platform(conn: Any) -> None:
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
for tier in seed.get("tiers", []):
conn.execute(
@@ -171,7 +264,7 @@ def _seed_platform(conn: sqlite3.Connection) -> None:
)
def _seed_prompts(conn: sqlite3.Connection) -> None:
def _seed_prompts(conn: Any) -> None:
"""Prompts come from JSON/DB, never from Python string literals.
Untouched system prompts (template == default_template) receive the seed.
@@ -246,7 +339,7 @@ def _parse_legacy_ids(raw: str | None) -> list[str]:
def _insert_source_refs(
conn: sqlite3.Connection,
conn: Any,
table: str,
owner_col: str,
owner_id: str,
@@ -275,7 +368,7 @@ def _insert_source_refs(
)
def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
def migrate_journal_source_refs(conn: Any) -> None:
"""Copy JSON id lists into relational source-ref tables without touching Source."""
existing = {
row["name"]
@@ -323,7 +416,7 @@ def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
)
def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
def _migrate_writing_profile_dialogue_style(conn: Any) -> None:
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
row = row_to_dict(
conn.execute(
@@ -354,7 +447,7 @@ def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
)
def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
def _migrate_writing_profile_shell(conn: Any) -> None:
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
@@ -449,7 +542,31 @@ def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
)
def _seed_runtime(conn: Any) -> None:
_seed_platform(conn)
_seed_prompts(conn)
from provider_settings import seed_provider_settings
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
seed_provider_settings(conn)
seed_generation_instructions(conn)
backfill_missing_settings(conn)
seed_generation_instructions(conn)
seed_generation_instructions(conn)
def init_db() -> None:
if use_postgres():
from db_init import ensure_postgres_ready
ensure_postgres_ready()
with get_db() as conn:
_seed_runtime(conn)
from writing_profile_store import bootstrap_from_existing
bootstrap_from_existing()
return
schema = SCHEMA_PATH.read_text(encoding="utf-8")
with get_db() as conn:
conn.executescript(schema)
@@ -474,35 +591,16 @@ def init_db() -> None:
from identity_store import migrate_legacy_identity_rows
migrate_legacy_identity_rows(conn)
_mark(conn, "001_frame")
_mark(conn, "002_platform")
_mark(conn, "003_dialogue_memory")
_mark(conn, "004_mvp_journal")
_mark(conn, "005_provider_settings")
_mark(conn, "006_writing_profile_dialogue_style")
_mark(conn, "007_journal_day_scratch")
_mark(conn, "008_journal_source_refs")
_mark(conn, "009_conversation_signals")
_mark(conn, "010_profile_governance")
_mark(conn, "011_profile_review")
_mark(conn, "012_profile_shell")
_mark(conn, "013_journal_generate_narration")
_mark(conn, "014_identity_registry")
for migration_id in SQLITE_HISTORY_MIGRATIONS:
_mark(conn, migration_id)
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
seed_generation_instructions(conn)
backfill_missing_settings(conn)
_mark(conn, "015_journal_generation_settings")
_mark(conn, "016_generation_instruction_fragments")
_mark(conn, "017_generation_guidelines")
_mark(conn, "018_debug_runs")
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
_mark(conn, "019_debug_run_placement")
seed_generation_instructions(conn)
_mark(conn, "020_voice_style_context")
seed_generation_instructions(conn)
_mark(conn, "021_voice_legacy_immutable")
from writing_profile_store import bootstrap_from_existing
bootstrap_from_existing()
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Wait for PostgreSQL, load schema.sql, apply numbered SQL migrations. Fail-fast."""
from __future__ import annotations
import os
import re
import sys
import time
from pathlib import Path
from sql_compat import postgres_connect_kwargs, split_sql, sqlite_schema_to_postgres, use_postgres
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations"
SQLITE_HISTORY_MIGRATIONS = (
"001_frame",
"002_platform",
"003_dialogue_memory",
"004_mvp_journal",
"005_provider_settings",
"006_writing_profile_dialogue_style",
"007_journal_day_scratch",
"008_journal_source_refs",
"009_conversation_signals",
"010_profile_governance",
"011_profile_review",
"012_profile_shell",
"013_journal_generate_narration",
"014_identity_registry",
"015_journal_generation_settings",
"016_generation_instruction_fragments",
"017_generation_guidelines",
"018_debug_runs",
"019_debug_run_placement",
"020_voice_style_context",
"021_voice_legacy_immutable",
)
_LEADING_DIGITS = re.compile(r"^(\d{3})_.*\.sql$")
def _connect_raw():
import psycopg
kwargs = postgres_connect_kwargs()
if "conninfo" in kwargs:
return psycopg.connect(kwargs["conninfo"], autocommit=False)
return psycopg.connect(autocommit=False, **kwargs)
def wait_for_postgres(max_retries: int = 30) -> None:
print("Checking PostgreSQL connection...")
last_error = None
for attempt in range(1, max_retries + 1):
try:
conn = _connect_raw()
conn.close()
print("PostgreSQL ready")
return
except Exception as exc: # noqa: BLE001 — fail-fast after retries
last_error = exc
print(f" waiting ({attempt}/{max_retries})")
time.sleep(2)
print(f"PostgreSQL not ready: {last_error}")
sys.exit(1)
def _table_exists(conn, name: str) -> bool:
row = conn.execute(
"""
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
""",
(name,),
).fetchone()
return row is not None
def _load_schema(conn) -> None:
sql = sqlite_schema_to_postgres(SCHEMA_PATH.read_text(encoding="utf-8"))
for statement in split_sql(sql):
conn.execute(statement)
print("Schema loaded from schema.sql (Postgres dialect)")
def _applied_ids(conn) -> set[str]:
rows = conn.execute("SELECT id FROM schema_migrations").fetchall()
return {row[0] for row in rows}
def _record(conn, migration_id: str) -> None:
conn.execute(
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
(migration_id,),
)
def _migration_files() -> list[tuple[str, Path]]:
if not MIGRATIONS_DIR.is_dir():
return []
rows: list[tuple[str, Path]] = []
for path in sorted(MIGRATIONS_DIR.iterdir()):
match = _LEADING_DIGITS.match(path.name)
if not match or path.suffix != ".sql":
continue
rows.append((path.stem, path))
return rows
def apply_schema_and_migrations() -> None:
conn = _connect_raw()
try:
if not _table_exists(conn, "schema_migrations"):
_load_schema(conn)
conn.commit()
applied = _applied_ids(conn)
if "001_frame" not in applied:
for migration_id in SQLITE_HISTORY_MIGRATIONS:
_record(conn, migration_id)
conn.commit()
print("Recorded SQLite history 001-021 as applied (greenfield Postgres schema)")
applied = _applied_ids(conn)
for stem, path in _migration_files():
if stem in applied:
continue
sql = path.read_text(encoding="utf-8")
for statement in split_sql(sql):
conn.execute(statement)
_record(conn, stem)
conn.commit()
print(f"Applied migration {stem}")
except Exception as exc:
conn.rollback()
print(f"Database initialization failed: {exc}")
raise
finally:
conn.close()
def ensure_postgres_ready() -> None:
if not use_postgres():
return
wait_for_postgres()
try:
apply_schema_and_migrations()
except Exception:
sys.exit(1)
def main() -> None:
os.environ.setdefault("KANSHO_DB_BACKEND", "postgres")
ensure_postgres_ready()
print("db_init complete")
if __name__ == "__main__":
main()
+24 -1
View File
@@ -2,6 +2,8 @@ from env_loader import load_env_file
load_env_file()
import os
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
@@ -10,11 +12,32 @@ import data_layer_dialogue # noqa: F401
from routers import admin, auth, dialogue, generation_instructions, journal, placeholders, prompts, subscription, users
from version import APP_VERSION
def allowed_origins() -> list[str]:
origins = [
"http://localhost:5188",
"http://127.0.0.1:5188",
"https://kansho.jinkendo.de",
"https://dev.kansho.jinkendo.de",
]
extra = (os.environ.get("ALLOWED_ORIGINS") or os.environ.get("KANSHO_ALLOWED_ORIGINS") or "").strip()
if extra:
origins.extend(item.strip() for item in extra.split(",") if item.strip())
app_url = (os.environ.get("APP_URL") or "").strip().rstrip("/")
if app_url:
origins.append(app_url)
seen: list[str] = []
for item in origins:
if item not in seen:
seen.append(item)
return seen
app = FastAPI(title="Kanshō", version=APP_VERSION)
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:5188", "http://127.0.0.1:5188"],
allow_origins=allowed_origins(),
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
@@ -0,0 +1,7 @@
-- Postgres runtime baseline for Kanshō.
-- Greenfield schema is backend/schema.sql translated at startup
-- (datetime('now') → CURRENT_TIMESTAMP::text). Historical SQLite
-- inline migrations 001-021 are recorded as applied, not replayed.
-- Future incremental DDL belongs in 023_*.sql and later.
SELECT 1;
+1
View File
@@ -3,3 +3,4 @@ uvicorn[standard]==0.34.0
bcrypt==4.2.1
httpx==0.28.1
python-multipart==0.0.20
psycopg[binary]==3.2.6
+179
View File
@@ -0,0 +1,179 @@
"""Translate SQLite-oriented SQL so the same stores can run against PostgreSQL.
Local Windows development and tests keep SQLite. Docker/Server set
KANSHO_DB_BACKEND=postgres. Identity, keys and journal bodies are not logged here.
"""
from __future__ import annotations
import os
import re
_INSERT_OR_IGNORE = re.compile(r"INSERT\s+OR\s+IGNORE\s+INTO", re.IGNORECASE)
_ON_CONFLICT_PAREN = re.compile(r"ON\s+CONFLICT\s*\(", re.IGNORECASE)
_PRAGMA_TABLE = re.compile(
r"^\s*PRAGMA\s+table_info\(\s*['\"]?(\w+)['\"]?\s*\)\s*;?\s*$",
re.IGNORECASE,
)
_SQLITE_MASTER_TABLES = re.compile(
r"^\s*SELECT\s+name\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s*;?\s*$",
re.IGNORECASE,
)
_SQLITE_MASTER_SQL = re.compile(
r"^\s*SELECT\s+sql\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s+AND\s+name\s*=\s*\?\s*;?\s*$",
re.IGNORECASE,
)
def use_postgres() -> bool:
"""Decide the engine at connect time, not at import time.
Explicit KANSHO_DB_BACKEND wins. KANSHO_DB_PATH without an explicit postgres
backend keeps tests on isolated SQLite even inside a Compose container.
"""
raw = (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
if raw in {"sqlite", "sqlite3"}:
return False
if raw in {"postgres", "postgresql", "pg"}:
return True
if (os.environ.get("KANSHO_DB_PATH") or "").strip():
return False
url = (os.environ.get("DATABASE_URL") or "").strip().lower()
if url.startswith("postgres"):
return True
if (os.environ.get("DB_HOST") or "").strip():
return True
return False
def postgres_connect_kwargs() -> dict:
url = (os.environ.get("DATABASE_URL") or "").strip()
if url:
return {"conninfo": url}
host = (os.environ.get("DB_HOST") or "postgres").strip() or "postgres"
port = int((os.environ.get("DB_PORT") or "5432").strip() or "5432")
dbname = (os.environ.get("DB_NAME") or "kansho").strip() or "kansho"
user = (os.environ.get("DB_USER") or "kansho").strip() or "kansho"
password = os.environ.get("DB_PASSWORD") or ""
return {
"host": host,
"port": port,
"dbname": dbname,
"user": user,
"password": password,
}
def sqlite_schema_to_postgres(sql: str) -> str:
return sql.replace("datetime('now')", "CURRENT_TIMESTAMP::text")
def replace_placeholders(sql: str) -> str:
"""Replace SQLite `?` placeholders with psycopg `%s`, ignoring quoted text."""
out: list[str] = []
i = 0
in_single = False
in_double = False
while i < len(sql):
ch = sql[i]
if ch == "'" and not in_double:
if in_single and i + 1 < len(sql) and sql[i + 1] == "'":
out.append("''")
i += 2
continue
in_single = not in_single
out.append(ch)
i += 1
continue
if ch == '"' and not in_single:
in_double = not in_double
out.append(ch)
i += 1
continue
if ch == "?" and not in_single and not in_double:
out.append("%s")
i += 1
continue
out.append(ch)
i += 1
return "".join(out)
def adapt_sql(sql: str) -> str:
"""Make a SQLite-shaped statement runnable on PostgreSQL."""
sql = sqlite_schema_to_postgres(sql)
sql = _ON_CONFLICT_PAREN.sub("ON CONFLICT (", sql)
used_ignore = bool(_INSERT_OR_IGNORE.search(sql))
sql = _INSERT_OR_IGNORE.sub("INSERT INTO", sql)
if used_ignore and re.search(r"ON\s+CONFLICT", sql, re.IGNORECASE) is None:
stripped = sql.rstrip()
ended = stripped.endswith(";")
if ended:
stripped = stripped[:-1].rstrip()
stripped = f"{stripped} ON CONFLICT DO NOTHING"
sql = stripped + (";" if ended else "")
return replace_placeholders(sql)
def rewrite_catalog_sql(sql: str) -> tuple[str, tuple | None] | None:
"""Rewrite SQLite catalog queries. Returns (sql, extra_params) or None."""
match = _PRAGMA_TABLE.match(sql)
if match:
return (
"""
SELECT column_name AS name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = %s
ORDER BY ordinal_position
""",
(match.group(1).lower(),),
)
if _SQLITE_MASTER_TABLES.match(sql):
return (
"""
SELECT tablename AS name
FROM pg_catalog.pg_tables
WHERE schemaname = 'public'
""",
None,
)
if _SQLITE_MASTER_SQL.match(sql):
return ("SELECT NULL AS sql WHERE FALSE", None)
return None
def split_sql(script: str) -> list[str]:
"""Split a SQL script into statements, respecting quotes."""
statements: list[str] = []
buf: list[str] = []
in_single = False
in_double = False
i = 0
while i < len(script):
ch = script[i]
if ch == "'" and not in_double:
if in_single and i + 1 < len(script) and script[i + 1] == "'":
buf.append("''")
i += 2
continue
in_single = not in_single
buf.append(ch)
i += 1
continue
if ch == '"' and not in_single:
in_double = not in_double
buf.append(ch)
i += 1
continue
if ch == ";" and not in_single and not in_double:
stmt = "".join(buf).strip()
if stmt:
statements.append(stmt)
buf = []
i += 1
continue
buf.append(ch)
i += 1
tail = "".join(buf).strip()
if tail:
statements.append(tail)
return statements
+226
View File
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
"""One-shot SQLite → PostgreSQL copy for the laptop/home journal instance.
Probe on a copy, never on the only backup. Does not send data to providers.
Media files are copied separately into the destination media root.
"""
from __future__ import annotations
import argparse
import os
import shutil
import sqlite3
import sys
from pathlib import Path
from sql_compat import postgres_connect_kwargs
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
DEFAULT_SQLITE = Path(__file__).resolve().parent / "data" / "kansho.sqlite"
DEFAULT_MEDIA = Path(__file__).resolve().parent / "data" / "media"
# Parent tables first — matches schema.sql CREATE order.
TABLES = [
"schema_migrations",
"tiers",
"features",
"tier_limits",
"profiles",
"sessions",
"user_feature_restrictions",
"user_feature_usage",
"ai_prompts",
"usage_sessions",
"conversations",
"messages",
"threads",
"conversation_threads",
"spaces",
"thread_spaces",
"derived_records",
"handoffs",
"external_refs",
"re_grounding_events",
"identity_mappings",
"identity_review_proposals",
"journal_days",
"journal_drafts",
"journal_entries",
"journal_entry_versions",
"journal_draft_source_refs",
"journal_entry_version_source_refs",
"media_assets",
"writing_profiles",
"writing_profile_sources",
"writing_profile_facets",
"writing_profile_traits",
"writing_profile_trait_refs",
"writing_profile_suggestions",
"interaction_profiles",
"interaction_preferences",
"interaction_suggestions",
"writing_profile_evidence",
"writing_profile_reviews",
"writing_profile_versions",
"provider_settings",
"journal_generation_selection",
"generation_guidelines",
"app_settings",
"debug_runs",
]
def _sqlite_path() -> Path:
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
if env:
return Path(env)
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
if data_dir:
return Path(data_dir) / "kansho.sqlite"
return DEFAULT_SQLITE
def _media_root() -> Path:
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
if env:
return Path(env)
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
if data_dir:
return Path(data_dir) / "media"
return DEFAULT_MEDIA
def _connect_pg():
import psycopg
kwargs = postgres_connect_kwargs()
if "conninfo" in kwargs:
return psycopg.connect(kwargs["conninfo"])
return psycopg.connect(**kwargs)
def _pg_columns(conn, table: str) -> list[str]:
rows = conn.execute(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = %s
ORDER BY ordinal_position
""",
(table,),
).fetchall()
return [row[0] for row in rows]
def _sqlite_rows(sqlite_path: Path, table: str) -> list[dict]:
conn = sqlite3.connect(str(sqlite_path))
conn.row_factory = sqlite3.Row
try:
rows = conn.execute(f"SELECT * FROM {table}").fetchall()
return [dict(row) for row in rows]
except sqlite3.OperationalError:
return []
finally:
conn.close()
def _copy_media(src: Path, dest: Path) -> int:
if not src.is_dir():
print(f"No media directory at {src}")
return 0
dest.mkdir(parents=True, exist_ok=True)
count = 0
for path in src.rglob("*"):
if not path.is_file():
continue
relative = path.relative_to(src)
target = dest / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
count += 1
return count
def migrate(sqlite_path: Path, media_src: Path, media_dest: Path, replace: bool) -> dict[str, tuple[int, int]]:
if not sqlite_path.is_file():
raise SystemExit(f"SQLite not found: {sqlite_path}")
pg = _connect_pg()
stats: dict[str, tuple[int, int]] = {}
try:
existing = pg.execute("SELECT COUNT(*) FROM profiles").fetchone()[0]
if existing and not replace:
raise SystemExit(
f"PostgreSQL already has {existing} profiles. Pass --replace after a probe copy, not on the only backup."
)
if existing and replace:
print(f"Replacing {existing} existing Postgres profiles")
for table in reversed(TABLES):
pg.execute(f"TRUNCATE TABLE {table} CASCADE")
for table in TABLES:
rows = _sqlite_rows(sqlite_path, table)
columns = _pg_columns(pg, table)
if not columns:
print(f" skip {table} (missing in Postgres schema)")
stats[table] = (len(rows), 0)
continue
if not rows:
print(f" {table}: empty")
stats[table] = (0, 0)
continue
usable = [col for col in columns if col in rows[0]]
placeholders = ", ".join(["%s"] * len(usable))
col_sql = ", ".join(usable)
sql = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
values = [tuple(row.get(col) for col in usable) for row in rows]
with pg.cursor() as cur:
cur.executemany(sql, values)
count = pg.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
print(f" {table}: {len(rows)} → {count}")
stats[table] = (len(rows), int(count))
pg.commit()
except Exception:
pg.rollback()
raise
finally:
pg.close()
copied = _copy_media(media_src, media_dest)
print(f"Media files copied: {copied} → {media_dest}")
return stats
def verify(stats: dict[str, tuple[int, int]]) -> None:
failed = False
print("\nVerification")
for table, (src, dest) in stats.items():
mark = "ok" if src == dest else "MISMATCH"
if src != dest:
failed = True
print(f" {mark:8} {table:36} sqlite={src:5} postgres={dest:5}")
if failed:
raise SystemExit("Row counts do not match")
print("Row counts match")
def main() -> None:
parser = argparse.ArgumentParser(description="Copy Kanshō SQLite data into PostgreSQL")
parser.add_argument("--sqlite", type=Path, default=_sqlite_path())
parser.add_argument("--media-src", type=Path, default=_media_root())
parser.add_argument("--media-dest", type=Path, default=Path(os.environ.get("KANSHO_MEDIA_DEST") or "/app/data/media"))
parser.add_argument("--confirm", action="store_true", help="Required. Refuse a silent copy.")
parser.add_argument("--replace", action="store_true", help="Truncate Postgres tables first")
args = parser.parse_args()
if not args.confirm:
raise SystemExit("Refusing to copy personal data without --confirm")
print(f"SQLite source: {args.sqlite}")
print(f"Media source: {args.media_src}")
print(f"Media dest: {args.media_dest}")
stats = migrate(args.sqlite, args.media_src, args.media_dest, args.replace)
verify(stats)
print("Import complete. Check entry/message counts and one media GET before Prod.")
if __name__ == "__main__":
main()
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
echo "=== Kanshō backend startup ==="
python /app/db_init.py
echo "=== starting uvicorn ==="
exec uvicorn main:app --host 0.0.0.0 --port 8000
+75
View File
@@ -0,0 +1,75 @@
"""SQL dialect adapter for the SQLite/Postgres dual backend."""
from __future__ import annotations
import os
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sql_compat import adapt_sql, split_sql, sqlite_schema_to_postgres, use_postgres
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def main() -> None:
os.environ.pop("KANSHO_DB_BACKEND", None)
os.environ.pop("KANSHO_DB_PATH", None)
os.environ.pop("DB_HOST", None)
os.environ.pop("DATABASE_URL", None)
expect(use_postgres() is False, "default engine is sqlite")
os.environ["KANSHO_DB_PATH"] = "/tmp/isolated.sqlite"
os.environ["DB_HOST"] = "postgres"
expect(use_postgres() is False, "KANSHO_DB_PATH keeps tests on sqlite")
os.environ["KANSHO_DB_BACKEND"] = "postgres"
expect(use_postgres() is True, "explicit postgres backend wins")
os.environ["KANSHO_DB_BACKEND"] = "sqlite"
expect(use_postgres() is False, "explicit sqlite backend wins")
os.environ.pop("KANSHO_DB_BACKEND", None)
os.environ.pop("KANSHO_DB_PATH", None)
expect(use_postgres() is True, "DB_HOST alone selects postgres")
os.environ.pop("DB_HOST", None)
translated = sqlite_schema_to_postgres("created TEXT NOT NULL DEFAULT (datetime('now'))")
expect("CURRENT_TIMESTAMP::text" in translated, "schema datetime default")
expect("datetime('now')" not in translated, "no sqlite datetime left in schema")
ignore = adapt_sql("INSERT OR IGNORE INTO tiers (id) VALUES (?)")
expect("INSERT INTO tiers" in ignore, "insert or ignore becomes insert")
expect("ON CONFLICT DO NOTHING" in ignore, "conflict do nothing appended")
expect("%s" in ignore and "?" not in ignore, "placeholders converted")
upsert = adapt_sql(
"INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
)
expect("ON CONFLICT (key)" in upsert, "on conflict parenthesis spacing")
expect(upsert.count("ON CONFLICT") == 1, "existing on conflict not duplicated")
expect("%s" in upsert, "upsert placeholders")
quoted = adapt_sql("SELECT '?' AS q, name FROM profiles WHERE id = ?")
expect("'?'" in quoted, "question mark inside quotes kept")
expect(quoted.endswith("%s") or quoted.rstrip().endswith("%s"), "trailing placeholder converted")
parts = split_sql("CREATE TABLE a (id TEXT); CREATE TABLE b (id TEXT);")
expect(parts == ["CREATE TABLE a (id TEXT)", "CREATE TABLE b (id TEXT)"], "split two statements")
schema = (ROOT / "schema.sql").read_text(encoding="utf-8")
pg_schema = sqlite_schema_to_postgres(schema)
expect("datetime('now')" not in pg_schema, "translated schema has no sqlite datetime")
expect("CURRENT_TIMESTAMP::text" in pg_schema, "translated schema uses timestamp text default")
expect(len(split_sql(pg_schema)) > 40, "schema splits into many statements")
from sqlite_to_postgres import TABLES
tables = re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", schema)
expect(tables == list(TABLES), "import table order matches schema.sql")
print("sql compat: OK")
if __name__ == "__main__":
main()