fix: migration 020 SQL syntax - correlated subquery issue

Fixed Step 3 pipeline_configs migration:
- Simplified JSONB aggregation logic
- Properly scope pc alias in subqueries
- Use UNNEST with FROM clause for array expansion

Previous version had correlation issues with nested subqueries.

backend/migrations/020_unified_prompt_system.sql

@@ -44,70 +44,70 @@ INSERT INTO ai_prompts (
     category
 )
 SELECT
-    'pipeline_config_' || LOWER(REPLACE(name, ' ', '_')) || '_' || SUBSTRING(id::TEXT FROM 1 FOR 8),
+    'pipeline_config_' || LOWER(REPLACE(pc.name, ' ', '_')) || '_' || SUBSTRING(pc.id::TEXT FROM 1 FOR 8) as slug,
-    name,
+    pc.name,
-    description,
+    pc.description,
-    'pipeline',
+    'pipeline' as type,
-    -- Build stages JSONB structure
+    -- Build stages JSONB: combine stage1_prompts, stage2_prompt, stage3_prompt
     (
+        -- Stage 1: Convert array to prompts
+        SELECT jsonb_agg(stage_obj ORDER BY stage_num)
+        FROM (
+            SELECT 1 as stage_num,
+                   jsonb_build_object(
+                       'stage', 1,
+                       'prompts', (
                            SELECT jsonb_agg(
-            jsonb_build_object(
-                'stage', stage_num,
-                'prompts', stage_prompts
-            )
-            ORDER BY stage_num
-        )
-        FROM (
-            -- Stage 1: All parallel prompts
-            SELECT
-                1 as stage_num,
-                jsonb_agg(
                                jsonb_build_object(
                                    'source', 'reference',
-                        'slug', stage1_slug,
+                                   'slug', s1.slug_val,
-                        'output_key', REPLACE(stage1_slug, 'pipeline_', 'stage1_'),
+                                   'output_key', REPLACE(s1.slug_val, 'pipeline_', 'stage1_'),
                                    'output_format', 'json'
                                )
-                ) as stage_prompts
+                           )
-            FROM UNNEST(pc.stage1_prompts) as stage1_slug
+                           FROM UNNEST(pc.stage1_prompts) AS s1(slug_val)
+                       )
+                   ) as stage_obj
+            WHERE array_length(pc.stage1_prompts, 1) > 0
 
             UNION ALL
 
-            -- Stage 2: Synthesis prompt
+            SELECT 2 as stage_num,
-            SELECT
+                   jsonb_build_object(
-                2 as stage_num,
+                       'stage', 2,
-                jsonb_build_array(
+                       'prompts', jsonb_build_array(
                            jsonb_build_object(
                                'source', 'reference',
                                'slug', pc.stage2_prompt,
                                'output_key', 'synthesis',
                                'output_format', 'text'
                            )
-                ) as stage_prompts
+                       )
+                   ) as stage_obj
             WHERE pc.stage2_prompt IS NOT NULL
 
             UNION ALL
 
-            -- Stage 3: Optional goals/analysis prompt
+            SELECT 3 as stage_num,
-            SELECT
+                   jsonb_build_object(
-                3 as stage_num,
+                       'stage', 3,
-                jsonb_build_array(
+                       'prompts', jsonb_build_array(
                            jsonb_build_object(
                                'source', 'reference',
                                'slug', pc.stage3_prompt,
                                'output_key', 'goals',
                                'output_format', 'text'
                            )
-                ) as stage_prompts
+                       )
+                   ) as stage_obj
             WHERE pc.stage3_prompt IS NOT NULL
-        ) stages_union
+        ) all_stages
     ) as stages,
-    'text',
+    'text' as output_format,
     pc.active,
-    pc.is_default,
+    pc.is_default as is_system_default,
-    'pipeline'
+    'pipeline' as category
-FROM pipeline_configs pc
+FROM pipeline_configs pc;
-WHERE pc.id IS NOT NULL;
 
 -- Step 4: Add indices for performance
 CREATE INDEX IF NOT EXISTS idx_ai_prompts_type ON ai_prompts(type);

test-unified-migration.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Test Migration 020: Unified Prompt System (Issue #28)
+
+echo "═══════════════════════════════════════════════════════════"
+echo "Migration 020: Unified Prompt System - Verification Tests"
+echo "═══════════════════════════════════════════════════════════"
+echo ""
+
+POSTGRES_CONTAINER="dev-mitai-postgres"
+DB_USER="mitai_dev"
+DB_NAME="mitai_dev"
+
+echo "Test 1: Migration 020 applied"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT version, applied_at FROM schema_migrations WHERE version = '020';"
+echo ""
+
+echo "Test 2: New columns exist in ai_prompts"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT column_name, data_type
+   FROM information_schema.columns
+   WHERE table_name = 'ai_prompts'
+   AND column_name IN ('type', 'stages', 'output_format', 'output_schema')
+   ORDER BY column_name;"
+echo ""
+
+echo "Test 3: Existing prompts migrated to pipeline type"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT slug, type,
+   CASE WHEN stages IS NOT NULL THEN 'Has stages' ELSE 'No stages' END as stages_status,
+   output_format
+   FROM ai_prompts
+   WHERE slug LIKE 'pipeline_%'
+   LIMIT 5;"
+echo ""
+
+echo "Test 4: Pipeline configs migrated to ai_prompts"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT slug, name, type,
+   jsonb_array_length(stages) as num_stages
+   FROM ai_prompts
+   WHERE slug LIKE 'pipeline_config_%';"
+echo ""
+
+echo "Test 5: Stages JSONB structure (sample)"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT slug, jsonb_pretty(stages) as stages_structure
+   FROM ai_prompts
+   WHERE slug LIKE 'pipeline_config_%'
+   LIMIT 1;"
+echo ""
+
+echo "Test 6: Backup table created"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT COUNT(*) as backup_count FROM pipeline_configs_backup_pre_020;"
+echo ""
+
+echo "Test 7: Indices created"
+echo "─────────────────────────────────────────────────────────"
+docker exec $POSTGRES_CONTAINER psql -U $DB_USER -d $DB_NAME -c \
+  "SELECT indexname FROM pg_indexes
+   WHERE tablename = 'ai_prompts'
+   AND indexname LIKE 'idx_ai_prompts_%';"
+echo ""
+
+echo "═══════════════════════════════════════════════════════════"
+echo "Migration 020 Tests Complete!"
+echo "═══════════════════════════════════════════════════════════"