feat: debug system for prompt execution (Issue #28)
- Backend: debug mode in prompt_executor with placeholder tracking - Backend: show resolved/unresolved placeholders, final prompts, AI responses - Frontend: test button in UnifiedPromptModal for saved prompts - Frontend: debug output viewer with JSON preview - Frontend: wider placeholder example fields in PlaceholderPicker Resolves pipeline execution debugging issues. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
4ba03c2a94
commit
7f2ba4fbad
4 files changed
+211
-33
No files matched your search
+108
-19
@@ -14,29 +14,50 @@ from db import get_db, get_cursor, r2d
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def resolve_placeholders(template: str, variables: Dict[str, Any]) -> str:
|
||||
def resolve_placeholders(template: str, variables: Dict[str, Any], debug_info: Optional[Dict] = None) -> str:
|
||||
"""
|
||||
Replace {{placeholder}} with values from variables dict.
|
||||
|
||||
Args:
|
||||
template: String with {{key}} placeholders
|
||||
variables: Dict of key -> value mappings
|
||||
debug_info: Optional dict to collect debug information
|
||||
|
||||
Returns:
|
||||
Template with placeholders replaced
|
||||
"""
|
||||
resolved = {}
|
||||
unresolved = []
|
||||
|
||||
def replacer(match):
|
||||
key = match.group(1).strip()
|
||||
if key in variables:
|
||||
value = variables[key]
|
||||
# Convert dict/list to JSON string
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
return str(value)
|
||||
# Keep placeholder if no value found
|
||||
return match.group(0)
|
||||
resolved_value = json.dumps(value, ensure_ascii=False)
|
||||
else:
|
||||
resolved_value = str(value)
|
||||
|
||||
return re.sub(r'\{\{([^}]+)\}\}', replacer, template)
|
||||
# Track resolution for debug
|
||||
if debug_info is not None:
|
||||
resolved[key] = resolved_value[:100] + ('...' if len(resolved_value) > 100 else '')
|
||||
|
||||
return resolved_value
|
||||
else:
|
||||
# Keep placeholder if no value found
|
||||
if debug_info is not None:
|
||||
unresolved.append(key)
|
||||
return match.group(0)
|
||||
|
||||
result = re.sub(r'\{\{([^}]+)\}\}', replacer, template)
|
||||
|
||||
# Store debug info
|
||||
if debug_info is not None:
|
||||
debug_info['resolved_placeholders'] = resolved
|
||||
debug_info['unresolved_placeholders'] = unresolved
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def validate_json_output(output: str, schema: Optional[Dict] = None) -> Dict:
|
||||
@@ -67,7 +88,8 @@ def validate_json_output(output: str, schema: Optional[Dict] = None) -> Dict:
|
||||
async def execute_prompt(
|
||||
prompt_slug: str,
|
||||
variables: Dict[str, Any],
|
||||
openrouter_call_func
|
||||
openrouter_call_func,
|
||||
enable_debug: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute a single prompt (base or pipeline type).
|
||||
@@ -76,6 +98,7 @@ async def execute_prompt(
|
||||
prompt_slug: Slug of prompt to execute
|
||||
variables: Dict of variables for placeholder replacement
|
||||
openrouter_call_func: Async function(prompt_text) -> response_text
|
||||
enable_debug: If True, include debug information in response
|
||||
|
||||
Returns:
|
||||
Dict with execution results:
|
||||
@@ -84,6 +107,7 @@ async def execute_prompt(
|
||||
"slug": "...",
|
||||
"output": "..." | {...}, # String or parsed JSON
|
||||
"stages": [...] # Only for pipeline type
|
||||
"debug": {...} # Only if enable_debug=True
|
||||
}
|
||||
"""
|
||||
# Load prompt from database
|
||||
@@ -104,11 +128,11 @@ async def execute_prompt(
|
||||
|
||||
if prompt_type == 'base':
|
||||
# Base prompt: single execution with template
|
||||
return await execute_base_prompt(prompt, variables, openrouter_call_func)
|
||||
return await execute_base_prompt(prompt, variables, openrouter_call_func, enable_debug)
|
||||
|
||||
elif prompt_type == 'pipeline':
|
||||
# Pipeline prompt: multi-stage execution
|
||||
return await execute_pipeline_prompt(prompt, variables, openrouter_call_func)
|
||||
return await execute_pipeline_prompt(prompt, variables, openrouter_call_func, enable_debug)
|
||||
|
||||
else:
|
||||
raise HTTPException(400, f"Unknown prompt type: {prompt_type}")
|
||||
@@ -117,19 +141,31 @@ async def execute_prompt(
|
||||
async def execute_base_prompt(
|
||||
prompt: Dict,
|
||||
variables: Dict[str, Any],
|
||||
openrouter_call_func
|
||||
openrouter_call_func,
|
||||
enable_debug: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute a base-type prompt (single template)."""
|
||||
template = prompt.get('template')
|
||||
if not template:
|
||||
raise HTTPException(400, f"Base prompt missing template: {prompt['slug']}")
|
||||
|
||||
debug_info = {} if enable_debug else None
|
||||
|
||||
# Resolve placeholders
|
||||
prompt_text = resolve_placeholders(template, variables)
|
||||
prompt_text = resolve_placeholders(template, variables, debug_info)
|
||||
|
||||
if enable_debug:
|
||||
debug_info['template'] = template
|
||||
debug_info['final_prompt'] = prompt_text[:500] + ('...' if len(prompt_text) > 500 else '')
|
||||
debug_info['available_variables'] = list(variables.keys())
|
||||
|
||||
# Call AI
|
||||
response = await openrouter_call_func(prompt_text)
|
||||
|
||||
if enable_debug:
|
||||
debug_info['ai_response_length'] = len(response)
|
||||
debug_info['ai_response_preview'] = response[:200] + ('...' if len(response) > 200 else '')
|
||||
|
||||
# Validate JSON if required
|
||||
output_format = prompt.get('output_format', 'text')
|
||||
if output_format == 'json':
|
||||
@@ -137,18 +173,24 @@ async def execute_base_prompt(
|
||||
else:
|
||||
output = response
|
||||
|
||||
return {
|
||||
result = {
|
||||
"type": "base",
|
||||
"slug": prompt['slug'],
|
||||
"output": output,
|
||||
"output_format": output_format
|
||||
}
|
||||
|
||||
if enable_debug:
|
||||
result['debug'] = debug_info
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def execute_pipeline_prompt(
|
||||
prompt: Dict,
|
||||
variables: Dict[str, Any],
|
||||
openrouter_call_func
|
||||
openrouter_call_func,
|
||||
enable_debug: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute a pipeline-type prompt (multi-stage).
|
||||
@@ -165,6 +207,7 @@ async def execute_pipeline_prompt(
|
||||
|
||||
stage_results = []
|
||||
context_vars = variables.copy()
|
||||
pipeline_debug = [] if enable_debug else None
|
||||
|
||||
# Execute stages in order
|
||||
for stage_def in sorted(stages, key=lambda s: s['stage']):
|
||||
@@ -174,6 +217,12 @@ async def execute_pipeline_prompt(
|
||||
if not stage_prompts:
|
||||
continue
|
||||
|
||||
stage_debug = {} if enable_debug else None
|
||||
if enable_debug:
|
||||
stage_debug['stage'] = stage_num
|
||||
stage_debug['available_variables'] = list(context_vars.keys())
|
||||
stage_debug['prompts'] = []
|
||||
|
||||
# Execute all prompts in this stage (parallel concept, sequential impl for now)
|
||||
stage_outputs = {}
|
||||
|
||||
@@ -182,25 +231,46 @@ async def execute_pipeline_prompt(
|
||||
output_key = prompt_def.get('output_key', f'stage{stage_num}')
|
||||
output_format = prompt_def.get('output_format', 'text')
|
||||
|
||||
prompt_debug = {} if enable_debug else None
|
||||
|
||||
if source == 'reference':
|
||||
# Reference to another prompt
|
||||
ref_slug = prompt_def.get('slug')
|
||||
if not ref_slug:
|
||||
raise HTTPException(400, f"Reference prompt missing slug in stage {stage_num}")
|
||||
|
||||
if enable_debug:
|
||||
prompt_debug['source'] = 'reference'
|
||||
prompt_debug['ref_slug'] = ref_slug
|
||||
|
||||
# Load referenced prompt
|
||||
result = await execute_prompt(ref_slug, context_vars, openrouter_call_func)
|
||||
result = await execute_prompt(ref_slug, context_vars, openrouter_call_func, enable_debug)
|
||||
output = result['output']
|
||||
|
||||
if enable_debug and 'debug' in result:
|
||||
prompt_debug['ref_debug'] = result['debug']
|
||||
|
||||
elif source == 'inline':
|
||||
# Inline template
|
||||
template = prompt_def.get('template')
|
||||
if not template:
|
||||
raise HTTPException(400, f"Inline prompt missing template in stage {stage_num}")
|
||||
|
||||
prompt_text = resolve_placeholders(template, context_vars)
|
||||
placeholder_debug = {} if enable_debug else None
|
||||
prompt_text = resolve_placeholders(template, context_vars, placeholder_debug)
|
||||
|
||||
if enable_debug:
|
||||
prompt_debug['source'] = 'inline'
|
||||
prompt_debug['template'] = template
|
||||
prompt_debug['final_prompt'] = prompt_text[:500] + ('...' if len(prompt_text) > 500 else '')
|
||||
prompt_debug.update(placeholder_debug)
|
||||
|
||||
response = await openrouter_call_func(prompt_text)
|
||||
|
||||
if enable_debug:
|
||||
prompt_debug['ai_response_length'] = len(response)
|
||||
prompt_debug['ai_response_preview'] = response[:200] + ('...' if len(response) > 200 else '')
|
||||
|
||||
# Validate JSON if required
|
||||
if output_format == 'json':
|
||||
output = validate_json_output(response, prompt_def.get('output_schema'))
|
||||
@@ -214,17 +284,26 @@ async def execute_pipeline_prompt(
|
||||
stage_outputs[output_key] = output
|
||||
|
||||
# Add to context for next stage
|
||||
context_vars[f'stage_{stage_num}_{output_key}'] = output
|
||||
context_var_key = f'stage_{stage_num}_{output_key}'
|
||||
context_vars[context_var_key] = output
|
||||
|
||||
if enable_debug:
|
||||
prompt_debug['output_key'] = output_key
|
||||
prompt_debug['context_var_key'] = context_var_key
|
||||
stage_debug['prompts'].append(prompt_debug)
|
||||
|
||||
stage_results.append({
|
||||
"stage": stage_num,
|
||||
"outputs": stage_outputs
|
||||
})
|
||||
|
||||
if enable_debug:
|
||||
pipeline_debug.append(stage_debug)
|
||||
|
||||
# Final output is last stage's first output
|
||||
final_output = stage_results[-1]['outputs'] if stage_results else {}
|
||||
|
||||
return {
|
||||
result = {
|
||||
"type": "pipeline",
|
||||
"slug": prompt['slug'],
|
||||
"stages": stage_results,
|
||||
@@ -232,13 +311,22 @@ async def execute_pipeline_prompt(
|
||||
"output_format": prompt.get('output_format', 'text')
|
||||
}
|
||||
|
||||
if enable_debug:
|
||||
result['debug'] = {
|
||||
'initial_variables': list(variables.keys()),
|
||||
'stages': pipeline_debug
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def execute_prompt_with_data(
|
||||
prompt_slug: str,
|
||||
profile_id: str,
|
||||
modules: Optional[Dict[str, bool]] = None,
|
||||
timeframes: Optional[Dict[str, int]] = None,
|
||||
openrouter_call_func = None
|
||||
openrouter_call_func = None,
|
||||
enable_debug: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute prompt with data loaded from database.
|
||||
@@ -249,6 +337,7 @@ async def execute_prompt_with_data(
|
||||
modules: Dict of module -> enabled (e.g., {"körper": true})
|
||||
timeframes: Dict of module -> days (e.g., {"körper": 30})
|
||||
openrouter_call_func: Async function for AI calls
|
||||
enable_debug: If True, include debug information in response
|
||||
|
||||
Returns:
|
||||
Execution result dict
|
||||
@@ -348,4 +437,4 @@ async def execute_prompt_with_data(
|
||||
variables['goals_data'] = []
|
||||
|
||||
# Execute prompt
|
||||
return await execute_prompt(prompt_slug, variables, openrouter_call_func)
|
||||
return await execute_prompt(prompt_slug, variables, openrouter_call_func, enable_debug)
|
||||
Reference in new issue
Block a user