Files
fewo-stommer/app/Console/Commands/EvaluateExtraction.php
T
LarsandClaude Opus 5.5 4170e3bd29 v0.11 U6: risikobasierte KI-Extraktion von Gast- und Portalnachrichten
- Kleines Modell (claude-haiku-5-5) extrahiert Fakten mit Belegstelle und
  Konfidenz; programmatische Prüfung (Gültigkeit, Belegstelle im Original,
  Widerspruch zum gespeicherten Wert oder in der Nachricht, Plausibilität,
  Kritikalität nach Feld und Phase). Widerspruch oder fehlgeschlagene
  Prüfung → Prüfmodell (claude-opus-5-5, serverseitiger Fallback bei
  Ablehnung) unabhängig von der Konfidenz; solche Fakten nie automatisch.
- Nur geprüfte, bisher fehlende Angaben gehen an den Fakteneingang (Quelle
  „llm“); Zeitraum/Personen nur als Änderungswunsch (M06); Rest als
  Vorschlag im Posteingang („Vorschläge übernehmen“).
- Versionierte Prompts mit Platzhaltern, JSON-Schema mit eigener
  Validierung und einer Wiederholung; Ausfall/Ablehnung → sicherer
  Rückfall zur Vermieterin. HTTP-Client, kein SDK (laut Plan).
- Dienst „KI: Gästekommunikation“ verdrahtet, standardmäßig aus;
  Schlüssel je Mandant oder FEWO_AI_API_KEY; Verbrauch in service_usages.
- messages:extract alle fünf Minuten; ai:evaluate misst Fehlwerte und
  übersehene Angaben getrennt (M05).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 13:49:51 +01:00

74 lines
3.6 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
namespace App\Console\Commands;
use App\Communication\Extraction\ExtractionFailed;
use App\Communication\Extraction\MessageExtractor;
use App\Models\Booking;
use App\Models\InboundMessage;
use App\Models\Property;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command;
/**
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
*/
#[Signature('ai:evaluate {--path= : JSON file with samples} {--key= : API key (default: FEWO_AI_API_KEY)}')]
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
class EvaluateExtraction extends Command
{
public function handle(MessageExtractor $extractor): int
{
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
$key = (string) ($this->option('key') ?: config('fewo.ai.api_key'));
if ($key === '') {
$this->error('Kein API-Schlüssel (FEWO_AI_API_KEY oder --key).');
return self::FAILURE;
}
$samples = (array) json_decode((string) file_get_contents($path), true);
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
foreach ($samples as $sample) {
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
$booking->setRelation('property', $property);
$booking->setRelation('guest', null);
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
try {
$result = $extractor->extract($key, $message, $booking)['data'];
} catch (ExtractionFailed $exception) {
$totals['failed']++;
$this->warn("{$sample['id']}: {$exception->getMessage()}");
continue;
}
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
foreach ($sample['expected'] as $field => $value) {
$totals['expected']++;
match (true) {
! isset($found[$field]) => $totals['missed']++,
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
default => $totals['found']++,
};
}
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
$changes = collect($result['change_requests'])->pluck('field')->all();
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
}
$this->table(array_keys($totals), [array_values($totals)]);
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
return self::SUCCESS;
}
}