- Kleines Modell (claude-haiku-5-5) extrahiert Fakten mit Belegstelle und Konfidenz; programmatische Prüfung (Gültigkeit, Belegstelle im Original, Widerspruch zum gespeicherten Wert oder in der Nachricht, Plausibilität, Kritikalität nach Feld und Phase). Widerspruch oder fehlgeschlagene Prüfung → Prüfmodell (claude-opus-5-5, serverseitiger Fallback bei Ablehnung) unabhängig von der Konfidenz; solche Fakten nie automatisch. - Nur geprüfte, bisher fehlende Angaben gehen an den Fakteneingang (Quelle „llm“); Zeitraum/Personen nur als Änderungswunsch (M06); Rest als Vorschlag im Posteingang („Vorschläge übernehmen“). - Versionierte Prompts mit Platzhaltern, JSON-Schema mit eigener Validierung und einer Wiederholung; Ausfall/Ablehnung → sicherer Rückfall zur Vermieterin. HTTP-Client, kein SDK (laut Plan). - Dienst „KI: Gästekommunikation“ verdrahtet, standardmäßig aus; Schlüssel je Mandant oder FEWO_AI_API_KEY; Verbrauch in service_usages. - messages:extract alle fünf Minuten; ai:evaluate misst Fehlwerte und übersehene Angaben getrennt (M05). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
3.6 KiB
PHP
74 lines
3.6 KiB
PHP
<?php
|
||
|
||
namespace App\Console\Commands;
|
||
|
||
use App\Communication\Extraction\ExtractionFailed;
|
||
use App\Communication\Extraction\MessageExtractor;
|
||
use App\Models\Booking;
|
||
use App\Models\InboundMessage;
|
||
use App\Models\Property;
|
||
use Illuminate\Console\Attributes\Description;
|
||
use Illuminate\Console\Attributes\Signature;
|
||
use Illuminate\Console\Command;
|
||
|
||
/**
|
||
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
|
||
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
|
||
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
|
||
*/
|
||
#[Signature('ai:evaluate {--path= : JSON file with samples} {--key= : API key (default: FEWO_AI_API_KEY)}')]
|
||
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
|
||
class EvaluateExtraction extends Command
|
||
{
|
||
public function handle(MessageExtractor $extractor): int
|
||
{
|
||
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
|
||
$key = (string) ($this->option('key') ?: config('fewo.ai.api_key'));
|
||
if ($key === '') {
|
||
$this->error('Kein API-Schlüssel (FEWO_AI_API_KEY oder --key).');
|
||
|
||
return self::FAILURE;
|
||
}
|
||
|
||
$samples = (array) json_decode((string) file_get_contents($path), true);
|
||
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
|
||
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
|
||
|
||
foreach ($samples as $sample) {
|
||
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
|
||
$booking->setRelation('property', $property);
|
||
$booking->setRelation('guest', null);
|
||
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
|
||
|
||
try {
|
||
$result = $extractor->extract($key, $message, $booking)['data'];
|
||
} catch (ExtractionFailed $exception) {
|
||
$totals['failed']++;
|
||
$this->warn("{$sample['id']}: {$exception->getMessage()}");
|
||
|
||
continue;
|
||
}
|
||
|
||
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
|
||
foreach ($sample['expected'] as $field => $value) {
|
||
$totals['expected']++;
|
||
match (true) {
|
||
! isset($found[$field]) => $totals['missed']++,
|
||
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
|
||
default => $totals['found']++,
|
||
};
|
||
}
|
||
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
|
||
$changes = collect($result['change_requests'])->pluck('field')->all();
|
||
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
|
||
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
|
||
}
|
||
|
||
$this->table(array_keys($totals), [array_values($totals)]);
|
||
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
|
||
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
|
||
|
||
return self::SUCCESS;
|
||
}
|
||
}
|