v0.11 U6: risikobasierte KI-Extraktion von Gast- und Portalnachrichten

- Kleines Modell (claude-haiku-5-5) extrahiert Fakten mit Belegstelle und
  Konfidenz; programmatische Prüfung (Gültigkeit, Belegstelle im Original,
  Widerspruch zum gespeicherten Wert oder in der Nachricht, Plausibilität,
  Kritikalität nach Feld und Phase). Widerspruch oder fehlgeschlagene
  Prüfung → Prüfmodell (claude-opus-5-5, serverseitiger Fallback bei
  Ablehnung) unabhängig von der Konfidenz; solche Fakten nie automatisch.
- Nur geprüfte, bisher fehlende Angaben gehen an den Fakteneingang (Quelle
  „llm“); Zeitraum/Personen nur als Änderungswunsch (M06); Rest als
  Vorschlag im Posteingang („Vorschläge übernehmen“).
- Versionierte Prompts mit Platzhaltern, JSON-Schema mit eigener
  Validierung und einer Wiederholung; Ausfall/Ablehnung → sicherer
  Rückfall zur Vermieterin. HTTP-Client, kein SDK (laut Plan).
- Dienst „KI: Gästekommunikation“ verdrahtet, standardmäßig aus;
  Schlüssel je Mandant oder FEWO_AI_API_KEY; Verbrauch in service_usages.
- messages:extract alle fünf Minuten; ai:evaluate misst Fehlwerte und
  übersehene Angaben getrennt (M05).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LarsandClaude Opus 5.5 committed 2026-11-03 13:49:51 +01:00
1 parent dff9e2344e
commit 4170e3bd29
18 files changed
+1130 -7

No files matched your search

@@ -0,0 +1,73 @@
<?php
namespace App\Console\Commands;
use App\Communication\Extraction\ExtractionFailed;
use App\Communication\Extraction\MessageExtractor;
use App\Models\Booking;
use App\Models\InboundMessage;
use App\Models\Property;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command;
/**
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
*/
#[Signature('ai:evaluate {--path= : JSON file with samples} {--key= : API key (default: FEWO_AI_API_KEY)}')]
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
class EvaluateExtraction extends Command
{
public function handle(MessageExtractor $extractor): int
{
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
$key = (string) ($this->option('key') ?: config('fewo.ai.api_key'));
if ($key === '') {
$this->error('Kein API-Schlüssel (FEWO_AI_API_KEY oder --key).');
return self::FAILURE;
}
$samples = (array) json_decode((string) file_get_contents($path), true);
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
foreach ($samples as $sample) {
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
$booking->setRelation('property', $property);
$booking->setRelation('guest', null);
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
try {
$result = $extractor->extract($key, $message, $booking)['data'];
} catch (ExtractionFailed $exception) {
$totals['failed']++;
$this->warn("{$sample['id']}: {$exception->getMessage()}");
continue;
}
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
foreach ($sample['expected'] as $field => $value) {
$totals['expected']++;
match (true) {
! isset($found[$field]) => $totals['missed']++,
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
default => $totals['found']++,
};
}
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
$changes = collect($result['change_requests'])->pluck('field')->all();
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
}
$this->table(array_keys($totals), [array_values($totals)]);
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
return self::SUCCESS;
}
}