Files
fewo-stommer/app/Console/Commands/EvaluateExtraction.php
LarsandClaude Opus 5.5 6862dec201
Deploy FeWo mit PipelineManager / deploy (push) Successful in 5m11s
v0.11 U6: Bring your own key für die KI-Auswertung (Anthropic, OpenRouter)
- Jeder Mandant verbindet unter „System › KI-Anbindung“ seinen eigenen
  Zugang: Anbieter (Anthropic oder OpenRouter), API-Schlüssel, Modell für
  die erste Auswertung und für die Prüfung – keine Modellkennung im Code,
  kein zentraler Betreiberschlüssel (FEWO_AI_API_KEY entfällt).
- Gespeichert im bestehenden TenantService (ai_guest_mail): Schlüssel
  verschlüsselt, nie wieder angezeigt, nie in Meldungen; Anbieter/Modelle
  in settings, Prüfergebnis in check_status/check_message/checked_at.
- Feste Endpunkte je Anbieter (AiProvider), nicht konfigurierbar.
  OpenRouter mit response_format json_schema (strict) und
  provider.require_parameters.
- „Verbindung prüfen“ testet beide Modelle. Ein abgelehnter Zugang
  (401/402/403) schaltet die Auswertung für den Mandanten ab, meldet es im
  Postfach der Verwaltung; Nachrichten bleiben zur Prüfung im Posteingang.
- Zweistufige Extraktion und ihre Sicherheitsregeln unverändert.
- Zielarchitektur §3.8, Umsetzungsplan U6, Bericht, Drehbuch, Status.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 14:19:13 +01:00

79 lines
4.0 KiB
PHP
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
namespace App\Console\Commands;
use App\Communication\Extraction\AiAccess;
use App\Communication\Extraction\ExtractionFailed;
use App\Communication\Extraction\MessageExtractor;
use App\Models\Booking;
use App\Models\InboundMessage;
use App\Models\Property;
use App\Models\Tenant;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command;
/**
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
*/
#[Signature('ai:evaluate {tenant : ID of the landlord whose own AI access is used} {--path= : JSON file with samples}')]
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
class EvaluateExtraction extends Command
{
public function handle(MessageExtractor $extractor): int
{
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
// Bring your own key: the landlord's own access, as configured (System › KI-Anbindung).
$tenant = Tenant::query()->find((int) $this->argument('tenant'));
$service = $tenant === null ? null : AiAccess::service($tenant);
$access = $service === null ? null : AiAccess::fromService($service);
if ($access === null) {
$this->error('Für diesen Mandanten ist keine vollständige KI-Anbindung gespeichert.');
return self::FAILURE;
}
$samples = (array) json_decode((string) file_get_contents($path), true);
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
foreach ($samples as $sample) {
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
$booking->setRelation('property', $property);
$booking->setRelation('guest', null);
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
try {
$result = $extractor->extract($access, $message, $booking)['data'];
} catch (ExtractionFailed $exception) {
$totals['failed']++;
$this->warn("{$sample['id']}: {$exception->getMessage()}");
continue;
}
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
foreach ($sample['expected'] as $field => $value) {
$totals['expected']++;
match (true) {
! isset($found[$field]) => $totals['missed']++,
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
default => $totals['found']++,
};
}
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
$changes = collect($result['change_requests'])->pluck('field')->all();
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
}
$this->table(array_keys($totals), [array_values($totals)]);
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
return self::SUCCESS;
}
}