Deploy FeWo mit PipelineManager / deploy (push) Successful in 5m11s
- Jeder Mandant verbindet unter „System › KI-Anbindung“ seinen eigenen Zugang: Anbieter (Anthropic oder OpenRouter), API-Schlüssel, Modell für die erste Auswertung und für die Prüfung – keine Modellkennung im Code, kein zentraler Betreiberschlüssel (FEWO_AI_API_KEY entfällt). - Gespeichert im bestehenden TenantService (ai_guest_mail): Schlüssel verschlüsselt, nie wieder angezeigt, nie in Meldungen; Anbieter/Modelle in settings, Prüfergebnis in check_status/check_message/checked_at. - Feste Endpunkte je Anbieter (AiProvider), nicht konfigurierbar. OpenRouter mit response_format json_schema (strict) und provider.require_parameters. - „Verbindung prüfen“ testet beide Modelle. Ein abgelehnter Zugang (401/402/403) schaltet die Auswertung für den Mandanten ab, meldet es im Postfach der Verwaltung; Nachrichten bleiben zur Prüfung im Posteingang. - Zweistufige Extraktion und ihre Sicherheitsregeln unverändert. - Zielarchitektur §3.8, Umsetzungsplan U6, Bericht, Drehbuch, Status. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
4.0 KiB
PHP
79 lines
4.0 KiB
PHP
<?php
|
||
|
||
namespace App\Console\Commands;
|
||
|
||
use App\Communication\Extraction\AiAccess;
|
||
use App\Communication\Extraction\ExtractionFailed;
|
||
use App\Communication\Extraction\MessageExtractor;
|
||
use App\Models\Booking;
|
||
use App\Models\InboundMessage;
|
||
use App\Models\Property;
|
||
use App\Models\Tenant;
|
||
use Illuminate\Console\Attributes\Description;
|
||
use Illuminate\Console\Attributes\Signature;
|
||
use Illuminate\Console\Command;
|
||
|
||
/**
|
||
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
|
||
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
|
||
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
|
||
*/
|
||
#[Signature('ai:evaluate {tenant : ID of the landlord whose own AI access is used} {--path= : JSON file with samples}')]
|
||
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
|
||
class EvaluateExtraction extends Command
|
||
{
|
||
public function handle(MessageExtractor $extractor): int
|
||
{
|
||
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
|
||
// Bring your own key: the landlord's own access, as configured (System › KI-Anbindung).
|
||
$tenant = Tenant::query()->find((int) $this->argument('tenant'));
|
||
$service = $tenant === null ? null : AiAccess::service($tenant);
|
||
$access = $service === null ? null : AiAccess::fromService($service);
|
||
if ($access === null) {
|
||
$this->error('Für diesen Mandanten ist keine vollständige KI-Anbindung gespeichert.');
|
||
|
||
return self::FAILURE;
|
||
}
|
||
|
||
$samples = (array) json_decode((string) file_get_contents($path), true);
|
||
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
|
||
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
|
||
|
||
foreach ($samples as $sample) {
|
||
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
|
||
$booking->setRelation('property', $property);
|
||
$booking->setRelation('guest', null);
|
||
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
|
||
|
||
try {
|
||
$result = $extractor->extract($access, $message, $booking)['data'];
|
||
} catch (ExtractionFailed $exception) {
|
||
$totals['failed']++;
|
||
$this->warn("{$sample['id']}: {$exception->getMessage()}");
|
||
|
||
continue;
|
||
}
|
||
|
||
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
|
||
foreach ($sample['expected'] as $field => $value) {
|
||
$totals['expected']++;
|
||
match (true) {
|
||
! isset($found[$field]) => $totals['missed']++,
|
||
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
|
||
default => $totals['found']++,
|
||
};
|
||
}
|
||
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
|
||
$changes = collect($result['change_requests'])->pluck('field')->all();
|
||
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
|
||
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
|
||
}
|
||
|
||
$this->table(array_keys($totals), [array_values($totals)]);
|
||
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
|
||
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
|
||
|
||
return self::SUCCESS;
|
||
}
|
||
}
|