v0.11 U6: risikobasierte KI-Extraktion von Gast- und Portalnachrichten
- Kleines Modell (claude-haiku-5-5) extrahiert Fakten mit Belegstelle und Konfidenz; programmatische Prüfung (Gültigkeit, Belegstelle im Original, Widerspruch zum gespeicherten Wert oder in der Nachricht, Plausibilität, Kritikalität nach Feld und Phase). Widerspruch oder fehlgeschlagene Prüfung → Prüfmodell (claude-opus-5-5, serverseitiger Fallback bei Ablehnung) unabhängig von der Konfidenz; solche Fakten nie automatisch. - Nur geprüfte, bisher fehlende Angaben gehen an den Fakteneingang (Quelle „llm“); Zeitraum/Personen nur als Änderungswunsch (M06); Rest als Vorschlag im Posteingang („Vorschläge übernehmen“). - Versionierte Prompts mit Platzhaltern, JSON-Schema mit eigener Validierung und einer Wiederholung; Ausfall/Ablehnung → sicherer Rückfall zur Vermieterin. HTTP-Client, kein SDK (laut Plan). - Dienst „KI: Gästekommunikation“ verdrahtet, standardmäßig aus; Schlüssel je Mandant oder FEWO_AI_API_KEY; Verbrauch in service_usages. - messages:extract alle fünf Minuten; ai:evaluate misst Fehlwerte und übersehene Angaben getrennt (M05). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
dff9e2344e
commit
4170e3bd29
18 files changed
+1130
-7
No files matched your search
@@ -0,0 +1,73 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Communication\Extraction\ExtractionFailed;
|
||||
use App\Communication\Extraction\MessageExtractor;
|
||||
use App\Models\Booking;
|
||||
use App\Models\InboundMessage;
|
||||
use App\Models\Property;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
/**
|
||||
* Measures the extraction on a set of anonymised sample messages (v0.11, M05): wrong values and missed details are
|
||||
* counted separately – no system guarantees that it finds every detail, so the rate of missed ones is reported, not
|
||||
* hidden. Calls the configured models (costs money); meant for calibrating prompts and thresholds before go-live.
|
||||
*/
|
||||
#[Signature('ai:evaluate {--path= : JSON file with samples} {--key= : API key (default: FEWO_AI_API_KEY)}')]
|
||||
#[Description('Evaluate the guest message extraction against sample messages (wrong values and missed details)')]
|
||||
class EvaluateExtraction extends Command
|
||||
{
|
||||
public function handle(MessageExtractor $extractor): int
|
||||
{
|
||||
$path = (string) ($this->option('path') ?: resource_path('prompts/extraction/evaluation/samples.json'));
|
||||
$key = (string) ($this->option('key') ?: config('fewo.ai.api_key'));
|
||||
if ($key === '') {
|
||||
$this->error('Kein API-Schlüssel (FEWO_AI_API_KEY oder --key).');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$samples = (array) json_decode((string) file_get_contents($path), true);
|
||||
$property = Property::query()->withoutGlobalScopes()->first() ?? new Property(['name' => 'Ferienwohnung', 'max_guests' => 4]);
|
||||
$totals = ['expected' => 0, 'found' => 0, 'missed' => 0, 'wrong' => 0, 'extra' => 0, 'changes_missed' => 0, 'failed' => 0];
|
||||
|
||||
foreach ($samples as $sample) {
|
||||
$booking = new Booking(['arrival' => now()->addDays(14)->toDateString(), 'departure' => now()->addDays(17)->toDateString(), 'guests' => 2, 'status' => 'reserved']);
|
||||
$booking->setRelation('property', $property);
|
||||
$booking->setRelation('guest', null);
|
||||
$message = new InboundMessage(['subject' => $sample['subject'], 'body' => $sample['body']]);
|
||||
|
||||
try {
|
||||
$result = $extractor->extract($key, $message, $booking)['data'];
|
||||
} catch (ExtractionFailed $exception) {
|
||||
$totals['failed']++;
|
||||
$this->warn("{$sample['id']}: {$exception->getMessage()}");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$found = collect($result['facts'])->mapWithKeys(fn (array $fact): array => [$fact['field'] => mb_strtolower(trim($fact['value']))])->all();
|
||||
foreach ($sample['expected'] as $field => $value) {
|
||||
$totals['expected']++;
|
||||
match (true) {
|
||||
! isset($found[$field]) => $totals['missed']++,
|
||||
$found[$field] !== mb_strtolower($value) => $totals['wrong']++,
|
||||
default => $totals['found']++,
|
||||
};
|
||||
}
|
||||
$totals['extra'] += count(array_diff_key($found, $sample['expected']));
|
||||
$changes = collect($result['change_requests'])->pluck('field')->all();
|
||||
$totals['changes_missed'] += count(array_diff($sample['expected_changes'], $changes));
|
||||
$this->line("{$sample['id']}: ".json_encode(['found' => $found, 'changes' => $changes], JSON_UNESCAPED_UNICODE));
|
||||
}
|
||||
|
||||
$this->table(array_keys($totals), [array_values($totals)]);
|
||||
$rate = $totals['expected'] > 0 ? round(100 * $totals['missed'] / $totals['expected'], 1) : 0;
|
||||
$this->info("Übersehene Angaben: {$rate} % · Prompt-Version ".MessageExtractor::promptVersion());
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Actions\ExtractMessageFacts;
|
||||
use App\Models\InboundMessage;
|
||||
use App\Models\Tenant;
|
||||
use App\Tenancy\TenantRunner;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
use Throwable;
|
||||
|
||||
#[Signature('messages:extract')]
|
||||
#[Description('Evaluate assigned guest and portal messages (v0.11 extraction) – a limited number per landlord and run')]
|
||||
class ExtractInboundMessages extends Command
|
||||
{
|
||||
public function handle(ExtractMessageFacts $extract, TenantRunner $runner): int
|
||||
{
|
||||
$runner->each(function (Tenant $tenant) use ($extract): void {
|
||||
$messages = InboundMessage::query()
|
||||
->whereNotNull('booking_id')
|
||||
->whereNull('extraction_status')
|
||||
->whereIn('status', [InboundMessage::ASSIGNED, InboundMessage::PORTAL])
|
||||
->orderBy('received_at')
|
||||
->limit((int) config('fewo.ai.per_run', 10))
|
||||
->get();
|
||||
|
||||
foreach ($messages as $message) {
|
||||
try {
|
||||
$this->line("{$tenant->name}: {$message->subject} → {$extract->handle($message)}");
|
||||
} catch (Throwable $exception) {
|
||||
report($exception);
|
||||
$message->update(['extraction_status' => 'failed']);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user