- Kleines Modell (claude-haiku-5-5) extrahiert Fakten mit Belegstelle und Konfidenz; programmatische Prüfung (Gültigkeit, Belegstelle im Original, Widerspruch zum gespeicherten Wert oder in der Nachricht, Plausibilität, Kritikalität nach Feld und Phase). Widerspruch oder fehlgeschlagene Prüfung → Prüfmodell (claude-opus-5-5, serverseitiger Fallback bei Ablehnung) unabhängig von der Konfidenz; solche Fakten nie automatisch. - Nur geprüfte, bisher fehlende Angaben gehen an den Fakteneingang (Quelle „llm“); Zeitraum/Personen nur als Änderungswunsch (M06); Rest als Vorschlag im Posteingang („Vorschläge übernehmen“). - Versionierte Prompts mit Platzhaltern, JSON-Schema mit eigener Validierung und einer Wiederholung; Ausfall/Ablehnung → sicherer Rückfall zur Vermieterin. HTTP-Client, kein SDK (laut Plan). - Dienst „KI: Gästekommunikation“ verdrahtet, standardmäßig aus; Schlüssel je Mandant oder FEWO_AI_API_KEY; Verbrauch in service_usages. - messages:extract alle fünf Minuten; ai:evaluate misst Fehlwerte und übersehene Angaben getrennt (M05). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
276 lines
14 KiB
PHP
276 lines
14 KiB
PHP
<?php
|
|
|
|
namespace Tests\Feature;
|
|
|
|
use App\Actions\CreateReservation;
|
|
use App\Actions\ExtractMessageFacts;
|
|
use App\Actions\ReceiveInboundMessage;
|
|
use App\Channels\Mail\PortalMail;
|
|
use App\Communication\Extraction\MessageExtractor;
|
|
use App\Enums\BookingStatus;
|
|
use App\Enums\BookingTaskKind;
|
|
use App\Enums\FactSource;
|
|
use App\Enums\TenantServiceKey;
|
|
use App\Filament\Resources\InboundMessages\Pages\ListInboundMessages;
|
|
use App\Models\BookingTask;
|
|
use App\Models\CaseFact;
|
|
use App\Models\InboundMessage;
|
|
use App\Models\Property;
|
|
use App\Models\ServiceUsage;
|
|
use App\Models\TenantService;
|
|
use App\Models\User;
|
|
use App\Pricing\StayRequest;
|
|
use App\Support\Content;
|
|
use App\Support\Settings;
|
|
use App\Tenancy\TenantFeatures;
|
|
use Carbon\CarbonImmutable;
|
|
use Database\Seeders\PropertySeeder;
|
|
use Database\Seeders\SiteSeeder;
|
|
use Filament\Actions\Testing\TestAction;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Http\Client\Request;
|
|
use Illuminate\Support\Facades\Http;
|
|
use Illuminate\Support\Facades\Mail;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Livewire\Livewire;
|
|
use Tests\TestCase;
|
|
|
|
/**
|
|
* v0.11 package U6: risk-based two-stage extraction (CH-14, E-035, M05, M06). The model only proposes; rules decide
|
|
* whether a detail goes to the fact intake. Contradictions and failed checks are reviewed regardless of the model's
|
|
* confidence and never taken over automatically. No network: the API is faked.
|
|
*/
|
|
class MessageExtractionTest extends TestCase
|
|
{
|
|
use RefreshDatabase;
|
|
|
|
private const ANSWER = "Hallo,\nhier unsere Anschrift: Lindenallee 12, 10115 Berlin. Telefon 030 1234567.\nGesa Gast";
|
|
|
|
private Property $property;
|
|
|
|
/** @var list<array<string, mixed>> */
|
|
private array $calls = [];
|
|
|
|
protected function setUp(): void
|
|
{
|
|
parent::setUp();
|
|
|
|
$this->travelTo(CarbonImmutable::parse('2026-10-22 10:00'));
|
|
$this->seed([PropertySeeder::class, SiteSeeder::class]);
|
|
$this->property = Property::sole();
|
|
Mail::fake();
|
|
Storage::fake('local');
|
|
Content::save(['contract.terms' => '<p>Echte Bedingungen</p>', 'stay.cancellation' => '<p>Echtes Storno</p>']);
|
|
Settings::tenant()->update(['bank_iban' => 'DE02120300000000202051']);
|
|
config(['fewo.ai.api_key' => 'test-key']);
|
|
TenantService::query()->create(['tenant_id' => Settings::tenant()->id, 'service' => TenantServiceKey::AiGuestMail->value, 'is_enabled' => true]);
|
|
app(TenantFeatures::class)->forget();
|
|
}
|
|
|
|
public function test_without_the_service_nothing_is_sent_to_a_provider(): void
|
|
{
|
|
TenantService::query()->update(['is_enabled' => false]);
|
|
app(TenantFeatures::class)->forget();
|
|
Http::fake();
|
|
|
|
$this->assertSame('disabled', app(ExtractMessageFacts::class)->handle($this->reply()));
|
|
Http::assertNothingSent();
|
|
}
|
|
|
|
public function test_confident_checked_details_are_taken_over_through_the_fact_intake_without_review(): void
|
|
{
|
|
Settings::tenant()->update(['process_defaults' => ['processing' => 'automatic', 'data_request_mode' => 'automatic']]);
|
|
$this->fakeModels(small: $this->extraction([
|
|
$this->fact('guest.street', 'Lindenallee 12', 'Lindenallee 12'), $this->fact('guest.postal_code', '10115', '10115 Berlin'),
|
|
$this->fact('guest.city', 'Berlin', '10115 Berlin'), $this->fact('guest.country', 'DE', 'Lindenallee 12, 10115 Berlin', 0.9),
|
|
$this->fact('guest.phone', '030 1234567', 'Telefon 030 1234567'),
|
|
]));
|
|
$message = $this->reply();
|
|
|
|
$this->assertSame('done', app(ExtractMessageFacts::class)->handle($message));
|
|
|
|
$booking = $message->booking->fresh();
|
|
$this->assertSame('Lindenallee 12', $booking->guest->street);
|
|
$this->assertSame(FactSource::Extraction, CaseFact::query()->where('field', 'guest.street')->where('outcome', CaseFact::APPLIED)->sole()->source);
|
|
$this->assertSame(BookingStatus::OfferSent, $booking->status, 'complete now: the existing process sent the offer');
|
|
$this->assertCount(1, $this->calls, 'no review needed');
|
|
$this->assertSame(InboundMessage::PROCESSED, $message->fresh()->status);
|
|
$this->assertSame(1, ServiceUsage::query()->count());
|
|
}
|
|
|
|
/** Precision 2: a contradiction is reviewed even at high confidence and is never taken over automatically. */
|
|
public function test_a_contradiction_is_reviewed_regardless_of_the_confidence_and_only_proposed(): void
|
|
{
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.street', 'Lindenallee 12', 'Lindenallee 12', 0.99)]),
|
|
review: $this->extraction([$this->fact('guest.street', 'Lindenallee 12', 'Lindenallee 12', 0.99)]),
|
|
);
|
|
$message = $this->reply(['street' => 'Alter Weg 2']);
|
|
|
|
app(ExtractMessageFacts::class)->handle($message);
|
|
|
|
$this->assertSame('Alter Weg 2', $message->booking->guest->fresh()->street);
|
|
$this->assertCount(2, $this->calls, 'the review model was asked');
|
|
$this->assertSame(['weicht vom gespeicherten Wert ab'], $message->fresh()->extraction['proposed'][0]['problems']);
|
|
}
|
|
|
|
public function test_a_value_without_evidence_in_the_message_is_not_taken_over(): void
|
|
{
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.street', 'Musterstraße 1', 'Musterstraße 1', 0.99)]),
|
|
review: $this->extraction([$this->fact('guest.street', 'Musterstraße 1', 'Musterstraße 1', 0.99)]),
|
|
);
|
|
$message = $this->reply();
|
|
|
|
app(ExtractMessageFacts::class)->handle($message);
|
|
|
|
$this->assertNull($message->booking->guest->fresh()->street);
|
|
$this->assertContains('Belegstelle nicht in der Nachricht', $message->fresh()->extraction['proposed'][0]['problems']);
|
|
}
|
|
|
|
public function test_two_models_reading_a_value_differently_leave_it_to_the_landlord(): void
|
|
{
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.phone', '030 1234567', 'Telefon 030 1234567', 0.3)]),
|
|
review: $this->extraction([$this->fact('guest.phone', '030 123456', 'Telefon 030 1234567', 0.95)]),
|
|
);
|
|
$message = $this->reply();
|
|
|
|
app(ExtractMessageFacts::class)->handle($message);
|
|
|
|
$this->assertNull($message->booking->guest->fresh()->phone);
|
|
$this->assertContains('die beiden Modelle lesen den Wert unterschiedlich', $message->fresh()->extraction['proposed'][0]['problems']);
|
|
}
|
|
|
|
/** M06: a phone number is completed, a wish to stay longer only becomes a request. */
|
|
public function test_a_wish_to_change_the_stay_is_only_a_request(): void
|
|
{
|
|
$text = 'Unsere Nummer: 030 1234567. Können wir bis zum 11.11.2026 bleiben?';
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.phone', '030 1234567', 'Unsere Nummer: 030 1234567')], [['field' => 'booking.departure', 'value' => '2026-11-11', 'evidence' => 'Können wir bis zum 11.11.2026 bleiben?']], 'change'),
|
|
review: $this->extraction([$this->fact('guest.phone', '030 1234567', 'Unsere Nummer: 030 1234567')], [['field' => 'booking.departure', 'value' => '2026-11-11', 'evidence' => 'Können wir bis zum 11.11.2026 bleiben?']], 'change'),
|
|
);
|
|
$message = $this->reply(text: $text);
|
|
|
|
app(ExtractMessageFacts::class)->handle($message);
|
|
|
|
$booking = $message->booking->fresh();
|
|
$this->assertSame('030 1234567', $booking->guest->phone);
|
|
$this->assertSame('2026-11-10', $booking->departure->toDateString(), 'the stay is unchanged');
|
|
$this->assertSame(1, BookingTask::query()->where('booking_id', $booking->id)->where('kind', BookingTaskKind::ChangeRequest)->count());
|
|
}
|
|
|
|
public function test_an_answer_that_does_not_fit_the_schema_or_a_refusal_waits_for_the_landlord(): void
|
|
{
|
|
$this->fakeModels(small: ['facts' => 'kaputt']);
|
|
$broken = $this->reply();
|
|
$this->assertSame('failed', app(ExtractMessageFacts::class)->handle($broken));
|
|
$this->assertCount(2, $this->calls, 'one repetition');
|
|
|
|
$this->calls = [];
|
|
Http::fake(['*' => Http::response(['content' => [], 'stop_reason' => 'refusal', 'model' => 'claude-haiku-5-5'])]);
|
|
$refused = $this->reply(id: '<2@x>', email: 'zwei@example.org', arrival: '2026-11-20', departure: '2026-11-23');
|
|
$this->assertSame('failed', app(ExtractMessageFacts::class)->handle($refused));
|
|
$this->assertSame(1, BookingTask::query()->where('booking_id', $refused->booking_id)->where('kind', BookingTaskKind::InboundReview)->count(), 'the landlord still has the message to look at');
|
|
}
|
|
|
|
public function test_the_request_uses_structured_outputs_and_the_review_model_has_the_refusal_fallback(): void
|
|
{
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.street', 'Lindenallee 12', 'Lindenallee 12', 0.3)]),
|
|
review: $this->extraction([]),
|
|
);
|
|
|
|
app(ExtractMessageFacts::class)->handle($this->reply());
|
|
|
|
[$small, $review] = $this->calls;
|
|
$this->assertSame(['claude-haiku-5-5', 'json_schema'], [$small['body']['model'], $small['body']['output_config']['format']['type']]);
|
|
$this->assertArrayNotHasKey('fallbacks', $small['body']);
|
|
$this->assertSame(['claude-opus-5-5', 'default', 'server-side-fallback-2026-07-01'], [$review['body']['model'], $review['body']['fallbacks'], $review['beta']]);
|
|
$this->assertSame('2026-10-21', $small['version']);
|
|
$this->assertStringNotContainsString('{{', $small['body']['system']);
|
|
$this->assertStringContainsString('Lindenallee 12', $review['body']['system'], 'the review sees the first extraction');
|
|
}
|
|
|
|
public function test_the_landlord_takes_over_a_proposal_in_the_inbox(): void
|
|
{
|
|
$this->actingAs(User::factory()->create());
|
|
$this->fakeModels(
|
|
small: $this->extraction([$this->fact('guest.city', 'Berlin', '10115 Berlin', 0.4)]),
|
|
review: $this->extraction([$this->fact('guest.city', 'Berlin', '10115 Berlin', 0.6)]),
|
|
);
|
|
$message = $this->reply();
|
|
app(ExtractMessageFacts::class)->handle($message);
|
|
$this->assertNull($message->booking->guest->fresh()->city);
|
|
|
|
Livewire::test(ListInboundMessages::class)->callAction(TestAction::make('applyProposals')->table($message->fresh()), ['fields' => ['guest.city']]);
|
|
|
|
$this->assertSame('Berlin', $message->booking->guest->fresh()->city);
|
|
$this->assertSame(FactSource::Landlord, CaseFact::query()->where('field', 'guest.city')->latest('id')->first()->source);
|
|
}
|
|
|
|
/** M05: wrong values and missed details are counted separately. */
|
|
public function test_the_evaluation_reports_missed_details(): void
|
|
{
|
|
$this->fakeModels(small: $this->extraction([$this->fact('guest.street', 'Lindenallee 12', 'Lindenallee 12')]));
|
|
|
|
$this->artisan('ai:evaluate')->expectsOutputToContain('Übersehene Angaben:')->assertSuccessful();
|
|
}
|
|
|
|
public function test_the_prompt_is_versioned_and_every_placeholder_is_filled(): void
|
|
{
|
|
$booking = $this->reply()->booking;
|
|
|
|
$prompt = app(MessageExtractor::class)->prompt('', $booking);
|
|
|
|
$this->assertSame('v1', MessageExtractor::promptVersion());
|
|
$this->assertStringContainsString('non-binding request', $prompt);
|
|
$this->assertStringContainsString('guest.street', $prompt);
|
|
$this->assertStringContainsString('guest.email=g', $prompt, 'known values are masked');
|
|
$this->assertStringNotContainsString('gesa@example.org', $prompt);
|
|
}
|
|
|
|
/**
|
|
* @param array<string, string> $guest
|
|
*/
|
|
private function reply(array $guest = [], string $text = self::ANSWER, string $id = '<1@x>', string $email = 'gesa@example.org', string $arrival = '2026-11-07', string $departure = '2026-11-10'): InboundMessage
|
|
{
|
|
$booking = app(CreateReservation::class)->handle($this->property->fresh(), new StayRequest(CarbonImmutable::parse($arrival), CarbonImmutable::parse($departure), 2), ['first_name' => 'Gesa', 'last_name' => 'Gast', 'email' => $email, ...$guest]);
|
|
|
|
return app(ReceiveInboundMessage::class)->handle(new PortalMail($id, $email, 'Re: Ihre Anfrage '.$booking->reference, $text, CarbonImmutable::now()))->fresh();
|
|
}
|
|
|
|
/**
|
|
* @return array<string, mixed>
|
|
*/
|
|
private function fact(string $field, string $value, string $evidence, float $confidence = 0.95): array
|
|
{
|
|
return ['field' => $field, 'value' => $value, 'evidence' => $evidence, 'confidence' => $confidence];
|
|
}
|
|
|
|
/**
|
|
* @param list<array<string, mixed>> $facts
|
|
* @param list<array<string, mixed>> $changes
|
|
* @return array<string, mixed>
|
|
*/
|
|
private function extraction(array $facts, array $changes = [], string $intent = 'answer'): array
|
|
{
|
|
return ['facts' => $facts, 'change_requests' => $changes, 'intent' => $intent, 'uncertainties' => []];
|
|
}
|
|
|
|
/**
|
|
* @param array<string, mixed> $small
|
|
* @param array<string, mixed>|null $review
|
|
*/
|
|
private function fakeModels(array $small, ?array $review = null): void
|
|
{
|
|
Http::fake(function (Request $request) use ($small, $review) {
|
|
$body = $request->data();
|
|
$this->calls[] = ['body' => $body, 'beta' => $request->header('anthropic-beta')[0] ?? null, 'version' => $request->header('anthropic-version')[0] ?? null];
|
|
$data = str_starts_with((string) $body['model'], 'claude-opus') ? ($review ?? $small) : $small;
|
|
|
|
return Http::response(['id' => 'msg', 'type' => 'message', 'role' => 'assistant', 'model' => $body['model'], 'content' => [['type' => 'text', 'text' => json_encode($data)]], 'stop_reason' => 'end_turn', 'usage' => ['input_tokens' => 10, 'output_tokens' => 10]]);
|
|
});
|
|
}
|
|
}
|