package itemoccurrences import ( "context" "encoding/json" "errors" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape" sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs" relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness" "go.maximumdirect.net/promptkit" ) const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) { t.Helper() registry, err := registryvalidator.New(registryvalidator.Options{}, request.References) if err != nil { t.Fatal(err) } validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{ shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{}), registry, } var calls, guidance []string request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) { var assessment contracts.CandidateAssessment for _, validator := range validators { calls = append(calls, validator.Name()) result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{ Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value, }) if err != nil { return contracts.CandidateAssessment{}, err } decision := contracts.AssessmentRejected if result.Approved { decision = contracts.AssessmentApproved } assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) if result.CorrectionGuidance != "" { guidance = append(guidance, result.CorrectionGuidance) } } return assessment, nil } return &calls, &guidance } func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) { request := extractionRequest() request.References = itemRegistryReferences(t) request.Operation.Controls.MaxAdditionalTurns = 1 request.Operation.ReuseEligible = false request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"} var prepared []promptkit.PreparedRun request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) } calls, guidance := bindAssessment(t, &request) client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}} result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key} if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 { t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared)) } occurrence := result.Value.Occurrences[0] if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired || occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID { t.Fatalf("holder mapping = %#v", occurrence) } first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder { t.Fatal("native continuation did not preserve the exact defective response and fixed context") } correction := next[len(first)+1].Content for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} { if !strings.Contains(correction, fragment) { t.Fatalf("missing holder guidance %q", fragment) } } for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} { if strings.Contains(correction, forbidden) { t.Fatalf("operator identifier became guidance: %q", forbidden) } } for i, call := range client.requests { id := PromptID if i > 0 { id = CorrectionPromptID } if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID { t.Fatal("native profile/session/override lost") } if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` { t.Fatal("item registry was not fixed and names-only") } } content, err := occurrencecodec.New().Encode(result.Value) if err != nil { t.Fatal(err) } var wire struct { Occurrences []map[string]json.RawMessage `json:"occurrences"` } if err := json.Unmarshal(content, &wire); err != nil { t.Fatal(err) } if _, present := wire.Occurrences[0]["from"]; present { t.Fatal("private null holder became a durable field") } } func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) { failure := errors.New("provider unavailable") for _, test := range []struct { name string mode contracts.ExecutionMode response string failures []error assessments, runs int wantError bool }{ {"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true}, {"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false}, {"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true}, {"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false}, {"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, {"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false}, {"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false}, } { t.Run(test.name, func(t *testing.T) { request := extractionRequest() request.References = itemRegistryReferences(t) request.Operation.Controls.Mode = test.mode request.Operation.Controls.MaxAdditionalTurns = 1 calls, _ := bindAssessment(t, &request) runs := 0 request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ } client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures} result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs { t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs) } if test.wantError { if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 { t.Fatal("strict failure returned output") } return } if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { t.Fatalf("fallback invented an item or holder: %#v", result) } if _, err := occurrencecodec.New().Encode(result.Value); err != nil { t.Fatal(err) } }) } } func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() request := extractionRequest() request.References = itemRegistryReferences(t) request.Operation.Controls.Mode = contracts.ModePermissive client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} result, err := newExtractor(t, client, request.References).Extract(ctx, request) if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { t.Fatalf("canceled delivery = %#v %v", result, err) } } func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) { quantity := 3 client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{ {Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)}, {Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)}, {Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)}, {Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)}, {Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)}, }}} request := extractionRequest() request.References = itemRegistryReferences(t) bindAssessment(t, &request) result, err := newExtractor(t, client).Extract(context.Background(), request) if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 { t.Fatalf("kinds = %#v %v", result, err) } kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred} for i, occurrence := range result.Value.Occurrences { if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 { t.Fatalf("ordered occurrence = %#v", occurrence) } } quantity = 9 if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" { t.Fatal("holder or quantity interpretation changed") } if _, err := occurrencecodec.New().Encode(result.Value); err != nil { t.Fatal(err) } }