package itemregistry import ( "context" "errors" "reflect" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape" refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_refs" relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/source_relatedness" "go.maximumdirect.net/promptkit" ) const ungroundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` const groundedRegistry = `{"items":[{"name":"Star Compass","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` const emptyRegistry = `{"items":[]}` func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.ItemRegistry], value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { assessment := contracts.CandidateAssessment{} for _, validator := range []contracts.TypedValidator[dnd.ItemRegistry]{ shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}), } { result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) if err != nil { return contracts.CandidateAssessment{}, err } decision := contracts.AssessmentRejected if result.Approved { decision = contracts.AssessmentApproved } assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) } return assessment, nil } func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) { client := &fakeItemsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}} request := extractionRequest() request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}} request.Operation.Controls.MaxAdditionalTurns = 1 request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"} var prepared []promptkit.PreparedRun request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } assessments := 0 request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { assessments++ assessment, err := assessRegistry(ctx, request, value) if assessments == 1 { if assessment.Accepted() { t.Fatal("unresolvable citation was accepted") } // Changes after capture must not cause source/reference preparation again. request.SourceInput.Content[0] = '!' request.References.Slots["glossary"].Items[0].Content[0] = '!' } return assessment, err } result, err := newExtractor(t, client).Extract(context.Background(), request) if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Items) != 1 { t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared)) } record := result.Value.Items[0] if record.Name != "Star Compass" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID || record.SourceRefs[0].StartUnitID != 2 || record.SourceRefs[0].EndUnitID != 2 || record.ID != identity.DeriveID(record.Name) { t.Fatalf("canonical record identity/evidence=%#v", record) } if _, err := registrycodec.New().Encode(result.Value); err != nil { t.Fatalf("durable registry encoding: %v", err) } for i, call := range client.requests { id := PromptID if i > 0 { id = CorrectionPromptID } if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID { t.Fatal("native selection/overrides lost") } } initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry { t.Fatal("native correction lost fixed inputs or prior response") } messageContaining(t, client.requests[1], "fixed-reference") messageContaining(t, client.requests[1], string(request.Chunk.Content)) } func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) { failure := errors.New("provider unavailable") for _, test := range []struct { name string mode contracts.ExecutionMode responses []string providerErrors []error wantAssessments, wantRuns int wantError, wantFallback bool }{ {"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false}, {"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false}, {"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true}, {"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true}, {"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true}, {"structural failure", contracts.ModePermissive, []string{`{"items":null}`}, nil, 0, 1, false, true}, } { t.Run(test.name, func(t *testing.T) { client := &fakeItemsLLMClient{responses: test.responses, errors: test.providerErrors} if test.name == "structural failure" { client.content = []byte(test.responses[0]) } request := extractionRequest() request.Operation.Controls.Mode = test.mode request.Operation.Controls.MaxAdditionalTurns = 1 assessments, prepared := 0, 0 request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { assessments++ return assessRegistry(ctx, request, value) } result, err := newExtractor(t, client).Extract(context.Background(), request) if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns { t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared) } if test.wantError { if result.Outcome.Disposition != "" { t.Fatal("strict rejection produced an outcome") } return } if result.Value.Items == nil || len(result.Value.Items) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback { t.Fatalf("unqualified empty registry=%#v", result) } if _, err := registrycodec.New().Encode(result.Value); err != nil { t.Fatalf("empty durable registry invalid: %v", err) } }) } } func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := &fakeItemsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} request := extractionRequest() request.Operation.Controls.Mode = contracts.ModePermissive result, err := newExtractor(t, client).Extract(ctx, request) if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { t.Fatalf("cancellation=%#v %v", result, err) } }