diff --git a/assets/dnd/location-registry/normalize/prompts/prompt.yaml b/assets/dnd/location-registry/normalize/prompts/prompt.yaml index 9f361245..be16fb0f 100644 --- a/assets/dnd/location-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/location-registry/normalize/prompts/prompt.yaml @@ -11,9 +11,9 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../../generic/normalize/deduplication/prompts/protocol.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -27,3 +27,17 @@ prompts: format: json validation_mode: json_schema schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json + + - id: dnd.location_registry.normalize.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ../../../../generic/normalize/deduplication/prompts/correction.md + output: + format: json + validation_mode: json_schema + schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json diff --git a/assets/dnd/npc-registry/normalize/prompts/prompt.yaml b/assets/dnd/npc-registry/normalize/prompts/prompt.yaml index 9ec3d8c1..73158bdb 100644 --- a/assets/dnd/npc-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/npc-registry/normalize/prompts/prompt.yaml @@ -11,9 +11,9 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../../generic/normalize/deduplication/prompts/protocol.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -27,3 +27,17 @@ prompts: format: json validation_mode: json_schema schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json + + - id: dnd.npc_registry.normalize.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ../../../../generic/normalize/deduplication/prompts/correction.md + output: + format: json + validation_mode: json_schema + schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json diff --git a/assets/package_test.go b/assets/package_test.go index fb5580e5..f17dc3a8 100644 --- a/assets/package_test.go +++ b/assets/package_test.go @@ -146,6 +146,58 @@ func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) { } } +func TestRegistryNormalizationPromptPairsRetainTheirNativeContracts(t *testing.T) { + _, engine := loadNativeAssets(t) + for _, test := range []struct { + id string + guidance string + rules []string + }{ + {"notarius/dnd.npc_registry.normalize", "Preserve distinct individuals.", []string{"same individual", "unadorned proper name"}}, + {"notarius/dnd.location_registry.normalize", "Keep parent and child places separate.", []string{"same physical place", "parent and child places"}}, + } { + t.Run(test.id, func(t *testing.T) { + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: test.id, APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{ + "candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`), + "transcript": promptkit.Inline(`{"windows":[]}`), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: test.id + ".repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.guidance)}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID || + first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("normalization pair lost its shared native defaults/output contract") + } + var system string + for _, message := range first.Messages { + if message.Role == "system" { + system += message.Content + } + } + for _, rule := range append(test.rules, "candidate_number_range", "start_unit_id", "canonical_candidate_number") { + if !strings.Contains(system, rule) { + t.Fatalf("missing enduring system principle %q", rule) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, test.guidance) { + t.Fatal("correction did not supply only the findings and replacement assignment") + } + }) + } +} + func TestEmbeddedProfileRetainsSelectedExecutionDefaults(t *testing.T) { t.Parallel() _, engine := loadNativeAssets(t) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6f165a28..57ec42dd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -673,6 +673,27 @@ preservation, unchanged-candidate reassessment, and cancellation. ## Stage 15: Migrate NPC And Location Registry Normalization +**Status:** Complete. Both normalizers now use the shared native producer helper +and fixed one-time reconciliation preparation. Each response applies its own safe +groups afresh to the original records, before typed operation assessment. Added +same-schema/default-profile correction prompts and static dependencies; removed +retry transport, schema/policy fingerprints, and manifest-only asset registration. +Name-based NPC identity, evidence-anchored location identity, canonical naming, +source ordering, and domain application rules remain unchanged. Strict handling +does not silently fall back; permissive output uses only the final safe candidate +or the supplied merged artifact, with codec/identity/source construction checks. +Retained domain regressions and added native provider coverage for correction, +unchanged reassessment, final-only safe groups, failed continuation without prior +proposal recovery, structural failure, overrides/history, cancellation, and +unusable fallbacks. Native prompt-pair checks run in the content package without +depending on legacy module helpers. +Asset/reconciliation tests, vet/race checks, focused pipeline producer/runner +tests, production framework builds, and `git diff --check` pass. Both normalizer +package tests remain blocked at setup by shared legacy asset helpers importing +old ConfigKit's removed YAML `plugin/limit`; their new tests are not claimed +passing. Scheduled dependency cleanup and final whole-tree validation must run +them and close that blocker. + **Goal:** apply the proven normalizer path without domain-policy drift. **Inspect/change:** `internal/modules/dnd/normalize/{npcregistry,locationregistry}` diff --git a/internal/modules/dnd/normalize/locationregistry/native_execution_test.go b/internal/modules/dnd/normalize/locationregistry/native_execution_test.go new file mode 100644 index 00000000..2824a337 --- /dev/null +++ b/internal/modules/dnd/normalize/locationregistry/native_execution_test.go @@ -0,0 +1,199 @@ +package locationregistry + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" + "go.maximumdirect.net/promptkit" +) + +const safeProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}` +const unknownHandleProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + +func mergedRegistry() dnd.LocationRegistry { + doc := semanticDocument() + result := dnd.LocationRegistry{Locations: []dnd.Location{ + {Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Greencloak Refuge", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + for i := range result.Locations { + result.Locations[i].ID = identity.DeriveID(result.Locations[i].Name, result.Locations[i].SourceRefs) + } + return result +} + +func TestNativeCorrectionPreservesFixedHistoryAndAssessesEachRevision(t *testing.T) { + finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + rejectFirst, wantError, wantFallback bool + names []string + }{ + {"corrected", contracts.ModeStrict, []string{unknownHandleProposal, safeProposal}, false, false, false, []string{"Greencloak Refuge", "The Tavern"}}, + {"unchanged strict rejection", contracts.ModeStrict, []string{unknownHandleProposal, unknownHandleProposal}, false, true, false, nil}, + {"only final safe groups", contracts.ModePermissive, []string{safeProposal, finalPartial}, true, false, true, []string{"Old Mill", "The Tavern"}}, + } { + t.Run(test.name, func(t *testing.T) { + input := mergedRegistry() + before := cloneRegistry(input) + client := &recordingLocationNormalizerClient{responses: test.responses} + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + request.SessionID = "normalization-session" + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "normalization-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + var assessed []dnd.LocationRegistry + request.Operation.Assess = func(_ context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessed = append(assessed, cloneRegistry(value)) + if test.rejectFirst && len(assessed) == 1 { + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "domain-policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the proposed grouping."}, + }}}, nil + } + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if (err != nil) != test.wantError || len(assessed) != 2 || len(client.requests) != 2 || len(prepared) != 2 { + t.Fatalf("result=%#v error=%v assessments/provider/prepared=%d/%d/%d", result, err, len(assessed), len(client.requests), len(prepared)) + } + if !reflect.DeepEqual(input, before) { + t.Fatal("normalization mutated fixed originals") + } + if test.name == "unchanged strict rejection" && !reflect.DeepEqual(assessed[0], assessed[1]) { + t.Fatal("fixture did not assess an unchanged candidate") + } + for i, call := range client.requests { + wantID := PromptID + if i > 0 { + wantID = CorrectionPromptID + } + if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "normalization-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native request selection/overrides lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || + revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != test.responses[0] { + t.Fatal("continuation changed original evidence or lost the prior response") + } + if strings.Contains(revised[len(initial)+1].Content, "operator-only-rejection") { + t.Fatal("operator text became correction guidance") + } + if test.wantError { + return + } + names := make([]string, len(result.Value.Locations)) + for i, record := range result.Value.Locations { + names[i] = record.Name + } + if !reflect.DeepEqual(names, test.names) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("unexpected final output: %#v", result) + } + if test.wantFallback { + refs := result.Value.Locations[1].SourceRefs + if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 { + t.Fatalf("earlier proposal evidence leaked into final output: %#v", refs) + } + } + }) + } +} + +func TestNativeFailureCannotRecoverEarlierProposal(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + response string + providerErrors []error + wantAssessments int + }{ + {"initial provider failure", "", []error{failure}, 0}, + {"failed correction", safeProposal, []error{nil, failure}, 1}, + {"structural failure", `{"duplicate_groups":null}`, nil, 0}, + } { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(test.name+"/"+string(mode), func(t *testing.T) { + input := mergedRegistry() + client := &recordingLocationNormalizerClient{response: test.response, errors: test.providerErrors} + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct records."}, + }}}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if assessments != test.wantAssessments { + t.Fatalf("assessments=%d, want %d", assessments, test.wantAssessments) + } + if test.providerErrors != nil && len(client.requests) != len(test.providerErrors) { + t.Fatalf("provider restarted: %d calls", len(client.requests)) + } + if mode == contracts.ModeStrict { + if err == nil || result.Outcome.Disposition != "" { + t.Fatalf("strict failure=%#v %v", result, err) + } + if test.providerErrors != nil && !errors.Is(err, failure) { + t.Fatalf("lost native cause: %v", err) + } + return + } + if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible { + t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err) + } + for _, diagnostic := range result.Diagnostics { + if diagnostic.ReasonCode == ReasonCodeDuplicateLocationCollapsed { + t.Fatal("discarded proposal left collapse diagnostics") + } + } + }) + } + } +} + +func TestCancellationDoesNotDeliverFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &recordingLocationNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := normalizeRequestWithSource(mergedRegistry(), semanticDocument()) + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newNormalizer(t, client).Normalize(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} + +func TestFallbackRequiresUsableMergedIdentityAndSource(t *testing.T) { + for _, invalidIdentity := range []bool{false, true} { + input := mergedRegistry() + if invalidIdentity { + input.Locations[0].ID = "wrong" + } else { + input.Locations[0].SourceRefs[0].SourceID = "foreign-source" + } + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = contracts.ModePermissive + failure := errors.New("provider unavailable") + result, err := newNormalizer(t, &recordingLocationNormalizerClient{err: failure}).Normalize(context.Background(), request) + if !errors.Is(err, failure) || result.Outcome.Disposition != "" { + t.Fatalf("unusable fallback=%#v %v", result, err) + } + } +} diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer.go b/internal/modules/dnd/normalize/locationregistry/normalizer.go index 13708e49..a4fb989c 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer.go @@ -15,23 +15,22 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + "go.maximumdirect.net/promptkit" ) const ( - Key = "dnd/location-registry" - PromptID = "dnd.location_registry.normalize" - PromptVersion = "v1" - normalizationPolicy = "dnd.location_registry.normalize.v4" - NormalizationPolicy = normalizationPolicy + Key = "dnd/location-registry" + PromptID = "notarius/dnd.location_registry.normalize" + CorrectionPromptID = "notarius/dnd.location_registry.normalize.repair" ReasonCodeLocationFieldsNormalized = "location_fields_normalized" ReasonCodeLocationIDRecomputed = "location_id_recomputed" ReasonCodeSourceReferencesNormalized = "source_references_normalized" ReasonCodeDuplicateLocationCollapsed = "duplicate_location_collapsed" - ReasonCodeLocationSemanticProposalInvalid = "location_semantic_proposal_invalid" ReasonCodeLocationSemanticReconciliationExhausted = "location_semantic_reconciliation_exhausted" ) @@ -39,28 +38,16 @@ var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.LocationRegistry] = (*Normalizer)(nil) -var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} type Normalizer struct { - engine *semanticreconcile.Engine + engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) { - if llmClient == nil { - return nil, normalizerErrorf("LLM client must not be nil") - } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, normalizerErrorf("load prompt metadata: %w", err) - } - engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{ - ID: PromptID, Version: PromptVersion, SHA256: promptSHA, - }, semanticreconcile.DefaultLimits()) - if err != nil { - return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err) +func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) { + if engine == nil { + return nil, normalizerErrorf("native engine must not be nil") } return &Normalizer{engine: engine}, nil } @@ -68,33 +55,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (n *Normalizer) ManifestMetadata() map[string]any { - if n == nil || n.engine == nil { - return nil - } - metadata := n.engine.ManifestMetadata() - metadata["identity_policy"] = identity.Policy - metadata["normalization_policy"] = normalizationPolicy - return metadata -} - -func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if n == nil || n.engine == nil { - return nil - } - fingerprints := n.engine.CheckpointFingerprints() - return append(fingerprints, - pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy}, - pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy}, - ) -} - func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.LocationRegistry]) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { if n == nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalizer must not be nil") } if n.engine == nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil") + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("native engine must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context must not be nil") @@ -106,68 +72,139 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("source document must not be nil") } + // Keep the already available merged artifact separate from preprocessed + // originals. A failed native response must not recover an earlier proposal. + merged := cloneRegistry(req.MergeOutput.Value) order := shared.NewSourceRefOrder(req.Source) - records, findings := preprocessRecords(req.MergeOutput.Value, order) + records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) - if len(records) < 2 { - return normalizationResult(deterministic, findings, nil) - } + findings := append([]diagnostics.Finding(nil), baseFindings...) + discardedGroups := -1 + var omissionDetails []string + fallback := func(_ context.Context, final *dnd.LocationRegistry) (dnd.LocationRegistry, error) { + value := cloneRegistry(merged) + if final != nil { + // Each interpreted candidate was built afresh from fixed original records, + // using only the final proposal's compatible, non-overlapping safe groups. + value = cloneRegistry(*final) + } else { + findings = nil + discardedGroups = -1 + omissionDetails = nil + } + if err := validateFallback(value, req.Source); err != nil { + return dnd.LocationRegistry{}, err + } + return value, nil + } + finish := func(value dnd.LocationRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { + if err != nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalize locations: %w", err) + } + result, err := normalizationResult(value, findings) + if err != nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err + } + result.Outcome = outcome + if outcome.Disposition == contracts.ProducerFallback { + details := append([]string(nil), omissionDetails...) + details = append(details, outcome.FallbackReason) + groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) + if err != nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) + } + result.Diagnostics = append(result.Diagnostics, groups...) + } + return result, nil + } + noModel := func() (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { + value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) + } + if len(records) < 2 { + return noModel() + } candidates, envelopes, err := reconciliationInputs(records) if err != nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err) } - reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{ - StageName: Key, Source: req.Source, Candidates: candidates, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction, - }) + preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits()) if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err) + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } - - switch reconciliation.Disposition() { - case semanticreconcile.SkippedInsufficientCandidates: - return normalizationResult(deterministic, findings, nil) - case semanticreconcile.SkippedLimitExceeded: - return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits")) - case semanticreconcile.RetryableInvalidStructuredOutput: - return n.invalidStructuredResult(deterministic, findings) - case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups: - default: - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition()) + switch preparation.Disposition() { + case semanticreconcile.InsufficientCandidates: + return noModel() + case semanticreconcile.LimitExceeded: + value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) } - - applied, semanticFindings, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order) + initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID) if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err) + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build native request: %w", err) } - findings = append(findings, semanticFindings...) - if reconciliation.Disposition() == semanticreconcile.Complete { - return normalizationResult(recordList(applied), findings, reconciliation.ModelCandidate()) - } - return retryResult(recordList(applied), findings, reconciliation) + value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) { + response, err := semanticreconcile.DecodeProposal(result.RawOutput) + if err != nil { + return nil, nil, err + } + assessment := preparation.Assess(response) + applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) + if err != nil { + return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) + } + findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...) + discardedGroups = assessment.DiscardedGroupCount() + omissionDetails = semanticreconcile.IssueDetails(assessment.Issues()) + guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange()) + if err != nil { + return nil, nil, err + } + candidate := recordList(applied) + return &candidate, guidance, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil + }, fallback) + return finish(value, outcome, err) } -func (n *Normalizer) invalidStructuredResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { - return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract")) +// validateFallback checks the durable artifact and the identity/source invariants +// used by registry consumers, not the configured ordinary validator chain. +func validateFallback(value dnd.LocationRegistry, document *source.SourceDocument) error { + if _, err := registrycodec.New().Encode(value); err != nil { + return normalizerErrorf("construct locations fallback: %w", err) + } + if issues := identity.ValidateList(value); len(issues) > 0 { + return normalizerErrorf("construct locations fallback: registry identity is not usable downstream") + } + index := source.NewDocumentIndex(document) + for _, location := range value.Locations { + for _, reference := range location.SourceRefs { + if err := index.ValidateRef(reference); err != nil { + return normalizerErrorf("construct locations fallback: %w", err) + } + } + } + return nil } -func retryResult(value dnd.LocationRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { - issues := reconciliation.Issues() - correctionDetails, err := reconciliation.CorrectionDetails() - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction details: %w", err) +func cloneRegistry(value dnd.LocationRegistry) dnd.LocationRegistry { + if value.Locations == nil { + return dnd.LocationRegistry{} } - correctionGuidance, err := reconciliation.CorrectionGuidance() - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err) + result := dnd.LocationRegistry{Locations: make([]dnd.Location, len(value.Locations))} + for i, location := range value.Locations { + result.Locations[i] = cloneLocation(location) } - return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...)) + return result } func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding { - message := "semantic reconciliation could not be completed; all affected candidates were preserved separately" - if discarded >= 0 { + message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output" + if discarded > 0 { message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded) } if len(details) > 0 { @@ -176,38 +213,12 @@ func semanticFallbackFinding(discarded int, details ...string) diagnostics.Findi return diagnostics.Finding{Scope: "locations", ReasonCode: ReasonCodeLocationSemanticReconciliationExhausted, Message: message} } -func normalizationResult(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { - diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) +func normalizationResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { + groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) if err != nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err) } - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil -} - -func fallbackResult(value dnd.LocationRegistry, findings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { - result, err := normalizationResult(value, findings, nil) - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Diagnostics = append(result.Diagnostics, fallbackGroups...) - return result, nil -} - -func retryResultWithFallback(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) { - result, err := normalizationResult(value, findings, candidate) - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups} - return result, nil + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Diagnostics: groups}, nil } type normalizedRecord struct { @@ -373,7 +384,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding { func locationScope(index int) string { return fmt.Sprintf("locations[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { - return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind} + return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind} } func Register(registry *pipeline.NormalizerRegistry) error { return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.LocationRegistry], error) { @@ -381,7 +392,7 @@ func Register(registry *pipeline.NormalizerRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options) + return New(request.Dependencies.Engine, options) }) } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer_test.go b/internal/modules/dnd/normalize/locationregistry/normalizer_test.go index 470bf7b3..ef812141 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer_test.go @@ -3,7 +3,6 @@ package locationregistry import ( "context" "encoding/json" - "errors" "fmt" "reflect" "strconv" @@ -18,8 +17,8 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity" ) -func TestModuleContractAndMetadata(t *testing.T) { - want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind} +func TestModuleContractAndNativeDependencies(t *testing.T) { + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } @@ -28,19 +27,14 @@ func TestModuleContractAndMetadata(t *testing.T) { t.Fatal(err) } if _, err := New(nil, Options{}); err == nil { - t.Fatal("New() accepted nil client") + t.Fatal("New() accepted nil native engine") } - normalizer := newNormalizer(t, &recordingLocationNormalizerClient{}) - metadata := normalizer.ManifestMetadata() - limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any) - if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 { - t.Fatalf("metadata = %#v", metadata) - } - for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} { - if !hasFingerprint(normalizer.CheckpointFingerprints(), name) { - t.Fatalf("fingerprints missing %q", name) - } + spec := ModuleSpec() + spec.Prompts[0] = "mutated" + if !reflect.DeepEqual(ModuleSpec(), want) { + t.Fatal("static dependencies are not owned") } + } func TestNormalizeRejectsNilSourceDocument(t *testing.T) { @@ -103,14 +97,14 @@ func TestNormalizeAppliesSafeAliasGroupAndUsesContextualInputs(t *testing.T) { {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry != nil || len(result.Value.Locations) != 2 { + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Locations) != 2 { t.Fatalf("Normalize() = %#v, %v", result, err) } merged := result.Value.Locations[0] if merged.Name != "the Greencloak's refuge" || merged.ID != identity.DeriveID(merged.Name, merged.SourceRefs) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateLocationCollapsed, contracts.DiagnosticDispositionObservation) { t.Fatalf("merged location = %#v, diagnostics = %#v", merged, result.Diagnostics) } - encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content) + encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript")) if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) { t.Fatalf("private inputs = %s", encoded) } @@ -124,16 +118,31 @@ func TestNormalizeRejectsUnsafeAndOverlappingGroupsWithoutLosingCandidates(t *te {Name: "Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3}]}`} - result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil || len(result.Value.Locations) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") { - t.Fatalf("Normalize() = %#v, %v; want safe retry fallback", result, err) + request := normalizeRequestWithSource(input, doc) + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 1 + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Locations) != 3 { + t.Fatalf("safe partial fallback = %#v %v", result, err) } - if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate groups 1, 2") || !strings.Contains(result.Retry.CorrectionGuidance, "at most one duplicate group") || strings.Contains(result.Retry.CorrectionGuidance, "overlapping_member") { - t.Fatalf("correction guidance = %q, want model-facing overlap explanation", result.Retry.CorrectionGuidance) + if len(client.requests) != 2 { + t.Fatalf("native calls = %d, want initial/correction", len(client.requests)) } - if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") { - t.Fatalf("fallback diagnostics = %#v", result.Retry.FallbackDiagnostics) + messages := client.requests[1].Prompt.Messages + guidance := messages[len(messages)-1].Content + if !strings.Contains(guidance, "Duplicate groups 1, 2") || !strings.Contains(guidance, "at most one duplicate group") || strings.Contains(guidance, "overlapping_member") { + t.Fatalf("model-facing overlap guidance = %q", guidance) } + var warning string + for _, diagnostic := range result.Diagnostics { + if diagnostic.ReasonCode == ReasonCodeLocationSemanticReconciliationExhausted && len(diagnostic.Samples) > 0 { + warning = diagnostic.Samples[0].Message + } + } + if !strings.Contains(warning, "2 proposal group") || !strings.Contains(warning, "preserved separately") { + t.Fatalf("fallback warning = %q", warning) + } + } func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) { @@ -156,7 +165,7 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) { Label string `json:"label"` } `json:"candidates"` } - if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil { + if err := json.Unmarshal(preparation.Materials()["candidates"], &candidateInput); err != nil { t.Fatal(err) } if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Tavern" || candidateInput.Candidates[1].Label != "The Tavern" { @@ -164,39 +173,36 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) { } } -func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) { - client := &recordingLocationNormalizerClient{} - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A sprawling city"}}} +func TestPreparationLimitUsesResolvedModeWithoutExecutionOrAssessment(t *testing.T) { + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "speech", Text: "A crowded hall"}}} limit := semanticreconcile.DefaultLimits().MaximumCandidates input := dnd.LocationRegistry{Locations: make([]dnd.Location, limit+1)} - for index := range input.Locations { - input.Locations[index] = dnd.Location{Name: fmt.Sprintf("Place %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + for i := range input.Locations { + input.Locations[i] = dnd.Location{Name: fmt.Sprintf("Fixture %d", i), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + input.Locations[i].ID = identity.DeriveID(input.Locations[i].Name, input.Locations[i].SourceRefs) } - result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry != nil { - t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err) - } - if len(client.requests) != 0 || len(result.Value.Locations) != limit+1 { - t.Fatalf("completion calls = %d, locations = %d; want no call and all records", len(client.requests), len(result.Value.Locations)) - } - if !hasDiagnostic(result.Diagnostics, ReasonCodeLocationSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) { - t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics) - } -} - -func TestNormalizeHandlesRetryFallbackAndErrors(t *testing.T) { - doc := semanticDocument() - input := dnd.LocationRegistry{Locations: []dnd.Location{{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}} - invalid, err := newNormalizer(t, &recordingLocationNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeLocationSemanticProposalInvalid { - t.Fatalf("invalid result = %#v, %v", invalid, err) - } - if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") { - t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", invalid.Retry) - } - _, err = newNormalizer(t, &recordingLocationNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err == nil || !strings.Contains(err.Error(), "provider unavailable") { - t.Fatalf("provider error = %v", err) + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(string(mode), func(t *testing.T) { + client := &recordingLocationNormalizerClient{} + req := normalizeRequestWithSource(input, doc) + req.Operation.Controls.Mode = mode + assessed := 0 + req.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), req) + if len(client.requests) != 0 || assessed != 0 { + t.Fatal("unready preparation performed native execution or ordinary assessment") + } + if mode == contracts.ModeStrict { + if err == nil { + t.Fatal("strict preparation limit succeeded") + } + } else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) { + t.Fatalf("permissive preparation limit = %#v %v", result, err) + } + }) } } @@ -217,12 +223,3 @@ func TestNormalizeOrdersEvidenceAndIsIdempotent(t *testing.T) { t.Fatalf("second Normalize() = %#v, %v; want idempotent value %#v", second, err, first.Value) } } - -func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool { - for _, fingerprint := range fingerprints { - if fingerprint.Name == name && fingerprint.Value != "" { - return true - } - } - return false -} diff --git a/internal/modules/dnd/normalize/locationregistry/prompt_assets.go b/internal/modules/dnd/normalize/locationregistry/prompt_assets.go deleted file mode 100644 index da11b2c4..00000000 --- a/internal/modules/dnd/normalize/locationregistry/prompt_assets.go +++ /dev/null @@ -1,78 +0,0 @@ -package locationregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -func promptAssetManifest() (shared.PromptAssetManifest, error) { - sharedFiles, err := semanticreconcile.SharedPromptFiles() - if err != nil { - return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err) - } - return shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{"common-dnd-system.md"}, - ExternalSharedFiles: sharedFiles, - }, nil -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/location-registry/normalize") - if err != nil { - return nil, fmt.Errorf("scope location normalization assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - manifest, err := promptAssetManifest() - if err != nil { - return err - } - promptFS, err := manifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare location normalization prompt assets: %w", err) - } - return registry.RegisterPromptFS(promptFS, promptAssetRoot) -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - manifest, err := promptAssetManifest() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = manifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/normalize/locationregistry/prompt_assets_test.go b/internal/modules/dnd/normalize/locationregistry/prompt_assets_test.go deleted file mode 100644 index 54eb53e2..00000000 --- a/internal/modules/dnd/normalize/locationregistry/prompt_assets_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package locationregistry - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesLocationNormalizationPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := semanticreconcile.RegisterAssets(registry); err != nil { - t.Fatal(err) - } - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "location-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "location-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"The Tavern","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}}) - if err != nil { - t.Fatal(err) - } - if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "same physical place") || !strings.Contains(prepared.Messages[2].Content, "parent and child places") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") { - t.Fatalf("prepared prompt = %#v", prepared) - } - for _, index := range []int{2, 4} { - if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != promptkit.CacheControlEphemeral { - t.Fatalf("message %d cache = %#v", index, prepared.Messages[index].CacheControl) - } - } - if !strings.Contains(prepared.Messages[3].Content, `"The Tavern"`) || strings.Contains(prepared.Messages[3].Content, `"candidate-`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) { - t.Fatalf("candidate message = %q", prepared.Messages[3].Content) - } - if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"The Tavern"`) { - t.Fatalf("transcript message = %q", prepared.Messages[4].Content) - } - for index, message := range prepared.Messages { - if index != 3 && strings.Contains(message.Content, `"The Tavern"`) { - t.Errorf("message %d unexpectedly rendered candidate input", index) - } - if index != 4 && strings.Contains(message.Content, `"windows"`) { - t.Errorf("message %d unexpectedly rendered transcript input", index) - } - } -} diff --git a/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go b/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go index 398cff07..d02acd46 100644 --- a/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go +++ b/internal/modules/dnd/normalize/locationregistry/test_helpers_test.go @@ -3,54 +3,110 @@ package locationregistry import ( "context" "encoding/json" - "fmt" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) type recordingLocationNormalizerClient struct { - response string - err error - requests []contracts.StructuredCompletionRequest + response string + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (c *recordingLocationNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { - snapshot, err := contracts.CloneStructuredCompletionRequest(request) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err) +func (c *recordingLocationNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(c.requests) + c.requests = append(c.requests, request) + if c.onGenerate != nil { + c.onGenerate(ctx, request) } - c.requests = append(c.requests, snapshot) if c.err != nil { - return contracts.StructuredCompletionResponse{}, c.err + return nil, c.err + } + if index < len(c.errors) && c.errors[index] != nil { + return nil, c.errors[index] } response := c.response + if index < len(c.responses) { + response = c.responses[index] + } if response == "" { response = `{"duplicate_groups":[]}` } - content := []byte(response) - if err := json.Unmarshal(content, output); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: response}, nil } -func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer { +func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer { t.Helper() - normalizer, err := New(client, Options{}) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit: + profiles: + - id: location-normalize-test + endpoint: https://example.invalid/v1 + model: normalization-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + normalizer, err := New(engine, Options{}) + if err != nil { + t.Fatal(err) } return normalizer } + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := "candidate_number_range" + if name == "transcript" { + key = "windows" + } + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} func normalizeRequest(value dnd.LocationRegistry) contracts.TypedNormalizeRequest[dnd.LocationRegistry] { return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{ - Source: &source.SourceDocument{}, + Source: &source.SourceDocument{}, LLMProfile: "location-normalize-test", + Operation: contracts.ProducerOperation[dnd.LocationRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value}, } } + func normalizeRequestWithSource(value dnd.LocationRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.LocationRegistry] { request := normalizeRequest(value) request.Source = doc diff --git a/internal/modules/dnd/normalize/npcregistry/native_execution_test.go b/internal/modules/dnd/normalize/npcregistry/native_execution_test.go new file mode 100644 index 00000000..f8dce9e9 --- /dev/null +++ b/internal/modules/dnd/normalize/npcregistry/native_execution_test.go @@ -0,0 +1,199 @@ +package npcregistry + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "go.maximumdirect.net/promptkit" +) + +const safeProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}` +const unknownHandleProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + +func mergedRegistry() dnd.NPCRegistry { + doc := semanticDocument() + result := dnd.NPCRegistry{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + for i := range result.NPCs { + result.NPCs[i].ID = identity.DeriveID(result.NPCs[i].Name) + } + return result +} + +func TestNativeCorrectionPreservesFixedHistoryAndAssessesEachRevision(t *testing.T) { + finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + rejectFirst, wantError, wantFallback bool + names []string + }{ + {"corrected", contracts.ModeStrict, []string{unknownHandleProposal, safeProposal}, false, false, false, []string{"Mira Thorn", "Captain Vale"}}, + {"unchanged strict rejection", contracts.ModeStrict, []string{unknownHandleProposal, unknownHandleProposal}, false, true, false, nil}, + {"only final safe groups", contracts.ModePermissive, []string{safeProposal, finalPartial}, true, false, true, []string{"Mira", "Captain Vale"}}, + } { + t.Run(test.name, func(t *testing.T) { + input := mergedRegistry() + before := cloneRegistry(input) + client := &recordingNPCNormalizerClient{responses: test.responses} + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + request.SessionID = "normalization-session" + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "normalization-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + var assessed []dnd.NPCRegistry + request.Operation.Assess = func(_ context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessed = append(assessed, cloneRegistry(value)) + if test.rejectFirst && len(assessed) == 1 { + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "domain-policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the proposed grouping."}, + }}}, nil + } + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if (err != nil) != test.wantError || len(assessed) != 2 || len(client.requests) != 2 || len(prepared) != 2 { + t.Fatalf("result=%#v error=%v assessments/provider/prepared=%d/%d/%d", result, err, len(assessed), len(client.requests), len(prepared)) + } + if !reflect.DeepEqual(input, before) { + t.Fatal("normalization mutated fixed originals") + } + if test.name == "unchanged strict rejection" && !reflect.DeepEqual(assessed[0], assessed[1]) { + t.Fatal("fixture did not assess an unchanged candidate") + } + for i, call := range client.requests { + wantID := PromptID + if i > 0 { + wantID = CorrectionPromptID + } + if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "normalization-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native request selection/overrides lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || + revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != test.responses[0] { + t.Fatal("continuation changed original evidence or lost the prior response") + } + if strings.Contains(revised[len(initial)+1].Content, "operator-only-rejection") { + t.Fatal("operator text became correction guidance") + } + if test.wantError { + return + } + names := make([]string, len(result.Value.NPCs)) + for i, record := range result.Value.NPCs { + names[i] = record.Name + } + if !reflect.DeepEqual(names, test.names) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("unexpected final output: %#v", result) + } + if test.wantFallback { + refs := result.Value.NPCs[1].SourceRefs + if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 { + t.Fatalf("earlier proposal evidence leaked into final output: %#v", refs) + } + } + }) + } +} + +func TestNativeFailureCannotRecoverEarlierProposal(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + response string + providerErrors []error + wantAssessments int + }{ + {"initial provider failure", "", []error{failure}, 0}, + {"failed correction", safeProposal, []error{nil, failure}, 1}, + {"structural failure", `{"duplicate_groups":null}`, nil, 0}, + } { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(test.name+"/"+string(mode), func(t *testing.T) { + input := mergedRegistry() + client := &recordingNPCNormalizerClient{response: test.response, errors: test.providerErrors} + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ + ValidatorName: "policy", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct records."}, + }}}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if assessments != test.wantAssessments { + t.Fatalf("assessments=%d, want %d", assessments, test.wantAssessments) + } + if test.providerErrors != nil && len(client.requests) != len(test.providerErrors) { + t.Fatalf("provider restarted: %d calls", len(client.requests)) + } + if mode == contracts.ModeStrict { + if err == nil || result.Outcome.Disposition != "" { + t.Fatalf("strict failure=%#v %v", result, err) + } + if test.providerErrors != nil && !errors.Is(err, failure) { + t.Fatalf("lost native cause: %v", err) + } + return + } + if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible { + t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err) + } + for _, diagnostic := range result.Diagnostics { + if diagnostic.ReasonCode == ReasonCodeDuplicateNPCCollapsed { + t.Fatal("discarded proposal left collapse diagnostics") + } + } + }) + } + } +} + +func TestCancellationDoesNotDeliverFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &recordingNPCNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + request := normalizeRequestWithSource(mergedRegistry(), semanticDocument()) + request.Operation.Controls.Mode = contracts.ModePermissive + result, err := newNormalizer(t, client).Normalize(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} + +func TestFallbackRequiresUsableMergedIdentityAndSource(t *testing.T) { + for _, invalidIdentity := range []bool{false, true} { + input := mergedRegistry() + if invalidIdentity { + input.NPCs[0].ID = "wrong" + } else { + input.NPCs[0].SourceRefs[0].SourceID = "foreign-source" + } + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = contracts.ModePermissive + failure := errors.New("provider unavailable") + result, err := newNormalizer(t, &recordingNPCNormalizerClient{err: failure}).Normalize(context.Background(), request) + if !errors.Is(err, failure) || result.Outcome.Disposition != "" { + t.Fatalf("unusable fallback=%#v %v", result, err) + } + } +} diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer.go b/internal/modules/dnd/normalize/npcregistry/normalizer.go index f5bbd8ab..e08bce64 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer.go @@ -14,23 +14,22 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + "go.maximumdirect.net/promptkit" ) const ( - Key = "dnd/npc-registry" - PromptID = "dnd.npc_registry.normalize" - PromptVersion = "v1" - normalizationPolicy = "dnd.npc_registry.normalize.v6" - NormalizationPolicy = normalizationPolicy + Key = "dnd/npc-registry" + PromptID = "notarius/dnd.npc_registry.normalize" + CorrectionPromptID = "notarius/dnd.npc_registry.normalize.repair" ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" ReasonCodeNPCIDRecomputed = "npc_id_recomputed" ReasonCodeSourceReferencesNormalized = "source_references_normalized" ReasonCodeDuplicateNPCCollapsed = "duplicate_npc_collapsed" - ReasonCodeNPCSemanticProposalInvalid = "npc_semantic_proposal_invalid" ReasonCodeNPCSemanticReconciliationExhausted = "npc_semantic_reconciliation_exhausted" ) @@ -38,28 +37,16 @@ var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.NPCRegistry] = (*Normalizer)(nil) -var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} type Normalizer struct { - engine *semanticreconcile.Engine + engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) { - if llmClient == nil { - return nil, normalizerErrorf("LLM client must not be nil") - } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, normalizerErrorf("load prompt metadata: %w", err) - } - engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{ - ID: PromptID, Version: PromptVersion, SHA256: promptSHA, - }, semanticreconcile.DefaultLimits()) - if err != nil { - return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err) +func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) { + if engine == nil { + return nil, normalizerErrorf("native engine must not be nil") } return &Normalizer{engine: engine}, nil } @@ -67,33 +54,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (n *Normalizer) ManifestMetadata() map[string]any { - if n == nil || n.engine == nil { - return nil - } - metadata := n.engine.ManifestMetadata() - metadata["identity_policy"] = identity.Policy - metadata["normalization_policy"] = normalizationPolicy - return metadata -} - -func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if n == nil || n.engine == nil { - return nil - } - fingerprints := n.engine.CheckpointFingerprints() - return append(fingerprints, - pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy}, - pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy}, - ) -} - func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.NPCRegistry]) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { if n == nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("normalizer must not be nil") } if n.engine == nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil") + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("native engine must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("context must not be nil") @@ -105,69 +71,140 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("source document must not be nil") } + // Keep the already available merged artifact separate from preprocessed + // originals. A failed native response must not recover an earlier proposal. + merged := cloneRegistry(req.MergeOutput.Value) order := shared.NewSourceRefOrder(req.Source) - records, findings := preprocessRecords(req.MergeOutput.Value, order) + records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) - if len(records) < 2 { - return normalizationResult(deterministic, findings, nil) - } + findings := append([]diagnostics.Finding(nil), baseFindings...) + discardedGroups := -1 + var omissionDetails []string + fallback := func(_ context.Context, final *dnd.NPCRegistry) (dnd.NPCRegistry, error) { + value := cloneRegistry(merged) + if final != nil { + // Each interpreted candidate was built afresh from fixed original records, + // using only the final proposal's compatible, non-overlapping safe groups. + value = cloneRegistry(*final) + } else { + findings = nil + discardedGroups = -1 + omissionDetails = nil + } + if err := validateFallback(value, req.Source); err != nil { + return dnd.NPCRegistry{}, err + } + return value, nil + } + finish := func(value dnd.NPCRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { + if err != nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("normalize NPCs: %w", err) + } + result, err := normalizationResult(value, findings) + if err != nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err + } + result.Outcome = outcome + if outcome.Disposition == contracts.ProducerFallback { + details := append([]string(nil), omissionDetails...) + details = append(details, outcome.FallbackReason) + groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) + if err != nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) + } + result.Diagnostics = append(result.Diagnostics, groups...) + } + return result, nil + } + noModel := func() (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { + value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.NPCRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) + } + if len(records) < 2 { + return noModel() + } candidates, envelopes, err := reconciliationInputs(records) if err != nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err) } - reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{ - StageName: Key, Source: req.Source, Candidates: candidates, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction, - }) + preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits()) if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err) + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } - - switch reconciliation.Disposition() { - case semanticreconcile.SkippedInsufficientCandidates: - return normalizationResult(deterministic, findings, nil) - case semanticreconcile.SkippedLimitExceeded: - return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits")) - case semanticreconcile.RetryableInvalidStructuredOutput: - return n.invalidStructuredResult(deterministic, findings) - case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups: - default: - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition()) + switch preparation.Disposition() { + case semanticreconcile.InsufficientCandidates: + return noModel() + case semanticreconcile.LimitExceeded: + value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.NPCRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) } - - applied, semanticFindings, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order) + initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID) if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err) + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build native request: %w", err) } - findings = append(findings, semanticFindings...) - if reconciliation.Disposition() == semanticreconcile.Complete { - return normalizationResult(recordList(applied), findings, reconciliation.ModelCandidate()) - } - return retryResult(recordList(applied), findings, reconciliation) + value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCRegistry, []string, error) { + response, err := semanticreconcile.DecodeProposal(result.RawOutput) + if err != nil { + return nil, nil, err + } + assessment := preparation.Assess(response) + applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) + if err != nil { + return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) + } + findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...) + discardedGroups = assessment.DiscardedGroupCount() + omissionDetails = semanticreconcile.IssueDetails(assessment.Issues()) + guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange()) + if err != nil { + return nil, nil, err + } + candidate := recordList(applied) + return &candidate, guidance, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil + }, fallback) + return finish(value, outcome, err) } -func (n *Normalizer) invalidStructuredResult(value dnd.NPCRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { - return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract")) +// validateFallback checks the durable artifact and the identity/source invariants +// used by registry consumers, not the configured ordinary validator chain. +func validateFallback(value dnd.NPCRegistry, document *source.SourceDocument) error { + if _, err := registrycodec.New().Encode(value); err != nil { + return normalizerErrorf("construct NPCs fallback: %w", err) + } + if issues := identity.ValidateList(value); len(issues) > 0 { + return normalizerErrorf("construct NPCs fallback: registry identity is not usable downstream") + } + index := source.NewDocumentIndex(document) + for _, npc := range value.NPCs { + for _, reference := range npc.SourceRefs { + if err := index.ValidateRef(reference); err != nil { + return normalizerErrorf("construct NPCs fallback: %w", err) + } + } + } + return nil } -func retryResult(value dnd.NPCRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { - issues := reconciliation.Issues() - correctionDetails, err := reconciliation.CorrectionDetails() - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction details: %w", err) +func cloneRegistry(value dnd.NPCRegistry) dnd.NPCRegistry { + if value.NPCs == nil { + return dnd.NPCRegistry{} } - correctionGuidance, err := reconciliation.CorrectionGuidance() - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err) + result := dnd.NPCRegistry{NPCs: make([]dnd.NPC, len(value.NPCs))} + for i, npc := range value.NPCs { + result.NPCs[i] = cloneNPC(npc) } - return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...)) + return result } -func semanticFallbackFinding(discardedGroups int, details ...string) diagnostics.Finding { - message := "semantic reconciliation could not be completed; all affected candidates were preserved separately" - if discardedGroups >= 0 { - message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discardedGroups) +func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding { + message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output" + if discarded > 0 { + message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded) } if len(details) > 0 { message = diagnostics.Aggregate(message, details) @@ -175,38 +212,12 @@ func semanticFallbackFinding(discardedGroups int, details ...string) diagnostics return diagnostics.Finding{Scope: "npcs", ReasonCode: ReasonCodeNPCSemanticReconciliationExhausted, Message: message} } -func normalizationResult(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { - diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) +func normalizationResult(value dnd.NPCRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { + groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) if err != nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err) } - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil -} - -func fallbackResult(value dnd.NPCRegistry, findings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { - result, err := normalizationResult(value, findings, nil) - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Diagnostics = append(result.Diagnostics, fallbackGroups...) - return result, nil -} - -func retryResultWithFallback(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) { - result, err := normalizationResult(value, findings, candidate) - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups} - return result, nil + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: value, Diagnostics: groups}, nil } type normalizedRecord struct { @@ -372,7 +383,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding { func npcScope(index int) string { return fmt.Sprintf("npcs[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { - return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.NPCRegistryKind} + return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.NPCRegistryKind} } func Register(registry *pipeline.NormalizerRegistry) error { @@ -381,7 +392,7 @@ func Register(registry *pipeline.NormalizerRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options) + return New(request.Dependencies.Engine, options) }) } diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer_test.go b/internal/modules/dnd/normalize/npcregistry/normalizer_test.go index a1c05265..20ebf073 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer_test.go @@ -3,15 +3,17 @@ package npcregistry import ( "context" "encoding/json" - "fmt" "reflect" "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" ) @@ -23,7 +25,7 @@ func TestModuleContractAndIdentity(t *testing.T) { if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown option") } - want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.NPCRegistryKind} + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.NPCRegistryKind} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } @@ -32,20 +34,14 @@ func TestModuleContractAndIdentity(t *testing.T) { t.Fatalf("Register() error = %v", err) } if _, err := New(nil, Options{}); err == nil { - t.Fatal("New(nil, Options{}) error = nil, want nil client rejection") + t.Fatal("New(nil, Options{}) error = nil, want nil native engine rejection") } - normalizer := newNormalizer(t, &recordingNPCNormalizerClient{}) - metadata := normalizer.ManifestMetadata() - limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any) - if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 { - t.Fatalf("metadata = %#v", metadata) - } - fingerprints := normalizer.CheckpointFingerprints() - for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} { - if !hasFingerprint(fingerprints, name) { - t.Fatalf("fingerprints = %#v, want %q", fingerprints, name) - } + spec := ModuleSpec() + spec.Prompts[0] = "mutated" + if !reflect.DeepEqual(ModuleSpec(), want) { + t.Fatal("static dependencies are not owned") } + } func TestNormalizeNamesEvidenceAndIDs(t *testing.T) { @@ -142,51 +138,93 @@ func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) { } type recordingNPCNormalizerClient struct { - response string - responses []string - err error - requests []contracts.StructuredCompletionRequest + response string + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (c *recordingNPCNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { - snapshot, err := contracts.CloneStructuredCompletionRequest(request) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err) +func (c *recordingNPCNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(c.requests) + c.requests = append(c.requests, request) + if c.onGenerate != nil { + c.onGenerate(ctx, request) } - c.requests = append(c.requests, snapshot) if c.err != nil { - return contracts.StructuredCompletionResponse{}, c.err + return nil, c.err + } + if index < len(c.errors) && c.errors[index] != nil { + return nil, c.errors[index] } response := c.response - if len(c.responses) > 0 { - responseIndex := len(c.requests) - 1 - if responseIndex >= len(c.responses) { - responseIndex = len(c.responses) - 1 - } - response = c.responses[responseIndex] + if index < len(c.responses) { + response = c.responses[index] } if response == "" { response = `{"duplicate_groups":[]}` } - content := []byte(response) - if err := json.Unmarshal(content, output); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: response}, nil } -func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer { +func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer { t.Helper() - normalizer, err := New(client, Options{}) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit: + profiles: + - id: normalizer-profile + endpoint: https://example.invalid/v1 + model: normalization-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + normalizer, err := New(engine, Options{}) + if err != nil { + t.Fatal(err) } return normalizer } +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := "candidate_number_range" + if name == "transcript" { + key = "windows" + } + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} func normalizeRequest(value dnd.NPCRegistry) contracts.TypedNormalizeRequest[dnd.NPCRegistry] { return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{ - Source: &source.SourceDocument{}, + Source: &source.SourceDocument{}, LLMProfile: "normalizer-profile", + Operation: contracts.ProducerOperation[dnd.NPCRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value}, } } @@ -210,12 +248,3 @@ func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason, scope str } return false } - -func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool { - for _, fingerprint := range fingerprints { - if fingerprint.Name == name && fingerprint.Value != "" { - return true - } - } - return false -} diff --git a/internal/modules/dnd/normalize/npcregistry/prompt_assets.go b/internal/modules/dnd/normalize/npcregistry/prompt_assets.go deleted file mode 100644 index 75c5505f..00000000 --- a/internal/modules/dnd/normalize/npcregistry/prompt_assets.go +++ /dev/null @@ -1,78 +0,0 @@ -package npcregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -func promptAssetManifest() (shared.PromptAssetManifest, error) { - sharedFiles, err := semanticreconcile.SharedPromptFiles() - if err != nil { - return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err) - } - return shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{"common-dnd-system.md"}, - ExternalSharedFiles: sharedFiles, - }, nil -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/npc-registry/normalize") - if err != nil { - return nil, fmt.Errorf("scope NPC normalization assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - manifest, err := promptAssetManifest() - if err != nil { - return err - } - promptFS, err := manifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare NPC normalization prompt assets: %w", err) - } - return registry.RegisterPromptFS(promptFS, promptAssetRoot) -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - manifest, err := promptAssetManifest() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = manifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/normalize/npcregistry/prompt_assets_test.go b/internal/modules/dnd/normalize/npcregistry/prompt_assets_test.go deleted file mode 100644 index d0babc16..00000000 --- a/internal/modules/dnd/normalize/npcregistry/prompt_assets_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package npcregistry - -import ( - "context" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) { - if promptHash, err := promptAssetMetadata(); err != nil || promptHash == "" { - t.Fatalf("promptAssetMetadata() = %q, %v; want prompt fingerprint", promptHash, err) - } - registry := llm.NewAssetRegistry() - if err := semanticreconcile.RegisterAssets(registry); err != nil { - t.Fatalf("RegisterAssets() error = %v", err) - } - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "normalize-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "normalize-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "normalize-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Mira","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), - "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want normalization prompt identity and schema", prepared) - } - if prepared.Messages[0].Role != "system" { - t.Fatalf("initial message role = %q, want system", prepared.Messages[0].Role) - } - if !strings.Contains(prepared.Messages[1].Content, "candidate_number") || !strings.Contains(prepared.Messages[1].Content, "integer") { - t.Fatalf("protocol message = %q, want shared integer-handle protocol", prepared.Messages[1].Content) - } - if !strings.Contains(prepared.Messages[2].Content, "same individual") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || strings.Contains(prepared.Messages[2].Content, "source ranges") { - t.Fatalf("NPC policy message = %q, want domain distinctions without copied ranges", prepared.Messages[2].Content) - } - for _, index := range []int{2, 4} { - if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral { - t.Errorf("message %d cache control = %#v, want ephemeral", index, cache) - } - } - for _, index := range []int{0, 1, 3} { - if cache := prepared.Messages[index].CacheControl; cache != nil { - t.Errorf("message %d cache control = %#v, want nil", index, cache) - } - } - if !strings.Contains(prepared.Messages[3].Content, `"Mira"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) { - t.Fatalf("candidate message = %q, want only rendered candidates", prepared.Messages[3].Content) - } - if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Mira"`) { - t.Fatalf("transcript message = %q, want only rendered transcript", prepared.Messages[4].Content) - } - for index, message := range prepared.Messages { - if index != 3 && strings.Contains(message.Content, `"Mira"`) { - t.Errorf("message %d unexpectedly rendered candidate input", index) - } - if index != 4 && strings.Contains(message.Content, `"windows"`) { - t.Errorf("message %d unexpectedly rendered transcript input", index) - } - } -} diff --git a/internal/modules/dnd/normalize/npcregistry/semantic_normalizer_test.go b/internal/modules/dnd/normalize/npcregistry/semantic_normalizer_test.go index 5b66e985..8ee81f61 100644 --- a/internal/modules/dnd/normalize/npcregistry/semantic_normalizer_test.go +++ b/internal/modules/dnd/normalize/npcregistry/semantic_normalizer_test.go @@ -3,13 +3,14 @@ package npcregistry import ( "context" "encoding/json" - "errors" "fmt" "math" "reflect" "strings" "testing" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" @@ -23,12 +24,10 @@ func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing doc := semanticDocument() input := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}}} result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || len(client.requests) != 0 || result.Retry != nil { + if err != nil || len(client.requests) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted { t.Fatalf("Normalize() = %#v, %v; calls = %d, want deterministic no-call result", result, err, len(client.requests)) } - if result.ModelCandidate != nil { - t.Fatalf("model candidate = %#v, want nil for no-call result", result.ModelCandidate) - } + if result.Value.NPCs[0].Name != "Mira Thorn" { t.Fatalf("NPCs = %#v, want deterministic record", result.Value.NPCs) } @@ -47,13 +46,10 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { request := normalizeRequestWithSource(input, doc) request.LLMProfile = "normalizer-profile" request.SessionID = "normalizer-session" - correction, err := contracts.NewSemanticCorrection([]byte(`{"duplicate_groups":[]}`), "keep the distinct captain") - if err != nil { - t.Fatal(err) - } - request.Correction = correction + var prepared promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value } result, err := normalizer.Normalize(context.Background(), request) - if err != nil || result.Retry != nil { + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted { t.Fatalf("Normalize() = %#v, %v; want accepted semantic result", result, err) } if !reflect.DeepEqual(input, before) { @@ -73,16 +69,10 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { t.Fatalf("completion calls = %d, want one", len(client.requests)) } completion := client.requests[0] - if !reflect.DeepEqual(completion.Correction, correction) { - t.Fatalf("completion correction = %#v, want %#v", completion.Correction, correction) + if prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completion.Prompt.SessionID != request.SessionID { + t.Fatal("native request routing lost") } - if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != client.response { - t.Fatalf("model candidate = %#v, want exact semantic response", result.ModelCandidate) - } - if completion.StageName != Key || completion.PromptID != PromptID || completion.PromptVersion != PromptVersion || completion.ProfileID != request.LLMProfile || completion.SessionID != request.SessionID || len(completion.Inputs) != 2 { - t.Fatalf("completion request = %#v, want normalize request identity and exactly two inputs", completion) - } - encoded := string(completion.Inputs["candidates"].Content) + string(completion.Inputs["transcript"].Content) + encoded := string(renderedInput(t, completion, "candidates")) + string(renderedInput(t, completion, "transcript")) if strings.Contains(encoded, "npc:sha256:") || strings.Contains(encoded, doc.ID) { t.Fatalf("completion inputs leaked private identifiers: %s", encoded) } @@ -91,7 +81,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { CandidateNumber int `json:"candidate_number"` } `json:"candidates"` } - if err := json.Unmarshal(completion.Inputs["candidates"].Content, &visible); err != nil { + if err := json.Unmarshal(renderedInput(t, completion, "candidates"), &visible); err != nil { t.Fatalf("decode candidates: %v", err) } if got := []int{visible.Candidates[0].CandidateNumber, visible.Candidates[1].CandidateNumber, visible.Candidates[2].CandidateNumber}; !reflect.DeepEqual(got, []int{1, 2, 3}) { @@ -99,7 +89,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { } } -func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) { +func TestNormalizeUnsafeProposalReturnsQualifiedSafePartial(t *testing.T) { client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`} normalizer := newNormalizer(t, client) doc := semanticDocument() @@ -108,41 +98,38 @@ func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) { {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} - result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil { - t.Fatalf("Normalize() = %#v, %v; want retryable safe fallback", result, err) + request := normalizeRequestWithSource(input, doc) + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 1 + result, err := normalizer.Normalize(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible { + t.Fatalf("qualified fallback = %#v %v", result, err) } - if result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || !strings.Contains(result.Retry.Message, "group 1: member_unknown") { - t.Fatalf("retry = %#v, want bounded invalid-proposal diagnostics", result.Retry) + if len(client.requests) != 2 { + t.Fatalf("native calls=%d, want initial/correction", len(client.requests)) } - if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate group 2") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 3") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeNPCSemanticProposalInvalid) { - t.Fatalf("correction guidance = %q, want contextual prose without internal labels", result.Retry.CorrectionGuidance) + messages := client.requests[1].Prompt.Messages + guidance := messages[len(messages)-1].Content + for _, expected := range []string{"Duplicate group 2", "candidate-number range 1 through 3", "source positions"} { + if !strings.Contains(guidance, expected) { + t.Fatalf("missing correction detail %q", expected) + } } + if strings.Contains(guidance, "member_unknown") || strings.Contains(guidance, "npc_semantic_proposal_invalid") { + t.Fatal("internal labels copied into correction") + } + if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Mira Thorn" || result.Value.NPCs[1].Name != "Captain Vale" { t.Fatalf("fallback NPCs = %#v, want independently safe group applied", result.Value.NPCs) } - if len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeNPCSemanticReconciliationExhausted || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "1 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "candidate-number range 1 through 3") { - t.Fatalf("fallback diagnostics = %#v, want exact omitted-group warning", result.Retry.FallbackDiagnostics) + var warning string + for _, diagnostic := range result.Diagnostics { + if diagnostic.ReasonCode == ReasonCodeNPCSemanticReconciliationExhausted && len(diagnostic.Samples) > 0 { + warning = diagnostic.Samples[0].Message + } } -} - -func TestNormalizeInvalidStructuredOutputAndOperationalErrorsRemainDistinct(t *testing.T) { - doc := semanticDocument() - input := dnd.NPCRegistry{NPCs: []dnd.NPC{ - {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, - {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, - }} - invalid := newNormalizer(t, &recordingNPCNormalizerClient{err: contracts.ErrInvalidStructuredOutput}) - result, err := invalid.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil || result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || len(result.Value.NPCs) != 2 { - t.Fatalf("invalid structured result = %#v, %v; want deterministic retry fallback", result, err) - } - if result.Retry.CorrectionGuidance != "" || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") { - t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", result.Retry) - } - operational := errors.New("provider unavailable") - if _, err := newNormalizer(t, &recordingNPCNormalizerClient{err: operational}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)); !errors.Is(err, operational) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { - t.Fatalf("operational completion error = %v, want ordinary error", err) + if !strings.Contains(warning, "1 proposal group") || !strings.Contains(warning, "preserved separately") { + t.Fatalf("fallback warning = %q", warning) } } @@ -182,7 +169,7 @@ func TestNormalizeRedactsContextMaterialFailures(t *testing.T) { } } -func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) { +func TestNormalizeDoesNotAccumulateSafeGroupsAcrossNativeRevisions(t *testing.T) { client := &recordingNPCNormalizerClient{responses: []string{ `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`, `{"duplicate_groups":[{"candidate_numbers":[1,3],"canonical_candidate_number":3}]}`, @@ -194,20 +181,27 @@ func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) { {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} - first, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || first.Retry == nil || len(first.Value.NPCs) != 2 { - t.Fatalf("first Normalize() = %#v, %v; want partial retry fallback", first, err) + request := normalizeRequestWithSource(input, doc) + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{}, nil } - second, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || second.Retry != nil { - t.Fatalf("second Normalize() = %#v, %v; want accepted independent retry result", second, err) + result, err := normalizer.Normalize(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(client.requests) != 2 || assessments != 2 { + t.Fatalf("native correction = %#v %v, assessments %d", result, err, assessments) } - if got := []string{second.Value.NPCs[0].Name, second.Value.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"Captain Vale", "Mira Thorn"}) { - t.Fatalf("second NPCs = %#v, want proposal applied to original merge output", second.Value.NPCs) + if got := []string{result.Value.NPCs[0].Name, result.Value.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"Captain Vale", "Mira Thorn"}) { + t.Fatalf("revised NPCs = %#v", result.Value.NPCs) } + if refs := result.Value.NPCs[0].SourceRefs; len(refs) != 2 || refs[0].StartUnitID != 10 || refs[1].StartUnitID != 30 { + t.Fatalf("earlier proposal evidence accumulated: %#v", refs) + } + } -func TestNormalizeRetainsRecordsWhenAProposalUsesAnUnknownHandle(t *testing.T) { +func TestNormalizeRetainsInvalidRecordsForAssessmentAfterAnUnknownHandle(t *testing.T) { client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`} normalizer := newNormalizer(t, client) doc := semanticDocument() @@ -216,13 +210,17 @@ func TestNormalizeRetainsRecordsWhenAProposalUsesAnUnknownHandle(t *testing.T) { {Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 999, EndUnitID: 999}}}, {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} - result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil || len(result.Value.NPCs) != 3 { - t.Fatalf("Normalize() = %#v, %v; want unchanged retry fallback", result, err) + request := normalizeRequestWithSource(input, doc) + var assessed dnd.NPCRegistry + request.Operation.Assess = func(_ context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessed = cloneRegistry(value) + return contracts.CandidateAssessment{}, nil } - if !strings.Contains(result.Retry.Message, "member_unknown") || result.Value.NPCs[1].Name != "Broken" { - t.Fatalf("result = %#v, want unknown descriptor rejected without dropping a record", result) + result, err := normalizer.Normalize(context.Background(), request) + if err == nil || result.Outcome.Disposition != "" || len(assessed.NPCs) != 3 || assessed.NPCs[1].Name != "Broken" || assessed.NPCs[1].SourceRefs[0].StartUnitID != 999 { + t.Fatalf("candidate was dropped before assessment: %#v %v", assessed, err) } + } func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testing.T) { @@ -245,7 +243,7 @@ func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testi Label string `json:"label"` } `json:"candidates"` } - if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil { + if err := json.Unmarshal(preparation.Materials()["candidates"], &candidateInput); err != nil { t.Fatal(err) } if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Guard" || candidateInput.Candidates[1].Label != "The Guard" { @@ -253,24 +251,36 @@ func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testi } } -func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) { - client := &recordingNPCNormalizerClient{} - normalizer := newNormalizer(t, client) +func TestPreparationLimitUsesResolvedModeWithoutExecutionOrAssessment(t *testing.T) { doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "speech", Text: "A crowded hall"}}} limit := semanticreconcile.DefaultLimits().MaximumCandidates input := dnd.NPCRegistry{NPCs: make([]dnd.NPC, limit+1)} - for index := range input.NPCs { - input.NPCs[index] = dnd.NPC{Name: fmt.Sprintf("Person %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + for i := range input.NPCs { + input.NPCs[i] = dnd.NPC{Name: fmt.Sprintf("Fixture %d", i), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + input.NPCs[i].ID = identity.DeriveID(input.NPCs[i].Name) } - result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry != nil { - t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err) - } - if len(client.requests) != 0 || len(result.Value.NPCs) != limit+1 { - t.Fatalf("completion calls = %d, NPCs = %d; want no call and all records", len(client.requests), len(result.Value.NPCs)) - } - if !hasDiagnostic(result.Diagnostics, ReasonCodeNPCSemanticReconciliationExhausted, "npcs", contracts.DiagnosticDispositionWarning) { - t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics) + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(string(mode), func(t *testing.T) { + client := &recordingNPCNormalizerClient{} + req := normalizeRequestWithSource(input, doc) + req.Operation.Controls.Mode = mode + assessed := 0 + req.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), req) + if len(client.requests) != 0 || assessed != 0 { + t.Fatal("unready preparation performed native execution or ordinary assessment") + } + if mode == contracts.ModeStrict { + if err == nil { + t.Fatal("strict preparation limit succeeded") + } + } else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) { + t.Fatalf("permissive preparation limit = %#v %v", result, err) + } + }) } }