Migrate NPC and location normalization to native correction and explicit fallback handling
This commit is contained in:
@@ -3,7 +3,6 @@ package locationregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
@@ -18,8 +17,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
)
|
||||
|
||||
func TestModuleContractAndMetadata(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind}
|
||||
func TestModuleContractAndNativeDependencies(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -28,19 +27,14 @@ func TestModuleContractAndMetadata(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(nil, Options{}); err == nil {
|
||||
t.Fatal("New() accepted nil client")
|
||||
t.Fatal("New() accepted nil native engine")
|
||||
}
|
||||
normalizer := newNormalizer(t, &recordingLocationNormalizerClient{})
|
||||
metadata := normalizer.ManifestMetadata()
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
||||
if !hasFingerprint(normalizer.CheckpointFingerprints(), name) {
|
||||
t.Fatalf("fingerprints missing %q", name)
|
||||
}
|
||||
spec := ModuleSpec()
|
||||
spec.Prompts[0] = "mutated"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("static dependencies are not owned")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
|
||||
@@ -103,14 +97,14 @@ func TestNormalizeAppliesSafeAliasGroupAndUsesContextualInputs(t *testing.T) {
|
||||
{Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Locations) != 2 {
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Locations) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
merged := result.Value.Locations[0]
|
||||
if merged.Name != "the Greencloak's refuge" || merged.ID != identity.DeriveID(merged.Name, merged.SourceRefs) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateLocationCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("merged location = %#v, diagnostics = %#v", merged, result.Diagnostics)
|
||||
}
|
||||
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
||||
encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript"))
|
||||
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
||||
t.Fatalf("private inputs = %s", encoded)
|
||||
}
|
||||
@@ -124,16 +118,31 @@ func TestNormalizeRejectsUnsafeAndOverlappingGroupsWithoutLosingCandidates(t *te
|
||||
{Name: "Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Locations) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") {
|
||||
t.Fatalf("Normalize() = %#v, %v; want safe retry fallback", result, err)
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Locations) != 3 {
|
||||
t.Fatalf("safe partial fallback = %#v %v", result, err)
|
||||
}
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate groups 1, 2") || !strings.Contains(result.Retry.CorrectionGuidance, "at most one duplicate group") || strings.Contains(result.Retry.CorrectionGuidance, "overlapping_member") {
|
||||
t.Fatalf("correction guidance = %q, want model-facing overlap explanation", result.Retry.CorrectionGuidance)
|
||||
if len(client.requests) != 2 {
|
||||
t.Fatalf("native calls = %d, want initial/correction", len(client.requests))
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("fallback diagnostics = %#v", result.Retry.FallbackDiagnostics)
|
||||
messages := client.requests[1].Prompt.Messages
|
||||
guidance := messages[len(messages)-1].Content
|
||||
if !strings.Contains(guidance, "Duplicate groups 1, 2") || !strings.Contains(guidance, "at most one duplicate group") || strings.Contains(guidance, "overlapping_member") {
|
||||
t.Fatalf("model-facing overlap guidance = %q", guidance)
|
||||
}
|
||||
var warning string
|
||||
for _, diagnostic := range result.Diagnostics {
|
||||
if diagnostic.ReasonCode == ReasonCodeLocationSemanticReconciliationExhausted && len(diagnostic.Samples) > 0 {
|
||||
warning = diagnostic.Samples[0].Message
|
||||
}
|
||||
}
|
||||
if !strings.Contains(warning, "2 proposal group") || !strings.Contains(warning, "preserved separately") {
|
||||
t.Fatalf("fallback warning = %q", warning)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
@@ -156,7 +165,7 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil {
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"], &candidateInput); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Tavern" || candidateInput.Candidates[1].Label != "The Tavern" {
|
||||
@@ -164,39 +173,36 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
||||
client := &recordingLocationNormalizerClient{}
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A sprawling city"}}}
|
||||
func TestPreparationLimitUsesResolvedModeWithoutExecutionOrAssessment(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "speech", Text: "A crowded hall"}}}
|
||||
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
||||
input := dnd.LocationRegistry{Locations: make([]dnd.Location, limit+1)}
|
||||
for index := range input.Locations {
|
||||
input.Locations[index] = dnd.Location{Name: fmt.Sprintf("Place %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
for i := range input.Locations {
|
||||
input.Locations[i] = dnd.Location{Name: fmt.Sprintf("Fixture %d", i), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
input.Locations[i].ID = identity.DeriveID(input.Locations[i].Name, input.Locations[i].SourceRefs)
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(result.Value.Locations) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, locations = %d; want no call and all records", len(client.requests), len(result.Value.Locations))
|
||||
}
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeLocationSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHandlesRetryFallbackAndErrors(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.LocationRegistry{Locations: []dnd.Location{{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
invalid, err := newNormalizer(t, &recordingLocationNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeLocationSemanticProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", invalid.Retry)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingLocationNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
client := &recordingLocationNormalizerClient{}
|
||||
req := normalizeRequestWithSource(input, doc)
|
||||
req.Operation.Controls.Mode = mode
|
||||
assessed := 0
|
||||
req.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), req)
|
||||
if len(client.requests) != 0 || assessed != 0 {
|
||||
t.Fatal("unready preparation performed native execution or ordinary assessment")
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil {
|
||||
t.Fatal("strict preparation limit succeeded")
|
||||
}
|
||||
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||||
t.Fatalf("permissive preparation limit = %#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,12 +223,3 @@ func TestNormalizeOrdersEvidenceAndIsIdempotent(t *testing.T) {
|
||||
t.Fatalf("second Normalize() = %#v, %v; want idempotent value %#v", second, err, first.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user