Migrate NPC and location normalization to native correction and explicit fallback handling
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const safeProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`
|
||||
const unknownHandleProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
|
||||
func mergedRegistry() dnd.LocationRegistry {
|
||||
doc := semanticDocument()
|
||||
result := dnd.LocationRegistry{Locations: []dnd.Location{
|
||||
{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Greencloak Refuge", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
for i := range result.Locations {
|
||||
result.Locations[i].ID = identity.DeriveID(result.Locations[i].Name, result.Locations[i].SourceRefs)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestNativeCorrectionPreservesFixedHistoryAndAssessesEachRevision(t *testing.T) {
|
||||
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
rejectFirst, wantError, wantFallback bool
|
||||
names []string
|
||||
}{
|
||||
{"corrected", contracts.ModeStrict, []string{unknownHandleProposal, safeProposal}, false, false, false, []string{"Greencloak Refuge", "The Tavern"}},
|
||||
{"unchanged strict rejection", contracts.ModeStrict, []string{unknownHandleProposal, unknownHandleProposal}, false, true, false, nil},
|
||||
{"only final safe groups", contracts.ModePermissive, []string{safeProposal, finalPartial}, true, false, true, []string{"Old Mill", "The Tavern"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
input := mergedRegistry()
|
||||
before := cloneRegistry(input)
|
||||
client := &recordingLocationNormalizerClient{responses: test.responses}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.SessionID = "normalization-session"
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "normalization-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
var assessed []dnd.LocationRegistry
|
||||
request.Operation.Assess = func(_ context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed = append(assessed, cloneRegistry(value))
|
||||
if test.rejectFirst && len(assessed) == 1 {
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "domain-policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the proposed grouping."},
|
||||
}}}, nil
|
||||
}
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if (err != nil) != test.wantError || len(assessed) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
|
||||
t.Fatalf("result=%#v error=%v assessments/provider/prepared=%d/%d/%d", result, err, len(assessed), len(client.requests), len(prepared))
|
||||
}
|
||||
if !reflect.DeepEqual(input, before) {
|
||||
t.Fatal("normalization mutated fixed originals")
|
||||
}
|
||||
if test.name == "unchanged strict rejection" && !reflect.DeepEqual(assessed[0], assessed[1]) {
|
||||
t.Fatal("fixture did not assess an unchanged candidate")
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
wantID := PromptID
|
||||
if i > 0 {
|
||||
wantID = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "normalization-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native request selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != test.responses[0] {
|
||||
t.Fatal("continuation changed original evidence or lost the prior response")
|
||||
}
|
||||
if strings.Contains(revised[len(initial)+1].Content, "operator-only-rejection") {
|
||||
t.Fatal("operator text became correction guidance")
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
names := make([]string, len(result.Value.Locations))
|
||||
for i, record := range result.Value.Locations {
|
||||
names[i] = record.Name
|
||||
}
|
||||
if !reflect.DeepEqual(names, test.names) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unexpected final output: %#v", result)
|
||||
}
|
||||
if test.wantFallback {
|
||||
refs := result.Value.Locations[1].SourceRefs
|
||||
if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 {
|
||||
t.Fatalf("earlier proposal evidence leaked into final output: %#v", refs)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeFailureCannotRecoverEarlierProposal(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response string
|
||||
providerErrors []error
|
||||
wantAssessments int
|
||||
}{
|
||||
{"initial provider failure", "", []error{failure}, 0},
|
||||
{"failed correction", safeProposal, []error{nil, failure}, 1},
|
||||
{"structural failure", `{"duplicate_groups":null}`, nil, 0},
|
||||
} {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(test.name+"/"+string(mode), func(t *testing.T) {
|
||||
input := mergedRegistry()
|
||||
client := &recordingLocationNormalizerClient{response: test.response, errors: test.providerErrors}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct records."},
|
||||
}}}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if assessments != test.wantAssessments {
|
||||
t.Fatalf("assessments=%d, want %d", assessments, test.wantAssessments)
|
||||
}
|
||||
if test.providerErrors != nil && len(client.requests) != len(test.providerErrors) {
|
||||
t.Fatalf("provider restarted: %d calls", len(client.requests))
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("strict failure=%#v %v", result, err)
|
||||
}
|
||||
if test.providerErrors != nil && !errors.Is(err, failure) {
|
||||
t.Fatalf("lost native cause: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||||
t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err)
|
||||
}
|
||||
for _, diagnostic := range result.Diagnostics {
|
||||
if diagnostic.ReasonCode == ReasonCodeDuplicateLocationCollapsed {
|
||||
t.Fatal("discarded proposal left collapse diagnostics")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationDoesNotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &recordingLocationNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := normalizeRequestWithSource(mergedRegistry(), semanticDocument())
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newNormalizer(t, client).Normalize(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFallbackRequiresUsableMergedIdentityAndSource(t *testing.T) {
|
||||
for _, invalidIdentity := range []bool{false, true} {
|
||||
input := mergedRegistry()
|
||||
if invalidIdentity {
|
||||
input.Locations[0].ID = "wrong"
|
||||
} else {
|
||||
input.Locations[0].SourceRefs[0].SourceID = "foreign-source"
|
||||
}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
failure := errors.New("provider unavailable")
|
||||
result, err := newNormalizer(t, &recordingLocationNormalizerClient{err: failure}).Normalize(context.Background(), request)
|
||||
if !errors.Is(err, failure) || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,23 +15,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/location-registry"
|
||||
PromptID = "dnd.location_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.location_registry.normalize.v4"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
Key = "dnd/location-registry"
|
||||
PromptID = "notarius/dnd.location_registry.normalize"
|
||||
CorrectionPromptID = "notarius/dnd.location_registry.normalize.repair"
|
||||
|
||||
ReasonCodeLocationFieldsNormalized = "location_fields_normalized"
|
||||
ReasonCodeLocationIDRecomputed = "location_id_recomputed"
|
||||
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
||||
ReasonCodeDuplicateLocationCollapsed = "duplicate_location_collapsed"
|
||||
ReasonCodeLocationSemanticProposalInvalid = "location_semantic_proposal_invalid"
|
||||
ReasonCodeLocationSemanticReconciliationExhausted = "location_semantic_reconciliation_exhausted"
|
||||
)
|
||||
|
||||
@@ -39,28 +38,16 @@ var requiredCapabilities = []string{"merged"}
|
||||
var providedCapabilities = []string{"normalized"}
|
||||
|
||||
var _ contracts.Normalizer[dnd.LocationRegistry] = (*Normalizer)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Normalizer struct {
|
||||
engine *semanticreconcile.Engine
|
||||
engine *promptkit.Engine
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) {
|
||||
if llmClient == nil {
|
||||
return nil, normalizerErrorf("LLM client must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{
|
||||
ID: PromptID, Version: PromptVersion, SHA256: promptSHA,
|
||||
}, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err)
|
||||
func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) {
|
||||
if engine == nil {
|
||||
return nil, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
return &Normalizer{engine: engine}, nil
|
||||
}
|
||||
@@ -68,33 +55,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error
|
||||
func (n *Normalizer) Key() string { return Key }
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := n.engine.ManifestMetadata()
|
||||
metadata["identity_policy"] = identity.Policy
|
||||
metadata["normalization_policy"] = normalizationPolicy
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
fingerprints := n.engine.CheckpointFingerprints()
|
||||
return append(fingerprints,
|
||||
pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy},
|
||||
pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy},
|
||||
)
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.LocationRegistry]) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
if n == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if n.engine == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil")
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context must not be nil")
|
||||
@@ -106,68 +72,139 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("source document must not be nil")
|
||||
}
|
||||
|
||||
// Keep the already available merged artifact separate from preprocessed
|
||||
// originals. A failed native response must not recover an earlier proposal.
|
||||
merged := cloneRegistry(req.MergeOutput.Value)
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, findings := preprocessRecords(req.MergeOutput.Value, order)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if len(records) < 2 {
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
discardedGroups := -1
|
||||
var omissionDetails []string
|
||||
|
||||
fallback := func(_ context.Context, final *dnd.LocationRegistry) (dnd.LocationRegistry, error) {
|
||||
value := cloneRegistry(merged)
|
||||
if final != nil {
|
||||
// Each interpreted candidate was built afresh from fixed original records,
|
||||
// using only the final proposal's compatible, non-overlapping safe groups.
|
||||
value = cloneRegistry(*final)
|
||||
} else {
|
||||
findings = nil
|
||||
discardedGroups = -1
|
||||
omissionDetails = nil
|
||||
}
|
||||
if err := validateFallback(value, req.Source); err != nil {
|
||||
return dnd.LocationRegistry{}, err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
finish := func(value dnd.LocationRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalize locations: %w", err)
|
||||
}
|
||||
result, err := normalizationResult(value, findings)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
}
|
||||
result.Outcome = outcome
|
||||
if outcome.Disposition == contracts.ProducerFallback {
|
||||
details := append([]string(nil), omissionDetails...)
|
||||
details = append(details, outcome.FallbackReason)
|
||||
groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, groups...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
noModel := func() (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
if len(records) < 2 {
|
||||
return noModel()
|
||||
}
|
||||
candidates, envelopes, err := reconciliationInputs(records)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err)
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction,
|
||||
})
|
||||
preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
|
||||
switch reconciliation.Disposition() {
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
default:
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition())
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
case semanticreconcile.LimitExceeded:
|
||||
value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
applied, semanticFindings, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
|
||||
initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build native request: %w", err)
|
||||
}
|
||||
findings = append(findings, semanticFindings...)
|
||||
if reconciliation.Disposition() == semanticreconcile.Complete {
|
||||
return normalizationResult(recordList(applied), findings, reconciliation.ModelCandidate())
|
||||
}
|
||||
return retryResult(recordList(applied), findings, reconciliation)
|
||||
value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) {
|
||||
response, err := semanticreconcile.DecodeProposal(result.RawOutput)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
}
|
||||
findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...)
|
||||
discardedGroups = assessment.DiscardedGroupCount()
|
||||
omissionDetails = semanticreconcile.IssueDetails(assessment.Issues())
|
||||
guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
candidate := recordList(applied)
|
||||
return &candidate, guidance, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil
|
||||
}, fallback)
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
// validateFallback checks the durable artifact and the identity/source invariants
|
||||
// used by registry consumers, not the configured ordinary validator chain.
|
||||
func validateFallback(value dnd.LocationRegistry, document *source.SourceDocument) error {
|
||||
if _, err := registrycodec.New().Encode(value); err != nil {
|
||||
return normalizerErrorf("construct locations fallback: %w", err)
|
||||
}
|
||||
if issues := identity.ValidateList(value); len(issues) > 0 {
|
||||
return normalizerErrorf("construct locations fallback: registry identity is not usable downstream")
|
||||
}
|
||||
index := source.NewDocumentIndex(document)
|
||||
for _, location := range value.Locations {
|
||||
for _, reference := range location.SourceRefs {
|
||||
if err := index.ValidateRef(reference); err != nil {
|
||||
return normalizerErrorf("construct locations fallback: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryResult(value dnd.LocationRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
issues := reconciliation.Issues()
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
func cloneRegistry(value dnd.LocationRegistry) dnd.LocationRegistry {
|
||||
if value.Locations == nil {
|
||||
return dnd.LocationRegistry{}
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
result := dnd.LocationRegistry{Locations: make([]dnd.Location, len(value.Locations))}
|
||||
for i, location := range value.Locations {
|
||||
result.Locations[i] = cloneLocation(location)
|
||||
}
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...))
|
||||
return result
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discarded >= 0 {
|
||||
message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output"
|
||||
if discarded > 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
@@ -176,38 +213,12 @@ func semanticFallbackFinding(discarded int, details ...string) diagnostics.Findi
|
||||
return diagnostics.Finding{Scope: "locations", ReasonCode: ReasonCodeLocationSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
|
||||
func normalizationResult(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
func normalizationResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
|
||||
}
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func fallbackResult(value dnd.LocationRegistry, findings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, nil)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, fallbackGroups...)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.LocationRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Diagnostics: groups}, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
@@ -373,7 +384,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding {
|
||||
func locationScope(index int) string { return fmt.Sprintf("locations[%d]", index) }
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind}
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind}
|
||||
}
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.LocationRegistry], error) {
|
||||
@@ -381,7 +392,7 @@ func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
return New(request.Dependencies.Engine, options)
|
||||
})
|
||||
}
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
@@ -3,7 +3,6 @@ package locationregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
@@ -18,8 +17,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
)
|
||||
|
||||
func TestModuleContractAndMetadata(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind}
|
||||
func TestModuleContractAndNativeDependencies(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.LocationRegistryKind}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -28,19 +27,14 @@ func TestModuleContractAndMetadata(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(nil, Options{}); err == nil {
|
||||
t.Fatal("New() accepted nil client")
|
||||
t.Fatal("New() accepted nil native engine")
|
||||
}
|
||||
normalizer := newNormalizer(t, &recordingLocationNormalizerClient{})
|
||||
metadata := normalizer.ManifestMetadata()
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
||||
if !hasFingerprint(normalizer.CheckpointFingerprints(), name) {
|
||||
t.Fatalf("fingerprints missing %q", name)
|
||||
}
|
||||
spec := ModuleSpec()
|
||||
spec.Prompts[0] = "mutated"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("static dependencies are not owned")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
|
||||
@@ -103,14 +97,14 @@ func TestNormalizeAppliesSafeAliasGroupAndUsesContextualInputs(t *testing.T) {
|
||||
{Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Locations) != 2 {
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Locations) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
merged := result.Value.Locations[0]
|
||||
if merged.Name != "the Greencloak's refuge" || merged.ID != identity.DeriveID(merged.Name, merged.SourceRefs) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateLocationCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("merged location = %#v, diagnostics = %#v", merged, result.Diagnostics)
|
||||
}
|
||||
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
||||
encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript"))
|
||||
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
||||
t.Fatalf("private inputs = %s", encoded)
|
||||
}
|
||||
@@ -124,16 +118,31 @@ func TestNormalizeRejectsUnsafeAndOverlappingGroupsWithoutLosingCandidates(t *te
|
||||
{Name: "Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingLocationNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Locations) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") {
|
||||
t.Fatalf("Normalize() = %#v, %v; want safe retry fallback", result, err)
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Locations) != 3 {
|
||||
t.Fatalf("safe partial fallback = %#v %v", result, err)
|
||||
}
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate groups 1, 2") || !strings.Contains(result.Retry.CorrectionGuidance, "at most one duplicate group") || strings.Contains(result.Retry.CorrectionGuidance, "overlapping_member") {
|
||||
t.Fatalf("correction guidance = %q, want model-facing overlap explanation", result.Retry.CorrectionGuidance)
|
||||
if len(client.requests) != 2 {
|
||||
t.Fatalf("native calls = %d, want initial/correction", len(client.requests))
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("fallback diagnostics = %#v", result.Retry.FallbackDiagnostics)
|
||||
messages := client.requests[1].Prompt.Messages
|
||||
guidance := messages[len(messages)-1].Content
|
||||
if !strings.Contains(guidance, "Duplicate groups 1, 2") || !strings.Contains(guidance, "at most one duplicate group") || strings.Contains(guidance, "overlapping_member") {
|
||||
t.Fatalf("model-facing overlap guidance = %q", guidance)
|
||||
}
|
||||
var warning string
|
||||
for _, diagnostic := range result.Diagnostics {
|
||||
if diagnostic.ReasonCode == ReasonCodeLocationSemanticReconciliationExhausted && len(diagnostic.Samples) > 0 {
|
||||
warning = diagnostic.Samples[0].Message
|
||||
}
|
||||
}
|
||||
if !strings.Contains(warning, "2 proposal group") || !strings.Contains(warning, "preserved separately") {
|
||||
t.Fatalf("fallback warning = %q", warning)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
@@ -156,7 +165,7 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil {
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"], &candidateInput); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Tavern" || candidateInput.Candidates[1].Label != "The Tavern" {
|
||||
@@ -164,39 +173,36 @@ func TestReconciliationCandidatesKeepSameNameEvidenceDistinct(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
||||
client := &recordingLocationNormalizerClient{}
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A sprawling city"}}}
|
||||
func TestPreparationLimitUsesResolvedModeWithoutExecutionOrAssessment(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "speech", Text: "A crowded hall"}}}
|
||||
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
||||
input := dnd.LocationRegistry{Locations: make([]dnd.Location, limit+1)}
|
||||
for index := range input.Locations {
|
||||
input.Locations[index] = dnd.Location{Name: fmt.Sprintf("Place %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
for i := range input.Locations {
|
||||
input.Locations[i] = dnd.Location{Name: fmt.Sprintf("Fixture %d", i), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
input.Locations[i].ID = identity.DeriveID(input.Locations[i].Name, input.Locations[i].SourceRefs)
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(result.Value.Locations) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, locations = %d; want no call and all records", len(client.requests), len(result.Value.Locations))
|
||||
}
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeLocationSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHandlesRetryFallbackAndErrors(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.LocationRegistry{Locations: []dnd.Location{{Name: "Old Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Mill", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
invalid, err := newNormalizer(t, &recordingLocationNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeLocationSemanticProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", invalid.Retry)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingLocationNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
client := &recordingLocationNormalizerClient{}
|
||||
req := normalizeRequestWithSource(input, doc)
|
||||
req.Operation.Controls.Mode = mode
|
||||
assessed := 0
|
||||
req.Operation.Assess = func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), req)
|
||||
if len(client.requests) != 0 || assessed != 0 {
|
||||
t.Fatal("unready preparation performed native execution or ordinary assessment")
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil {
|
||||
t.Fatal("strict preparation limit succeeded")
|
||||
}
|
||||
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||||
t.Fatalf("permissive preparation limit = %#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,12 +223,3 @@ func TestNormalizeOrdersEvidenceAndIsIdempotent(t *testing.T) {
|
||||
t.Fatalf("second Normalize() = %#v, %v; want idempotent value %#v", second, err, first.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
func promptAssetManifest() (shared.PromptAssetManifest, error) {
|
||||
sharedFiles, err := semanticreconcile.SharedPromptFiles()
|
||||
if err != nil {
|
||||
return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err)
|
||||
}
|
||||
return shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{"common-dnd-system.md"},
|
||||
ExternalSharedFiles: sharedFiles,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/location-registry/normalize")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope location normalization assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := manifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare location normalization prompt assets: %w", err)
|
||||
}
|
||||
return registry.RegisterPromptFS(promptFS, promptAssetRoot)
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = manifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,52 +0,0 @@
|
||||
package locationregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesLocationNormalizationPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := semanticreconcile.RegisterAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"location-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "location-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"The Tavern","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "same physical place") || !strings.Contains(prepared.Messages[2].Content, "parent and child places") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
for _, index := range []int{2, 4} {
|
||||
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache = %#v", index, prepared.Messages[index].CacheControl)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[3].Content, `"The Tavern"`) || strings.Contains(prepared.Messages[3].Content, `"candidate-`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) {
|
||||
t.Fatalf("candidate message = %q", prepared.Messages[3].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"The Tavern"`) {
|
||||
t.Fatalf("transcript message = %q", prepared.Messages[4].Content)
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
if index != 3 && strings.Contains(message.Content, `"The Tavern"`) {
|
||||
t.Errorf("message %d unexpectedly rendered candidate input", index)
|
||||
}
|
||||
if index != 4 && strings.Contains(message.Content, `"windows"`) {
|
||||
t.Errorf("message %d unexpectedly rendered transcript input", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,54 +3,110 @@ package locationregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
type recordingLocationNormalizerClient struct {
|
||||
response string
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response string
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (c *recordingLocationNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err)
|
||||
func (c *recordingLocationNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(c.requests)
|
||||
c.requests = append(c.requests, request)
|
||||
if c.onGenerate != nil {
|
||||
c.onGenerate(ctx, request)
|
||||
}
|
||||
c.requests = append(c.requests, snapshot)
|
||||
if c.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, c.err
|
||||
return nil, c.err
|
||||
}
|
||||
if index < len(c.errors) && c.errors[index] != nil {
|
||||
return nil, c.errors[index]
|
||||
}
|
||||
response := c.response
|
||||
if index < len(c.responses) {
|
||||
response = c.responses[index]
|
||||
}
|
||||
if response == "" {
|
||||
response = `{"duplicate_groups":[]}`
|
||||
}
|
||||
content := []byte(response)
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: response}, nil
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
||||
func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(client, Options{})
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: location-normalize-test
|
||||
endpoint: https://example.invalid/v1
|
||||
model: normalization-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizer, err := New(engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return normalizer
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := "candidate_number_range"
|
||||
if name == "transcript" {
|
||||
key = "windows"
|
||||
}
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
func normalizeRequest(value dnd.LocationRegistry) contracts.TypedNormalizeRequest[dnd.LocationRegistry] {
|
||||
return contracts.TypedNormalizeRequest[dnd.LocationRegistry]{
|
||||
Source: &source.SourceDocument{},
|
||||
Source: &source.SourceDocument{}, LLMProfile: "location-normalize-test",
|
||||
Operation: contracts.ProducerOperation[dnd.LocationRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
MergeOutput: contracts.MergeArtifact[dnd.LocationRegistry]{Value: value},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRequestWithSource(value dnd.LocationRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.LocationRegistry] {
|
||||
request := normalizeRequest(value)
|
||||
request.Source = doc
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const safeProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`
|
||||
const unknownHandleProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
|
||||
func mergedRegistry() dnd.NPCRegistry {
|
||||
doc := semanticDocument()
|
||||
result := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
{Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
for i := range result.NPCs {
|
||||
result.NPCs[i].ID = identity.DeriveID(result.NPCs[i].Name)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestNativeCorrectionPreservesFixedHistoryAndAssessesEachRevision(t *testing.T) {
|
||||
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
rejectFirst, wantError, wantFallback bool
|
||||
names []string
|
||||
}{
|
||||
{"corrected", contracts.ModeStrict, []string{unknownHandleProposal, safeProposal}, false, false, false, []string{"Mira Thorn", "Captain Vale"}},
|
||||
{"unchanged strict rejection", contracts.ModeStrict, []string{unknownHandleProposal, unknownHandleProposal}, false, true, false, nil},
|
||||
{"only final safe groups", contracts.ModePermissive, []string{safeProposal, finalPartial}, true, false, true, []string{"Mira", "Captain Vale"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
input := mergedRegistry()
|
||||
before := cloneRegistry(input)
|
||||
client := &recordingNPCNormalizerClient{responses: test.responses}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.SessionID = "normalization-session"
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "normalization-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
var assessed []dnd.NPCRegistry
|
||||
request.Operation.Assess = func(_ context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed = append(assessed, cloneRegistry(value))
|
||||
if test.rejectFirst && len(assessed) == 1 {
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "domain-policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the proposed grouping."},
|
||||
}}}, nil
|
||||
}
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if (err != nil) != test.wantError || len(assessed) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
|
||||
t.Fatalf("result=%#v error=%v assessments/provider/prepared=%d/%d/%d", result, err, len(assessed), len(client.requests), len(prepared))
|
||||
}
|
||||
if !reflect.DeepEqual(input, before) {
|
||||
t.Fatal("normalization mutated fixed originals")
|
||||
}
|
||||
if test.name == "unchanged strict rejection" && !reflect.DeepEqual(assessed[0], assessed[1]) {
|
||||
t.Fatal("fixture did not assess an unchanged candidate")
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
wantID := PromptID
|
||||
if i > 0 {
|
||||
wantID = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "normalization-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native request selection/overrides lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
|
||||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != test.responses[0] {
|
||||
t.Fatal("continuation changed original evidence or lost the prior response")
|
||||
}
|
||||
if strings.Contains(revised[len(initial)+1].Content, "operator-only-rejection") {
|
||||
t.Fatal("operator text became correction guidance")
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
names := make([]string, len(result.Value.NPCs))
|
||||
for i, record := range result.Value.NPCs {
|
||||
names[i] = record.Name
|
||||
}
|
||||
if !reflect.DeepEqual(names, test.names) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("unexpected final output: %#v", result)
|
||||
}
|
||||
if test.wantFallback {
|
||||
refs := result.Value.NPCs[1].SourceRefs
|
||||
if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 {
|
||||
t.Fatalf("earlier proposal evidence leaked into final output: %#v", refs)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeFailureCannotRecoverEarlierProposal(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response string
|
||||
providerErrors []error
|
||||
wantAssessments int
|
||||
}{
|
||||
{"initial provider failure", "", []error{failure}, 0},
|
||||
{"failed correction", safeProposal, []error{nil, failure}, 1},
|
||||
{"structural failure", `{"duplicate_groups":null}`, nil, 0},
|
||||
} {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(test.name+"/"+string(mode), func(t *testing.T) {
|
||||
input := mergedRegistry()
|
||||
client := &recordingNPCNormalizerClient{response: test.response, errors: test.providerErrors}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct records."},
|
||||
}}}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if assessments != test.wantAssessments {
|
||||
t.Fatalf("assessments=%d, want %d", assessments, test.wantAssessments)
|
||||
}
|
||||
if test.providerErrors != nil && len(client.requests) != len(test.providerErrors) {
|
||||
t.Fatalf("provider restarted: %d calls", len(client.requests))
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("strict failure=%#v %v", result, err)
|
||||
}
|
||||
if test.providerErrors != nil && !errors.Is(err, failure) {
|
||||
t.Fatalf("lost native cause: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||||
t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err)
|
||||
}
|
||||
for _, diagnostic := range result.Diagnostics {
|
||||
if diagnostic.ReasonCode == ReasonCodeDuplicateNPCCollapsed {
|
||||
t.Fatal("discarded proposal left collapse diagnostics")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationDoesNotDeliverFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &recordingNPCNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := normalizeRequestWithSource(mergedRegistry(), semanticDocument())
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newNormalizer(t, client).Normalize(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFallbackRequiresUsableMergedIdentityAndSource(t *testing.T) {
|
||||
for _, invalidIdentity := range []bool{false, true} {
|
||||
input := mergedRegistry()
|
||||
if invalidIdentity {
|
||||
input.NPCs[0].ID = "wrong"
|
||||
} else {
|
||||
input.NPCs[0].SourceRefs[0].SourceID = "foreign-source"
|
||||
}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
failure := errors.New("provider unavailable")
|
||||
result, err := newNormalizer(t, &recordingNPCNormalizerClient{err: failure}).Normalize(context.Background(), request)
|
||||
if !errors.Is(err, failure) || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,23 +14,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/npc-registry"
|
||||
PromptID = "dnd.npc_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.npc_registry.normalize.v6"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
Key = "dnd/npc-registry"
|
||||
PromptID = "notarius/dnd.npc_registry.normalize"
|
||||
CorrectionPromptID = "notarius/dnd.npc_registry.normalize.repair"
|
||||
|
||||
ReasonCodeNPCFieldsNormalized = "npc_fields_normalized"
|
||||
ReasonCodeNPCIDRecomputed = "npc_id_recomputed"
|
||||
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
||||
ReasonCodeDuplicateNPCCollapsed = "duplicate_npc_collapsed"
|
||||
ReasonCodeNPCSemanticProposalInvalid = "npc_semantic_proposal_invalid"
|
||||
ReasonCodeNPCSemanticReconciliationExhausted = "npc_semantic_reconciliation_exhausted"
|
||||
)
|
||||
|
||||
@@ -38,28 +37,16 @@ var requiredCapabilities = []string{"merged"}
|
||||
var providedCapabilities = []string{"normalized"}
|
||||
|
||||
var _ contracts.Normalizer[dnd.NPCRegistry] = (*Normalizer)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Normalizer struct {
|
||||
engine *semanticreconcile.Engine
|
||||
engine *promptkit.Engine
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) {
|
||||
if llmClient == nil {
|
||||
return nil, normalizerErrorf("LLM client must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{
|
||||
ID: PromptID, Version: PromptVersion, SHA256: promptSHA,
|
||||
}, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err)
|
||||
func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) {
|
||||
if engine == nil {
|
||||
return nil, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
return &Normalizer{engine: engine}, nil
|
||||
}
|
||||
@@ -67,33 +54,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error
|
||||
func (n *Normalizer) Key() string { return Key }
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := n.engine.ManifestMetadata()
|
||||
metadata["identity_policy"] = identity.Policy
|
||||
metadata["normalization_policy"] = normalizationPolicy
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
fingerprints := n.engine.CheckpointFingerprints()
|
||||
return append(fingerprints,
|
||||
pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy},
|
||||
pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy},
|
||||
)
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.NPCRegistry]) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
if n == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if n.engine == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil")
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("context must not be nil")
|
||||
@@ -105,69 +71,140 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("source document must not be nil")
|
||||
}
|
||||
|
||||
// Keep the already available merged artifact separate from preprocessed
|
||||
// originals. A failed native response must not recover an earlier proposal.
|
||||
merged := cloneRegistry(req.MergeOutput.Value)
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, findings := preprocessRecords(req.MergeOutput.Value, order)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if len(records) < 2 {
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
discardedGroups := -1
|
||||
var omissionDetails []string
|
||||
|
||||
fallback := func(_ context.Context, final *dnd.NPCRegistry) (dnd.NPCRegistry, error) {
|
||||
value := cloneRegistry(merged)
|
||||
if final != nil {
|
||||
// Each interpreted candidate was built afresh from fixed original records,
|
||||
// using only the final proposal's compatible, non-overlapping safe groups.
|
||||
value = cloneRegistry(*final)
|
||||
} else {
|
||||
findings = nil
|
||||
discardedGroups = -1
|
||||
omissionDetails = nil
|
||||
}
|
||||
if err := validateFallback(value, req.Source); err != nil {
|
||||
return dnd.NPCRegistry{}, err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
finish := func(value dnd.NPCRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("normalize NPCs: %w", err)
|
||||
}
|
||||
result, err := normalizationResult(value, findings)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
}
|
||||
result.Outcome = outcome
|
||||
if outcome.Disposition == contracts.ProducerFallback {
|
||||
details := append([]string(nil), omissionDetails...)
|
||||
details = append(details, outcome.FallbackReason)
|
||||
groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, groups...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
noModel := func() (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.NPCRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
if len(records) < 2 {
|
||||
return noModel()
|
||||
}
|
||||
candidates, envelopes, err := reconciliationInputs(records)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err)
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction,
|
||||
})
|
||||
preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
|
||||
switch reconciliation.Disposition() {
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
default:
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition())
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
case semanticreconcile.LimitExceeded:
|
||||
value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.NPCRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
applied, semanticFindings, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
|
||||
initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build native request: %w", err)
|
||||
}
|
||||
findings = append(findings, semanticFindings...)
|
||||
if reconciliation.Disposition() == semanticreconcile.Complete {
|
||||
return normalizationResult(recordList(applied), findings, reconciliation.ModelCandidate())
|
||||
}
|
||||
return retryResult(recordList(applied), findings, reconciliation)
|
||||
value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCRegistry, []string, error) {
|
||||
response, err := semanticreconcile.DecodeProposal(result.RawOutput)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
}
|
||||
findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...)
|
||||
discardedGroups = assessment.DiscardedGroupCount()
|
||||
omissionDetails = semanticreconcile.IssueDetails(assessment.Issues())
|
||||
guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
candidate := recordList(applied)
|
||||
return &candidate, guidance, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil
|
||||
}, fallback)
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.NPCRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
// validateFallback checks the durable artifact and the identity/source invariants
|
||||
// used by registry consumers, not the configured ordinary validator chain.
|
||||
func validateFallback(value dnd.NPCRegistry, document *source.SourceDocument) error {
|
||||
if _, err := registrycodec.New().Encode(value); err != nil {
|
||||
return normalizerErrorf("construct NPCs fallback: %w", err)
|
||||
}
|
||||
if issues := identity.ValidateList(value); len(issues) > 0 {
|
||||
return normalizerErrorf("construct NPCs fallback: registry identity is not usable downstream")
|
||||
}
|
||||
index := source.NewDocumentIndex(document)
|
||||
for _, npc := range value.NPCs {
|
||||
for _, reference := range npc.SourceRefs {
|
||||
if err := index.ValidateRef(reference); err != nil {
|
||||
return normalizerErrorf("construct NPCs fallback: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryResult(value dnd.NPCRegistry, findings []diagnostics.Finding, reconciliation semanticreconcile.Result) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
issues := reconciliation.Issues()
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
func cloneRegistry(value dnd.NPCRegistry) dnd.NPCRegistry {
|
||||
if value.NPCs == nil {
|
||||
return dnd.NPCRegistry{}
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
result := dnd.NPCRegistry{NPCs: make([]dnd.NPC, len(value.NPCs))}
|
||||
for i, npc := range value.NPCs {
|
||||
result.NPCs[i] = cloneNPC(npc)
|
||||
}
|
||||
return retryResultWithFallback(value, findings, reconciliation.ModelCandidate(), correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(issues)), semanticFallbackFinding(reconciliation.DiscardedGroupCount(), correctionDetails...))
|
||||
return result
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discardedGroups int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discardedGroups >= 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discardedGroups)
|
||||
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output"
|
||||
if discarded > 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
message = diagnostics.Aggregate(message, details)
|
||||
@@ -175,38 +212,12 @@ func semanticFallbackFinding(discardedGroups int, details ...string) diagnostics
|
||||
return diagnostics.Finding{Scope: "npcs", ReasonCode: ReasonCodeNPCSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
|
||||
func normalizationResult(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
func normalizationResult(value dnd.NPCRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
|
||||
}
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func fallbackResult(value dnd.NPCRegistry, findings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, nil)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, fallbackGroups...)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.NPCRegistry, findings []diagnostics.Finding, candidate *contracts.ModelCandidate, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.NPCRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{Value: value, Diagnostics: groups}, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
@@ -372,7 +383,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding {
|
||||
func npcScope(index int) string { return fmt.Sprintf("npcs[%d]", index) }
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.NPCRegistryKind}
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.NPCRegistryKind}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
@@ -381,7 +392,7 @@ func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
return New(request.Dependencies.Engine, options)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -3,15 +3,17 @@ package npcregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
)
|
||||
@@ -23,7 +25,7 @@ func TestModuleContractAndIdentity(t *testing.T) {
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.NPCRegistryKind}
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.NPCRegistryKind}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -32,20 +34,14 @@ func TestModuleContractAndIdentity(t *testing.T) {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if _, err := New(nil, Options{}); err == nil {
|
||||
t.Fatal("New(nil, Options{}) error = nil, want nil client rejection")
|
||||
t.Fatal("New(nil, Options{}) error = nil, want nil native engine rejection")
|
||||
}
|
||||
normalizer := newNormalizer(t, &recordingNPCNormalizerClient{})
|
||||
metadata := normalizer.ManifestMetadata()
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
fingerprints := normalizer.CheckpointFingerprints()
|
||||
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
||||
if !hasFingerprint(fingerprints, name) {
|
||||
t.Fatalf("fingerprints = %#v, want %q", fingerprints, name)
|
||||
}
|
||||
spec := ModuleSpec()
|
||||
spec.Prompts[0] = "mutated"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("static dependencies are not owned")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestNormalizeNamesEvidenceAndIDs(t *testing.T) {
|
||||
@@ -142,51 +138,93 @@ func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) {
|
||||
}
|
||||
|
||||
type recordingNPCNormalizerClient struct {
|
||||
response string
|
||||
responses []string
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response string
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (c *recordingNPCNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err)
|
||||
func (c *recordingNPCNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(c.requests)
|
||||
c.requests = append(c.requests, request)
|
||||
if c.onGenerate != nil {
|
||||
c.onGenerate(ctx, request)
|
||||
}
|
||||
c.requests = append(c.requests, snapshot)
|
||||
if c.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, c.err
|
||||
return nil, c.err
|
||||
}
|
||||
if index < len(c.errors) && c.errors[index] != nil {
|
||||
return nil, c.errors[index]
|
||||
}
|
||||
response := c.response
|
||||
if len(c.responses) > 0 {
|
||||
responseIndex := len(c.requests) - 1
|
||||
if responseIndex >= len(c.responses) {
|
||||
responseIndex = len(c.responses) - 1
|
||||
}
|
||||
response = c.responses[responseIndex]
|
||||
if index < len(c.responses) {
|
||||
response = c.responses[index]
|
||||
}
|
||||
if response == "" {
|
||||
response = `{"duplicate_groups":[]}`
|
||||
}
|
||||
content := []byte(response)
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: response}, nil
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
||||
func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(client, Options{})
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: normalizer-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: normalization-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizer, err := New(engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return normalizer
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := "candidate_number_range"
|
||||
if name == "transcript" {
|
||||
key = "windows"
|
||||
}
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
func normalizeRequest(value dnd.NPCRegistry) contracts.TypedNormalizeRequest[dnd.NPCRegistry] {
|
||||
return contracts.TypedNormalizeRequest[dnd.NPCRegistry]{
|
||||
Source: &source.SourceDocument{},
|
||||
Source: &source.SourceDocument{}, LLMProfile: "normalizer-profile",
|
||||
Operation: contracts.ProducerOperation[dnd.NPCRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
MergeOutput: contracts.MergeArtifact[dnd.NPCRegistry]{Value: value},
|
||||
}
|
||||
}
|
||||
@@ -210,12 +248,3 @@ func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason, scope str
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
func promptAssetManifest() (shared.PromptAssetManifest, error) {
|
||||
sharedFiles, err := semanticreconcile.SharedPromptFiles()
|
||||
if err != nil {
|
||||
return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err)
|
||||
}
|
||||
return shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{"common-dnd-system.md"},
|
||||
ExternalSharedFiles: sharedFiles,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/npc-registry/normalize")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope NPC normalization assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := manifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare NPC normalization prompt assets: %w", err)
|
||||
}
|
||||
return registry.RegisterPromptFS(promptFS, promptAssetRoot)
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = manifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,75 +0,0 @@
|
||||
package npcregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
if promptHash, err := promptAssetMetadata(); err != nil || promptHash == "" {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v; want prompt fingerprint", promptHash, err)
|
||||
}
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := semanticreconcile.RegisterAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterAssets() error = %v", err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"normalize-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "normalize-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "normalize-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Mira","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
"transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want normalization prompt identity and schema", prepared)
|
||||
}
|
||||
if prepared.Messages[0].Role != "system" {
|
||||
t.Fatalf("initial message role = %q, want system", prepared.Messages[0].Role)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[1].Content, "candidate_number") || !strings.Contains(prepared.Messages[1].Content, "integer") {
|
||||
t.Fatalf("protocol message = %q, want shared integer-handle protocol", prepared.Messages[1].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "same individual") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || strings.Contains(prepared.Messages[2].Content, "source ranges") {
|
||||
t.Fatalf("NPC policy message = %q, want domain distinctions without copied ranges", prepared.Messages[2].Content)
|
||||
}
|
||||
for _, index := range []int{2, 4} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
for _, index := range []int{0, 1, 3} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, cache)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[3].Content, `"Mira"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) {
|
||||
t.Fatalf("candidate message = %q, want only rendered candidates", prepared.Messages[3].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Mira"`) {
|
||||
t.Fatalf("transcript message = %q, want only rendered transcript", prepared.Messages[4].Content)
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
if index != 3 && strings.Contains(message.Content, `"Mira"`) {
|
||||
t.Errorf("message %d unexpectedly rendered candidate input", index)
|
||||
}
|
||||
if index != 4 && strings.Contains(message.Content, `"windows"`) {
|
||||
t.Errorf("message %d unexpectedly rendered transcript input", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,14 @@ package npcregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
@@ -23,12 +24,10 @@ func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing
|
||||
doc := semanticDocument()
|
||||
input := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}}}
|
||||
result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || len(client.requests) != 0 || result.Retry != nil {
|
||||
if err != nil || len(client.requests) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted {
|
||||
t.Fatalf("Normalize() = %#v, %v; calls = %d, want deterministic no-call result", result, err, len(client.requests))
|
||||
}
|
||||
if result.ModelCandidate != nil {
|
||||
t.Fatalf("model candidate = %#v, want nil for no-call result", result.ModelCandidate)
|
||||
}
|
||||
|
||||
if result.Value.NPCs[0].Name != "Mira Thorn" {
|
||||
t.Fatalf("NPCs = %#v, want deterministic record", result.Value.NPCs)
|
||||
}
|
||||
@@ -47,13 +46,10 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.LLMProfile = "normalizer-profile"
|
||||
request.SessionID = "normalizer-session"
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"duplicate_groups":[]}`), "keep the distinct captain")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Correction = correction
|
||||
var prepared promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
|
||||
result, err := normalizer.Normalize(context.Background(), request)
|
||||
if err != nil || result.Retry != nil {
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted {
|
||||
t.Fatalf("Normalize() = %#v, %v; want accepted semantic result", result, err)
|
||||
}
|
||||
if !reflect.DeepEqual(input, before) {
|
||||
@@ -73,16 +69,10 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
t.Fatalf("completion calls = %d, want one", len(client.requests))
|
||||
}
|
||||
completion := client.requests[0]
|
||||
if !reflect.DeepEqual(completion.Correction, correction) {
|
||||
t.Fatalf("completion correction = %#v, want %#v", completion.Correction, correction)
|
||||
if prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completion.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native request routing lost")
|
||||
}
|
||||
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != client.response {
|
||||
t.Fatalf("model candidate = %#v, want exact semantic response", result.ModelCandidate)
|
||||
}
|
||||
if completion.StageName != Key || completion.PromptID != PromptID || completion.PromptVersion != PromptVersion || completion.ProfileID != request.LLMProfile || completion.SessionID != request.SessionID || len(completion.Inputs) != 2 {
|
||||
t.Fatalf("completion request = %#v, want normalize request identity and exactly two inputs", completion)
|
||||
}
|
||||
encoded := string(completion.Inputs["candidates"].Content) + string(completion.Inputs["transcript"].Content)
|
||||
encoded := string(renderedInput(t, completion, "candidates")) + string(renderedInput(t, completion, "transcript"))
|
||||
if strings.Contains(encoded, "npc:sha256:") || strings.Contains(encoded, doc.ID) {
|
||||
t.Fatalf("completion inputs leaked private identifiers: %s", encoded)
|
||||
}
|
||||
@@ -91,7 +81,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
CandidateNumber int `json:"candidate_number"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(completion.Inputs["candidates"].Content, &visible); err != nil {
|
||||
if err := json.Unmarshal(renderedInput(t, completion, "candidates"), &visible); err != nil {
|
||||
t.Fatalf("decode candidates: %v", err)
|
||||
}
|
||||
if got := []int{visible.Candidates[0].CandidateNumber, visible.Candidates[1].CandidateNumber, visible.Candidates[2].CandidateNumber}; !reflect.DeepEqual(got, []int{1, 2, 3}) {
|
||||
@@ -99,7 +89,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) {
|
||||
func TestNormalizeUnsafeProposalReturnsQualifiedSafePartial(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
@@ -108,41 +98,38 @@ func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) {
|
||||
{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want retryable safe fallback", result, err)
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
result, err := normalizer.Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||||
t.Fatalf("qualified fallback = %#v %v", result, err)
|
||||
}
|
||||
if result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || !strings.Contains(result.Retry.Message, "group 1: member_unknown") {
|
||||
t.Fatalf("retry = %#v, want bounded invalid-proposal diagnostics", result.Retry)
|
||||
if len(client.requests) != 2 {
|
||||
t.Fatalf("native calls=%d, want initial/correction", len(client.requests))
|
||||
}
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "Duplicate group 2") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 3") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeNPCSemanticProposalInvalid) {
|
||||
t.Fatalf("correction guidance = %q, want contextual prose without internal labels", result.Retry.CorrectionGuidance)
|
||||
messages := client.requests[1].Prompt.Messages
|
||||
guidance := messages[len(messages)-1].Content
|
||||
for _, expected := range []string{"Duplicate group 2", "candidate-number range 1 through 3", "source positions"} {
|
||||
if !strings.Contains(guidance, expected) {
|
||||
t.Fatalf("missing correction detail %q", expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(guidance, "member_unknown") || strings.Contains(guidance, "npc_semantic_proposal_invalid") {
|
||||
t.Fatal("internal labels copied into correction")
|
||||
}
|
||||
|
||||
if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Mira Thorn" || result.Value.NPCs[1].Name != "Captain Vale" {
|
||||
t.Fatalf("fallback NPCs = %#v, want independently safe group applied", result.Value.NPCs)
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeNPCSemanticReconciliationExhausted || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "1 proposal group") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "candidate-number range 1 through 3") {
|
||||
t.Fatalf("fallback diagnostics = %#v, want exact omitted-group warning", result.Retry.FallbackDiagnostics)
|
||||
var warning string
|
||||
for _, diagnostic := range result.Diagnostics {
|
||||
if diagnostic.ReasonCode == ReasonCodeNPCSemanticReconciliationExhausted && len(diagnostic.Samples) > 0 {
|
||||
warning = diagnostic.Samples[0].Message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeInvalidStructuredOutputAndOperationalErrorsRemainDistinct(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.NPCRegistry{NPCs: []dnd.NPC{
|
||||
{Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
}}
|
||||
invalid := newNormalizer(t, &recordingNPCNormalizerClient{err: contracts.ErrInvalidStructuredOutput})
|
||||
result, err := invalid.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || len(result.Value.NPCs) != 2 {
|
||||
t.Fatalf("invalid structured result = %#v, %v; want deterministic retry fallback", result, err)
|
||||
}
|
||||
if result.Retry.CorrectionGuidance != "" || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry and informative fallback", result.Retry)
|
||||
}
|
||||
operational := errors.New("provider unavailable")
|
||||
if _, err := newNormalizer(t, &recordingNPCNormalizerClient{err: operational}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)); !errors.Is(err, operational) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
t.Fatalf("operational completion error = %v, want ordinary error", err)
|
||||
if !strings.Contains(warning, "1 proposal group") || !strings.Contains(warning, "preserved separately") {
|
||||
t.Fatalf("fallback warning = %q", warning)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +169,7 @@ func TestNormalizeRedactsContextMaterialFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) {
|
||||
func TestNormalizeDoesNotAccumulateSafeGroupsAcrossNativeRevisions(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{responses: []string{
|
||||
`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,99],"canonical_candidate_number":3}]}`,
|
||||
`{"duplicate_groups":[{"candidate_numbers":[1,3],"canonical_candidate_number":3}]}`,
|
||||
@@ -194,20 +181,27 @@ func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) {
|
||||
{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
first, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || first.Retry == nil || len(first.Value.NPCs) != 2 {
|
||||
t.Fatalf("first Normalize() = %#v, %v; want partial retry fallback", first, err)
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
second, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || second.Retry != nil {
|
||||
t.Fatalf("second Normalize() = %#v, %v; want accepted independent retry result", second, err)
|
||||
result, err := normalizer.Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(client.requests) != 2 || assessments != 2 {
|
||||
t.Fatalf("native correction = %#v %v, assessments %d", result, err, assessments)
|
||||
}
|
||||
if got := []string{second.Value.NPCs[0].Name, second.Value.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"Captain Vale", "Mira Thorn"}) {
|
||||
t.Fatalf("second NPCs = %#v, want proposal applied to original merge output", second.Value.NPCs)
|
||||
if got := []string{result.Value.NPCs[0].Name, result.Value.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"Captain Vale", "Mira Thorn"}) {
|
||||
t.Fatalf("revised NPCs = %#v", result.Value.NPCs)
|
||||
}
|
||||
if refs := result.Value.NPCs[0].SourceRefs; len(refs) != 2 || refs[0].StartUnitID != 10 || refs[1].StartUnitID != 30 {
|
||||
t.Fatalf("earlier proposal evidence accumulated: %#v", refs)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestNormalizeRetainsRecordsWhenAProposalUsesAnUnknownHandle(t *testing.T) {
|
||||
func TestNormalizeRetainsInvalidRecordsForAssessmentAfterAnUnknownHandle(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`}
|
||||
normalizer := newNormalizer(t, client)
|
||||
doc := semanticDocument()
|
||||
@@ -216,13 +210,17 @@ func TestNormalizeRetainsRecordsWhenAProposalUsesAnUnknownHandle(t *testing.T) {
|
||||
{Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 999, EndUnitID: 999}}},
|
||||
{Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.NPCs) != 3 {
|
||||
t.Fatalf("Normalize() = %#v, %v; want unchanged retry fallback", result, err)
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
var assessed dnd.NPCRegistry
|
||||
request.Operation.Assess = func(_ context.Context, value dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed = cloneRegistry(value)
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
if !strings.Contains(result.Retry.Message, "member_unknown") || result.Value.NPCs[1].Name != "Broken" {
|
||||
t.Fatalf("result = %#v, want unknown descriptor rejected without dropping a record", result)
|
||||
result, err := normalizer.Normalize(context.Background(), request)
|
||||
if err == nil || result.Outcome.Disposition != "" || len(assessed.NPCs) != 3 || assessed.NPCs[1].Name != "Broken" || assessed.NPCs[1].SourceRefs[0].StartUnitID != 999 {
|
||||
t.Fatalf("candidate was dropped before assessment: %#v %v", assessed, err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testing.T) {
|
||||
@@ -245,7 +243,7 @@ func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testi
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"].Content, &candidateInput); err != nil {
|
||||
if err := json.Unmarshal(preparation.Materials()["candidates"], &candidateInput); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidateInput.Candidates) != 2 || candidateInput.Candidates[0].CandidateNumber != 1 || candidateInput.Candidates[1].CandidateNumber != 2 || candidateInput.Candidates[0].Label != "The Guard" || candidateInput.Candidates[1].Label != "The Guard" {
|
||||
@@ -253,24 +251,36 @@ func TestReconciliationCandidatesKeepEqualContextualDescriptorsDistinct(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{}
|
||||
normalizer := newNormalizer(t, client)
|
||||
func TestPreparationLimitUsesResolvedModeWithoutExecutionOrAssessment(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "speech", Text: "A crowded hall"}}}
|
||||
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
||||
input := dnd.NPCRegistry{NPCs: make([]dnd.NPC, limit+1)}
|
||||
for index := range input.NPCs {
|
||||
input.NPCs[index] = dnd.NPC{Name: fmt.Sprintf("Person %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
for i := range input.NPCs {
|
||||
input.NPCs[i] = dnd.NPC{Name: fmt.Sprintf("Fixture %d", i), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
input.NPCs[i].ID = identity.DeriveID(input.NPCs[i].Name)
|
||||
}
|
||||
result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(result.Value.NPCs) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, NPCs = %d; want no call and all records", len(client.requests), len(result.Value.NPCs))
|
||||
}
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeNPCSemanticReconciliationExhausted, "npcs", contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
client := &recordingNPCNormalizerClient{}
|
||||
req := normalizeRequestWithSource(input, doc)
|
||||
req.Operation.Controls.Mode = mode
|
||||
assessed := 0
|
||||
req.Operation.Assess = func(context.Context, dnd.NPCRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), req)
|
||||
if len(client.requests) != 0 || assessed != 0 {
|
||||
t.Fatal("unready preparation performed native execution or ordinary assessment")
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil {
|
||||
t.Fatal("strict preparation limit succeeded")
|
||||
}
|
||||
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||||
t.Fatalf("permissive preparation limit = %#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user