Migrate item normalization to native correction and explicit fallback handling
This commit is contained in:
@@ -11,9 +11,9 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../../generic/normalize/deduplication/prompts/protocol.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -27,3 +27,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json
|
||||
|
||||
- id: dnd.item_registry.normalize.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ../../../../generic/normalize/deduplication/prompts/correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json
|
||||
|
||||
@@ -625,6 +625,29 @@ as a request to restart the core.
|
||||
|
||||
## Stage 14: Migrate Item Registry Normalization
|
||||
|
||||
**Status:** Complete. Item normalization now uses one fixed preprocessing and
|
||||
reconciliation preparation, native initial/correction requests, and the shared
|
||||
producer helper. Each response applies its own safe proposal to original records
|
||||
before operation-scoped assessment. Currency compatibility, identity derivation,
|
||||
evidence ordering, and domain diagnostics remain. Added the same-schema/default-
|
||||
profile correction declaration and static dependencies; removed module retries,
|
||||
completion/asset registration, and manifest/fingerprint metadata.
|
||||
Permissive output uses only the final safe candidate or the supplied merged
|
||||
artifact after a result-less failure. Fallback construction checks codec, registry
|
||||
identity, and source usability without ordinary revalidation. Preparation limits
|
||||
honor resolved mode; insufficient candidates use the existing assessment path
|
||||
without native requests. Exposed/factored the existing terminal/no-model assessment
|
||||
functions for these real callers rather than duplicating acceptance policy.
|
||||
Retained domain regressions and added native-fake correction, final safe partial,
|
||||
unchanged-candidate, failed-continuation/no-earlier-recovery, structural-failure,
|
||||
mode/limit, no-model, fallback-usability, override, and cancellation tests.
|
||||
Focused pipeline tests/vet/race, terminal tests, complete reconciliation/asset and
|
||||
item-shape tests/vet/race, production framework builds, and `git diff --check`
|
||||
pass. Item-normalizer/codec/identity/source-validator tests remain blocked at
|
||||
setup by shared legacy asset helpers importing old ConfigKit's removed YAML
|
||||
`plugin/limit`; the new module tests are not claimed passing. Their full execution
|
||||
remains required after scheduled dependency cleanup and in final verification.
|
||||
|
||||
**Goal:** prove a complete native normalization operation and safe fallback.
|
||||
|
||||
**Inspect/change:** `internal/modules/dnd/normalize/itemregistry`, its validators/
|
||||
|
||||
@@ -155,7 +155,7 @@ func RunProducer[T any](
|
||||
if fallback != nil {
|
||||
construct = func(ctx context.Context) (T, error) { return fallback(ctx, candidate) }
|
||||
}
|
||||
return finishProducerFailure(ctx, operation.Controls.Mode, failure, construct)
|
||||
return FinishProducerFailure(ctx, operation.Controls.Mode, failure, construct)
|
||||
}
|
||||
|
||||
// producerRequest applies native options to a value copy. The invocation
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// finishProducerFailure applies the resolved terminal mode without retrying
|
||||
// FinishProducerFailure applies the resolved terminal mode without retrying
|
||||
// production or running ordinary validators against a supported fallback.
|
||||
func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
|
||||
func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
|
||||
var zero T
|
||||
if ctx == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
|
||||
@@ -40,3 +40,30 @@ func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM
|
||||
return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode)
|
||||
}
|
||||
}
|
||||
|
||||
// AssessProducer assesses a no-model result once, including deterministic
|
||||
// short-circuits inside otherwise model-backed modules. It never constructs a
|
||||
// native request or gives deterministic work a correction loop.
|
||||
func AssessProducer[T any](ctx context.Context, operation contracts.ProducerOperation[T], value T, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
|
||||
var zero T
|
||||
if ctx == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if operation.Assess == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer assessment must be supplied")
|
||||
}
|
||||
assessment, err := operation.Assess(ctx, value)
|
||||
if err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if !assessment.Accepted() {
|
||||
return FinishProducerFailure(ctx, operation.Controls.Mode, assessment.Err(), fallback)
|
||||
}
|
||||
return value, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) {
|
||||
failure := errors.New("candidate not accepted")
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
calls := 0
|
||||
value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil })
|
||||
value, outcome, err := FinishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil })
|
||||
if mode == contracts.ModeStrict {
|
||||
if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" {
|
||||
t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls)
|
||||
@@ -25,11 +25,11 @@ func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) {
|
||||
|
||||
func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) {
|
||||
original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback")
|
||||
_, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction })
|
||||
_, outcome, err := FinishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction })
|
||||
if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" {
|
||||
t.Fatalf("outcome = %#v, error = %v", outcome, err)
|
||||
}
|
||||
if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) {
|
||||
if _, _, err := FinishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) {
|
||||
t.Fatalf("unsupported fallback = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func TestProducerCancellationNeverDeliversFallback(t *testing.T) {
|
||||
if !cancelDuringFallback {
|
||||
cancel()
|
||||
}
|
||||
value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) {
|
||||
value, outcome, err := FinishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return "not delivered", nil
|
||||
@@ -57,8 +57,54 @@ func TestProducerCancellationNeverDeliversFallback(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) {
|
||||
value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil })
|
||||
value, outcome, err := FinishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil })
|
||||
if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback {
|
||||
t.Fatalf("local timeout = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoModelProducerIsAssessedOnceWithoutRevalidatingFallback(t *testing.T) {
|
||||
for _, approve := range []bool{true, false} {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
assessed, fallbacks := 0, 0
|
||||
operation := contracts.ProducerOperation[string]{
|
||||
Controls: contracts.ProducerControls{Mode: mode}, ReuseEligible: true,
|
||||
Assess: func(context.Context, string) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
if approve {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "policy", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{Message: "not accepted"}}}}, nil
|
||||
},
|
||||
}
|
||||
value, outcome, err := AssessProducer(context.Background(), operation, "candidate", func(context.Context) (string, error) { fallbacks++; return "supported", nil })
|
||||
if assessed != 1 {
|
||||
t.Fatalf("assessed %d times", assessed)
|
||||
}
|
||||
if approve {
|
||||
if err != nil || value != "candidate" || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 {
|
||||
t.Fatalf("acceptance = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
} else if mode == contracts.ModeStrict {
|
||||
if err == nil || value != "" || outcome.Disposition != "" || fallbacks != 0 {
|
||||
t.Fatalf("strict failure = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
} else if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 {
|
||||
t.Fatalf("fallback = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoModelProducerStopsAfterAssessmentCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive},
|
||||
Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
cancel()
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}}
|
||||
_, outcome, err := AssessProducer(ctx, operation, 1, func(context.Context) (int, error) { t.Fatal("fallback after cancellation"); return 0, nil })
|
||||
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
|
||||
t.Fatalf("cancellation = %#v %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,17 +17,7 @@ func completeProducerOperation[T any](ctx context.Context, class contracts.Execu
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if class == contracts.ExecutionClassDeterministic {
|
||||
assessment, err := operation.Assess(ctx, value)
|
||||
if err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if !assessment.Accepted() {
|
||||
return finishProducerFailure(ctx, operation.Controls.Mode, assessment.Err(), fallback)
|
||||
}
|
||||
outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}
|
||||
return AssessProducer(ctx, operation, value, fallback)
|
||||
} else if class != contracts.ExecutionClassLLMBacked {
|
||||
return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer execution class %q", class)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const safeFirstProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`
|
||||
const invalidItemProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
|
||||
func mergedItems() dnd.ItemRegistry {
|
||||
doc := semanticDocument()
|
||||
result := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Astral Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
for i := range result.Items {
|
||||
result.Items[i].ID = identity.DeriveID(result.Items[i].Name)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func itemNames(value dnd.ItemRegistry) []string {
|
||||
names := make([]string, len(value.Items))
|
||||
for i, item := range value.Items {
|
||||
names[i] = item.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) {
|
||||
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
rejectFirst bool
|
||||
wantError bool
|
||||
wantFallback bool
|
||||
wantNames []string
|
||||
}{
|
||||
{"corrected proposal accepted", contracts.ModeStrict, []string{invalidItemProposal, safeFirstProposal}, false, false, false, []string{"Compass of the Stars", "Astral Compass"}},
|
||||
{"unchanged candidate reassessed", contracts.ModeStrict, []string{invalidItemProposal, invalidItemProposal}, false, true, false, nil},
|
||||
{"final partial proposal only", contracts.ModePermissive, []string{safeFirstProposal, finalPartial}, true, false, true, []string{"Star Compass", "Astral Compass"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
input := mergedItems()
|
||||
before := cloneRegistry(input)
|
||||
client := &recordingNormalizerClient{responses: test.responses}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = len(test.responses) - 1
|
||||
request.SessionID = "normalization-session"
|
||||
model := "run-wide-normalization-model"
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
base := request.Operation.Assess
|
||||
var assessed []dnd.ItemRegistry
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed = append(assessed, cloneRegistry(value))
|
||||
assessment, err := base(ctx, value)
|
||||
if test.rejectFirst {
|
||||
finding := contracts.ValidatorAssessment{ValidatorName: "configured-domain-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}}
|
||||
if len(assessed) == 1 {
|
||||
finding.Decision = contracts.AssessmentRejected
|
||||
finding.Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the first compass grouping."}
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, finding)
|
||||
}
|
||||
return assessment, err
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("result = %#v %v", result, err)
|
||||
}
|
||||
if len(client.requests) != len(test.responses) || len(assessed) != len(test.responses) || len(prepared) != len(test.responses) {
|
||||
t.Fatalf("provider/assessment/preparation = %d/%d/%d, want %d", len(client.requests), len(assessed), len(prepared), len(test.responses))
|
||||
}
|
||||
if !reflect.DeepEqual(input, before) {
|
||||
t.Fatal("proposal mutated fixed originals")
|
||||
}
|
||||
if test.name == "unchanged candidate reassessed" && !reflect.DeepEqual(assessed[0], assessed[1]) {
|
||||
t.Fatal("fixture did not exercise an unchanged typed candidate")
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
if call.Target.Model != model || call.Prompt.SessionID != "normalization-session" || prepared[i].SelectedProfileID != "item-normalize-test" {
|
||||
t.Fatal("native override lost")
|
||||
}
|
||||
wantID := PromptID
|
||||
if i > 0 {
|
||||
wantID = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != wantID {
|
||||
t.Fatalf("prompt = %q, want %q", prepared[i].PromptID, wantID)
|
||||
}
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
old := client.requests[i-1].Prompt.Messages
|
||||
current := call.Prompt.Messages
|
||||
if len(current) != len(old)+2 || !reflect.DeepEqual(current[:len(old)], old) {
|
||||
t.Fatal("native continuation changed fixed evidence/history")
|
||||
}
|
||||
if current[len(old)].Role != "assistant" || current[len(old)].Content != test.responses[i-1] {
|
||||
t.Fatal("native continuation lost the prior response")
|
||||
}
|
||||
if strings.Contains(current[len(old)+1].Content, "operator-only-rejection") {
|
||||
t.Fatal("operator diagnostic became model guidance")
|
||||
}
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(itemNames(result.Value), test.wantNames) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback {
|
||||
t.Fatalf("final outcome = %#v", result)
|
||||
}
|
||||
if _, err := itemcodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable artifact encoding: %v", err)
|
||||
}
|
||||
if test.wantFallback {
|
||||
refs := result.Value.Items[1].SourceRefs
|
||||
if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 {
|
||||
t.Fatalf("final proposal carried earlier evidence: %#v", refs)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeFailureUsesMergedInputWithoutRecoveringEarlierProposal(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, failCorrection := range []bool{false, true} {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
input := mergedItems()
|
||||
client := &recordingNormalizerClient{err: failure}
|
||||
if failCorrection {
|
||||
client.err = nil
|
||||
client.responses = []string{safeFirstProposal}
|
||||
client.errors = []error{nil, failure}
|
||||
}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.Mode = mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "policy", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct compass records."}}}}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
wantCalls, wantAssess := 1, 0
|
||||
if failCorrection {
|
||||
wantCalls, wantAssess = 2, 1
|
||||
}
|
||||
if len(client.requests) != wantCalls || assessments != wantAssess {
|
||||
t.Fatalf("calls = %d/%d, want %d/%d", len(client.requests), assessments, wantCalls, wantAssess)
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if !errors.Is(err, failure) {
|
||||
t.Fatalf("strict failure = %v", err)
|
||||
}
|
||||
} else if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeStructuralFailureProducesNoSemanticCandidate(t *testing.T) {
|
||||
input := mergedItems()
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":null}`}
|
||||
request := permissiveRequestWithSource(input, semanticDocument())
|
||||
request.Operation.Controls.MaxAdditionalTurns = 2
|
||||
assessed := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || assessed != 0 || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||||
t.Fatalf("native structural failure = %#v %v, assessed %d", result, err, assessed)
|
||||
}
|
||||
for _, call := range client.requests {
|
||||
for _, message := range call.Prompt.Messages {
|
||||
if strings.Contains(message.Content, "Revise the previous duplicate-entity proposal using these findings:") {
|
||||
t.Fatal("native structural failure started semantic continuation")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsufficientCandidatesUseAssessmentWithoutNativeExecution(t *testing.T) {
|
||||
input := mergedItems()
|
||||
input.Items = input.Items[:1]
|
||||
client := &recordingNormalizerClient{}
|
||||
request := normalizeRequestWithSource(input, semanticDocument())
|
||||
base := request.Operation.Assess
|
||||
assessed := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return base(ctx, value)
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || len(client.requests) != 0 || assessed != 1 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||||
t.Fatalf("no-model outcome = %#v %v, assessed %d", result, err, assessed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationNeverDeliversNormalizationFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &recordingNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newNormalizer(t, client).Normalize(ctx, permissiveRequestWithSource(mergedItems(), semanticDocument()))
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation = %#v %v, calls %d", result, err, len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFallbackCannotPublishAnUnusableMergedRegistry(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*dnd.ItemRegistry)
|
||||
}{
|
||||
{"identity mismatch", func(value *dnd.ItemRegistry) { value.Items[0].ID = identity.DeriveID("Wrong identity") }},
|
||||
{"foreign source", func(value *dnd.ItemRegistry) { value.Items[0].SourceRefs[0].SourceID = "other-session" }},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
input := mergedItems()
|
||||
test.mutate(&input)
|
||||
result, err := newNormalizer(t, &recordingNormalizerClient{err: failure}).Normalize(context.Background(), permissiveRequestWithSource(input, semanticDocument()))
|
||||
if !errors.Is(err, failure) || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("invalid fallback = %#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -14,24 +14,23 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/item-registry"
|
||||
PromptID = "dnd.item_registry.normalize"
|
||||
PromptVersion = "v1"
|
||||
normalizationPolicy = "dnd.item_registry.normalize.v4"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
Key = "dnd/item-registry"
|
||||
PromptID = "notarius/dnd.item_registry.normalize"
|
||||
CorrectionPromptID = "notarius/dnd.item_registry.normalize.repair"
|
||||
|
||||
ReasonCodeItemFieldsNormalized = "item_fields_normalized"
|
||||
ReasonCodeItemIDRecomputed = "item_id_recomputed"
|
||||
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
||||
ReasonCodeDuplicateItemCollapsed = "duplicate_item_collapsed"
|
||||
ReasonCodeItemSemanticProposalInvalid = "item_semantic_proposal_invalid"
|
||||
ReasonCodeItemSemanticRetryProposalInvalid = "item_semantic_retry_proposal_invalid"
|
||||
ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted"
|
||||
)
|
||||
|
||||
@@ -39,28 +38,16 @@ var requiredCapabilities = []string{"merged"}
|
||||
var providedCapabilities = []string{"normalized"}
|
||||
|
||||
var _ contracts.Normalizer[dnd.ItemRegistry] = (*Normalizer)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Normalizer struct {
|
||||
engine *semanticreconcile.Engine
|
||||
engine *promptkit.Engine
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) {
|
||||
if llmClient == nil {
|
||||
return nil, normalizerErrorf("LLM client must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{
|
||||
ID: PromptID, Version: PromptVersion, SHA256: promptSHA,
|
||||
}, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err)
|
||||
func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) {
|
||||
if engine == nil {
|
||||
return nil, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
return &Normalizer{engine: engine}, nil
|
||||
}
|
||||
@@ -68,33 +55,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error
|
||||
func (n *Normalizer) Key() string { return Key }
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := n.engine.ManifestMetadata()
|
||||
metadata["identity_policy"] = identity.Policy
|
||||
metadata["normalization_policy"] = normalizationPolicy
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if n == nil || n.engine == nil {
|
||||
return nil
|
||||
}
|
||||
fingerprints := n.engine.CheckpointFingerprints()
|
||||
return append(fingerprints,
|
||||
pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy},
|
||||
pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy},
|
||||
)
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemRegistry]) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
if n == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if n.engine == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil")
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("native engine must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context must not be nil")
|
||||
@@ -106,78 +72,145 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil")
|
||||
}
|
||||
|
||||
// Keep the already available merged artifact separate from preprocessed
|
||||
// originals. A failed native response must not recover an earlier proposal.
|
||||
merged := cloneRegistry(req.MergeOutput.Value)
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, findings := preprocessRecords(req.MergeOutput.Value, order)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if len(records) < 2 {
|
||||
return normalizationResult(deterministic, findings, nil, nil)
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
var advisories []diagnostics.Finding
|
||||
discardedGroups := -1
|
||||
var omissionDetails []string
|
||||
|
||||
fallback := func(_ context.Context, final *dnd.ItemRegistry) (dnd.ItemRegistry, error) {
|
||||
value := cloneRegistry(merged)
|
||||
if final != nil {
|
||||
// Each interpreted candidate was built afresh from fixed original records,
|
||||
// using only the final proposal's compatible, non-overlapping safe groups.
|
||||
value = cloneRegistry(*final)
|
||||
} else {
|
||||
findings, advisories = nil, nil
|
||||
discardedGroups = -1
|
||||
omissionDetails = nil
|
||||
}
|
||||
if err := validateFallback(value, req.Source); err != nil {
|
||||
return dnd.ItemRegistry{}, err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
finish := func(value dnd.ItemRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalize items: %w", err)
|
||||
}
|
||||
result, err := normalizationResult(value, findings, advisories)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
result.Outcome = outcome
|
||||
if outcome.Disposition == contracts.ProducerFallback {
|
||||
details := append([]string(nil), omissionDetails...)
|
||||
details = append(details, outcome.FallbackReason)
|
||||
groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, groups...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
noModel := func() (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
if len(records) < 2 {
|
||||
return noModel()
|
||||
}
|
||||
candidates, envelopes, err := reconciliationInputs(records)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err)
|
||||
}
|
||||
reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
|
||||
StageName: Key, Source: req.Source, Candidates: candidates,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction,
|
||||
})
|
||||
preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits())
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
|
||||
switch reconciliation.Disposition() {
|
||||
case semanticreconcile.SkippedInsufficientCandidates:
|
||||
return normalizationResult(deterministic, findings, nil, nil)
|
||||
case semanticreconcile.SkippedLimitExceeded:
|
||||
return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
|
||||
case semanticreconcile.RetryableInvalidStructuredOutput:
|
||||
return n.invalidStructuredResult(deterministic, findings)
|
||||
case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
|
||||
default:
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition())
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
case semanticreconcile.LimitExceeded:
|
||||
value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) })
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
|
||||
initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build native request: %w", err)
|
||||
}
|
||||
findings = append(findings, semanticFindings...)
|
||||
discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
|
||||
if discardedGroups == 0 {
|
||||
return normalizationResult(recordList(applied), findings, advisoryFindings, reconciliation.ModelCandidate())
|
||||
}
|
||||
return retryResult(recordList(applied), findings, advisoryFindings, reconciliation, rejectedGroups)
|
||||
value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemRegistry, []string, error) {
|
||||
response, err := semanticreconcile.DecodeProposal(result.RawOutput)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
}
|
||||
findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...)
|
||||
advisories = advisoryFindings
|
||||
discardedGroups = assessment.DiscardedGroupCount() + rejectedGroups
|
||||
omissionDetails = semanticreconcile.IssueDetails(assessment.Issues())
|
||||
guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if rejectedGroups > 0 {
|
||||
guidance = append(guidance, "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items.")
|
||||
omissionDetails = append(omissionDetails, "currency may only be consolidated with aliases of one denomination")
|
||||
}
|
||||
candidate := recordList(applied)
|
||||
return &candidate, guidance, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil
|
||||
}, fallback)
|
||||
return finish(value, outcome, err)
|
||||
}
|
||||
|
||||
func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
|
||||
// validateFallback checks the durable artifact and the identity/source invariants
|
||||
// used by registry consumers, not the configured ordinary validator chain.
|
||||
func validateFallback(value dnd.ItemRegistry, document *source.SourceDocument) error {
|
||||
if _, err := itemcodec.New().Encode(value); err != nil {
|
||||
return normalizerErrorf("construct item fallback: %w", err)
|
||||
}
|
||||
if issues := identity.ValidateList(value); len(issues) > 0 {
|
||||
return normalizerErrorf("construct item fallback: registry identity is not usable downstream")
|
||||
}
|
||||
index := source.NewDocumentIndex(document)
|
||||
for _, item := range value.Items {
|
||||
for _, reference := range item.SourceRefs {
|
||||
if err := index.ValidateRef(reference); err != nil {
|
||||
return normalizerErrorf("construct item fallback: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, reconciliation semanticreconcile.Result, rejectedGroups int) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
issues := reconciliation.Issues()
|
||||
operatorDetails := semanticreconcile.IssueDetails(issues)
|
||||
correctionDetails, err := reconciliation.CorrectionDetails()
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
|
||||
func cloneRegistry(value dnd.ItemRegistry) dnd.ItemRegistry {
|
||||
if value.Items == nil {
|
||||
return dnd.ItemRegistry{}
|
||||
}
|
||||
var additionalGuidance []string
|
||||
if rejectedGroups > 0 {
|
||||
const currencyGuidance = "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items."
|
||||
operatorDetails = append(operatorDetails, "currency may only be consolidated with aliases of one denomination")
|
||||
correctionDetails = append(correctionDetails, currencyGuidance)
|
||||
additionalGuidance = append(additionalGuidance, currencyGuidance)
|
||||
result := dnd.ItemRegistry{Items: make([]dnd.Item, len(value.Items))}
|
||||
for i, item := range value.Items {
|
||||
result.Items[i] = cloneItem(item)
|
||||
}
|
||||
correctionGuidance, err := reconciliation.CorrectionGuidance(additionalGuidance...)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
|
||||
}
|
||||
discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
|
||||
return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", operatorDetails), semanticFallbackFinding(discardedGroups, correctionDetails...))
|
||||
return result
|
||||
}
|
||||
|
||||
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
||||
message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
|
||||
if discarded >= 0 {
|
||||
message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output"
|
||||
if discarded > 0 {
|
||||
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
||||
}
|
||||
if len(details) > 0 {
|
||||
@@ -186,8 +219,8 @@ func semanticFallbackFinding(discarded int, details ...string) diagnostics.Findi
|
||||
return diagnostics.Finding{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message}
|
||||
}
|
||||
|
||||
func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
|
||||
}
|
||||
@@ -195,34 +228,7 @@ func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []di
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect data-quality diagnostics: %w", err)
|
||||
}
|
||||
diagnosticGroups = append(diagnosticGroups, advisoryGroups...)
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func fallbackResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, advisoryFindings, nil)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Diagnostics = append(result.Diagnostics, fallbackGroups...)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate, reasonCode, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
|
||||
result, err := normalizationResult(value, findings, advisoryFindings, candidate)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
||||
}
|
||||
result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
|
||||
return result, nil
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: append(groups, advisoryGroups...)}, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
@@ -363,7 +369,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding {
|
||||
func itemScope(index int) string { return fmt.Sprintf("items[%d]", index) }
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind}
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind}
|
||||
}
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemRegistry], error) {
|
||||
@@ -371,7 +377,7 @@ func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
return New(request.Dependencies.Engine, options)
|
||||
})
|
||||
}
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
@@ -3,28 +3,26 @@ package itemregistry
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestModuleContractAndMetadata(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
||||
func TestModuleContractAndNativeDependencies(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -33,17 +31,12 @@ func TestModuleContractAndMetadata(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(nil, Options{}); err == nil {
|
||||
t.Fatal("New() accepted nil client")
|
||||
t.Fatal("New() accepted nil native engine")
|
||||
}
|
||||
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
||||
if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) {
|
||||
t.Fatalf("fingerprints missing %q", name)
|
||||
}
|
||||
got := ModuleSpec()
|
||||
got.Prompts[0] = "caller mutation"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("static dependency declaration is not owned")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,7 +86,7 @@ func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
|
||||
if err := json.Unmarshal(renderedInput(t, client.requests[0], "candidates"), &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
|
||||
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
|
||||
}
|
||||
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
|
||||
@@ -125,14 +118,14 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Items) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
merged := result.Value.Items[0]
|
||||
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
|
||||
t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics)
|
||||
}
|
||||
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
||||
encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript"))
|
||||
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
||||
t.Fatalf("private inputs = %s", encoded)
|
||||
}
|
||||
@@ -149,7 +142,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response string
|
||||
wantNames []string
|
||||
wantRefCounts []int
|
||||
wantRetry bool
|
||||
wantFallback bool
|
||||
reasonCode string
|
||||
disposition contracts.DiagnosticDisposition
|
||||
}{
|
||||
@@ -175,7 +168,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
wantFallback: true,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
@@ -188,7 +181,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
wantFallback: true,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
@@ -213,7 +206,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
wantFallback: true,
|
||||
reasonCode: ReasonCodeItemSemanticProposalInvalid,
|
||||
disposition: contracts.DiagnosticDispositionAdvisory,
|
||||
},
|
||||
@@ -228,8 +221,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
before.Items[index] = cloneItem(item)
|
||||
}
|
||||
|
||||
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
|
||||
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), permissiveRequestWithSource(input, doc))
|
||||
if err != nil || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
if len(result.Value.Items) != len(test.wantNames) {
|
||||
@@ -240,8 +233,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
|
||||
}
|
||||
}
|
||||
if test.wantRetry && (len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted || result.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid) {
|
||||
t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry)
|
||||
if test.wantFallback && (result.Outcome.ReuseEligible || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning)) {
|
||||
t.Fatalf("fallback lacks qualification: %#v", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -255,9 +248,9 @@ func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
||||
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,3],"canonical_candidate_number":99}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), permissiveRequestWithSource(input, doc))
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Items) != 3 || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("Normalize() = %#v, %v; want qualified safe fallback", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,9 +267,11 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
{Name: "Rope", SourceRefs: ref(10)},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,4],"canonical_candidate_number":3},{"candidate_numbers":[5,99],"canonical_candidate_number":5}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err)
|
||||
request := permissiveRequestWithSource(input, doc)
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback {
|
||||
t.Fatalf("Normalize() = %#v, %v; want qualified safe partial output", result, err)
|
||||
}
|
||||
wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"}
|
||||
if len(result.Value.Items) != len(wantNames) {
|
||||
@@ -290,115 +285,214 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
|
||||
t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
|
||||
}
|
||||
if !strings.Contains(result.Retry.CorrectionGuidance, "aliases of the same denomination") || !strings.Contains(result.Retry.CorrectionGuidance, "do not combine currency with non-currency items") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 5") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeItemSemanticRetryProposalInvalid) {
|
||||
t.Fatalf("correction guidance = %q, want shared and item-specific prose only", result.Retry.CorrectionGuidance)
|
||||
|
||||
if len(client.requests) != 2 {
|
||||
t.Fatalf("native calls = %d, want initial and correction", len(client.requests))
|
||||
}
|
||||
if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "same denomination") {
|
||||
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
|
||||
messages := client.requests[1].Prompt.Messages
|
||||
guidance := messages[len(messages)-1].Content
|
||||
for _, expected := range []string{"aliases of the same denomination", "do not combine currency with non-currency items", "candidate-number range 1 through 5", "source positions"} {
|
||||
if !strings.Contains(guidance, expected) {
|
||||
t.Fatalf("missing domain/shared guidance %q", expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(guidance, "member_unknown") || strings.Contains(guidance, ReasonCodeItemSemanticProposalInvalid) {
|
||||
t.Fatal("internal diagnostics copied into correction")
|
||||
}
|
||||
var warning string
|
||||
for _, group := range result.Diagnostics {
|
||||
if group.ReasonCode == ReasonCodeItemSemanticReconciliationExhausted && len(group.Samples) > 0 {
|
||||
warning = group.Samples[0].Message
|
||||
}
|
||||
}
|
||||
if !strings.Contains(warning, "2 proposal group(s)") || !strings.Contains(warning, "preserved separately") || !strings.Contains(warning, "same denomination") {
|
||||
t.Fatalf("fallback warning = %q", warning)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
||||
client := &recordingNormalizerClient{}
|
||||
func TestPreparationLimitsFollowResolvedModeWithoutModelOrAssessment(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}}
|
||||
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
||||
input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)}
|
||||
for index := range input.Items {
|
||||
input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
for i := range input.Items {
|
||||
name := fmt.Sprintf("Item %d", i)
|
||||
input.Items[i] = dnd.Item{ID: identity.DeriveID(name), Name: name, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
|
||||
}
|
||||
if !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
|
||||
t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
client := &recordingNormalizerClient{}
|
||||
request := normalizeRequestWithSource(input, doc)
|
||||
request.Operation.Controls.Mode = mode
|
||||
assessed := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if len(client.requests) != 0 || assessed != 0 {
|
||||
t.Fatal("unready preparation caused model execution or ordinary validation")
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil {
|
||||
t.Fatal("strict preparation-limit failure succeeded")
|
||||
}
|
||||
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) {
|
||||
t.Fatalf("permissive preparation limit: %#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRetryFallbackErrorsAndIdempotence(t *testing.T) {
|
||||
func TestNormalizationRemainsIdempotent(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") {
|
||||
t.Fatalf("invalid structured retry = %#v, want feedback-free retry with final-cause fallback", invalid.Retry)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
}
|
||||
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
||||
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
||||
t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := semanticreconcile.RegisterAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"item-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
second, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
||||
if err != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
||||
t.Fatalf("repeated normalization changed output: %#v %v", second, err)
|
||||
}
|
||||
if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) {
|
||||
t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared)
|
||||
}
|
||||
|
||||
func TestNativeNormalizationPromptPairPreparesWithSameContract(t *testing.T) {
|
||||
client := &recordingNormalizerClient{}
|
||||
normalizer := newNormalizer(t, client)
|
||||
var initial promptkit.PreparedRun
|
||||
for _, id := range []string{PromptID, CorrectionPromptID} {
|
||||
inputs := map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep currency denominations separate.")}
|
||||
if id == PromptID {
|
||||
inputs = map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`), "transcript": promptkit.Inline(`{"windows":[]}`)}
|
||||
}
|
||||
handle, err := normalizer.engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: id, APIKey: "test-only", Inputs: inputs})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared := handle.Details()
|
||||
handle.Discard()
|
||||
if prepared.SelectedProfileID != "notarius/dnd-extraction" || prepared.StructuredOutput == nil {
|
||||
t.Fatal("missing native defaults/schema")
|
||||
}
|
||||
if id == PromptID {
|
||||
initial = prepared
|
||||
var system string
|
||||
for _, message := range prepared.Messages {
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range []string{"candidate_number_range", "start_unit_id", "currency denominations", "materially different item", "canonical_candidate_number"} {
|
||||
if !strings.Contains(system, rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
} else if !reflect.DeepEqual(initial.OutputContract, prepared.OutputContract) || len(prepared.Messages) != 1 || prepared.Messages[0].Role != "user" || !strings.Contains(prepared.Messages[0].Content, "Keep currency denominations separate.") {
|
||||
t.Fatal("correction did not preserve the output contract or supply only findings/assignment")
|
||||
}
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatal("offline preparation invoked generation")
|
||||
}
|
||||
}
|
||||
|
||||
type recordingNormalizerClient struct {
|
||||
response string
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response string
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err)
|
||||
func (c *recordingNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(c.requests)
|
||||
c.requests = append(c.requests, request)
|
||||
if c.onGenerate != nil {
|
||||
c.onGenerate(ctx, request)
|
||||
}
|
||||
c.requests = append(c.requests, snapshot)
|
||||
if c.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, c.err
|
||||
return nil, c.err
|
||||
}
|
||||
if index < len(c.errors) && c.errors[index] != nil {
|
||||
return nil, c.errors[index]
|
||||
}
|
||||
response := c.response
|
||||
if index < len(c.responses) {
|
||||
response = c.responses[index]
|
||||
}
|
||||
if response == "" {
|
||||
response = `{"duplicate_groups":[]}`
|
||||
}
|
||||
content := []byte(response)
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: response}, nil
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
||||
func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(client, Options{})
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: item-normalize-test
|
||||
endpoint: https://example.invalid/v1
|
||||
model: normalization-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizer, err := New(engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return normalizer
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := "candidate_number_range"
|
||||
if name == "transcript" {
|
||||
key = "windows"
|
||||
}
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
||||
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{
|
||||
Source: &source.SourceDocument{},
|
||||
Source: &source.SourceDocument{},
|
||||
LLMProfile: "item-normalize-test",
|
||||
Operation: contracts.ProducerOperation[dnd.ItemRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) {
|
||||
result, err := identityvalidator.New(identityvalidator.Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value, Stage: string(pipeline.StageNormalize)})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentApproved
|
||||
if !result.Approved {
|
||||
decision = contracts.AssessmentRejected
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: identityvalidator.Key, Decision: decision, Result: result}}}, nil
|
||||
}},
|
||||
MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value},
|
||||
}
|
||||
}
|
||||
@@ -419,11 +513,8 @@ func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, di
|
||||
return false
|
||||
}
|
||||
|
||||
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
func permissiveRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
||||
request := normalizeRequestWithSource(value, doc)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
return request
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
func promptAssetManifest() (shared.PromptAssetManifest, error) {
|
||||
sharedFiles, err := semanticreconcile.SharedPromptFiles()
|
||||
if err != nil {
|
||||
return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err)
|
||||
}
|
||||
return shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{"common-dnd-system.md"},
|
||||
ExternalSharedFiles: sharedFiles,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/normalize")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope item normalization assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := manifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare item normalization prompt assets: %w", err)
|
||||
}
|
||||
return registry.RegisterPromptFS(promptFS, promptAssetRoot)
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
manifest, err := promptAssetManifest()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = manifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -35,7 +35,7 @@ func TestValidatorApprovesAndRegisters(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user