Capture native execution facts and complete scoped debug evidence
This commit is contained in:
@@ -2,13 +2,18 @@ package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
@@ -39,6 +44,43 @@ func itemNames(value dnd.ItemRegistry) []string {
|
||||
return names
|
||||
}
|
||||
|
||||
func TestNormalizationCaptureRetainsFixedMappingAndEveryProposal(t *testing.T) {
|
||||
recorder, err := frameworkdebug.Start(t.TempDir(), "normalization", time.Unix(1, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &recordingNormalizerClient{responses: []string{invalidItemProposal, safeFirstProposal}}
|
||||
request := normalizeRequestWithSource(mergedItems(), semanticDocument())
|
||||
request.Operation.Execution.Capture = recorder
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted {
|
||||
t.Fatalf("normalization: %#v %v", result, err)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(recorder.Path(), "normalization", "originals.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var originals dnd.ItemRegistry
|
||||
if err := json.Unmarshal(content, &originals); err != nil || len(originals.Items) != 3 {
|
||||
t.Fatalf("fixed originals missing: %#v %v", originals, err)
|
||||
}
|
||||
content, err = os.ReadFile(filepath.Join(recorder.Path(), "normalization", "preparation.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var preparation struct {
|
||||
Mappings []any `json:"candidate_mappings"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &preparation); err != nil || len(preparation.Mappings) != 3 {
|
||||
t.Fatalf("fixed handle mapping missing: %s %v", content, err)
|
||||
}
|
||||
proposals, err := os.ReadDir(filepath.Join(recorder.Path(), "normalization", "proposals"))
|
||||
if err != nil || len(proposals) != 2 {
|
||||
t.Fatalf("revised proposals lost: %v %v", proposals, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) {
|
||||
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
for _, test := range []struct {
|
||||
|
||||
@@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
var advisories []diagnostics.Finding
|
||||
discardedGroups := -1
|
||||
@@ -134,6 +137,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
|
||||
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
|
||||
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
|
||||
}); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
|
||||
}
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
@@ -152,6 +161,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
|
||||
@@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
discardedGroups := -1
|
||||
var omissionDetails []string
|
||||
@@ -133,6 +136,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
|
||||
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
|
||||
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
|
||||
}); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
||||
}
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
@@ -151,6 +160,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
|
||||
@@ -77,6 +77,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
records, baseFindings := preprocessRecords(merged, order)
|
||||
deterministic := recordList(records)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
}
|
||||
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
||||
discardedGroups := -1
|
||||
var omissionDetails []string
|
||||
@@ -132,6 +135,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
||||
}
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
|
||||
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
|
||||
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
|
||||
}); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
|
||||
}
|
||||
switch preparation.Disposition() {
|
||||
case semanticreconcile.InsufficientCandidates:
|
||||
return noModel()
|
||||
@@ -150,6 +159,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
|
||||
return nil, nil, err
|
||||
}
|
||||
assessment := preparation.Assess(response)
|
||||
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
||||
if err != nil {
|
||||
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user