diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 318fbb8a..d2f3c112 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1706,6 +1706,18 @@ legacy ConfigKit YAML import. Remaining CLI test migration, dependency tidy/ old-path removal, notice reconciliation, and whole-tree validation are required before completion. This stage is not marked finished. +Assembled spell tests now exercise the real native extractor and continuation +history rather than a synthetic model-candidate/correction protocol. Retained +merged-cast ordering, canonicalization, evidence, explicit validator overrides, +and process/advisory distinctions. Native catalog correction tests cover selected +turn exhaustion and corrected overlay acceptance; strict terminal rejection +publishes no artifact. Catalog materialization retains raw-reference provenance +and semantic recognition across reordered overlay bytes, and the native CLI +publishes the grounded overlay spell. Deleted only superseded checkpoint-identity/ +resume and manifest/sidecar assertions and their helpers. Focused assembled spell/ +catalog tests, vet, and repeated race checks pass; remaining CLI fixtures and final +compilation are still required. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/cli/assembled_spell_pipeline_contract_test.go b/internal/cli/assembled_spell_pipeline_contract_test.go index 95ffe103..1dc1704c 100644 --- a/internal/cli/assembled_spell_pipeline_contract_test.go +++ b/internal/cli/assembled_spell_pipeline_contract_test.go @@ -10,19 +10,17 @@ import ( "sync" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" - spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" + "go.maximumdirect.net/promptkit" ) const assembledSpellExtractorKey = "test/dnd/spell-casts" -const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts" - const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction" func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { @@ -33,9 +31,8 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, + Prepared: prepared, + RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) @@ -46,17 +43,10 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) { t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes) } - if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { - t.Fatalf("run output = %#v, want approved normalized output without rejections", output) - } - if output.NormalizeOutputs[0].NormalizerKey != spellnormalize.Key { - t.Fatalf("normalized output module = %q, want %q", output.NormalizeOutputs[0].NormalizerKey, spellnormalize.Key) - } - - var normalized dnd.SpellList - if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil { - t.Fatalf("decode normalized output: %v", err) + if len(output.OutputFiles) != 1 || output.FallbackCount != 0 { + t.Fatalf("output=%#v, want accepted spell artifact", output) } + normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json") if len(normalized.SpellCasts) != 2 { t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts) } @@ -88,55 +78,60 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) { t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2]) } - warningsFile := decodeAssembledOutput[struct { - Groups []contracts.DiagnosticGroup `json:"groups"` - }](t, output.OutputFiles, "warnings.json") - if len(warningsFile.Groups) != 0 { - t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups) - } - diagnosticsFile := decodeAssembledOutput[struct { - SchemaVersion string `json:"schema_version"` - GroupCount int `json:"group_count"` - OccurrenceCount int `json:"occurrence_count"` - Truncated bool `json:"truncated"` - UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"` - Groups []contracts.DiagnosticGroup `json:"groups"` - }](t, output.OutputFiles, "diagnostics.json") - if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount { - t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics) - } - manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json") - if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key { - t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes) - } - normalizerMetadata, ok := manifest.ArtifactLanes[0].Metadata["normalizer"].(map[string]any) - _, hasOverlayIDs := normalizerMetadata["catalog_overlay_ids"] - if !ok || normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !hasOverlayIDs { - t.Fatalf("normalizer manifest metadata = %#v, want base ID, digest, and overlay IDs", manifest.ArtifactLanes[0].Metadata) + for _, group := range output.Diagnostics.Groups { + if group.Disposition == contracts.DiagnosticDispositionWarning { + t.Fatalf("advisory-only normalization emitted a process warning: %#v", group) + } } } func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) { - registries, resolved, extractor := assembledCorrectingSpellPipeline(t) - prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) + components := productionTestComponents(t) + if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { return assembledDirectSpellValidator{}, nil }); err != nil { + t.Fatal(err) + } + turns := 1 + extract := pipeline.Binding(spells.Key) + extract.MaxAdditionalTurns = &turns + extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(assembledDirectSpellValidatorKey)}} + resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{ + ID: "assembled-dnd-correcting-spells", Input: pipeline.Binding("seriatim"), Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 2}}, + Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)}}, Output: pipeline.Binding("json"), + }, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries)) if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) + t.Fatal(err) } - - output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, - }) + client := &assembledSpellCorrectionClient{} + prepared, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)}) if err != nil { - t.Fatalf("Run() error = %v, want nil", err) + t.Fatal(err) } - if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 { - t.Fatalf("run output = %#v, want corrected accepted spell output", output) + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")}) + if err != nil { + t.Fatal(err) } - correction := extractor.correctionSnapshot() - if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") { - t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction) + value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json") + if output.FallbackCount != 0 || len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" { + t.Fatalf("output=%#v spells=%#v", output, value) + } + client.mu.Lock() + requests := append([]promptkit.GenerateRequest(nil), client.requests...) + client.mu.Unlock() + if len(requests) != 2 { + t.Fatalf("provider calls=%d, want rejected and revised candidates", len(requests)) + } + initial, revised := requests[0].Prompt.Messages, requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) { + t.Fatal("correction lost native growing history") + } + assistant, guidance := revised[len(initial)], revised[len(initial)+1] + if assistant.Role != promptkit.RoleAssistant || assistant.Content != assembledRejectedSpellResponse || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "use a known spell name") || !strings.Contains(guidance.Content, "complete replacement") { + t.Fatalf("native correction=%#v", revised[len(initial):]) + } + for _, fragment := range []string{"unknown_spell", "spell is not in the catalog", assembledDirectSpellValidatorKey} { + if strings.Contains(guidance.Content, fragment) { + t.Fatalf("correction contains diagnostic prose or identifier: %s", guidance.Content) + } } } @@ -159,11 +154,10 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) { t.Fatalf("Prepare() error = %v, want nil", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, + Prepared: prepared, + RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), }) - if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { + if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 { t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output) } for _, group := range output.Diagnostics.Groups { @@ -173,36 +167,20 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) { } } -func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) { +func TestAssembledSpellPipelineRejectsUnknownSpellDuringNormalization(t *testing.T) { registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true}) - resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{}) if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) + t.Fatal(err) } - output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, - }) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")}) + for _, fragment := range []string{"normalize", `lane "spells"`, "extract/dnd/spells/catalog"} { + if err == nil || !strings.Contains(err.Error(), fragment) { + t.Fatalf("error=%v, want scoped catalog rejection (%s)", err, fragment) + } } - if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 || len(output.Rejected) != 1 { - t.Fatalf("run output = %#v, want one rejected normalize candidate and no normalized output", output) - } - rejection := output.Rejected[0] - if rejection.Stage != string(pipeline.StageNormalize) || rejection.LaneID != "spells" || rejection.ModuleKey != spellnormalize.Key || rejection.ValidatorName != "extract/dnd/spells/catalog" || rejection.ReasonCode != "unknown_spell" { - t.Fatalf("rejection = %#v, want durable normalize catalog rejection", rejection) - } - rejectedFile := decodeAssembledOutput[struct { - Rejected []contracts.RejectedOutput `json:"rejected"` - }](t, output.OutputFiles, "rejected.json") - if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) { - t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected) - } - if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" { - t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics) + if len(output.OutputFiles) != 0 || output.FallbackCount != 0 { + t.Fatalf("strict normalization failure published output: %#v", output) } } @@ -213,37 +191,24 @@ func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t t.Fatalf("Prepare() error = %v, want nil", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, + Prepared: prepared, + RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } - if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { - t.Fatalf("run output = %#v, want accepted unknown spell with explicit validator override", output) - } - var normalized dnd.SpellList - if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil { - t.Fatalf("decode normalized output: %v", err) + if output.FallbackCount != 0 || len(output.OutputFiles) != 1 { + t.Fatalf("output=%#v, want accepted explicit validator override", output) } + normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json") if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" { t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts) } if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" { t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics) } - warningsFile := decodeAssembledOutput[struct { - Groups []contracts.DiagnosticGroup `json:"groups"` - }](t, output.OutputFiles, "warnings.json") - if len(warningsFile.Groups) != 0 { - t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups) - } - diagnosticsFile := decodeAssembledOutput[struct { - Groups []contracts.DiagnosticGroup `json:"groups"` - }](t, output.OutputFiles, "diagnostics.json") - if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) { - t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics) + if output.Diagnostics.Groups[0].Disposition == contracts.DiagnosticDispositionWarning { + t.Fatal("unknown-spell advisory became a process warning") } } @@ -252,47 +217,6 @@ type assembledSpellPipelineOptions struct { unknownSpell bool } -func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) { - t.Helper() - components := productionTestComponents(t) - extractor := &assembledCorrectingSpellExtractor{} - if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{ - Key: assembledCorrectingSpellExtractorKey, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: []string{"chunks", "source.transcript"}, - Provides: []string{"dnd.spell_casts"}, - ArtifactKind: dnd.SpellListKind, - }, func() (contracts.Extractor[dnd.SpellList], error) { - return extractor, nil - }); err != nil { - t.Fatalf("register correcting extractor: %v", err) - } - if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { - return assembledDirectSpellValidator{}, nil - }); err != nil { - t.Fatalf("register direct spell validator: %v", err) - } - - extract := pipeline.Binding(assembledCorrectingSpellExtractorKey) - extract.Retries = 1 - extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}} - resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{ - ID: "assembled-dnd-correcting-spells", - Input: pipeline.Binding("seriatim"), - Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}}, - Artifacts: map[string]pipeline.ArtifactLaneProfile{ - "spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)}, - }, - Output: pipeline.Binding("json"), - }, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries)) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - return components.registries, resolved, extractor -} - func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) { t.Helper() components := productionTestComponents(t) @@ -338,50 +262,28 @@ type assembledSpellExtractor struct { unknownSpell bool } -type assembledCorrectingSpellExtractor struct { - mu sync.Mutex - correction *contracts.SemanticCorrection +const assembledRejectedSpellResponse = `{"spell_casts":[{"caster":"Aria","spell":"Mysterious Burst","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` + +type assembledSpellCorrectionClient struct { + mu sync.Mutex + requests []promptkit.GenerateRequest } -func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey } - -func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } - -func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) { - if req.Source == nil || req.Chunk == nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk") +func (client *assembledSpellCorrectionClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if err := ctx.Err(); err != nil { + return nil, err } - response := `{"spell":"accepted"}` - value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}} - if req.Chunk.Index == 0 && req.Correction == nil { - response = `{"spell":"Mysterious Burst"}` - value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}} + client.mu.Lock() + defer client.mu.Unlock() + if !productionSchemaHasProperty(request, "spell_casts") { + return nil, fmt.Errorf("unexpected native response contract") } - if req.Chunk.Index == 0 && req.Correction != nil { - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, err - } - e.mu.Lock() - e.correction = correction - e.mu.Unlock() - value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}} + client.requests = append(client.requests, request) + content := assembledRejectedSpellResponse + if len(client.requests) > 1 { + content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` } - candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1) - if err != nil { - return contracts.TypedExtractionResult[dnd.SpellList]{}, err - } - return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil -} - -func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection { - e.mu.Lock() - defer e.mu.Unlock() - correction, err := contracts.CloneSemanticCorrection(e.correction) - if err != nil { - return nil - } - return correction + return &promptkit.GenerateResponse{Content: content}, nil } type assembledDirectSpellValidator struct{} @@ -405,7 +307,7 @@ func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorK func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) { +func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) { if err := ctx.Err(); err != nil { return contracts.TypedExtractionResult[dnd.SpellList]{}, err } @@ -446,14 +348,6 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int { return append([]int(nil), e.chunkIndexes...) } -func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int { - count := 0 - for _, group := range groups { - count += group.OccurrenceCount - } - return count -} - func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T { t.Helper() for _, file := range files { diff --git a/internal/cli/spell_catalog_retry_contract_test.go b/internal/cli/spell_catalog_correction_contract_test.go similarity index 50% rename from internal/cli/spell_catalog_retry_contract_test.go rename to internal/cli/spell_catalog_correction_contract_test.go index b88b0f31..51d4d0b5 100644 --- a/internal/cli/spell_catalog_retry_contract_test.go +++ b/internal/cli/spell_catalog_correction_contract_test.go @@ -5,19 +5,19 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" + "go.maximumdirect.net/promptkit" ) -func TestProductionSpellCatalogValidationRetries(t *testing.T) { - const retries = 2 +func TestProductionSpellCatalogValidationCoordinatesCorrections(t *testing.T) { + const additionalTurns = 2 tests := []struct { name string responses []string @@ -33,11 +33,11 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) { productionSpellResponse("Unknown Spell"), productionSpellResponse("Unknown Spell"), }, - wantCalls: retries + 1, + wantCalls: additionalTurns + 1, wantRejected: true, }, { - name: "overlay spell becomes valid on retry", + name: "overlay spell becomes valid after correction", responses: []string{ productionSpellResponse("Unknown Spell"), productionSpellResponse("Aegis of Emberfall"), @@ -53,6 +53,12 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) cfg := loadMaintainedExample(t, configPath) + profile := cfg.Pipelines["dnd-session"] + lane := profile.Artifacts["spells"] + turns := additionalTurns + lane.Extract.MaxAdditionalTurns = &turns + profile.Artifacts["spells"] = lane + cfg.Pipelines["dnd-session"] = profile effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve production configuration: %v", err) @@ -64,56 +70,36 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) { if err != nil { t.Fatalf("materialize production references: %v", err) } - materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries - materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput - llmClient := &catalogRetryLLMClient{responses: tt.responses} - prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient}) + llmClient := &catalogCorrectionLLMClient{responses: tt.responses} + prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient)}) if err != nil { t.Fatalf("prepare production pipeline: %v", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, + Prepared: prepared, + RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), }) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if calls := llmClient.CallCount(); calls > retries+1 || calls != tt.wantCalls { - t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, retries+1) + if calls := llmClient.CallCount(); calls > additionalTurns+1 || calls != tt.wantCalls { + t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, additionalTurns+1) } if tt.wantRejected { - if len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 { - t.Fatalf("rejected = %#v normalized = %#v, want one nonfatal rejection and no merge output", output.Rejected, output.NormalizeOutputs) - } - rejection := output.Rejected[0] - if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 { - t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection) - } - if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" { - t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics) + if err == nil || !strings.Contains(err.Error(), "extract/dnd/spells/catalog") || len(output.OutputFiles) != 0 || output.FallbackCount != 0 { + t.Fatalf("output=%#v error=%v, want strict terminal catalog rejection", output, err) } return } - - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { - t.Fatalf("rejected = %#v normalized = %#v, want only accepted output", output.Rejected, output.NormalizeOutputs) - } - var value dnd.SpellList - if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &value); err != nil { - t.Fatalf("decode normalized spell list: %v", err) + if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 { + t.Fatalf("output=%#v error=%v", output, err) } + value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json") if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell { t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value) } if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode { t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics) } - if len(output.Diagnostics.Groups) != 2 { - t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics) - } for _, diagnostic := range output.Diagnostics.Groups { if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode { t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics) @@ -123,47 +109,43 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) { } } -type catalogRetryLLMClient struct { +type catalogCorrectionLLMClient struct { mu sync.Mutex responses []string calls int } -func (client *catalogRetryLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (client *catalogCorrectionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if req.PromptID != spells.PromptID { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) + if !productionSchemaHasProperty(request, "spell_casts") { + return nil, fmt.Errorf("unexpected native spell response contract") } client.mu.Lock() index := client.calls client.calls++ client.mu.Unlock() if index >= len(client.responses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index) + return nil, fmt.Errorf("missing fake response %d", index) } - content := []byte(client.responses[index]) - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil + return &promptkit.GenerateResponse{Content: client.responses[index]}, nil } -func (client *catalogRetryLLMClient) CallCount() int { +func (client *catalogCorrectionLLMClient) CallCount() int { client.mu.Lock() defer client.mu.Unlock() return client.calls } func productionSpellResponse(name string) string { - content, err := json.Marshal(dnd.SpellList{SpellCasts: []dnd.SpellCast{{ - Caster: "Aria", - Spell: name, - SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}}, + content, err := json.Marshal(map[string]any{"spell_casts": []any{map[string]any{ + "caster": "Aria", "spell": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}, }}}) if err != nil { panic(err) } return string(content) } + +var _ promptkit.LLMClient = (*catalogCorrectionLLMClient)(nil) diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go index 8309eb69..6eadb40c 100644 --- a/internal/cli/spell_catalog_identity_contract_test.go +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -1,8 +1,6 @@ package cli import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "os" "path/filepath" @@ -10,100 +8,69 @@ import ( "strings" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" - spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" ) -func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) { +func TestMaterializedSpellCatalogOrderingPreservesSemanticContents(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) - effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + catalog := catalogFromRegistries(components.registries) + effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog}) if err != nil { - t.Fatalf("resolve production configuration: %v", err) + t.Fatal(err) } - overlayPath := filepath.Join(t.TempDir(), "catalog.json") resolved := effective.ResolvedPipeline - bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings - catalogBindingIndex := -1 - for index, binding := range bindings { - if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { - catalogBindingIndex = index - break + overlayPath := filepath.Join(t.TempDir(), "catalog.json") + for _, target := range []*pipeline.ResolvedReferenceTarget{&resolved.Steps[0].ArtifactLanes[0].ExtractReferences, &resolved.Steps[0].ArtifactLanes[0].NormalizeReferences} { + matched := false + for index := range target.Bindings { + if target.Bindings[index].SlotName == spellcatalog.SpellCatalogReferenceSlot { + target.Bindings[index].Source = overlayPath + matched = true + } + } + if !matched { + t.Fatal("missing configured spell catalog binding") } } - if catalogBindingIndex < 0 { - t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings) - } - resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath - normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings - normalizeCatalogBindingIndex := -1 - for index, binding := range normalizeBindings { - if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { - normalizeCatalogBindingIndex = index - break + materialize := func(contents string) pipeline.ResolvedPipeline { + t.Helper() + if err := os.WriteFile(overlayPath, []byte(contents), 0600); err != nil { + t.Fatal(err) } + value, _, err := pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) + if err != nil { + t.Fatal(err) + } + return value } - if normalizeCatalogBindingIndex < 0 { - t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings) - } - resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath - - if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil { - t.Fatal(err) - } - materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) - if err != nil { - t.Fatalf("materialize first catalog: %v", err) - } - identityA := catalogCheckpointIdentity(t, materializedA) - metadataA := catalogExtractorMetadata(t, materializedA) - normalizerMetadataA := catalogNormalizerMetadata(t, materializedA) - referenceA := catalogReference(t, materializedA) - - if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil { - t.Fatal(err) - } - materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) - if err != nil { - t.Fatalf("materialize reordered catalog: %v", err) - } - identityB := catalogCheckpointIdentity(t, materializedB) - metadataB := catalogExtractorMetadata(t, materializedB) - normalizerMetadataB := catalogNormalizerMetadata(t, materializedB) - referenceB := catalogReference(t, materializedB) - - if identityA.Digest == identityB.Digest { - t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest) - } - if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI { - t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB) - } - digestA, ok := metadataA["catalog_digest"].(string) - if !ok { - t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA) - } - digestB, ok := metadataB["catalog_digest"].(string) - if !ok || digestA != digestB { - t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB) - } - if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) { - t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want) - } - normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string) - normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string) - if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB { - t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB) - } - if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) { - t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want) + first, second := materialize(reorderedOverlayA), materialize(reorderedOverlayB) + laneA, laneB := first.Steps[0].ArtifactLanes[0], second.Steps[0].ArtifactLanes[0] + for _, pair := range [][2]pipeline.ResolvedReferenceTarget{ + {laneA.ExtractReferences, laneB.ExtractReferences}, + {laneA.NormalizeReferences, laneB.NormalizeReferences}, + } { + itemsA, itemsB := pair[0].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items, pair[1].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items + if len(itemsA) != 1 || len(itemsB) != 1 { + t.Fatal("materialized catalog missing") + } + if itemsA[0].Digest == itemsB[0].Digest || itemsA[0].Origin != itemsB[0].Origin { + t.Fatalf("raw catalog provenance=%#v/%#v", itemsA, itemsB) + } + catalogA, err := spellcatalog.ResolveEffectiveCatalog(pair[0].ReferenceSet) + if err != nil { + t.Fatal(err) + } + catalogB, err := spellcatalog.ResolveEffectiveCatalog(pair[1].ReferenceSet) + if err != nil { + t.Fatal(err) + } + if catalogA.Digest() != catalogB.Digest() || !reflect.DeepEqual(catalogA.OverlayIDs(), []string{"campaign.a", "campaign.b"}) || !reflect.DeepEqual(catalogB.OverlayIDs(), catalogA.OverlayIDs()) || !reflect.DeepEqual(catalogA.PromptSpells(), catalogB.PromptSpells()) { + t.Fatal("catalog ordering changed semantic recognition") + } } } @@ -123,11 +90,11 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing t.Fatal(err) } components := productionTestComponents(t) - first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + first, err := loadMaintainedExample(t, firstPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve first configuration: %v", err) } - second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + second, err := loadMaintainedExample(t, secondPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve changed configuration: %v", err) } @@ -136,360 +103,39 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing } } -func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) { - components := productionTestComponents(t) - configPath := writeProductionSpellCatalogContractConfig(t) - effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) - if err != nil { - t.Fatal(err) - } - materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) - if err != nil { - t.Fatal(err) - } - prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) - if err != nil { - t.Fatal(err) - } - fingerprints := prepared.CheckpointFingerprints() - wantNames := map[string]struct{}{ - "extract:spells:" + spells.Key + ":effective_catalog": {}, - "extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {}, - "normalize:spells:" + spellnormalize.Key + ":effective_catalog": {}, - "normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {}, - } - seen := make(map[string]string, len(fingerprints)) - for _, fingerprint := range fingerprints { - if _, ok := wantNames[fingerprint.Name]; ok { - seen[fingerprint.Name] = fingerprint.Value - } - } - if len(seen) != len(wantNames) { - t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints) - } - var catalogDigest string - for name, value := range seen { - if catalogDigest == "" { - catalogDigest = value - } else if value != catalogDigest { - t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest) - } - } - - identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity { - identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{ - Pipeline: materialized, - InputKey: materialized.Input.Module, - RawInputDigest: "sha256:unchanged-input", - References: pipeline.ReferenceProvenance(materialized), - ProvenanceFingerprints: checkpointIdentityFingerprints(values), - }) - if identityErr != nil { - t.Fatal(identityErr) - } - return identity - } - first := identityFor(fingerprints) - changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") - assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) - second := identityFor(changed) - if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) { - t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second) - } -} - -func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) { - components := productionTestComponents(t) - configPath := writeProductionSpellCatalogContractConfig(t) - effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) - if err != nil { - t.Fatal(err) - } - materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) - if err != nil { - t.Fatal(err) - } - prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) - if err != nil { - t.Fatal(err) - } - fingerprints := prepared.CheckpointFingerprints() - llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}} - settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} - recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false) - if err != nil { - t.Fatal(err) - } - doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"} - doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} - doc.Digest, err = source.DigestDocument(&doc) - if err != nil { - t.Fatal(err) - } - if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil { - t.Fatal(err) - } - normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}} - normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"} - normalizeArtifact := pipeline.CheckpointArtifact{ - LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID, - SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema), - Artifact: contracts.SerializedArtifact{ - Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), - }, - } - if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil { - t.Fatal(err) - } - - _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) - if err != nil { - t.Fatal(err) - } - if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused { - t.Fatalf("same fingerprint decision = %#v, want reuse", decision) - } - if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` { - t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision) - } - changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") - assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) - _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) - if err != nil { - t.Fatal(err) - } - if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused { - t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision) - } - if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { - t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision) - } - changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed") - assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) - _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) - if err != nil { - t.Fatal(err) - } - if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused { - t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision) - } - - changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}} - _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) - if err != nil { - t.Fatal(err) - } - if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused { - t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision) - } - if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { - t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision) - } -} - -func normalizeSpellCatalogFingerprintName() string { - return "normalize:spells:" + spellnormalize.Key + ":effective_catalog" -} - -func extractSpellMappingFingerprintName() string { - return "extract:spells:" + spells.Key + ":mapping_policy" -} - -func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint { - t.Helper() - changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) - matches := 0 - for index := range changed { - if changed[index].Name == name { - changed[index].Value = value - matches++ - } - } - if matches != 1 { - t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name) - } - return changed -} - -func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) { - t.Helper() - if len(before) != len(after) { - t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after)) - } - changes := 0 - for index := range before { - if before[index].Name != after[index].Name { - t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name) - } - if before[index].Value == after[index].Value { - continue - } - changes++ - if before[index].Name != changedName { - t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name) - } - } - if changes != 1 { - t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName) - } -} - -func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) { +func TestProductionOverlayRunAlignsGroundingAndValidation(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") - fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} - options := productionRunOptions(t, fake) + client := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} + configPath := writeProductionContractConfig(t, productionSpellCatalogContractConfig(t)+productionProviderConfiguration) var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", writeProductionSpellCatalogContractConfig(t), - "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), - "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, - }, &stdout, &stderr, options) + code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client)) if code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } - - runRoot := filepath.Join(outputRoot, productionRunID) - manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json")) - if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 { - t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest) - } - lane := manifest.ArtifactLanes[0] - extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) - if !ok { - t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata) - } - if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") { - t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata) - } - if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) { - t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got) - } - normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) - if !ok { - t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata) - } - if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) { - t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata) - } - if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) { - t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata) - } - - var catalogProvenances []artifacts.ReferenceProvenance - for index := range manifest.References { - reference := &manifest.References[index] - if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot { - catalogProvenances = append(catalogProvenances, *reference) - } - } - if len(catalogProvenances) != 2 { - t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References) - } - overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json") - for _, catalogProvenance := range catalogProvenances { - if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" { - t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance) - } - if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") { - t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance) - } - } - manifestBytes, err := json.Marshal(manifest) - if err != nil { + var report runResult + if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil { t.Fatal(err) } - for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} { - if strings.Contains(string(manifestBytes), leaked) { - t.Fatalf("manifest leaked overlay content %q", leaked) - } + if report.ArtifactCount != 1 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) { + t.Fatalf("completion=%#v", report) } - - requests := fake.requestsFor(spells.PromptID) + requests := client.requestsWithProperty("spell_casts") if len(requests) != 1 { - t.Fatalf("spell requests = %d, want one", len(requests)) + t.Fatalf("native spell requests=%d", len(requests)) } - catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] - if !ok || !strings.Contains(string(catalogInput.Content), `"canonical_name":"Aegis of Emberfall"`) || !strings.Contains(string(catalogInput.Content), `"aliases":["Emberfall Aegis"]`) { - t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name and recognition alias", catalogInput) + var prompt strings.Builder + for _, message := range requests[0].Prompt.Messages { + prompt.WriteString(message.Content) } - artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json")) - if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" { - t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact) - } - rejected := readProductionJSON[struct { - Rejected []json.RawMessage `json:"rejected"` - }](t, filepath.Join(runRoot, "rejected.json")) - if len(rejected.Rejected) != 0 { - t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected) - } -} - -func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity { - t.Helper() - identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ - Pipeline: resolved, - InputKey: resolved.Input.Module, - RawInputDigest: "sha256:catalog-test-input", - References: pipeline.ReferenceProvenance(resolved), - }) - if err != nil { - t.Fatalf("create checkpoint identity: %v", err) - } - return identity -} - -func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { - t.Helper() - lane := resolved.Steps[0].ArtifactLanes[0] - extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet) - if err != nil { - t.Fatalf("construct extractor: %v", err) - } - return extractor.ManifestMetadata() -} - -func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { - t.Helper() - lane := resolved.Steps[0].ArtifactLanes[0] - normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet) - if err != nil { - t.Fatalf("construct normalizer: %v", err) - } - return normalizer.ManifestMetadata() -} - -func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance { - t.Helper() - for _, reference := range pipeline.ReferenceProvenance(resolved) { - if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" { - return reference + for _, fragment := range []string{`"canonical_name":"Aegis of Emberfall"`, `"aliases":["Emberfall Aegis"]`} { + if !strings.Contains(prompt.String(), fragment) { + t.Fatalf("catalog prompt omitted overlay recognition %q", fragment) } } - t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved)) - return artifacts.ReferenceProvenance{} -} - -func stringValue(value any) string { - result, _ := value.(string) - return result -} - -func stringValues(value any) []string { - raw, err := json.Marshal(value) - if err != nil { - return nil + artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(report.OutputDirectory, "lanes", "spells.json")) + if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" { + t.Fatalf("spell artifact=%#v", artifact) } - var values []string - if err := json.Unmarshal(raw, &values); err != nil { - return nil - } - return values -} - -func digestBytes(value []byte) string { - sum := sha256.Sum256(value) - return "sha256:" + hex.EncodeToString(sum[:]) } const reorderedOverlayA = `{