Migrate framework tests to native producer and artifact contracts

This commit is contained in:
2026-11-22 04:28:48 +00:00
parent 0b46f2d37c
commit 93d94fdd0a
22 changed files with 184 additions and 3309 deletions

View File

@@ -1,11 +1,13 @@
package pipeline
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) {
@@ -73,3 +75,46 @@ func TestDebugChunkEnvelopeClonesAnnotationScopes(t *testing.T) {
t.Fatal("debugSourceChunkEnvelope() shares annotation bytes")
}
}
type observingChunkValidator struct {
request contracts.ChunkValidationRequest
}
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
v.request = request
return contracts.ValidationResult{Approved: true}, nil
}
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
plan := prepared.chunker.(*typedTestChunker).plan
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
prepared.chunker = &typedTestChunker{key: prepared.resolved.Chunk.Module, plan: plan}
validator := &observingChunkValidator{}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
chunk: validator,
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(validator.request.Chunks) != 1 {
t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
}
chunk := validator.request.Chunks[0]
if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
t.Fatalf("materialized chunk identity = %#v", chunk)
}
if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
}
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
t.Fatalf("materialized metadata = %#v", chunk.Metadata)
}
}

View File

@@ -1,117 +0,0 @@
package pipeline
import (
"bytes"
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type executionDebugClient struct {
response contracts.StructuredCompletionResponse
err error
}
func (c executionDebugClient) CompleteStructured(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) {
return c.response, c.err
}
func TestDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
providerErr := contracts.NewLLMGenerationError(503, "provider unavailable")
const content = `{"ok":true}`
for _, test := range []struct {
name string
err error
content string
debugResponse bool
}{
{name: "provider failure without body", err: providerErr},
{name: "canceled without body", err: context.Canceled},
{name: "split debug response", content: content, debugResponse: true},
{name: "split fallback response", content: content},
} {
t.Run(test.name, func(t *testing.T) {
want := contracts.LLMExecutionSummary{
Attempts: 2, TransientRetries: 1, Outcome: "success",
CapacityWaitMS: 1, GenerationMS: 3, RetryDelayMS: 5, DurationMS: 9,
Usage: contracts.LLMObservedTokenUsage{
PromptTokens: contracts.LLMObservedTokenCount{Value: 7, Known: true},
CompletionTokens: contracts.LLMObservedTokenCount{Known: true},
},
}
if test.err != nil {
want.Outcome = "error"
want.FailureKind = "provider_status"
want.StatusCode = 503
want.RetriesExhausted = true
if errors.Is(test.err, context.Canceled) {
want.FailureKind = "canceled"
want.StatusCode = 0
want.RetriesExhausted = false
}
}
execution := want
response := contracts.StructuredCompletionResponse{Execution: &execution}
if test.debugResponse {
response.Debug = &contracts.LLMDebugMaterial{Response: &contracts.LLMDebugResponse{Content: test.content}}
} else {
response.Content = json.RawMessage(test.content)
}
recorder := newCapturedDebugRecorder()
client := WithDebugLLMRecording(executionDebugClient{response, test.err}, recorder)
ctx, scope := withDebugLLMScope(context.Background(), "attempt")
got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{})
if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want {
t.Fatalf("debug completion = (%+v, %v), want summary and original error", got, err)
}
terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"})
if err := terminal.record(nil, test.err); !errors.Is(err, test.err) {
t.Fatalf("terminal recording error = %v", err)
}
envelope := recorder.envelope(t, "attempt/response-0001.json")
encoded, err := json.Marshal(envelope.Payload)
if err != nil {
t.Fatal(err)
}
var artifact debugLLMResponseArtifact
if err := json.Unmarshal(encoded, &artifact); err != nil {
t.Fatal(err)
}
if artifact.Execution == nil || *artifact.Execution != want {
t.Fatalf("persisted response lost execution summary: %+v", artifact)
}
if !test.debugResponse && (artifact.Fallback == nil || artifact.Fallback.Execution == nil || *artifact.Fallback.Execution != want) {
t.Fatalf("persisted fallback lost execution summary: %+v", artifact.Fallback)
}
if test.content == "" {
if artifact.ContentPath != "" {
t.Fatalf("bodyless failure has content path %q", artifact.ContentPath)
}
} else {
var compact bytes.Buffer
if artifact.ContentPath == "" || json.Compact(&compact, recorder.bytes[artifact.ContentPath]) != nil || compact.String() != test.content {
t.Fatalf("split content not preserved at %q", artifact.ContentPath)
}
if (artifact.Fallback != nil && artifact.Fallback.Content != "") || (artifact.Response != nil && artifact.Response.Content != "") {
t.Fatal("split content remains in response metadata")
}
}
refs := recorder.envelope(t, "attempt/terminal.json").LLMCalls
if len(refs) != 1 || refs[0].Execution == nil || *refs[0].Execution != want || refs[0].Error != (test.err != nil) {
t.Fatalf("terminal references = %+v", refs)
}
// Decode the persisted JSON, not just Go defaults: omitted false and
// zero fields would erase the presence-aware usage contract.
usage := envelope.Payload.(map[string]any)["execution"].(map[string]any)["usage"].(map[string]any)
unknown := usage["cached_tokens"].(map[string]any)
for _, key := range []string{"value", "known", "complete"} {
if _, ok := unknown[key]; !ok {
t.Fatalf("unknown token usage omits %q: %+v", key, unknown)
}
}
})
}
}

View File

@@ -1,78 +1,12 @@
package pipeline
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestCleanDebugPathPreservesRawComponents(t *testing.T) {
for _, test := range []struct {
value string
want string
}{
{value: "", want: "%"},
{value: ".", want: "%2E"},
{value: "..", want: "%2E%2E"},
{value: "a//b", want: "a/%/b"},
{value: "/a/", want: "%/a/%"},
{value: " a ", want: "%20a%20"},
} {
if got := cleanDebugPath(test.value); got != test.want {
t.Errorf("cleanDebugPath(%q) = %q, want %q", test.value, got, test.want)
}
}
}
func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) {
recorder := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, recorder)
for _, test := range []struct {
stageName string
path string
}{
{stageName: ".", path: "%2E/response-0001.json"},
{stageName: "..", path: "%2E%2E/response-0002.json"},
} {
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{StageName: test.stageName}, nil); err != nil {
t.Fatalf("CompleteStructured(%q): %v", test.stageName, err)
}
if !recorder.has(test.path) {
t.Errorf("debug artifact %q was not written; names = %#v", test.path, recorder.names())
}
}
}
func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) {
const assistantResponse = `{"secret":"assistant response"}`
const userGuidance = "secret user guidance"
correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance)
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
summary := debugCompletionRequest(contracts.StructuredCompletionRequest{
Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
Vars: map[string]any{"custom": "value"},
Correction: correction,
})
encoded, err := json.Marshal(summary)
if err != nil {
t.Fatalf("marshal completion summary: %v", err)
}
for _, secret := range []string{assistantResponse, userGuidance} {
if strings.Contains(string(encoded), secret) {
t.Fatalf("debug completion summary leaked %q: %s", secret, encoded)
}
}
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary)
}
}
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
doc := validSourceDocument()
envelope := debugSourceDocumentEnvelope(doc)
@@ -95,35 +29,6 @@ func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
}
}
func TestDebugSourceUnitsRedactSecrets(t *testing.T) {
units := []source.SourceUnit{{
ID: 1,
Kind: "paragraph",
Text: "application text Bearer secretvalue sk-secretvalue",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Metadata: map[string]any{"api_key": "sk-secretvalue"},
}}
got := cloneSourceUnitsForDebug(units)
if len(got) != 1 {
t.Fatalf("debug unit count = %d, want 1", len(got))
}
if !strings.Contains(got[0].Text, "application text") {
t.Fatalf("debug unit text = %q, want application content retained", got[0].Text)
}
for _, forbidden := range []string{"secretvalue", "sk-secretvalue"} {
if strings.Contains(got[0].Text, forbidden) {
t.Fatalf("debug unit text contains %q: %q", forbidden, got[0].Text)
}
}
if got, want := got[0].Metadata["api_key"], "[REDACTED]"; got != want {
t.Fatalf("debug unit metadata api_key = %#v, want %q", got, want)
}
if units[0].Text != "application text Bearer secretvalue sk-secretvalue" {
t.Fatalf("source unit text was mutated: %q", units[0].Text)
}
}
func TestDebugSourceChunkPreservesReference(t *testing.T) {
doc := validSourceDocument()
chunk := source.Chunk{

View File

@@ -1,48 +1,19 @@
package pipeline
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTerminalDiagnosticGroupsDiscardSupersededAttemptDiagnostics(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "first", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded")}, Candidate: attemptCandidate(t, "first")}, nil
}
return producerAttemptOutput{Value: "second", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("terminal", "terminal")}}, nil
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "first" {
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: "invalid", correctionGuidance: "Correct it."}}}, nil
}
return validationReport{}, nil
})
func TestPromoteProducerDiagnosticsPreservesValidatorOrigin(t *testing.T) {
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells", ValidatorKey: "dnd/spells/source-relatedness"}
groups, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")}, origin, nil)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
t.Fatal(err)
}
groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil)
if err != nil {
t.Fatalf("terminalDiagnosticGroups() error = %v", err)
}
if len(groups) != 1 || groups[0].Samples[0].Scope != "terminal" {
t.Fatalf("groups = %#v, want only terminal attempt diagnostics", groups)
}
}
func TestTerminalDiagnosticGroupsPreserveValidatorOrigin(t *testing.T) {
terminal := producerAttemptTerminal{Validation: validationReport{records: []validationRecord{{
validatorName: "dnd/spells/source-relatedness",
outcome: validationApproved,
diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")},
}}}}
groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells"}, nil)
if err != nil {
t.Fatalf("terminalDiagnosticGroups() error = %v", err)
}
if len(groups) != 1 || groups[0].Origin.ValidatorKey != "dnd/spells/source-relatedness" {
t.Fatalf("groups = %#v, want validator origin", groups)
if len(groups) != 1 || groups[0].Origin != origin {
t.Fatalf("groups = %#v, want validator origin %#v", groups, origin)
}
}

View File

@@ -43,9 +43,9 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS
installGeneratedTarget(&consumer.resolved.MergeReferences, StageMerge, consumer.resolved.Merge.Module)
installGeneratedTarget(&consumer.resolved.NormalizeReferences, StageNormalize, consumer.resolved.Normalize.Module)
artifact, err := checkpointArtifact(prepared.Steps[0].lanes[0].typed.codec, "notes", "typed/normalize", "source", value)
artifact, err := serializeArtifact(prepared.Steps[0].lanes[0].typed.codec, value, false)
if err != nil {
t.Fatalf("checkpointArtifact() error = %v", err)
t.Fatalf("serializeArtifact() error = %v", err)
}
input := RunInput{Prepared: prepared}
input.pipeline = prepared.resolved
@@ -54,23 +54,19 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS
LaneID: "notes",
NormalizerKey: "typed/normalize",
SourceID: "source",
Artifact: contracts.CloneSerializedArtifact(artifact.Artifact),
Artifact: contracts.CloneSerializedArtifact(artifact),
}
}
func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) {
input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}})
sets, provenance, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
if err != nil {
t.Fatalf("buildStepReferenceSets() error = %v", err)
}
if got, want := len(sets), 3; got != want {
t.Fatalf("reference target set count = %d, want %d", got, want)
}
if got, want := len(provenance), 3; got != want {
t.Fatalf("generated provenance count = %d, want %d", got, want)
}
var first []byte
for key, set := range sets {
item := set.Slots["producer-output"].Items[0]
@@ -91,20 +87,6 @@ func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) {
if got := len(step.lanes[0].resolved.ExtractReferences.ReferenceSet.Slots["producer-output"].Items); got != 0 {
t.Fatalf("prepared extract reference items = %d, want zero", got)
}
encoded, err := json.Marshal(provenance)
if err != nil {
t.Fatalf("marshal generated provenance: %v", err)
}
text := string(encoded)
for _, forbidden := range []string{"first", "file://", "content_base64"} {
if strings.Contains(text, forbidden) {
t.Fatalf("generated provenance contains forbidden %q: %s", forbidden, text)
}
}
if !strings.Contains(text, "producer_pipeline_id") || !strings.Contains(text, "schema_digest") {
t.Fatalf("generated provenance = %s, want bounded producer and schema identity", text)
}
}
func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
@@ -115,12 +97,12 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
}
outputs = append(outputs, producerOutput)
sets, provenance, err := buildStepReferenceSets(input, step, outputs)
sets, err := buildStepReferenceSets(input, step, outputs)
if err != nil {
t.Fatalf("buildStepReferenceSets() error = %v", err)
}
if len(sets) != 3 || len(provenance) != 3 {
t.Fatalf("reference sets/provenance = %d/%d, want 3/3", len(sets), len(provenance))
if len(sets) != 3 {
t.Fatalf("reference set count = %d, want 3", len(sets))
}
for target, set := range sets {
item := set.Slots["producer-output"].Items[0]
@@ -132,7 +114,7 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
func TestGeneratedReferenceAcceptsTypedEmptyCollection(t *testing.T) {
input, step, producerOutput := handoffFixture(t, codecNotes{})
sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
if err != nil {
t.Fatalf("buildStepReferenceSets() error = %v, want nil for accepted empty collection", err)
}
@@ -189,7 +171,7 @@ func TestGeneratedReferenceRejectsInvalidProducerOutputsDeterministically(t *tes
target := &step.lanes[0].resolved.ExtractReferences
test.mutate(target, &output)
}
_, _, err := buildStepReferenceSets(input, step, test.outputs(output))
_, err := buildStepReferenceSets(input, step, test.outputs(output))
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("buildStepReferenceSets() error = %v, want %q", err, test.want)
}
@@ -208,142 +190,45 @@ func BenchmarkIndexGeneratedOutputs(b *testing.B) {
}
}
func TestGeneratedReferenceFingerprintChangesWithCanonicalContent(t *testing.T) {
input, step, first := handoffFixture(t, codecNotes{Items: []string{"first"}})
firstSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{first})
if err != nil {
t.Fatalf("build first reference set: %v", err)
}
_, _, second := handoffFixture(t, codecNotes{Items: []string{"second"}})
secondSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{second})
if err != nil {
t.Fatalf("build second reference set: %v", err)
}
firstDeps := generatedReferenceDependencies(firstSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)])
secondDeps := generatedReferenceDependencies(secondSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)])
if reflect.DeepEqual(firstDeps, secondDeps) {
t.Fatalf("generated dependencies = %#v, want content-sensitive fingerprint", firstDeps)
}
}
func TestGeneratedReferenceFingerprintChangesWithCanonicalSize(t *testing.T) {
input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}})
sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
if err != nil {
t.Fatalf("build reference set: %v", err)
}
key := keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)
firstDeps := generatedReferenceDependencies(sets[key])
changed := CloneReferenceSet(sets[key])
item := changed.Slots["producer-output"].Items[0]
item.SizeBytes++
changed.Slots["producer-output"] = contracts.ResolvedReferenceSlot{
Slot: changed.Slots["producer-output"].Slot,
Items: []contracts.ReferenceItem{item},
}
secondDeps := generatedReferenceDependencies(changed)
if reflect.DeepEqual(firstDeps, secondDeps) {
t.Fatalf("generated dependencies = %#v, want size-sensitive fingerprint", firstDeps)
}
}
func TestRunnerHandsOffAcceptedNormalizedOutputBeforeConsumerLanes(t *testing.T) {
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"first"}})
prepared := input.Prepared
var received contracts.ReferenceSet
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
received := make(map[ModuleStage]contracts.ReferenceSet)
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
}
prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
received = CloneReferenceSet(request.References)
prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
received[StageExtract] = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
output, err := New().Run(context.Background(), input)
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
}
prepared.Steps[1].lanes[0].typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
received[StageMerge] = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
prepared.Steps[1].lanes[0].typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
received[StageNormalize] = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
_, err := New().Run(context.Background(), input)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if got, want := len(output.NormalizeOutputs), 2; got != want {
t.Fatalf("normalized output count = %d, want %d", got, want)
}
if output.NormalizeOutputs[0].StepID != "step-1" || output.NormalizeOutputs[1].StepID != "step-2" {
t.Fatalf("normalized output step IDs = %#v, want step-1 and step-2", output.NormalizeOutputs)
}
item := received.Slots["producer-output"].Items[0]
if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" {
t.Fatalf("consumer reference = %#v, want generated producer reference", item)
}
if len(output.Manifest.References) == 0 || output.Manifest.References[len(output.Manifest.References)-1].OriginType != "generated" {
t.Fatalf("manifest references = %#v, want generated provenance", output.Manifest.References)
}
}
type handoffDependencyLoader struct {
CheckpointLoader
extract [][]CheckpointFingerprint
merge [][]CheckpointFingerprint
normalize [][]CheckpointFingerprint
}
func (l *handoffDependencyLoader) Enabled() bool { return true }
func (l *handoffDependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.extract = append(l.extract, append([]CheckpointFingerprint(nil), dependencies...))
return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func (l *handoffDependencyLoader) Merge(_ string, _ string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.merge = append(l.merge, append([]CheckpointFingerprint(nil), dependencies...))
return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func (l *handoffDependencyLoader) Normalize(_ string, _ string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.normalize = append(l.normalize, append([]CheckpointFingerprint(nil), dependencies...))
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func runHandoffWithProducerValue(t *testing.T, value codecNotes) *handoffDependencyLoader {
t.Helper()
input, _, _ := handoffFixture(t, value)
prepared := input.Prepared
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
prepared.Steps[1].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
loader := &handoffDependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader}); err != nil {
t.Fatalf("Run() error = %v", err)
}
return loader
}
func TestRunnerAddsGeneratedDependencyToEveryReceivingCheckpoint(t *testing.T) {
first := runHandoffWithProducerValue(t, codecNotes{Items: []string{"first"}})
second := runHandoffWithProducerValue(t, codecNotes{Items: []string{"second"}})
if len(first.extract) != 2 || len(first.merge) != 2 || len(first.normalize) != 2 {
t.Fatalf("checkpoint load calls = extract %d merge %d normalize %d, want two each", len(first.extract), len(first.merge), len(first.normalize))
}
for name, calls := range map[string][][]CheckpointFingerprint{"extract": first.extract, "merge": first.merge, "normalize": first.normalize} {
if got := generatedFingerprintCount(calls[1]); got != 1 {
t.Fatalf("%s consumer dependency count = %d, want one", name, got)
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
set, found := received[target]
if !found || len(set.Slots["producer-output"].Items) != 1 {
t.Fatalf("%s reference set = %#v, want producer artifact", target, set)
}
item := set.Slots["producer-output"].Items[0]
if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" {
t.Fatalf("%s reference = %#v, want generated producer reference", target, item)
}
var value codecNotes
if err := json.Unmarshal(item.Content, &value); err != nil || !reflect.DeepEqual(value, codecNotes{Items: []string{"first"}}) {
t.Fatalf("%s reference value = %#v, %v", target, value, err)
}
}
if reflect.DeepEqual(first.extract[1], second.extract[1]) || reflect.DeepEqual(first.merge[1], second.merge[1]) || reflect.DeepEqual(first.normalize[1], second.normalize[1]) {
t.Fatalf("consumer checkpoint dependencies did not change with producer content")
}
}
func generatedFingerprintCount(dependencies []CheckpointFingerprint) int {
count := 0
for _, dependency := range dependencies {
if strings.HasPrefix(dependency.Name, "generated-reference:") {
count++
}
}
return count
}

View File

@@ -1,49 +0,0 @@
package pipeline
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type retryingNotesNormalizer struct {
retry *contracts.NormalizeRetry
}
func (retryingNotesNormalizer) Key() string { return "test/retry-normalize" }
func (retryingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (n retryingNotesNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
return contracts.TypedNormalizeResult[codecNotes]{Value: req.MergeOutput.Value, Retry: n.retry}, nil
}
func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
retry := &contracts.NormalizeRetry{
ReasonCode: "retryable",
Message: "safe fallback available",
CorrectionGuidance: "return a complete corrected proposal",
}
registry := NewNormalizerRegistry()
if err := RegisterNormalizer(registry, ModuleSpec{Key: "test/retry-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
return retryingNotesNormalizer{retry: retry}, nil
}); err != nil {
t.Fatalf("RegisterNormalizer() error = %v", err)
}
entry, ok := registry.typedEntry("test/retry-normalize", "test/notes")
if !ok {
t.Fatal("typed normalizer entry missing")
}
implementation, err := entry.builder(BuildRequest{})
if err != nil {
t.Fatalf("builder() error = %v", err)
}
result, err := entry.normalize(context.Background(), implementation, contracts.TypedNormalizeRequest[any]{MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"safe"}}}})
if err != nil {
t.Fatalf("normalize() error = %v", err)
}
retry.Message = "mutated"
retry.CorrectionGuidance = "mutated guidance"
if result.Retry == nil || result.Retry.Message != "safe fallback available" || result.Retry.CorrectionGuidance != "return a complete corrected proposal" {
t.Fatalf("erased retry result = %#v, want independent retry data", result)
}
}

View File

@@ -132,168 +132,6 @@ func TestPrepareRetainsGeneratedSelectorsWithoutReferenceBytes(t *testing.T) {
}
}
func TestPreparedCheckpointFingerprintsCollectEveryComponentInStableScopeOrder(t *testing.T) {
provider := func(value string) checkpointFingerprintTestProvider {
return checkpointFingerprintTestProvider{{Name: "identity", Value: value}}
}
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{
ID: "fingerprints",
Input: Binding("input"),
Chunk: Binding("chunk"),
Output: Binding("output"),
},
input: provider("input"),
chunker: provider("chunk"),
chunkValidators: preparedValidatorChain{validators: []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("chunk-validator"), Target: ValidatorTargetChunk},
chunk: provider("chunk-validator"),
}}},
output: provider("output"),
Steps: []PreparedPipelineStep{{ID: "default"}},
}
lane := preparedLaneExecutor{
resolved: ResolvedArtifactLane{
ID: "spells",
Extract: Binding("extract"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
typed: &preparedTypedLane{
extractor: provider("extract"),
merger: provider("merge"),
normalizer: provider("normalize"),
},
extractValidators: fingerprintTestValidatorChain("extract-validator", provider("extract-validator")),
mergeValidators: fingerprintTestValidatorChain("merge-validator", provider("merge-validator")),
normalizeValidators: fingerprintTestValidatorChain("normalize-validator", provider("normalize-validator")),
}
prepared.Steps[0].lanes = []preparedLaneExecutor{lane}
first, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("collectPreparedCheckpointFingerprints() error = %v, want nil", err)
}
second, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("second collection error = %v, want nil", err)
}
if !reflect.DeepEqual(first, second) {
t.Fatalf("fingerprints are not deterministic: %#v / %#v", first, second)
}
if len(first) != 10 {
t.Fatalf("fingerprints = %#v, want all 10 prepared components", first)
}
for index := 1; index < len(first); index++ {
if first[index-1].Name >= first[index].Name {
t.Fatalf("fingerprints are not sorted: %#v", first)
}
}
prepared.checkpointFingerprints = first
copy := prepared.CheckpointFingerprints()
copy[0].Value = "mutated"
if reflect.DeepEqual(copy, prepared.CheckpointFingerprints()) {
t.Fatal("CheckpointFingerprints() exposed mutable backing storage")
}
}
func TestPreparedCheckpointFingerprintsValidateProviderValues(t *testing.T) {
tests := []struct {
name string
fingerprints []CheckpointFingerprint
want string
}{
{name: "empty name", fingerprints: []CheckpointFingerprint{{Value: "value"}}, want: "non-empty name and value"},
{name: "empty value", fingerprints: []CheckpointFingerprint{{Name: "name"}}, want: "non-empty name and value"},
{name: "duplicate after trimming", fingerprints: []CheckpointFingerprint{{Name: "same", Value: "one"}, {Name: " same ", Value: "two"}}, want: "duplicated"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
input: checkpointFingerprintTestProvider(test.fingerprints),
}
_, err := collectPreparedCheckpointFingerprints(prepared)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("error = %v, want substring %q", err, test.want)
}
})
}
}
func TestPreparedCheckpointFingerprintsRemainEmptyWithoutProviders(t *testing.T) {
registries, _ := constructionRegistries(t, nil, nil)
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
if err != nil {
t.Fatal(err)
}
prepared, err := Prepare(resolved, registries, ModuleDependencies{})
if err != nil {
t.Fatal(err)
}
if got := prepared.CheckpointFingerprints(); len(got) != 0 {
t.Fatalf("CheckpointFingerprints() = %#v, want empty", got)
}
}
func TestPreparedCheckpointFingerprintsScopeRepeatedValidatorsByPosition(t *testing.T) {
provider := checkpointFingerprintTestProvider{{Name: "identity", Value: "same"}}
validator := func() preparedValidator {
return preparedValidator{resolved: ResolvedValidator{Binding: Binding("configured"), Target: ValidatorTargetChunk}, chunk: provider}
}
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
chunkValidators: preparedValidatorChain{validators: []preparedValidator{validator(), validator()}},
}
got, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("collect repeated validators: %v", err)
}
if len(got) != 2 || got[0].Name == got[1].Name {
t.Fatalf("fingerprints = %#v, want independently scoped repeated validators", got)
}
}
type checkpointFingerprintTestProvider []CheckpointFingerprint
func (p checkpointFingerprintTestProvider) CheckpointFingerprints() []CheckpointFingerprint {
return append([]CheckpointFingerprint(nil), p...)
}
func (checkpointFingerprintTestProvider) Key() string { return "fingerprint-test" }
func (checkpointFingerprintTestProvider) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (checkpointFingerprintTestProvider) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
return typedTestDocument(), nil
}
func (checkpointFingerprintTestProvider) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{}, nil
}
func (checkpointFingerprintTestProvider) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func (checkpointFingerprintTestProvider) Name() string { return "fingerprint-test" }
func (checkpointFingerprintTestProvider) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (checkpointFingerprintTestProvider) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type checkpointFingerprintOutput struct {
contracts.OutputEncoder
fingerprints []CheckpointFingerprint
}
func (o checkpointFingerprintOutput) CheckpointFingerprints() []CheckpointFingerprint {
return append([]CheckpointFingerprint(nil), o.fingerprints...)
}
func fingerprintTestValidatorChain(key string, provider checkpointFingerprintTestProvider) preparedValidatorChain {
return preparedValidatorChain{validators: []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding(key), Target: ValidatorTargetTyped},
typed: provider,
}}}
}
func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) {
var built []string
var observations []constructionBuildObservation
@@ -351,7 +189,7 @@ func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) {
t.Fatalf("prepared extract options = %#v, want original nested value", got)
}
_, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest{
_, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest[any]{
References: CloneReferenceSet(resolved.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet),
})
if err != nil {
@@ -531,9 +369,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) {
wantBuilt []string
}{
{
name: "missing required llm dependency",
name: "missing required engine dependency",
configure: func(failure *constructionFailure) {
failure.requireExtractorLLM = true
failure.requireExtractorEngine = true
},
want: `lane "artifact" extract module "extract"`,
wantBuilt: []string{"input", "chunk", "validator", "extract"},
@@ -573,26 +411,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) {
}
}
func TestPrepareRejectsInvalidComponentCheckpointFingerprint(t *testing.T) {
failure := &constructionFailure{outputFingerprints: []CheckpointFingerprint{{Name: "identity"}}}
registries, input := constructionRegistries(t, nil, failure)
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
if err != nil {
t.Fatal(err)
}
_, err = Prepare(resolved, registries, ModuleDependencies{})
if err == nil || !strings.Contains(err.Error(), "checkpoint fingerprint") || !strings.Contains(err.Error(), "output:output") {
t.Fatalf("Prepare() error = %v, want scoped checkpoint fingerprint error", err)
}
if len(input.requests) != 0 {
t.Fatalf("input Parse calls = %d, want zero", len(input.requests))
}
}
type constructionFailure struct {
requireExtractorLLM bool
output error
outputFingerprints []CheckpointFingerprint
requireExtractorEngine bool
output error
}
func constructionProfile() PipelineProfile {
@@ -664,8 +485,8 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
extractSpec.ArtifactKind = "test/notes"
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
record("extract", &request)
if failure.requireExtractorLLM && request.Dependencies.LLM == nil {
return nil, errors.New("structured LLM client is required")
if failure.requireExtractorEngine && request.Dependencies.Engine == nil {
return nil, errors.New("native engine is required")
}
return &constructionExtractor{key: "extract", requests: &input.extractRequests}, nil
}); err != nil {
@@ -705,11 +526,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
if failure.output != nil {
return nil, failure.output
}
output := contracts.OutputEncoder(&typedTestOutput{key: "output"})
if failure.outputFingerprints != nil {
output = checkpointFingerprintOutput{OutputEncoder: output, fingerprints: failure.outputFingerprints}
}
return output, nil
return &typedTestOutput{key: "output"}, nil
}); err != nil {
t.Fatal(err)
}
@@ -719,17 +536,17 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
type constructionInput struct {
key string
requests []contracts.ParseRequest
extractRequests []contracts.TypedExtractionRequest
extractRequests []contracts.TypedExtractionRequest[codecNotes]
}
type constructionExtractor struct {
key string
requests *[]contracts.TypedExtractionRequest
requests *[]contracts.TypedExtractionRequest[codecNotes]
}
func (extractor *constructionExtractor) Key() string { return extractor.key }
func (*constructionExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[codecNotes], error) {
func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) {
if extractor.requests != nil {
*extractor.requests = append(*extractor.requests, request)
}

View File

@@ -126,148 +126,6 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) {
}
}
func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) {
correction := contracts.CorrectionProtocolSingleResponseV1
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.ChunkCorrectionProtocol != correction {
t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction {
t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction)
}
withoutCapability := cloneResolvedPipeline(resolved)
withoutCapability.ChunkCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = ""
withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = ""
withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability)
if err != nil {
t.Fatalf("resolvedPipelineDigest() error = %v", err)
}
if withoutCapabilityDigest == resolved.Digest {
t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if prepared.ChunkCorrectionProtocol != correction {
t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction)
}
preparedLane := prepared.Steps[0].ArtifactLanes[0]
if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction {
t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction)
}
}
func TestPrepareRequiresCorrectionCapabilityForValidatorRetries(t *testing.T) {
for _, test := range []struct {
name string
protocol contracts.CorrectionProtocol
retries int
validators bool
want string
}{
{name: "supported", protocol: contracts.CorrectionProtocolSingleResponseV1, retries: 1, validators: true},
{name: "unsupported", retries: 1, validators: true, want: "does not declare correction protocol"},
{name: "no validators", retries: 1},
{name: "no retries", validators: true},
} {
t.Run(test.name, func(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: test.protocol, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
)
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) {
return llmProfileTestChunkValidator{key: "chunk-validator"}, nil
}); err != nil {
t.Fatal(err)
}
profile := llmProfilePipeline()
profile.Chunk.Retries = test.retries
if test.validators {
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "chunk-validator"}}}
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
_, err = Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if test.want != "" {
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
}
return
}
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
})
}
}
type llmProfileTestChunkValidator struct{ key string }
func (validator llmProfileTestChunkValidator) Name() string { return validator.key }
func (llmProfileTestChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (llmProfileTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) {
for _, test := range []struct {
name string
class contracts.ExecutionClass
want string
}{
{name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"},
{name: "LLM validator", class: contracts.ExecutionClassLLMBacked},
} {
t.Run(test.name, func(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class})
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}}
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if test.want != "" {
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
}
return
}
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 {
t.Fatalf("validator retries = %d, want 1", got)
}
})
}
}
func TestResolvePipelineAppliesDefaults(t *testing.T) {
resolved, err := ResolvePipeline(PipelineProfile{
ID: "defaulted",
@@ -552,148 +410,6 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
}
}
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
pipelineAttempts := 2
chunkAttempts := 1
extractAttempts := 0
validatorAttempts := 3
profile := llmProfilePipeline()
profile.StructuredOutputRepairAttempts = &pipelineAttempts
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
lane := profile.Artifacts["events"]
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
encoded, err := json.Marshal(resolved)
if err != nil {
t.Fatalf("json.Marshal(resolved) error = %v", err)
}
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
}
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
for _, test := range []struct {
name string
got *int
want int
}{
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
} {
t.Run(test.name, func(t *testing.T) {
if test.got == nil || *test.got != test.want {
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
}
})
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
if extractChain == nil || len(extractChain.Validators) != 1 {
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
}
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
}
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
if chunkChain == nil || len(chunkChain.Validators) != 1 {
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
}
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
}
pipelineAttempts = 1
chunkAttempts = 2
extractAttempts = 3
validatorAttempts = 0
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
}
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
}
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
}
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
}
}
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*PipelineProfile)
}{
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
{name: "validator", mutate: func(profile *PipelineProfile) {
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
profile.Artifacts["events"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := baselineProfile()
test.mutate(&profile)
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
}
})
}
}
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
profile := baselineProfile()
profile.StructuredOutputRepairAttempts = repairAttempts(2)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
}
lane := resolved.Steps[0].ArtifactLanes[0]
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
}
}
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
digests := make(map[string]string)
for _, test := range []struct {
name string
attempts *int
}{
{name: "prompt owned", attempts: nil},
{name: "disabled", attempts: repairAttempts(0)},
{name: "configured", attempts: repairAttempts(1)},
} {
t.Run(test.name, func(t *testing.T) {
profile := llmProfilePipeline()
profile.StructuredOutputRepairAttempts = test.attempts
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
digests[test.name] = resolved.Digest
})
}
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
}
}
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
@@ -1811,7 +1527,6 @@ func TestResolvedPipelineDigestIncludesCompleteValidatorPolicy(t *testing.T) {
value.ValidatorChains[0].Validators[0].Binding.Module = "other-validator"
}},
{name: "llm profile", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.LLMProfile = "fast" }},
{name: "retries", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.Retries++ }},
{name: "options", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].Binding.Options["threshold"] = 0.75
}},
@@ -1877,7 +1592,6 @@ func validatorDigestFixture() ResolvedPipeline {
Binding: ModuleBinding{
Module: "semantic-validator",
LLMProfile: "careful",
Retries: 2,
Options: map[string]any{"mode": "strict", "threshold": 0.5},
References: ExternalReferenceMap(map[string]string{"rules": "rules.md", "examples": "examples.md"}),
},
@@ -2129,10 +1843,6 @@ func llmProfilePipeline() PipelineProfile {
}
}
func repairAttempts(value int) *int {
return &value
}
func llmProfileValues(profile string) map[string]string {
return map[string]string{
"input": profile,

View File

@@ -143,17 +143,6 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
}
provenance := ReferenceProvenance(first)
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
encoded, err := json.Marshal(first)
if err != nil {
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
@@ -163,7 +152,7 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
}
}
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
func TestMaterializeReferencesStoresSetsForAllTargets(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
@@ -215,22 +204,6 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T)
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
}
provenance := ReferenceProvenance(materialized)
if len(provenance) != 4 {
t.Fatalf("ReferenceProvenance() = %#v, want four entries", provenance)
}
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
}
if provenance[2].Stage != string(StageMerge) || provenance[2].LaneID != "events" || provenance[2].SlotName != "merge_notes" || provenance[2].Digest != mergeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want merge notes provenance", provenance[2])
}
if provenance[3].Stage != string(StageNormalize) || provenance[3].LaneID != "events" || provenance[3].SlotName != "normalization_notes" || provenance[3].Digest != normalizeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
}
}
func TestMaterializeReferencesUsesLaneArtifactVariant(t *testing.T) {

View File

@@ -1,478 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type capturedDebugRecorder struct {
mu sync.Mutex
json map[string][]byte
bytes map[string][]byte
failPath string
}
func newCapturedDebugRecorder() *capturedDebugRecorder {
return &capturedDebugRecorder{json: make(map[string][]byte), bytes: make(map[string][]byte)}
}
func (*capturedDebugRecorder) Enabled() bool { return true }
func (r *capturedDebugRecorder) WriteJSON(name string, payload any) error {
if name == r.failPath {
return errors.New("debug recorder failure")
}
data, err := json.Marshal(payload)
if err != nil {
return err
}
r.mu.Lock()
r.json[name] = data
r.mu.Unlock()
return nil
}
func (r *capturedDebugRecorder) WriteBytes(name string, data []byte) error {
if name == r.failPath {
return errors.New("debug recorder failure")
}
r.mu.Lock()
r.bytes[name] = append([]byte(nil), data...)
r.mu.Unlock()
return nil
}
func (r *capturedDebugRecorder) has(name string) bool {
r.mu.Lock()
defer r.mu.Unlock()
_, jsonOK := r.json[name]
_, bytesOK := r.bytes[name]
return jsonOK || bytesOK
}
func (r *capturedDebugRecorder) names() []string {
r.mu.Lock()
defer r.mu.Unlock()
names := make([]string, 0, len(r.json)+len(r.bytes))
for name := range r.json {
names = append(names, name)
}
for name := range r.bytes {
names = append(names, name)
}
sort.Strings(names)
return names
}
func (r *capturedDebugRecorder) envelope(t *testing.T, name string) debugTimedEnvelope {
t.Helper()
r.mu.Lock()
data := append([]byte(nil), r.json[name]...)
r.mu.Unlock()
if len(data) == 0 {
t.Fatalf("debug envelope %q was not written; names = %#v", name, r.names())
}
var envelope debugTimedEnvelope
if err := json.Unmarshal(data, &envelope); err != nil {
t.Fatalf("decode debug envelope %q: %v", name, err)
}
return envelope
}
type attemptDebugLLM struct{}
func (attemptDebugLLM) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, _ any) (contracts.StructuredCompletionResponse, error) {
return contracts.StructuredCompletionResponse{
Content: json.RawMessage(`{"accepted":true}`),
Provider: "test",
Model: "test-model",
ProfileID: request.ProfileID,
Debug: &contracts.LLMDebugMaterial{
Prompt: &contracts.LLMDebugPrompt{PromptID: request.PromptID, Messages: []contracts.LLMDebugMessage{{Role: "user", Content: "test"}}},
Response: &contracts.LLMDebugResponse{
Content: `{"accepted":true}`,
PromptID: request.PromptID,
ModelName: "test-model",
},
},
}, nil
}
func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
return preparedAttemptDebugPipelineWithChunks(t, 1)
}
func preparedAttemptDebugPipelineWithChunks(t *testing.T, chunkCount int) *PreparedPipeline {
t.Helper()
prepared := preparedConcurrentPipeline(t, chunkCount)
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
return prepared
}
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
producerAttempts := 2
validatorAttempts := 0
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
if len(lane.extractValidators.validators) == 0 {
t.Fatal("extract validators are empty")
}
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
var observedProducer, observedValidator *int
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
observedProducer = request.StructuredOutputRepairAttempts
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
})
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
observedValidator = target.structuredOutputRepairAttempts
return contracts.ValidationResult{Approved: true}, nil
}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
}
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
}
}
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
return err
}
func TestRunnerWritesAttemptScopedMergeAndNormalizeDebug(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
if err := callAttemptDebugLLM(ctx, client, "merge"); err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("merge", "merge diagnostic")}}, nil
}
prepared.Steps[0].lanes[0].typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
if err := callAttemptDebugLLM(ctx, client, "normalize"); err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{"normalized"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("normalize", "normalize diagnostic")}}, nil
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want one", len(output.NormalizeOutputs))
}
wantPaths := []string{
"merge/notes/input.json",
"merge/notes/attempt-01.json",
"merge/notes/attempt-01/prompt-0001.json",
"merge/notes/attempt-01/response-0001.json",
"merge/notes/attempt-01/response-content-0001.json",
"merge/notes/output.json",
"normalize/notes/input.json",
"normalize/notes/attempt-01.json",
"normalize/notes/attempt-01/prompt-0002.json",
"normalize/notes/attempt-01/response-0002.json",
"normalize/notes/attempt-01/response-content-0002.json",
"normalize/notes/output.json",
}
for _, name := range wantPaths {
if !debug.has(name) {
t.Errorf("debug artifact %q missing; names = %#v", name, debug.names())
}
}
for _, name := range debug.names() {
if strings.HasPrefix(name, "unscoped-merge/") || strings.HasPrefix(name, "unscoped-normalize/") {
t.Errorf("LLM call used fallback debug path %q", name)
}
}
mergeEnvelope := debug.envelope(t, "merge/notes/attempt-01.json")
normalizeEnvelope := debug.envelope(t, "normalize/notes/attempt-01.json")
if len(mergeEnvelope.LLMCalls) != 1 || mergeEnvelope.LLMCalls[0].ResponsePath != "merge/notes/attempt-01/response-0001.json" {
t.Fatalf("merge LLM calls = %#v, want attempt-scoped call", mergeEnvelope.LLMCalls)
}
if len(normalizeEnvelope.LLMCalls) != 1 || normalizeEnvelope.LLMCalls[0].ResponsePath != "normalize/notes/attempt-01/response-0002.json" {
t.Fatalf("normalize LLM calls = %#v, want attempt-scoped call", normalizeEnvelope.LLMCalls)
}
}
func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedDiagnosticsOnly(t *testing.T) {
for _, stage := range []ModuleStage{StageMerge, StageNormalize} {
t.Run(string(stage), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
lane := &prepared.Steps[0].lanes[0]
attempts := 0
operation := func(ctx context.Context) (erasedTypedResult, error) {
attempts++
if err := callAttemptDebugLLM(ctx, client, string(stage)); err != nil {
return erasedTypedResult{}, err
}
scope := "accepted"
if attempts == 1 {
scope = "discarded"
}
return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope))}, nil
}
validatorCalls := 0
validator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}
switch stage {
case StageMerge:
lane.resolved.Merge.Retries = 1
lane.mergeValidators.validators = []preparedValidator{validator}
lane.typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
return operation(ctx)
}
case StageNormalize:
lane.resolved.Normalize.Retries = 1
lane.normalizeValidators.validators = []preparedValidator{validator}
lane.typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return operation(ctx)
}
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
laneID := lane.resolved.ID
firstPath := fmt.Sprintf("%s/%s/attempt-01.json", stage, laneID)
secondPath := fmt.Sprintf("%s/%s/attempt-02.json", stage, laneID)
assertAttemptEnvelopeSequence(t, debug, fmt.Sprintf("%s/%s", stage, laneID), 1, 2)
first := debug.envelope(t, firstPath)
second := debug.envelope(t, secondPath)
if len(first.LLMCalls) != 1 || len(second.LLMCalls) != 1 || first.LLMCalls[0].CallID == second.LLMCalls[0].CallID {
t.Fatalf("retry LLM calls = first %#v, second %#v; want distinct calls", first.LLMCalls, second.LLMCalls)
}
if !strings.Contains(string(debug.json[firstPath]), "rejection") {
t.Fatalf("first attempt envelope = %s, want rejection", debug.json[firstPath])
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" {
t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics)
}
})
}
}
func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
tests := []struct {
name string
configure func(*PreparedPipeline)
path string
wantError string
wantBody string
attemptError bool
}{
{
name: "merge module error",
configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{}, errors.New("merge exploded")
}
},
path: "merge/notes/attempt-01.json",
wantError: "merge exploded",
attemptError: true,
},
{
name: "normalize validator error",
configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureFailRun
prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, errors.New("validator exploded")
},
}}
},
path: "normalize/notes/attempt-01.json",
wantError: "validator exploded",
wantBody: "output",
},
{
name: "merge final rejection",
configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
},
path: "merge/notes/attempt-01.json",
wantBody: "rejection",
},
{
name: "normalize serialization error",
configure: func(prepared *PreparedPipeline) {
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: "wrong artifact type"}, nil
}
},
path: "normalize/notes/attempt-01.json",
wantError: "serialize normalize candidate",
attemptError: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
debug := newCapturedDebugRecorder()
tc.configure(prepared)
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
envelope := debug.envelope(t, tc.path)
if tc.wantError != "" {
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else if runErr != nil {
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
}
if tc.wantBody != "" && !strings.Contains(string(debug.json[tc.path]), tc.wantBody) {
t.Fatalf("attempt envelope = %s, want %q", debug.json[tc.path], tc.wantBody)
}
if tc.name == "merge final rejection" && len(output.Rejected) != 1 {
t.Fatalf("rejected outputs = %#v, want one", output.Rejected)
}
})
}
}
func TestRunnerKeepsValidatorLLMCallsOutOfModuleAttempt(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
if err := callAttemptDebugLLM(ctx, client, "merge-module"); err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}}, nil
}
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
if err := callAttemptDebugLLM(ctx, client, "merge-validator"); err != nil {
return contracts.ValidationResult{}, err
}
return contracts.ValidationResult{Approved: true}, nil
},
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
moduleEnvelope := debug.envelope(t, "merge/notes/attempt-01.json")
validatorPath := "validate/merge/notes/typed%2Fmerge/01-llm-check-attempt-01.json"
validatorEnvelope := debug.envelope(t, validatorPath)
if len(moduleEnvelope.LLMCalls) != 1 || !strings.Contains(moduleEnvelope.LLMCalls[0].ResponsePath, "merge/notes/attempt-01/") {
t.Fatalf("module LLM calls = %#v, want module call only", moduleEnvelope.LLMCalls)
}
if len(validatorEnvelope.LLMCalls) != 1 || !strings.Contains(validatorEnvelope.LLMCalls[0].ResponsePath, "validate/merge/notes/") {
t.Fatalf("validator LLM calls = %#v, want validator call only", validatorEnvelope.LLMCalls)
}
if moduleEnvelope.LLMCalls[0].CallID == validatorEnvelope.LLMCalls[0].CallID {
t.Fatalf("module and validator envelopes reference the same call: %#v", moduleEnvelope.LLMCalls)
}
}
type attemptReuseLoader struct {
CheckpointLoader
laneID string
merge MergeCheckpoint
normalize NormalizeCheckpoint
}
func (l attemptReuseLoader) Enabled() bool { return true }
func (l attemptReuseLoader) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
if laneID == l.laneID {
return l.merge, CheckpointDecision{Reused: true, Reason: "test reuse"}
}
return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func (l attemptReuseLoader) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
if laneID == l.laneID {
return l.normalize, CheckpointDecision{Reused: true, Reason: "test reuse"}
}
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func TestRunnerCheckpointReuseDoesNotSynthesizeModuleAttempts(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := prepared.Steps[0].lanes[0]
merge, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Merge.Module, "source", codecNotes{Items: []string{"merged"}})
if err != nil {
t.Fatalf("checkpointArtifact(merge): %v", err)
}
normalize, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Normalize.Module, "source", codecNotes{Items: []string{"normalized"}})
if err != nil {
t.Fatalf("checkpointArtifact(normalize): %v", err)
}
loader := attemptReuseLoader{
CheckpointLoader: NoopCheckpointLoader(),
laneID: lane.resolved.ID,
merge: MergeCheckpoint{Output: merge},
normalize: NormalizeCheckpoint{Output: normalize},
}
debug := newCapturedDebugRecorder()
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoint: loader}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
for _, name := range debug.names() {
if (strings.HasPrefix(name, "merge/notes/attempt-") || strings.HasPrefix(name, "normalize/notes/attempt-")) && name != "" {
t.Fatalf("checkpoint reuse synthesized module attempt artifact %q", name)
}
}
for _, name := range []string{"merge/notes/input.json", "merge/notes/output.json", "normalize/notes/input.json", "normalize/notes/output.json"} {
if !debug.has(name) {
t.Errorf("checkpoint reuse missing stage-level artifact %q", name)
}
}
}
func TestRunnerTreatsModuleAttemptDebugWriteFailureAsFrameworkError(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
debug := newCapturedDebugRecorder()
debug.failPath = "merge/notes/attempt-01.json"
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err == nil || !strings.Contains(err.Error(), "write merge attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
t.Fatalf("Run() error = %v, want merge attempt debug failure", err)
}
}

View File

@@ -2,10 +2,8 @@ package pipeline
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
@@ -93,26 +91,6 @@ func matchingNotes(values []codecNotes, value string) int {
return count
}
type candidateCheckpointRecorder struct {
CheckpointRecorder
mergeSucceeded int
normalizeSucceeded int
mergeOutput CheckpointArtifact
normalizeOutput CheckpointArtifact
}
func (r *candidateCheckpointRecorder) MergeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error {
r.mergeSucceeded++
r.mergeOutput = cloneCheckpointArtifact(output)
return nil
}
func (r *candidateCheckpointRecorder) NormalizeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error {
r.normalizeSucceeded++
r.normalizeOutput = cloneCheckpointArtifact(output)
return nil
}
func installObservedNotesCodec(t *testing.T, prepared *PreparedPipeline, codec *observedNotesCodec) {
t.Helper()
registry := NewArtifactCodecRegistry()
@@ -130,7 +108,7 @@ func configureCandidateOperation(prepared *PreparedPipeline, target ModuleStage,
lane := &prepared.Steps[0].lanes[0]
switch target {
case StageExtract:
lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
@@ -176,12 +154,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
if target == StageMerge {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
} else {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
}
prepared := preparedConcurrentPipeline(t, 1)
prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{}
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
@@ -233,8 +206,7 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
}
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -251,23 +223,10 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
}
case StageNormalize:
var normalized codecNotes
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) {
if err := json.Unmarshal(output.normalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) {
t.Fatalf("normalized output = %#v, %v; want %#v", normalized, err, candidate)
}
}
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target)
if target == StageExtract {
attemptPath = "extract/notes/chunk-000001/attempt-01.json"
}
payload := debug.envelope(t, attemptPath).Payload.(map[string]any)
content, err := base64.StdEncoding.DecodeString(payload["output"].(map[string]any)["content"].(map[string]any)["content_base64"].(string))
var debugValue codecNotes
if err == nil {
err = json.Unmarshal(content, &debugValue)
}
if err != nil || !reflect.DeepEqual(debugValue, candidate) {
t.Fatalf("debug candidate = %#v, %v; want %#v", debugValue, err, candidate)
}
})
}
}
@@ -284,7 +243,7 @@ func setCandidateValidators(prepared *PreparedPipeline, target ModuleStage, vali
}
func TestRunnerReportsCandidateDecodeFailure(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedConcurrentPipeline(t, 1)
codec := &observedNotesCodec{candidateDecodeError: "decode-failure"}
installObservedNotesCodec(t, prepared, codec)
configureCandidateOperation(prepared, StageMerge, codecNotes{Items: []string{"decode-failure"}})
@@ -298,131 +257,74 @@ func TestRunnerReportsCandidateDecodeFailure(t *testing.T) {
}
func TestRunnerRejectsCandidatesBeforeFinalEncoding(t *testing.T) {
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
if target == StageMerge {
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
} else {
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
}
prepared := preparedConcurrentPipeline(t, 1)
codec := &observedNotesCodec{}
installObservedNotesCodec(t, prepared, codec)
configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}})
setCandidateValidator(prepared, target, false)
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder})
if err != nil {
t.Fatalf("Run() error = %v, want validator rejection", err)
}
if len(output.Rejected) != 1 || output.Rejected[0].Stage != string(target) || output.Rejected[0].ReasonCode != "candidate_rejected" {
t.Fatalf("rejected outputs = %#v, want %s validator rejection", output.Rejected, target)
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err == nil || !strings.Contains(err.Error(), "candidate-check") || len(output.OutputFiles) != 0 {
t.Fatalf("result=%#v error=%v, want rejection without publication", output, err)
}
if codec.candidateCalls("invalid") != 1 || codec.finalCalls("invalid") != 0 {
t.Fatalf("invalid candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid"))
}
if target == StageMerge && recorder.mergeSucceeded != 0 {
t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded)
}
if target == StageNormalize && recorder.normalizeSucceeded != 0 {
t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded)
t.Fatalf("rejected candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid"))
}
})
}
}
func TestRunnerFinalEncodesAcceptedCandidatesOnce(t *testing.T) {
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedConcurrentPipeline(t, 1)
codec := &observedNotesCodec{}
installObservedNotesCodec(t, prepared, codec)
value := "accepted-" + string(target)
configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}})
setCandidateValidator(prepared, target, true)
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
loader := &extractResultLoader{CheckpointLoader: NoopCheckpointLoader(), decision: CheckpointDecision{Reason: "not found"}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Checkpoint: loader})
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
t.Fatal(err)
}
if codec.candidateCalls(value) != 1 || codec.finalCalls(value) != 1 {
t.Fatalf("accepted candidate calls = candidate %d, final %d; want 1, 1", codec.candidateCalls(value), codec.finalCalls(value))
}
if target == StageMerge && recorder.mergeSucceeded != 1 {
t.Fatalf("merge checkpoints = %d, want one", recorder.mergeSucceeded)
}
if target == StageNormalize && recorder.normalizeSucceeded != 1 {
t.Fatalf("normalize checkpoints = %d, want one", recorder.normalizeSucceeded)
}
checkpoint := recorder.mergeOutput
if target == StageNormalize {
checkpoint = recorder.normalizeOutput
}
var stored codecNotes
if err := json.Unmarshal(checkpoint.Artifact.Content, &stored); err != nil || !reflect.DeepEqual(stored, codecNotes{Items: []string{value}}) {
t.Fatalf("checkpoint content = %s, %v; want accepted value %q", checkpoint.Artifact.Content, err, value)
}
gotStages := make([]string, len(output.CheckpointEvents))
for i, event := range output.CheckpointEvents {
gotStages[i] = event.Stage
}
wantStages := []string{"source", string(StageExtract), string(StageMerge), string(StageNormalize)}
if !reflect.DeepEqual(gotStages, wantStages) {
t.Fatalf("checkpoint event stages = %#v, want %#v", gotStages, wantStages)
}
})
}
}
func TestRunnerRecordsCandidateAndFinalEncodingFailures(t *testing.T) {
tests := []struct {
name string
target ModuleStage
value string
candidate bool
wantError string
}{
{name: "merge candidate", target: StageMerge, value: "merge-candidate-failure", candidate: true, wantError: "serialize merge candidate"},
{name: "normalize candidate", target: StageNormalize, value: "normalize-candidate-failure", candidate: true, wantError: "serialize normalize candidate"},
{name: "merge final", target: StageMerge, value: "merge-final-failure", wantError: "serialize accepted merge output"},
{name: "normalize final", target: StageNormalize, value: "normalize-final-failure", wantError: "serialize accepted normalize output"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
codec := &observedNotesCodec{}
if tc.candidate {
codec.candidateError = tc.value
} else {
codec.finalError = tc.value
func TestRunnerReportsCandidateAndFinalEncodingFailures(t *testing.T) {
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
for _, candidateFailure := range []bool{true, false} {
name := string(target) + "/final"
if candidateFailure {
name = string(target) + "/candidate"
}
installObservedNotesCodec(t, prepared, codec)
configureCandidateOperation(prepared, tc.target, codecNotes{Items: []string{tc.value}})
setCandidateValidator(prepared, tc.target, true)
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
debug := newCapturedDebugRecorder()
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, want %q", err, tc.wantError)
}
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", tc.target)
envelope := debug.envelope(t, attemptPath)
if !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
}
if tc.candidate && codec.finalCalls(tc.value) != 0 {
t.Fatalf("final encode calls = %d, want none after candidate failure", codec.finalCalls(tc.value))
}
if tc.target == StageMerge && recorder.mergeSucceeded != 0 {
t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded)
}
if tc.target == StageNormalize && recorder.normalizeSucceeded != 0 {
t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded)
}
})
t.Run(name, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
codec := &observedNotesCodec{}
value := "encoding-failure"
wantOperation := "encode"
if candidateFailure {
codec.candidateError = value
wantOperation = "encode candidate"
} else {
codec.finalError = value
}
installObservedNotesCodec(t, prepared, codec)
configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}})
setCandidateValidator(prepared, target, true)
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
var codecErr *ArtifactCodecOperationError
if !errors.As(err, &codecErr) || codecErr.Operation != wantOperation || codecErr.Kind != "test/notes" || len(output.OutputFiles) != 0 {
t.Fatalf("output=%#v error=%v codec error=%#v, want scoped %s failure without publication", output, err, codecErr, wantOperation)
}
if candidateFailure && codec.finalCalls(value) != 0 {
t.Fatal("candidate failure reached final encoding")
}
})
}
}
}

View File

@@ -1,129 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.Extract.Retries = 1
var mu sync.Mutex
corrections := make(map[int]*contracts.SemanticCorrection)
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
chunkIndex := request.Chunk.Index
if request.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return erasedTypedResult{}, err
}
mu.Lock()
corrections[chunkIndex] = correction
mu.Unlock()
candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("corrected-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("corrected-%d", chunkIndex)}}, ModelCandidate: candidate}, nil
}
candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("initial-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("invalid-%d", chunkIndex)}}, ModelCandidate: candidate}, nil
})
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
value := target.value.(codecNotes)
if value.Items[0] == fmt.Sprintf("invalid-%d", target.chunk.Index) {
return contracts.ValidationResult{Approved: false, ReasonCode: "incorrect_extract", Message: "candidate needs correction", CorrectionGuidance: fmt.Sprintf("correct chunk %d", target.chunk.Index)}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("run output = %#v, want corrected accepted extracts", output)
}
mu.Lock()
defer mu.Unlock()
if len(corrections) != 2 {
t.Fatalf("corrections = %#v, want one correction per chunk", corrections)
}
for index := 0; index < 2; index++ {
correction := corrections[index]
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || !strings.Contains(correction.UserGuidance, fmt.Sprintf("correct chunk %d", index)) || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "incorrect_extract") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
t.Fatalf("chunk %d correction = %#v, want its exact initial response and semantic replacement guidance only", index, correction)
}
}
terminal := string(debug.json["extract/notes/chunk-000001/terminal.json"])
if !strings.Contains(terminal, `"kind":"semantic_correction"`) || !strings.Contains(terminal, `"producer_attempt_count":2`) {
t.Fatalf("extract terminal debug = %s, want correction provenance", terminal)
}
if strings.Contains(terminal, "initial-response-0") || strings.Contains(terminal, "corrected-response-0") || strings.Contains(terminal, "correct chunk 0") {
t.Fatalf("extract terminal debug leaked correction content: %s", terminal)
}
manifest, err := json.Marshal(output.Manifest)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(manifest), "initial-response-0") || strings.Contains(string(manifest), "corrected-response-0") || strings.Contains(string(manifest), "correct chunk 0") {
t.Fatalf("manifest leaked correction content: %s", manifest)
}
}
func containsValidationSummary(summaries []artifacts.ValidationSummary, status string) bool {
for _, summary := range summaries {
if summary.Status == status {
return true
}
}
return false
}
func TestRunnerContinuesValidationIncompleteExtractWithoutCheckpoint(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
}
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
t.Fatalf("run output = %#v, want accepted incomplete extract", output)
}
if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") {
t.Fatalf("manifest validation = %#v, want incomplete extract provenance", output.Manifest)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics)
}
if len(recorder.checkpoint.Outputs) != 0 || len(recorder.checkpoint.Rejected) != 0 {
t.Fatalf("extract checkpoint = %#v, want no persisted incomplete output", recorder.checkpoint)
}
encoded := string(debug.json["extract/notes/output.json"])
if !strings.Contains(encoded, `"validation_incomplete_chunks":[0]`) {
t.Fatalf("extract output debug = %s, want incomplete chunk marker", encoded)
}
}

View File

@@ -3,8 +3,6 @@ package pipeline
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
@@ -12,11 +10,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type extractCaptureRecorder struct {
CheckpointRecorder
checkpoint ExtractCheckpoint
}
type pipelineTypedMetadata map[string]any
type mutatingMetadataValidator struct {
@@ -35,48 +28,14 @@ func (v *mutatingMetadataValidator) Validate(_ context.Context, request contract
return contracts.ValidationResult{Approved: true}, nil
}
func (r *extractCaptureRecorder) ExtractSucceeded(_ string, _ string, _ []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
r.checkpoint = ExtractCheckpoint{
Outputs: cloneCheckpointArtifacts(outputs),
Rejected: cloneRejectedOutputs(rejected),
}
return nil
}
type extractResultLoader struct {
CheckpointLoader
checkpoint ExtractCheckpoint
decision CheckpointDecision
}
func (l *extractResultLoader) Enabled() bool { return true }
func (l *extractResultLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{
Outputs: cloneCheckpointArtifacts(l.checkpoint.Outputs),
Rejected: cloneRejectedOutputs(l.checkpoint.Rejected),
}, l.decision
}
func cloneCheckpointArtifacts(values []CheckpointArtifact) []CheckpointArtifact {
if len(values) == 0 {
return nil
}
cloned := make([]CheckpointArtifact, len(values))
for i := range values {
cloned[i] = cloneCheckpointArtifact(values[i])
}
return cloned
}
func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedConcurrentPipeline(t, 1)
chunker := prepared.chunker.(*typedTestChunker)
chunker.plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)}
chunker.plan.Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)}
var gotRange, gotPlan string
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
gotRange = string(request.Chunk.Annotations["same"])
gotPlan = string(request.Chunk.PlanAnnotations["same"])
request.Chunk.Annotations["same"][0] = '['
@@ -95,7 +54,7 @@ func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
}
func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedConcurrentPipeline(t, 1)
input := prepared.input.(*typedTestInput)
input.doc.Units[0].Metadata = map[string]any{
"typed": pipelineTypedMetadata{"raw": json.RawMessage(`{"value":true}`)},
@@ -114,7 +73,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes
chunk: validator,
}}
var extractorSeen string
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
nested := request.Chunk.Units[0].Metadata["typed"].(pipelineTypedMetadata)
extractorSeen = string(nested["raw"].(json.RawMessage))
nested["raw"].(json.RawMessage)[0] = '['
@@ -135,7 +94,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes
}
func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared := preparedConcurrentPipeline(t, 1)
cyclic := make(map[string]any)
cyclic["self"] = cyclic
_, err := New().Run(context.Background(), RunInput{
@@ -147,144 +106,3 @@ func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) {
t.Fatalf("Run() error = %v, want contextual metadata clone failure", err)
}
}
func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
extractCalls := 0
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractCalls++
return erasedTypedResult{
Value: typedValueForLane(0, request.Chunk.Index),
Diagnostics: []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionObservation,
Category: contracts.DiagnosticCategoryNormalization,
ReasonCode: "accepted_extract_normalized",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "extract", Message: "accepted extract"}},
}},
}, nil
})
freshDebug := newCapturedDebugRecorder()
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
freshLoader := &extractResultLoader{
CheckpointLoader: NoopCheckpointLoader(),
decision: CheckpointDecision{Reason: "extract checkpoint not found"},
}
fresh, err := New().Run(context.Background(), RunInput{
Prepared: prepared,
RawInput: []byte("input"),
Checkpoints: recorder,
Checkpoint: freshLoader,
Debug: freshDebug,
})
if err != nil {
t.Fatalf("fresh Run() error = %v, want nil", err)
}
if extractCalls != 1 {
t.Fatalf("fresh extract calls = %d, want 1", extractCalls)
}
assertExtractDecision(t, fresh.CheckpointEvents, "executed", "extract checkpoint not found")
assertExtractDebugPaths(t, freshDebug, true)
reusedDebug := newCapturedDebugRecorder()
reusedLoader := &extractResultLoader{
CheckpointLoader: NoopCheckpointLoader(),
checkpoint: recorder.checkpoint,
decision: CheckpointDecision{Reused: true, Reason: "extract checkpoint matched"},
}
reused, err := New().Run(context.Background(), RunInput{
Prepared: prepared,
RawInput: []byte("input"),
Checkpoint: reusedLoader,
Debug: reusedDebug,
})
if err != nil {
t.Fatalf("reused Run() error = %v, want nil", err)
}
if extractCalls != 1 {
t.Fatalf("extract calls after reuse = %d, want 1", extractCalls)
}
assertExtractDecision(t, reused.CheckpointEvents, "reused", "extract checkpoint matched")
assertExtractDebugPaths(t, reusedDebug, false)
if !reflect.DeepEqual(reused.NormalizeOutputs, fresh.NormalizeOutputs) {
t.Fatalf("reused normalize outputs = %#v, want fresh outputs %#v", reused.NormalizeOutputs, fresh.NormalizeOutputs)
}
if !reflect.DeepEqual(reused.Diagnostics, fresh.Diagnostics) {
t.Fatalf("reused diagnostics = %#v, want fresh diagnostics %#v", reused.Diagnostics, fresh.Diagnostics)
}
}
func TestRunnerPromotesOnlyAcceptedExtractRetryDiagnostics(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared.Steps[0].lanes[0].resolved.Extract.Retries = 1
attempts := 0
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
attempts++
scope := "discarded"
if attempts == 2 {
scope = "accepted"
}
return erasedTypedResult{
Value: typedValueForLane(0, request.Chunk.Index),
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)},
ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope)),
}, nil
})
validatorCalls := 0
prepared.Steps[0].lanes[0].extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract", CorrectionGuidance: "return an acceptable extract"}, nil
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if attempts != 2 {
t.Fatalf("extract attempts = %d, want 2", attempts)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" {
t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics)
}
assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1, 2)
first := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
if first.Error != "" || !strings.Contains(string(debug.json["extract/notes/chunk-000001/attempt-01.json"]), "rejection") {
t.Fatalf("first extract attempt = %#v, want rejection without error", first)
}
name := "extract/notes/chunk-000001/attempt-02.json"
if !debug.has(name) {
t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names())
}
}
func assertExtractDecision(t *testing.T, events []CheckpointEvent, action CheckpointDecisionCategory, reason string) {
t.Helper()
for _, event := range events {
if event.Stage == string(StageExtract) {
if event.Action != action || event.Reason != reason {
t.Fatalf("extract checkpoint event = %#v, want action %q reason %q", event, action, reason)
}
return
}
}
t.Fatalf("extract checkpoint event missing from %#v", events)
}
func assertExtractDebugPaths(t *testing.T, debug *capturedDebugRecorder, wantAttempt bool) {
t.Helper()
for _, name := range []string{"extract/notes/input.json", "extract/notes/output.json"} {
if !debug.has(name) {
t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names())
}
}
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
if debug.has(attemptPath) != wantAttempt {
t.Fatalf("attempt debug path present = %t, want %t; names = %#v", debug.has(attemptPath), wantAttempt, debug.names())
}
input := debug.envelope(t, "extract/notes/input.json")
wantReuse := !wantAttempt
if !strings.Contains(string(debug.json["extract/notes/input.json"]), `"reused":`+map[bool]string{true: "true", false: "false"}[wantReuse]) {
t.Fatalf("extract input payload = %#v, want reused %t", input.Payload, wantReuse)
}
}

View File

@@ -1,29 +0,0 @@
package pipeline
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
)
func TestMergeLLMProfileManifestsDistinguishesEffectiveTargets(t *testing.T) {
got := mergeLLMProfileManifests(
[]artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
},
[]artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
},
)
want := []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("merged profiles = %#v, want %#v", got, want)
}
}

View File

@@ -1,380 +0,0 @@
package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
tests := []struct {
name string
retries int
operation func(int) erasedTypedResult
validator *preparedValidator
wantCalls int
wantItem string
wantMessages []string
wantRejected int
wantDebug []string
wantCheckpoint int
}{
{
name: "accepts zero-retry fallback",
retries: 0,
operation: func(int) erasedTypedResult {
return retryableNormalizeResult("fallback", "ordinary", "fallback-warning")
},
wantCalls: 1,
wantItem: "fallback",
wantMessages: []string{"ordinary", "fallback-warning"},
wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`},
wantCheckpoint: 1,
},
{
name: "retries before accepting ordinary result",
retries: 1,
operation: func(attempt int) erasedTypedResult {
if attempt == 1 {
return retryableNormalizeResult("discarded", "discarded-ordinary", "discarded-fallback")
}
return erasedTypedResult{Value: codecNotes{Items: []string{"accepted"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted-warning")}}
},
wantCalls: 2,
wantItem: "accepted",
wantMessages: []string{"accepted-warning"},
wantDebug: []string{`"another_attempt":true`, `"fallback_accepted":false`},
wantCheckpoint: 1,
},
{
name: "accepts final fallback after exhaustion",
retries: 1,
operation: func(attempt int) erasedTypedResult {
return retryableNormalizeResult(fmt.Sprintf("fallback-%d", attempt), fmt.Sprintf("ordinary-%d", attempt), fmt.Sprintf("fallback-warning-%d", attempt))
},
wantCalls: 2,
wantItem: "fallback-2",
wantMessages: []string{"ordinary-2", "fallback-warning-2"},
wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`},
wantCheckpoint: 1,
},
{
name: "keeps final fallback rejection terminal",
retries: 1,
operation: func(int) erasedTypedResult {
return retryableNormalizeResult("rejected", "ordinary", "fallback-warning")
},
validator: &preparedValidator{
resolved: ResolvedValidator{Binding: Binding("reject-final-fallback"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback", CorrectionGuidance: "return an acceptable normalized artifact"}, nil
},
},
wantCalls: 2,
wantRejected: 1,
wantDebug: []string{`"fallback_accepted":true`, `"rejection"`},
wantCheckpoint: 0,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.Normalize.Retries = tc.retries
lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
if tc.validator != nil {
lane.normalizeValidators.validators = []preparedValidator{*tc.validator}
}
calls := 0
lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
calls++
return tc.operation(calls), nil
}
debug := newCapturedDebugRecorder()
checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoints: checkpoints})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != tc.wantCalls {
t.Fatalf("normalize calls = %d, want %d", calls, tc.wantCalls)
}
if checkpoints.normalizeSucceeded != tc.wantCheckpoint {
t.Fatalf("normalize checkpoints = %d, want %d", checkpoints.normalizeSucceeded, tc.wantCheckpoint)
}
if len(output.Rejected) != tc.wantRejected {
t.Fatalf("rejected outputs = %#v, want %d", output.Rejected, tc.wantRejected)
}
var retryDebug strings.Builder
for _, name := range debug.names() {
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
retryDebug.Write(debug.json[name])
}
}
for _, fragment := range tc.wantDebug {
if !strings.Contains(retryDebug.String(), fragment) {
t.Fatalf("retry debug = %s, want %q", retryDebug.String(), fragment)
}
}
if tc.wantRejected != 0 {
if output.Rejected[0].AttemptCount != tc.wantCalls {
t.Fatalf("rejection attempt count = %d, want %d", output.Rejected[0].AttemptCount, tc.wantCalls)
}
return
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %#v, want one", output.NormalizeOutputs)
}
decoded, err := lane.typed.codec.decode(output.NormalizeOutputs[0].Artifact.Content)
if err != nil {
t.Fatalf("decode normalized output: %v", err)
}
normalized, ok := decoded.(codecNotes)
if !ok {
t.Fatalf("decoded normalized output = %T, want codecNotes", decoded)
}
if got := firstNote(normalized); got != tc.wantItem {
t.Fatalf("normalized item = %q, want %q", got, tc.wantItem)
}
var gotMessages []string
for _, group := range output.Diagnostics.Groups {
for _, sample := range group.Samples {
gotMessages = append(gotMessages, sample.Message)
}
}
if strings.Join(gotMessages, "|") != strings.Join(tc.wantMessages, "|") {
t.Fatalf("durable diagnostics = %#v, want %#v", gotMessages, tc.wantMessages)
}
})
}
}
func TestRunnerForwardsModuleRequestedNormalizeCorrection(t *testing.T) {
const (
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
guidance = "Duplicate group 1 must use only supplied candidate numbers. Return one complete corrected response."
)
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.Normalize.Retries = 1
var observed *contracts.SemanticCorrection
calls := 0
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
calls++
if request.Correction != nil {
clone, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return erasedTypedResult{}, err
}
observed = clone
}
if calls == 1 {
return erasedTypedResult{
Value: codecNotes{Items: []string{"safe fallback"}},
ModelCandidate: attemptCandidate(t, defective),
Retry: &contracts.NormalizeRetry{
ReasonCode: "semantic_proposal_invalid",
Message: "operator-facing proposal diagnostic",
CorrectionGuidance: guidance,
},
}, nil
}
return erasedTypedResult{Value: codecNotes{Items: []string{"corrected"}}}, nil
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 2 || observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
t.Fatalf("normalize calls = %d correction = %#v", calls, observed)
}
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
t.Fatalf("run output = %#v, want corrected accepted output", output)
}
var retryDebug strings.Builder
for _, name := range debug.names() {
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
retryDebug.Write(debug.json[name])
}
}
if !strings.Contains(retryDebug.String(), `"correction_available":true`) || strings.Contains(retryDebug.String(), guidance) || strings.Contains(retryDebug.String(), defective) {
t.Fatalf("retry debug = %s, want safe correction metadata without content", retryDebug.String())
}
}
func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
const (
reasonSentinel = "reason-diagnostic-sentinel"
messageSentinel = "message-diagnostic-sentinel"
)
reasonOverLimit := strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes-len(reasonSentinel)) + reasonSentinel + "x"
messageOverLimit := strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes-len(messageSentinel)) + messageSentinel + "x"
guidanceOverLimit := strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes-len(messageSentinel)) + messageSentinel + "x"
tests := []struct {
name string
retry contracts.NormalizeRetry
candidate *contracts.ModelCandidate
wantError string
hiddenValues []string
}{
{
name: "accepts byte limits",
retry: contracts.NormalizeRetry{
ReasonCode: strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes),
Message: strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes),
},
},
{
name: "accepts correction guidance byte limit with candidate",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel,
CorrectionGuidance: strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes),
},
candidate: attemptCandidate(t, `{"duplicate_groups":[]}`),
},
{
name: "rejects oversized reason code",
retry: contracts.NormalizeRetry{
ReasonCode: reasonOverLimit,
Message: messageSentinel,
},
wantError: "reason code exceeds maximum length",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects oversized message",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageOverLimit,
},
wantError: "message exceeds maximum length",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects invalid reason code UTF-8",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel + string([]byte{0xff}),
Message: messageSentinel,
},
wantError: "reason code has invalid UTF-8",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects invalid message UTF-8",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel + string([]byte{0xff}),
},
wantError: "message has invalid UTF-8",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects oversized correction guidance",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel,
CorrectionGuidance: guidanceOverLimit,
},
wantError: "correction guidance exceeds maximum length",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects invalid correction guidance UTF-8",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel,
CorrectionGuidance: messageSentinel + string([]byte{0xff}),
},
wantError: "correction guidance has invalid UTF-8",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects blank correction guidance",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel,
CorrectionGuidance: " \t\n ",
},
wantError: "correction guidance is blank",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects correction guidance without candidate",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: messageSentinel,
CorrectionGuidance: "Return one complete corrected response.",
},
wantError: "requires a model candidate",
hiddenValues: []string{reasonSentinel, messageSentinel},
},
{
name: "rejects blank reason code",
retry: contracts.NormalizeRetry{
ReasonCode: " \t\n ",
Message: messageSentinel,
},
wantError: "reason code is blank",
hiddenValues: []string{messageSentinel},
},
{
name: "rejects blank message",
retry: contracts.NormalizeRetry{
ReasonCode: reasonSentinel,
Message: " \t\n ",
},
wantError: "message is blank",
hiddenValues: []string{reasonSentinel},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry, ModelCandidate: tc.candidate}, nil
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if tc.wantError == "" {
if err != nil {
t.Fatalf("Run() error = %v, want accepted retry fallback", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %#v, want one accepted fallback", output.NormalizeOutputs)
}
return
}
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, want fixed %q error", err, tc.wantError)
}
var debugOutput strings.Builder
for _, name := range debug.names() {
debugOutput.Write(debug.json[name])
}
for _, value := range tc.hiddenValues {
if strings.Contains(err.Error(), value) || strings.Contains(debugOutput.String(), value) {
t.Fatalf("retry diagnostic leaked %q", value)
}
}
})
}
}
func retryableNormalizeResult(item, ordinary, fallback string) erasedTypedResult {
return erasedTypedResult{
Value: codecNotes{Items: []string{item}},
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("attempt", ordinary)},
Retry: &contracts.NormalizeRetry{
ReasonCode: "retryable_normalization",
Message: "safe fallback is available",
FallbackDiagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("fallback", fallback)},
},
}
}

View File

@@ -1,148 +0,0 @@
package pipeline
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type diagnosticChunker struct {
key string
plan source.ChunkPlan
calls int
}
func (c *diagnosticChunker) Key() string { return c.key }
func (*diagnosticChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c *diagnosticChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
return contracts.ChunkPlanResult{
Plan: source.CloneChunkPlan(c.plan),
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", c.calls), "operation diagnostic")},
}, nil
}
type chunkValidationFunc struct {
name string
validate func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
}
func (v chunkValidationFunc) Name() string { return v.name }
func (chunkValidationFunc) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v chunkValidationFunc) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return v.validate(ctx, request)
}
func TestRunnerPromotesOnlyTerminalRejectionDiagnostics(t *testing.T) {
for _, target := range []ModuleStage{StageChunk, StageExtract, StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
attempts := 0
first := func() contracts.ValidationResult {
return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", attempts), "validator diagnostic")}}
}
reject := func() contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected", CorrectionGuidance: "return an acceptable candidate"}
}
debug := newCapturedDebugRecorder()
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
switch target {
case StageChunk:
prepared.resolved.ChunkValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
chunker := prepared.chunker.(*typedTestChunker)
prepared.chunker = &diagnosticChunker{key: prepared.resolved.Chunk.Module, plan: source.CloneChunkPlan(chunker.plan)}
prepared.resolved.Chunk.Retries = 1
prepared.chunkValidators.validators = []preparedValidator{
{resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-approval", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return first(), nil
}}},
{resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-rejection", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return reject(), nil
}}},
}
chunkerWithDiagnostics := prepared.chunker.(*diagnosticChunker)
first = func() contracts.ValidationResult {
return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", chunkerWithDiagnostics.calls), "validator diagnostic")}}
}
case StageExtract:
lane.resolved.Extract.Retries = 1
lane.resolved.ExtractValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
attempts++
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
})
lane.extractValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
case StageMerge:
lane.resolved.Merge.Retries = 1
lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
attempts++
return erasedTypedResult{Value: codecNotes{Items: []string{"merge"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
}
lane.mergeValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
case StageNormalize:
lane.resolved.Normalize.Retries = 1
lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
attempts++
return erasedTypedResult{Value: codecNotes{Items: []string{"normalize"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
}
lane.normalizeValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
wantScopes := []string{"operation-2", "validator-2"}
wantAttempts := 2
if target == StageChunk {
wantScopes = []string{"operation-1", "validator-1"}
wantAttempts = 1
}
if got := rejectionDiagnosticScopes(output.Diagnostics.Groups); !reflect.DeepEqual(got, wantScopes) {
t.Fatalf("published diagnostic scopes = %#v, want %#v", got, wantScopes)
}
if len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != wantAttempts {
t.Fatalf("rejections = %#v, want terminal rejection after %d attempt(s)", output.Rejected, wantAttempts)
}
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target)
if target == StageChunk {
attemptPath = "chunk/attempt-01.json"
} else if target == StageExtract {
attemptPath = "extract/notes/chunk-000001/attempt-01.json"
}
if !strings.Contains(string(debug.json[attemptPath]), "rejection") {
t.Fatalf("first attempt debug = %s, want rejection", debug.json[attemptPath])
}
})
}
}
func rejectionDiagnosticTypedValidators(first func() contracts.ValidationResult, reject func() contracts.ValidationResult) []preparedValidator {
return []preparedValidator{
{resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return first(), nil
}},
{resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return reject(), nil
}},
}
}
func rejectionDiagnosticScopes(groups []contracts.DiagnosticGroup) []string {
scopes := make([]string, len(groups))
for index := range groups {
scopes[index] = groups[index].Samples[0].Scope
}
return scopes
}

View File

@@ -1,238 +0,0 @@
package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type reuseLineageCheckpointSpy struct {
CheckpointLoader
CheckpointRecorder
loads map[string]int
writes map[string]int
forbid map[string]struct{}
}
func newReuseLineageCheckpointSpy() *reuseLineageCheckpointSpy {
return &reuseLineageCheckpointSpy{
CheckpointLoader: NoopCheckpointLoader(),
CheckpointRecorder: NoopCheckpointRecorder(),
loads: make(map[string]int),
writes: make(map[string]int),
forbid: make(map[string]struct{}),
}
}
func (s *reuseLineageCheckpointSpy) Enabled() bool { return true }
func (s *reuseLineageCheckpointSpy) load(stage, laneID string) {
key := stage + "/" + laneID
if _, forbidden := s.forbid[key]; forbidden {
panic("checkpoint lookup crossed validation-incomplete lineage: " + key)
}
s.loads[key]++
}
func (s *reuseLineageCheckpointSpy) write(stage, action, laneID string) {
s.writes[stage+"/"+action+"/"+laneID]++
}
func (s *reuseLineageCheckpointSpy) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
s.load("extract", laneID)
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
s.load("merge", laneID)
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
s.load("normalize", laneID)
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (s *reuseLineageCheckpointSpy) ExtractRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("extract", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) ExtractSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ []CheckpointArtifact, _ []contracts.RejectedOutput) error {
s.write("extract", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) ExtractFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("extract", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("merge", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error {
s.write("merge", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
s.write("merge", "rejected", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) MergeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("merge", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
s.write("normalize", "running", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error {
s.write("normalize", "succeeded", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
s.write("normalize", "rejected", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) NormalizeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
s.write("normalize", "failed", laneID)
return nil
}
func (s *reuseLineageCheckpointSpy) writesFor(stage, laneID string) int {
total := 0
needle := stage + "/"
suffix := "/" + laneID
for key, count := range s.writes {
if strings.HasPrefix(key, needle) && strings.HasSuffix(key, suffix) {
total += count
}
}
return total
}
func unavailableTypedValidator(kind contracts.ArtifactKind) preparedValidator {
return preparedValidator{
resolved: ResolvedValidator{Binding: Binding("typed/check"), Target: ValidatorTargetTyped, ArtifactKind: kind},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
},
}
}
func TestValidationIncompleteExtractDisablesDownstreamCheckpointIO(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
}
if spy.loads["merge/notes"] != 0 || spy.loads["normalize/notes"] != 0 {
t.Fatalf("downstream loads = %#v, want none", spy.loads)
}
if spy.writesFor("merge", "notes") != 0 || spy.writesFor("normalize", "notes") != 0 {
t.Fatalf("downstream writes = %#v, want none", spy.writes)
}
if spy.writes["extract/succeeded/notes"] != 0 {
t.Fatalf("extract writes = %#v, want no reusable success", spy.writes)
}
}
func TestValidationIncompleteMergeDisablesNormalizeCheckpointIO(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.MergeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.mergeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
}
if spy.loads["merge/notes"] != 1 || spy.loads["normalize/notes"] != 0 {
t.Fatalf("loads = %#v, want merge lookup only", spy.loads)
}
if spy.writes["merge/succeeded/notes"] != 0 || spy.writesFor("normalize", "notes") != 0 {
t.Fatalf("writes = %#v, want no reusable merge or normalize state", spy.writes)
}
}
func TestValidationIncompleteNormalizeIsNotPublished(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
lane.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
spy := newReuseLineageCheckpointSpy()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 1 || spy.writes["normalize/succeeded/notes"] != 0 {
t.Fatalf("output/writes = %d / %#v, want in-memory output without normalize publication", len(output.NormalizeOutputs), spy.writes)
}
}
func TestGeneratedReferenceFromIncompleteValidationDisablesDependentCheckpointIO(t *testing.T) {
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"producer"}})
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
producer.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
producer.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(producer.resolved.ArtifactKind)}
referenceItems := 0
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
referenceItems = len(request.References.Slots["producer-output"].Items)
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
spy := newReuseLineageCheckpointSpy()
for _, stage := range []string{"extract", "merge", "normalize"} {
spy.forbid[stage+"/score"] = struct{}{}
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if referenceItems != 1 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("current-run handoff = items %d outputs %d, want one generated item and two outputs", referenceItems, len(output.NormalizeOutputs))
}
for _, stage := range []string{"extract", "merge", "normalize"} {
if spy.writesFor(stage, "score") != 0 {
t.Fatalf("dependent writes = %#v, want none for %s", spy.writes, stage)
}
}
}

View File

@@ -1,126 +0,0 @@
package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerCorrectsRejectedMergeAndNormalizeCandidates(t *testing.T) {
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
t.Run(string(target), func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
var correction *contracts.SemanticCorrection
validator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("correction-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
value := target.value.(codecNotes)
if firstNote(value) == "invalid" {
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "candidate needs correction", CorrectionGuidance: "produce the accepted value"}, nil
}
return contracts.ValidationResult{Approved: true}, nil
},
}
operation := func(observed *contracts.SemanticCorrection) (erasedTypedResult, error) {
if observed != nil {
cloned, err := contracts.CloneSemanticCorrection(observed)
if err != nil {
return erasedTypedResult{}, err
}
correction = cloned
}
item, response := "invalid", `{"items":["invalid"]}`
if observed != nil {
item, response = "accepted", `{"items":["accepted"]}`
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: codecNotes{Items: []string{item}}, ModelCandidate: candidate}, nil
}
switch target {
case StageMerge:
lane.resolved.Merge.Retries = 1
lane.mergeValidators.validators = []preparedValidator{validator}
lane.typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
return operation(request.Correction)
}
case StageNormalize:
lane.resolved.Normalize.Retries = 1
lane.normalizeValidators.validators = []preparedValidator{validator}
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return operation(request.Correction)
}
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || !strings.Contains(correction.UserGuidance, "produce the accepted value") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "invalid_candidate") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
t.Fatalf("%s correction = %#v, want exact rejected response and semantic replacement guidance only", target, correction)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want corrected accepted output", output)
}
})
}
}
func TestRunnerRejectsDeterministicMergeCandidateWithoutCorrection(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.Merge.Retries = 1
lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
calls := 0
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
calls++
return erasedTypedResult{Value: codecNotes{Items: []string{"invalid"}}}, nil
}
lane.mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejection", err)
}
if calls != 1 || len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != 1 {
t.Fatalf("merge calls = %d output = %#v, want one deterministic terminal rejection", calls, output)
}
}
func TestRunnerContinuesNormalizeAfterValidatorFailure(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
lane := &prepared.Steps[0].lanes[0]
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
lane.normalizeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("unavailable"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
},
}}
checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: checkpoints})
if err != nil {
t.Fatalf("Run() error = %v, want incomplete accepted output", err)
}
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 || checkpoints.normalizeSucceeded != 0 {
t.Fatalf("run output = %#v checkpoints = %#v, want non-checkpointed incomplete normalized output", output, checkpoints)
}
if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") {
t.Fatalf("manifest validation = %#v, want incomplete normalization provenance", output.Manifest)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics)
}
}

View File

@@ -3,34 +3,39 @@ package pipeline
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerUsesProvidedSessionWithoutSourceFallback(t *testing.T) {
for _, test := range []struct {
name string
session string
wantSet bool
want string
}{
{name: "provided", session: " supplied-session ", wantSet: true, want: "supplied-session"},
{name: "empty", wantSet: false},
{name: "provided", session: " supplied-session ", want: "supplied-session"},
{name: "empty"},
} {
t.Run(test.name, func(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
output, err := New().Run(context.Background(), RunInput{
prepared := preparedConcurrentPipeline(t, 1)
var session string
calls := 0
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
calls++
session = request.SessionID
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
_, err := New().Run(context.Background(), RunInput{
Prepared: prepared,
RawInput: []byte("input"),
SessionID: test.session,
Workers: 1,
})
if err != nil {
t.Fatal(err)
}
value, found := output.Manifest.Metadata["session_id"]
if found != test.wantSet {
t.Fatalf("manifest session presence = %t, want %t; metadata = %#v", found, test.wantSet, output.Manifest.Metadata)
}
if found && value != test.want {
t.Fatalf("manifest session = %#v, want %q", value, test.want)
if calls != 1 || session != test.want {
t.Fatalf("extract calls=%d session=%q, want one call and %q", calls, session, test.want)
}
})
}

View File

@@ -1,472 +0,0 @@
package pipeline
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type terminalChunker struct {
key string
plan source.ChunkPlan
diagnostics []contracts.ProducerDiagnostic
err error
calls *int
}
func (c terminalChunker) Key() string { return c.key }
func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
if c.calls != nil {
(*c.calls)++
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: contracts.CloneProducerDiagnostics(c.diagnostics)}, c.err
}
type terminalChunkValidator struct {
result contracts.ValidationResult
err error
}
type retryingLLMChunkValidator struct {
requests []contracts.ChunkValidationRequest
calls int
}
type observingChunkValidator struct {
request contracts.ChunkValidationRequest
}
type observingSerializedValidator struct {
request contracts.SerializedValidationRequest
}
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
v.request = request
return contracts.ValidationResult{Approved: true}, nil
}
func (*observingSerializedValidator) Name() string { return "observing/serialized-validator" }
func (*observingSerializedValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *observingSerializedValidator) Validate(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
v.request = request
return contracts.ValidationResult{Approved: true}, nil
}
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return v.result, v.err
}
func (*retryingLLMChunkValidator) Name() string { return "retrying/llm-chunk-validator" }
func (*retryingLLMChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (validator *retryingLLMChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, request)
if validator.calls == 1 && len(request.Chunks) > 0 {
request.Chunks[0].ID = "mutated"
}
if validator.calls == 1 {
return contracts.ValidationResult{}, errors.New("temporary validator failure")
}
return contracts.ValidationResult{Approved: true}, nil
}
func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) {
t.Helper()
marker := strings.TrimSuffix(prefix, "/") + "/attempt-"
var got []int
for _, name := range debug.names() {
if !strings.HasPrefix(name, marker) || !strings.HasSuffix(name, ".json") {
continue
}
remainder := strings.TrimSuffix(strings.TrimPrefix(name, marker), ".json")
if strings.Contains(remainder, "/") {
continue
}
value, err := strconv.Atoi(remainder)
if err != nil {
t.Fatalf("parse attempt index from %q: %v", name, err)
}
got = append(got, value)
}
sort.Ints(got)
if !reflect.DeepEqual(got, attempts) {
t.Fatalf("attempt envelopes under %q = %#v, want %#v; names = %#v", prefix, got, attempts, debug.names())
}
}
func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) {
t.Helper()
prepared := preparedAttemptDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy = ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionRejectOutput,
ValidatorFailure: ValidatorFailureFailRun,
}
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionRejectOutput,
ValidatorFailure: ValidatorFailureFailRun,
}
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy = ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionRejectOutput,
ValidatorFailure: ValidatorFailureFailRun,
}
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy = ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionRejectOutput,
ValidatorFailure: ValidatorFailureFailRun,
}
chunker, ok := prepared.chunker.(*typedTestChunker)
if !ok {
t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
}
return prepared, source.CloneChunkPlan(chunker.plan)
}
func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
tests := []struct {
name string
moduleError error
validator terminalChunkValidator
wantError string
wantRejection bool
attemptError bool
}{
{name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}},
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed", attemptError: true},
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected", CorrectionGuidance: "return an acceptable chunk plan"}}, wantRejection: true},
{name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("chunk", "chunk diagnostic")}, err: tc.moduleError}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
envelope := debug.envelope(t, "chunk/attempt-01.json")
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if tc.wantRejection {
if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "chunk_rejected" {
t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
}
}
if tc.name == "accepted" {
encoded := string(debug.json["chunk/attempt-01.json"])
if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) {
t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded)
}
}
})
}
}
func TestRunnerRetriesLLMValidatorsWithoutRegeneratingChunkCandidate(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
chunkCalls := 0
prepared.resolved.Chunk.Retries = 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &chunkCalls}
validator := &retryingLLMChunkValidator{}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: ModuleBinding{Module: validator.Name(), Retries: 1}, ExecutionClass: contracts.ExecutionClassLLMBacked, Target: ValidatorTargetChunk},
chunk: validator,
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v", err)
}
if chunkCalls != 1 || validator.calls != 2 {
t.Fatalf("producer calls = %d validator calls = %d, want 1 and 2", chunkCalls, validator.calls)
}
if len(validator.requests) != 2 || len(validator.requests[0].Chunks) == 0 || len(validator.requests[1].Chunks) == 0 || validator.requests[1].Chunks[0].ID == "mutated" {
t.Fatalf("validator requests = %#v, want independently owned immutable candidates", validator.requests)
}
}
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
validator := &observingChunkValidator{}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
chunk: validator,
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(validator.request.Chunks) != 1 {
t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
}
chunk := validator.request.Chunks[0]
if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
t.Fatalf("materialized chunk identity = %#v", chunk)
}
if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
}
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
t.Fatalf("materialized metadata = %#v", chunk.Metadata)
}
}
func TestRunnerForwardsDetachedRepairAttemptsToChunkValidators(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
chunkValidator := &observingChunkValidator{}
serializedValidator := &observingSerializedValidator{}
chunkAttempts := 2
serializedAttempts := 3
chunkBinding := Binding(chunkValidator.Name())
chunkBinding.StructuredOutputRepairAttempts = &chunkAttempts
serializedBinding := Binding(serializedValidator.Name())
serializedBinding.StructuredOutputRepairAttempts = &serializedAttempts
prepared.chunkValidators.validators = []preparedValidator{
{resolved: ResolvedValidator{Binding: chunkBinding, Target: ValidatorTargetChunk}, chunk: chunkValidator},
{resolved: ResolvedValidator{Binding: serializedBinding, Target: ValidatorTargetSerialized}, serialized: serializedValidator},
}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if got := chunkValidator.request.StructuredOutputRepairAttempts; got == nil || *got != chunkAttempts || got == chunkBinding.StructuredOutputRepairAttempts {
t.Fatalf("chunk validator repair attempts = %v, want detached value %d", got, chunkAttempts)
}
if got := serializedValidator.request.StructuredOutputRepairAttempts; got == nil || *got != serializedAttempts || got == serializedBinding.StructuredOutputRepairAttempts {
t.Fatalf("serialized validator repair attempts = %v, want detached value %d", got, serializedAttempts)
}
}
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
prepared.resolved.Chunk.Retries = 1
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
debug := newCapturedDebugRecorder()
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err == nil || !strings.Contains(err.Error(), "valid JSON") {
t.Fatalf("Run() error = %v, want malformed annotation error", err)
}
if calls != 2 {
t.Fatalf("plan calls = %d, want two attempts", calls)
}
assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2)
}
func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
tests := []struct {
name string
moduleError error
validatorErr error
reject bool
candidateFail bool
finalFail bool
wantError string
attemptError bool
}{
{name: "terminal rejection", reject: true},
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed", attemptError: true},
{name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"},
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate", attemptError: true},
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output", attemptError: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionRejectOutput,
ValidatorFailure: ValidatorFailureFailRun,
}
value := "extract-terminal"
codec := &observedNotesCodec{}
if tc.candidateFail {
codec.candidateError = value
}
if tc.finalFail {
codec.finalError = value
}
installObservedNotesCodec(t, prepared, codec)
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
if tc.moduleError != nil {
return erasedTypedResult{}, tc.moduleError
}
return erasedTypedResult{Value: codecNotes{Items: []string{value}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("extract", "extract diagnostic")}}, nil
})
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected", CorrectionGuidance: "return an acceptable extract"}, tc.validatorErr
},
}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1)
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
envelope := debug.envelope(t, attemptPath)
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
}
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
}
if !tc.attemptError && envelope.Error != "" {
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
}
} else {
if err != nil {
t.Fatalf("Run() error = %v, want nil rejection", err)
}
if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "extract_rejected" {
t.Fatalf("extract rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
}
}
})
}
}
func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
t.Run("chunk", func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
primary := errors.New("chunk operation failed")
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary}
debug := newCapturedDebugRecorder()
debug.failPath = "chunk/attempt-01.json"
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
}
})
t.Run("extract", func(t *testing.T) {
prepared := preparedAttemptDebugPipeline(t)
primary := errors.New("extract operation failed")
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{}, primary
})
debug := newCapturedDebugRecorder()
debug.failPath = "extract/notes/chunk-000001/attempt-01.json"
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write extract attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
}
})
}
func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Retries = 1
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
}}
debug := newCapturedDebugRecorder()
debug.failPath = "chunk/attempt-01.json"
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
if err == nil || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
t.Fatalf("Run() error = %v, want terminal attempt debug failure", err)
}
if calls != 1 {
t.Fatalf("chunk calls = %d, want one attempt without retry", calls)
}
if debug.has("chunk/attempt-02.json") {
t.Fatal("second chunk attempt envelope exists after non-retryable debug persistence failure")
}
}
func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
prepared := preparedAttemptDebugPipelineWithChunks(t, 2)
debug := newCapturedDebugRecorder()
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
if err := callAttemptDebugLLM(ctx, client, "extract-module"); err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
if err := callAttemptDebugLLM(ctx, client, "extract-validator"); err != nil {
return contracts.ValidationResult{}, err
}
return contracts.ValidationResult{Approved: true}, nil
},
}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
validatorResponses := make(map[string]struct{})
for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ {
chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber)
module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json")
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json")
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") {
t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls)
}
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") {
t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls)
}
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls)
}
validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{}
}
if len(validatorResponses) != 2 {
t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses)
}
}

View File

@@ -18,10 +18,10 @@ func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
invocation *contracts.CacheMode
want contracts.ProducerControls
}{
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheAuto}},
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModePermissive, 5, contracts.CacheRefresh}},
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{contracts.ModeStrict, 0, contracts.CacheAuto}},
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheBypass}},
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheAuto}},
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5, Cache: contracts.CacheRefresh}},
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheAuto}},
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
@@ -75,13 +75,13 @@ func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if resolved.ChunkControls != (contracts.ProducerControls{contracts.ModePermissive, 0, contracts.CacheBypass}) {
if resolved.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
t.Fatalf("chunk controls = %#v", resolved.ChunkControls)
}
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
for _, got := range []contracts.ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
if got != (contracts.ProducerControls{contracts.ModePermissive, 2, contracts.CacheBypass}) {
if got != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}) {
t.Fatalf("lane controls = %#v", got)
}
}