Migrate framework tests to native producer and artifact contracts
This commit is contained in:
@@ -1674,12 +1674,22 @@ the real executable route and text/JSON/YAML/fallback tests now pass with a loca
|
||||
fake HTTP provider, closing their earlier runtime-wrapper setup blocker. All D&D
|
||||
module and configuration tests, their focused vet checks, contracts/shared/helper
|
||||
tests, and focused native CLI tests pass. Whole-tree compilation remains pending:
|
||||
CLI, framework, and integration fixtures still reference removed APIs and legacy
|
||||
imports. Leaf-assembled module tests also need a proper integration-test home to
|
||||
CLI and integration fixtures still reference removed APIs and legacy imports.
|
||||
Leaf-assembled module tests also need a proper integration-test home to
|
||||
satisfy the existing import-boundary check. Dependency tidy/old-path removal,
|
||||
notice reconciliation, remaining fixture migration, and full validation are not
|
||||
complete; this stage is not marked finished.
|
||||
|
||||
All framework package tests and vet checks now pass; the complete pipeline
|
||||
package also passes race checks. Migrated resolution, construction, reference materialization,
|
||||
generated handoff, codec, source/annotation isolation, diagnostics, and session
|
||||
coverage to the current contracts. Removed tests tied solely to superseded
|
||||
outer retries, checkpoint provenance, manifests, and attempt-debug wrappers;
|
||||
native producer, cache, assessment, and capture tests retain their replacement
|
||||
behavioral coverage. Whole-tree compile-only validation still fails at the old
|
||||
ConfigKit YAML import in CLI tests and removed completion/checkpoint APIs in
|
||||
module integration fixtures; those blockers remain required work.
|
||||
|
||||
**Goal:** delete the old architecture rather than leave dormant alternatives.
|
||||
|
||||
**Inspect/change:** all production/test callers of the feature roadmap's removal
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) {
|
||||
@@ -73,3 +75,46 @@ func TestDebugChunkEnvelopeClonesAnnotationScopes(t *testing.T) {
|
||||
t.Fatal("debugSourceChunkEnvelope() shares annotation bytes")
|
||||
}
|
||||
}
|
||||
|
||||
type observingChunkValidator struct {
|
||||
request contracts.ChunkValidationRequest
|
||||
}
|
||||
|
||||
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
||||
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
v.request = request
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
plan := prepared.chunker.(*typedTestChunker).plan
|
||||
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
|
||||
prepared.chunker = &typedTestChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||
validator := &observingChunkValidator{}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
|
||||
chunk: validator,
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(validator.request.Chunks) != 1 {
|
||||
t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
|
||||
}
|
||||
chunk := validator.request.Chunks[0]
|
||||
if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
|
||||
t.Fatalf("materialized chunk identity = %#v", chunk)
|
||||
}
|
||||
if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
|
||||
t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
|
||||
}
|
||||
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
|
||||
t.Fatalf("materialized metadata = %#v", chunk.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type executionDebugClient struct {
|
||||
response contracts.StructuredCompletionResponse
|
||||
err error
|
||||
}
|
||||
|
||||
func (c executionDebugClient) CompleteStructured(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) {
|
||||
return c.response, c.err
|
||||
}
|
||||
|
||||
func TestDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
|
||||
providerErr := contracts.NewLLMGenerationError(503, "provider unavailable")
|
||||
const content = `{"ok":true}`
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
content string
|
||||
debugResponse bool
|
||||
}{
|
||||
{name: "provider failure without body", err: providerErr},
|
||||
{name: "canceled without body", err: context.Canceled},
|
||||
{name: "split debug response", content: content, debugResponse: true},
|
||||
{name: "split fallback response", content: content},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
want := contracts.LLMExecutionSummary{
|
||||
Attempts: 2, TransientRetries: 1, Outcome: "success",
|
||||
CapacityWaitMS: 1, GenerationMS: 3, RetryDelayMS: 5, DurationMS: 9,
|
||||
Usage: contracts.LLMObservedTokenUsage{
|
||||
PromptTokens: contracts.LLMObservedTokenCount{Value: 7, Known: true},
|
||||
CompletionTokens: contracts.LLMObservedTokenCount{Known: true},
|
||||
},
|
||||
}
|
||||
if test.err != nil {
|
||||
want.Outcome = "error"
|
||||
want.FailureKind = "provider_status"
|
||||
want.StatusCode = 503
|
||||
want.RetriesExhausted = true
|
||||
if errors.Is(test.err, context.Canceled) {
|
||||
want.FailureKind = "canceled"
|
||||
want.StatusCode = 0
|
||||
want.RetriesExhausted = false
|
||||
}
|
||||
}
|
||||
execution := want
|
||||
response := contracts.StructuredCompletionResponse{Execution: &execution}
|
||||
if test.debugResponse {
|
||||
response.Debug = &contracts.LLMDebugMaterial{Response: &contracts.LLMDebugResponse{Content: test.content}}
|
||||
} else {
|
||||
response.Content = json.RawMessage(test.content)
|
||||
}
|
||||
recorder := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(executionDebugClient{response, test.err}, recorder)
|
||||
ctx, scope := withDebugLLMScope(context.Background(), "attempt")
|
||||
got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{})
|
||||
if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want {
|
||||
t.Fatalf("debug completion = (%+v, %v), want summary and original error", got, err)
|
||||
}
|
||||
terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"})
|
||||
if err := terminal.record(nil, test.err); !errors.Is(err, test.err) {
|
||||
t.Fatalf("terminal recording error = %v", err)
|
||||
}
|
||||
envelope := recorder.envelope(t, "attempt/response-0001.json")
|
||||
encoded, err := json.Marshal(envelope.Payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var artifact debugLLMResponseArtifact
|
||||
if err := json.Unmarshal(encoded, &artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if artifact.Execution == nil || *artifact.Execution != want {
|
||||
t.Fatalf("persisted response lost execution summary: %+v", artifact)
|
||||
}
|
||||
if !test.debugResponse && (artifact.Fallback == nil || artifact.Fallback.Execution == nil || *artifact.Fallback.Execution != want) {
|
||||
t.Fatalf("persisted fallback lost execution summary: %+v", artifact.Fallback)
|
||||
}
|
||||
if test.content == "" {
|
||||
if artifact.ContentPath != "" {
|
||||
t.Fatalf("bodyless failure has content path %q", artifact.ContentPath)
|
||||
}
|
||||
} else {
|
||||
var compact bytes.Buffer
|
||||
if artifact.ContentPath == "" || json.Compact(&compact, recorder.bytes[artifact.ContentPath]) != nil || compact.String() != test.content {
|
||||
t.Fatalf("split content not preserved at %q", artifact.ContentPath)
|
||||
}
|
||||
if (artifact.Fallback != nil && artifact.Fallback.Content != "") || (artifact.Response != nil && artifact.Response.Content != "") {
|
||||
t.Fatal("split content remains in response metadata")
|
||||
}
|
||||
}
|
||||
refs := recorder.envelope(t, "attempt/terminal.json").LLMCalls
|
||||
if len(refs) != 1 || refs[0].Execution == nil || *refs[0].Execution != want || refs[0].Error != (test.err != nil) {
|
||||
t.Fatalf("terminal references = %+v", refs)
|
||||
}
|
||||
// Decode the persisted JSON, not just Go defaults: omitted false and
|
||||
// zero fields would erase the presence-aware usage contract.
|
||||
usage := envelope.Payload.(map[string]any)["execution"].(map[string]any)["usage"].(map[string]any)
|
||||
unknown := usage["cached_tokens"].(map[string]any)
|
||||
for _, key := range []string{"value", "known", "complete"} {
|
||||
if _, ok := unknown[key]; !ok {
|
||||
t.Fatalf("unknown token usage omits %q: %+v", key, unknown)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,78 +1,12 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestCleanDebugPathPreservesRawComponents(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
value string
|
||||
want string
|
||||
}{
|
||||
{value: "", want: "%"},
|
||||
{value: ".", want: "%2E"},
|
||||
{value: "..", want: "%2E%2E"},
|
||||
{value: "a//b", want: "a/%/b"},
|
||||
{value: "/a/", want: "%/a/%"},
|
||||
{value: " a ", want: "%20a%20"},
|
||||
} {
|
||||
if got := cleanDebugPath(test.value); got != test.want {
|
||||
t.Errorf("cleanDebugPath(%q) = %q, want %q", test.value, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) {
|
||||
recorder := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, recorder)
|
||||
for _, test := range []struct {
|
||||
stageName string
|
||||
path string
|
||||
}{
|
||||
{stageName: ".", path: "%2E/response-0001.json"},
|
||||
{stageName: "..", path: "%2E%2E/response-0002.json"},
|
||||
} {
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{StageName: test.stageName}, nil); err != nil {
|
||||
t.Fatalf("CompleteStructured(%q): %v", test.stageName, err)
|
||||
}
|
||||
if !recorder.has(test.path) {
|
||||
t.Errorf("debug artifact %q was not written; names = %#v", test.path, recorder.names())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) {
|
||||
const assistantResponse = `{"secret":"assistant response"}`
|
||||
const userGuidance = "secret user guidance"
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
summary := debugCompletionRequest(contracts.StructuredCompletionRequest{
|
||||
Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
Correction: correction,
|
||||
})
|
||||
encoded, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal completion summary: %v", err)
|
||||
}
|
||||
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||
if strings.Contains(string(encoded), secret) {
|
||||
t.Fatalf("debug completion summary leaked %q: %s", secret, encoded)
|
||||
}
|
||||
}
|
||||
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||
t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
|
||||
doc := validSourceDocument()
|
||||
envelope := debugSourceDocumentEnvelope(doc)
|
||||
@@ -95,35 +29,6 @@ func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugSourceUnitsRedactSecrets(t *testing.T) {
|
||||
units := []source.SourceUnit{{
|
||||
ID: 1,
|
||||
Kind: "paragraph",
|
||||
Text: "application text Bearer secretvalue sk-secretvalue",
|
||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
||||
Metadata: map[string]any{"api_key": "sk-secretvalue"},
|
||||
}}
|
||||
|
||||
got := cloneSourceUnitsForDebug(units)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("debug unit count = %d, want 1", len(got))
|
||||
}
|
||||
if !strings.Contains(got[0].Text, "application text") {
|
||||
t.Fatalf("debug unit text = %q, want application content retained", got[0].Text)
|
||||
}
|
||||
for _, forbidden := range []string{"secretvalue", "sk-secretvalue"} {
|
||||
if strings.Contains(got[0].Text, forbidden) {
|
||||
t.Fatalf("debug unit text contains %q: %q", forbidden, got[0].Text)
|
||||
}
|
||||
}
|
||||
if got, want := got[0].Metadata["api_key"], "[REDACTED]"; got != want {
|
||||
t.Fatalf("debug unit metadata api_key = %#v, want %q", got, want)
|
||||
}
|
||||
if units[0].Text != "application text Bearer secretvalue sk-secretvalue" {
|
||||
t.Fatalf("source unit text was mutated: %q", units[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugSourceChunkPreservesReference(t *testing.T) {
|
||||
doc := validSourceDocument()
|
||||
chunk := source.Chunk{
|
||||
|
||||
@@ -1,48 +1,19 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestTerminalDiagnosticGroupsDiscardSupersededAttemptDiagnostics(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "first", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded")}, Candidate: attemptCandidate(t, "first")}, nil
|
||||
}
|
||||
return producerAttemptOutput{Value: "second", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("terminal", "terminal")}}, nil
|
||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||
if output.Value == "first" {
|
||||
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: "invalid", correctionGuidance: "Correct it."}}}, nil
|
||||
}
|
||||
return validationReport{}, nil
|
||||
})
|
||||
func TestPromoteProducerDiagnosticsPreservesValidatorOrigin(t *testing.T) {
|
||||
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells", ValidatorKey: "dnd/spells/source-relatedness"}
|
||||
groups, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")}, origin, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("terminalDiagnosticGroups() error = %v", err)
|
||||
}
|
||||
if len(groups) != 1 || groups[0].Samples[0].Scope != "terminal" {
|
||||
t.Fatalf("groups = %#v, want only terminal attempt diagnostics", groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminalDiagnosticGroupsPreserveValidatorOrigin(t *testing.T) {
|
||||
terminal := producerAttemptTerminal{Validation: validationReport{records: []validationRecord{{
|
||||
validatorName: "dnd/spells/source-relatedness",
|
||||
outcome: validationApproved,
|
||||
diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")},
|
||||
}}}}
|
||||
groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("terminalDiagnosticGroups() error = %v", err)
|
||||
}
|
||||
if len(groups) != 1 || groups[0].Origin.ValidatorKey != "dnd/spells/source-relatedness" {
|
||||
t.Fatalf("groups = %#v, want validator origin", groups)
|
||||
if len(groups) != 1 || groups[0].Origin != origin {
|
||||
t.Fatalf("groups = %#v, want validator origin %#v", groups, origin)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -43,9 +43,9 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS
|
||||
installGeneratedTarget(&consumer.resolved.MergeReferences, StageMerge, consumer.resolved.Merge.Module)
|
||||
installGeneratedTarget(&consumer.resolved.NormalizeReferences, StageNormalize, consumer.resolved.Normalize.Module)
|
||||
|
||||
artifact, err := checkpointArtifact(prepared.Steps[0].lanes[0].typed.codec, "notes", "typed/normalize", "source", value)
|
||||
artifact, err := serializeArtifact(prepared.Steps[0].lanes[0].typed.codec, value, false)
|
||||
if err != nil {
|
||||
t.Fatalf("checkpointArtifact() error = %v", err)
|
||||
t.Fatalf("serializeArtifact() error = %v", err)
|
||||
}
|
||||
input := RunInput{Prepared: prepared}
|
||||
input.pipeline = prepared.resolved
|
||||
@@ -54,23 +54,19 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS
|
||||
LaneID: "notes",
|
||||
NormalizerKey: "typed/normalize",
|
||||
SourceID: "source",
|
||||
Artifact: contracts.CloneSerializedArtifact(artifact.Artifact),
|
||||
Artifact: contracts.CloneSerializedArtifact(artifact),
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) {
|
||||
input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}})
|
||||
sets, provenance, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
|
||||
sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
|
||||
if err != nil {
|
||||
t.Fatalf("buildStepReferenceSets() error = %v", err)
|
||||
}
|
||||
if got, want := len(sets), 3; got != want {
|
||||
t.Fatalf("reference target set count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(provenance), 3; got != want {
|
||||
t.Fatalf("generated provenance count = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
var first []byte
|
||||
for key, set := range sets {
|
||||
item := set.Slots["producer-output"].Items[0]
|
||||
@@ -91,20 +87,6 @@ func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) {
|
||||
if got := len(step.lanes[0].resolved.ExtractReferences.ReferenceSet.Slots["producer-output"].Items); got != 0 {
|
||||
t.Fatalf("prepared extract reference items = %d, want zero", got)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(provenance)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal generated provenance: %v", err)
|
||||
}
|
||||
text := string(encoded)
|
||||
for _, forbidden := range []string{"first", "file://", "content_base64"} {
|
||||
if strings.Contains(text, forbidden) {
|
||||
t.Fatalf("generated provenance contains forbidden %q: %s", forbidden, text)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(text, "producer_pipeline_id") || !strings.Contains(text, "schema_digest") {
|
||||
t.Fatalf("generated provenance = %s, want bounded producer and schema identity", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
|
||||
@@ -115,12 +97,12 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
|
||||
}
|
||||
outputs = append(outputs, producerOutput)
|
||||
|
||||
sets, provenance, err := buildStepReferenceSets(input, step, outputs)
|
||||
sets, err := buildStepReferenceSets(input, step, outputs)
|
||||
if err != nil {
|
||||
t.Fatalf("buildStepReferenceSets() error = %v", err)
|
||||
}
|
||||
if len(sets) != 3 || len(provenance) != 3 {
|
||||
t.Fatalf("reference sets/provenance = %d/%d, want 3/3", len(sets), len(provenance))
|
||||
if len(sets) != 3 {
|
||||
t.Fatalf("reference set count = %d, want 3", len(sets))
|
||||
}
|
||||
for target, set := range sets {
|
||||
item := set.Slots["producer-output"].Items[0]
|
||||
@@ -132,7 +114,7 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) {
|
||||
|
||||
func TestGeneratedReferenceAcceptsTypedEmptyCollection(t *testing.T) {
|
||||
input, step, producerOutput := handoffFixture(t, codecNotes{})
|
||||
sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
|
||||
sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
|
||||
if err != nil {
|
||||
t.Fatalf("buildStepReferenceSets() error = %v, want nil for accepted empty collection", err)
|
||||
}
|
||||
@@ -189,7 +171,7 @@ func TestGeneratedReferenceRejectsInvalidProducerOutputsDeterministically(t *tes
|
||||
target := &step.lanes[0].resolved.ExtractReferences
|
||||
test.mutate(target, &output)
|
||||
}
|
||||
_, _, err := buildStepReferenceSets(input, step, test.outputs(output))
|
||||
_, err := buildStepReferenceSets(input, step, test.outputs(output))
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("buildStepReferenceSets() error = %v, want %q", err, test.want)
|
||||
}
|
||||
@@ -208,142 +190,45 @@ func BenchmarkIndexGeneratedOutputs(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedReferenceFingerprintChangesWithCanonicalContent(t *testing.T) {
|
||||
input, step, first := handoffFixture(t, codecNotes{Items: []string{"first"}})
|
||||
firstSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{first})
|
||||
if err != nil {
|
||||
t.Fatalf("build first reference set: %v", err)
|
||||
}
|
||||
_, _, second := handoffFixture(t, codecNotes{Items: []string{"second"}})
|
||||
secondSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{second})
|
||||
if err != nil {
|
||||
t.Fatalf("build second reference set: %v", err)
|
||||
}
|
||||
firstDeps := generatedReferenceDependencies(firstSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)])
|
||||
secondDeps := generatedReferenceDependencies(secondSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)])
|
||||
if reflect.DeepEqual(firstDeps, secondDeps) {
|
||||
t.Fatalf("generated dependencies = %#v, want content-sensitive fingerprint", firstDeps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedReferenceFingerprintChangesWithCanonicalSize(t *testing.T) {
|
||||
input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}})
|
||||
sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput})
|
||||
if err != nil {
|
||||
t.Fatalf("build reference set: %v", err)
|
||||
}
|
||||
key := keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)
|
||||
firstDeps := generatedReferenceDependencies(sets[key])
|
||||
changed := CloneReferenceSet(sets[key])
|
||||
item := changed.Slots["producer-output"].Items[0]
|
||||
item.SizeBytes++
|
||||
changed.Slots["producer-output"] = contracts.ResolvedReferenceSlot{
|
||||
Slot: changed.Slots["producer-output"].Slot,
|
||||
Items: []contracts.ReferenceItem{item},
|
||||
}
|
||||
secondDeps := generatedReferenceDependencies(changed)
|
||||
if reflect.DeepEqual(firstDeps, secondDeps) {
|
||||
t.Fatalf("generated dependencies = %#v, want size-sensitive fingerprint", firstDeps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerHandsOffAcceptedNormalizedOutputBeforeConsumerLanes(t *testing.T) {
|
||||
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"first"}})
|
||||
prepared := input.Prepared
|
||||
var received contracts.ReferenceSet
|
||||
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
received := make(map[ModuleStage]contracts.ReferenceSet)
|
||||
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
|
||||
}
|
||||
prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
received = CloneReferenceSet(request.References)
|
||||
prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
received[StageExtract] = CloneReferenceSet(request.References)
|
||||
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), input)
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
|
||||
}
|
||||
prepared.Steps[1].lanes[0].typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
received[StageMerge] = CloneReferenceSet(request.References)
|
||||
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
|
||||
}
|
||||
prepared.Steps[1].lanes[0].typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
received[StageNormalize] = CloneReferenceSet(request.References)
|
||||
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
|
||||
}
|
||||
_, err := New().Run(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if got, want := len(output.NormalizeOutputs), 2; got != want {
|
||||
t.Fatalf("normalized output count = %d, want %d", got, want)
|
||||
}
|
||||
if output.NormalizeOutputs[0].StepID != "step-1" || output.NormalizeOutputs[1].StepID != "step-2" {
|
||||
t.Fatalf("normalized output step IDs = %#v, want step-1 and step-2", output.NormalizeOutputs)
|
||||
}
|
||||
item := received.Slots["producer-output"].Items[0]
|
||||
if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" {
|
||||
t.Fatalf("consumer reference = %#v, want generated producer reference", item)
|
||||
}
|
||||
if len(output.Manifest.References) == 0 || output.Manifest.References[len(output.Manifest.References)-1].OriginType != "generated" {
|
||||
t.Fatalf("manifest references = %#v, want generated provenance", output.Manifest.References)
|
||||
}
|
||||
}
|
||||
|
||||
type handoffDependencyLoader struct {
|
||||
CheckpointLoader
|
||||
extract [][]CheckpointFingerprint
|
||||
merge [][]CheckpointFingerprint
|
||||
normalize [][]CheckpointFingerprint
|
||||
}
|
||||
|
||||
func (l *handoffDependencyLoader) Enabled() bool { return true }
|
||||
|
||||
func (l *handoffDependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
l.extract = append(l.extract, append([]CheckpointFingerprint(nil), dependencies...))
|
||||
return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||
}
|
||||
|
||||
func (l *handoffDependencyLoader) Merge(_ string, _ string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
l.merge = append(l.merge, append([]CheckpointFingerprint(nil), dependencies...))
|
||||
return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||
}
|
||||
|
||||
func (l *handoffDependencyLoader) Normalize(_ string, _ string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
l.normalize = append(l.normalize, append([]CheckpointFingerprint(nil), dependencies...))
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||
}
|
||||
|
||||
func runHandoffWithProducerValue(t *testing.T, value codecNotes) *handoffDependencyLoader {
|
||||
t.Helper()
|
||||
input, _, _ := handoffFixture(t, value)
|
||||
prepared := input.Prepared
|
||||
prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: value}, nil
|
||||
}
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: value}, nil
|
||||
}
|
||||
prepared.Steps[1].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
|
||||
}
|
||||
loader := &handoffDependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader}); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
return loader
|
||||
}
|
||||
|
||||
func TestRunnerAddsGeneratedDependencyToEveryReceivingCheckpoint(t *testing.T) {
|
||||
first := runHandoffWithProducerValue(t, codecNotes{Items: []string{"first"}})
|
||||
second := runHandoffWithProducerValue(t, codecNotes{Items: []string{"second"}})
|
||||
if len(first.extract) != 2 || len(first.merge) != 2 || len(first.normalize) != 2 {
|
||||
t.Fatalf("checkpoint load calls = extract %d merge %d normalize %d, want two each", len(first.extract), len(first.merge), len(first.normalize))
|
||||
}
|
||||
for name, calls := range map[string][][]CheckpointFingerprint{"extract": first.extract, "merge": first.merge, "normalize": first.normalize} {
|
||||
if got := generatedFingerprintCount(calls[1]); got != 1 {
|
||||
t.Fatalf("%s consumer dependency count = %d, want one", name, got)
|
||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
set, found := received[target]
|
||||
if !found || len(set.Slots["producer-output"].Items) != 1 {
|
||||
t.Fatalf("%s reference set = %#v, want producer artifact", target, set)
|
||||
}
|
||||
item := set.Slots["producer-output"].Items[0]
|
||||
if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" {
|
||||
t.Fatalf("%s reference = %#v, want generated producer reference", target, item)
|
||||
}
|
||||
var value codecNotes
|
||||
if err := json.Unmarshal(item.Content, &value); err != nil || !reflect.DeepEqual(value, codecNotes{Items: []string{"first"}}) {
|
||||
t.Fatalf("%s reference value = %#v, %v", target, value, err)
|
||||
}
|
||||
}
|
||||
if reflect.DeepEqual(first.extract[1], second.extract[1]) || reflect.DeepEqual(first.merge[1], second.merge[1]) || reflect.DeepEqual(first.normalize[1], second.normalize[1]) {
|
||||
t.Fatalf("consumer checkpoint dependencies did not change with producer content")
|
||||
}
|
||||
}
|
||||
|
||||
func generatedFingerprintCount(dependencies []CheckpointFingerprint) int {
|
||||
count := 0
|
||||
for _, dependency := range dependencies {
|
||||
if strings.HasPrefix(dependency.Name, "generated-reference:") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type retryingNotesNormalizer struct {
|
||||
retry *contracts.NormalizeRetry
|
||||
}
|
||||
|
||||
func (retryingNotesNormalizer) Key() string { return "test/retry-normalize" }
|
||||
func (retryingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (n retryingNotesNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
||||
return contracts.TypedNormalizeResult[codecNotes]{Value: req.MergeOutput.Value, Retry: n.retry}, nil
|
||||
}
|
||||
|
||||
func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) {
|
||||
retry := &contracts.NormalizeRetry{
|
||||
ReasonCode: "retryable",
|
||||
Message: "safe fallback available",
|
||||
CorrectionGuidance: "return a complete corrected proposal",
|
||||
}
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: "test/retry-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return retryingNotesNormalizer{retry: retry}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
||||
}
|
||||
entry, ok := registry.typedEntry("test/retry-normalize", "test/notes")
|
||||
if !ok {
|
||||
t.Fatal("typed normalizer entry missing")
|
||||
}
|
||||
implementation, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
result, err := entry.normalize(context.Background(), implementation, contracts.TypedNormalizeRequest[any]{MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"safe"}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
retry.Message = "mutated"
|
||||
retry.CorrectionGuidance = "mutated guidance"
|
||||
if result.Retry == nil || result.Retry.Message != "safe fallback available" || result.Retry.CorrectionGuidance != "return a complete corrected proposal" {
|
||||
t.Fatalf("erased retry result = %#v, want independent retry data", result)
|
||||
}
|
||||
}
|
||||
@@ -132,168 +132,6 @@ func TestPrepareRetainsGeneratedSelectorsWithoutReferenceBytes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCheckpointFingerprintsCollectEveryComponentInStableScopeOrder(t *testing.T) {
|
||||
provider := func(value string) checkpointFingerprintTestProvider {
|
||||
return checkpointFingerprintTestProvider{{Name: "identity", Value: value}}
|
||||
}
|
||||
prepared := &PreparedPipeline{
|
||||
resolved: ResolvedPipeline{
|
||||
ID: "fingerprints",
|
||||
Input: Binding("input"),
|
||||
Chunk: Binding("chunk"),
|
||||
Output: Binding("output"),
|
||||
},
|
||||
input: provider("input"),
|
||||
chunker: provider("chunk"),
|
||||
chunkValidators: preparedValidatorChain{validators: []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("chunk-validator"), Target: ValidatorTargetChunk},
|
||||
chunk: provider("chunk-validator"),
|
||||
}}},
|
||||
output: provider("output"),
|
||||
Steps: []PreparedPipelineStep{{ID: "default"}},
|
||||
}
|
||||
lane := preparedLaneExecutor{
|
||||
resolved: ResolvedArtifactLane{
|
||||
ID: "spells",
|
||||
Extract: Binding("extract"),
|
||||
Merge: Binding("merge"),
|
||||
Normalize: Binding("normalize"),
|
||||
},
|
||||
typed: &preparedTypedLane{
|
||||
extractor: provider("extract"),
|
||||
merger: provider("merge"),
|
||||
normalizer: provider("normalize"),
|
||||
},
|
||||
extractValidators: fingerprintTestValidatorChain("extract-validator", provider("extract-validator")),
|
||||
mergeValidators: fingerprintTestValidatorChain("merge-validator", provider("merge-validator")),
|
||||
normalizeValidators: fingerprintTestValidatorChain("normalize-validator", provider("normalize-validator")),
|
||||
}
|
||||
prepared.Steps[0].lanes = []preparedLaneExecutor{lane}
|
||||
|
||||
first, err := collectPreparedCheckpointFingerprints(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("collectPreparedCheckpointFingerprints() error = %v, want nil", err)
|
||||
}
|
||||
second, err := collectPreparedCheckpointFingerprints(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("second collection error = %v, want nil", err)
|
||||
}
|
||||
if !reflect.DeepEqual(first, second) {
|
||||
t.Fatalf("fingerprints are not deterministic: %#v / %#v", first, second)
|
||||
}
|
||||
if len(first) != 10 {
|
||||
t.Fatalf("fingerprints = %#v, want all 10 prepared components", first)
|
||||
}
|
||||
for index := 1; index < len(first); index++ {
|
||||
if first[index-1].Name >= first[index].Name {
|
||||
t.Fatalf("fingerprints are not sorted: %#v", first)
|
||||
}
|
||||
}
|
||||
prepared.checkpointFingerprints = first
|
||||
copy := prepared.CheckpointFingerprints()
|
||||
copy[0].Value = "mutated"
|
||||
if reflect.DeepEqual(copy, prepared.CheckpointFingerprints()) {
|
||||
t.Fatal("CheckpointFingerprints() exposed mutable backing storage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCheckpointFingerprintsValidateProviderValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fingerprints []CheckpointFingerprint
|
||||
want string
|
||||
}{
|
||||
{name: "empty name", fingerprints: []CheckpointFingerprint{{Value: "value"}}, want: "non-empty name and value"},
|
||||
{name: "empty value", fingerprints: []CheckpointFingerprint{{Name: "name"}}, want: "non-empty name and value"},
|
||||
{name: "duplicate after trimming", fingerprints: []CheckpointFingerprint{{Name: "same", Value: "one"}, {Name: " same ", Value: "two"}}, want: "duplicated"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
prepared := &PreparedPipeline{
|
||||
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
|
||||
input: checkpointFingerprintTestProvider(test.fingerprints),
|
||||
}
|
||||
_, err := collectPreparedCheckpointFingerprints(prepared)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("error = %v, want substring %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCheckpointFingerprintsRemainEmptyWithoutProviders(t *testing.T) {
|
||||
registries, _ := constructionRegistries(t, nil, nil)
|
||||
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := Prepare(resolved, registries, ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := prepared.CheckpointFingerprints(); len(got) != 0 {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCheckpointFingerprintsScopeRepeatedValidatorsByPosition(t *testing.T) {
|
||||
provider := checkpointFingerprintTestProvider{{Name: "identity", Value: "same"}}
|
||||
validator := func() preparedValidator {
|
||||
return preparedValidator{resolved: ResolvedValidator{Binding: Binding("configured"), Target: ValidatorTargetChunk}, chunk: provider}
|
||||
}
|
||||
prepared := &PreparedPipeline{
|
||||
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
|
||||
chunkValidators: preparedValidatorChain{validators: []preparedValidator{validator(), validator()}},
|
||||
}
|
||||
got, err := collectPreparedCheckpointFingerprints(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("collect repeated validators: %v", err)
|
||||
}
|
||||
if len(got) != 2 || got[0].Name == got[1].Name {
|
||||
t.Fatalf("fingerprints = %#v, want independently scoped repeated validators", got)
|
||||
}
|
||||
}
|
||||
|
||||
type checkpointFingerprintTestProvider []CheckpointFingerprint
|
||||
|
||||
func (p checkpointFingerprintTestProvider) CheckpointFingerprints() []CheckpointFingerprint {
|
||||
return append([]CheckpointFingerprint(nil), p...)
|
||||
}
|
||||
func (checkpointFingerprintTestProvider) Key() string { return "fingerprint-test" }
|
||||
func (checkpointFingerprintTestProvider) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (checkpointFingerprintTestProvider) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
return typedTestDocument(), nil
|
||||
}
|
||||
func (checkpointFingerprintTestProvider) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
return contracts.ChunkPlanResult{}, nil
|
||||
}
|
||||
func (checkpointFingerprintTestProvider) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
func (checkpointFingerprintTestProvider) Name() string { return "fingerprint-test" }
|
||||
func (checkpointFingerprintTestProvider) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (checkpointFingerprintTestProvider) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
type checkpointFingerprintOutput struct {
|
||||
contracts.OutputEncoder
|
||||
fingerprints []CheckpointFingerprint
|
||||
}
|
||||
|
||||
func (o checkpointFingerprintOutput) CheckpointFingerprints() []CheckpointFingerprint {
|
||||
return append([]CheckpointFingerprint(nil), o.fingerprints...)
|
||||
}
|
||||
|
||||
func fingerprintTestValidatorChain(key string, provider checkpointFingerprintTestProvider) preparedValidatorChain {
|
||||
return preparedValidatorChain{validators: []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding(key), Target: ValidatorTargetTyped},
|
||||
typed: provider,
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) {
|
||||
var built []string
|
||||
var observations []constructionBuildObservation
|
||||
@@ -351,7 +189,7 @@ func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) {
|
||||
t.Fatalf("prepared extract options = %#v, want original nested value", got)
|
||||
}
|
||||
|
||||
_, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest{
|
||||
_, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest[any]{
|
||||
References: CloneReferenceSet(resolved.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet),
|
||||
})
|
||||
if err != nil {
|
||||
@@ -531,9 +369,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) {
|
||||
wantBuilt []string
|
||||
}{
|
||||
{
|
||||
name: "missing required llm dependency",
|
||||
name: "missing required engine dependency",
|
||||
configure: func(failure *constructionFailure) {
|
||||
failure.requireExtractorLLM = true
|
||||
failure.requireExtractorEngine = true
|
||||
},
|
||||
want: `lane "artifact" extract module "extract"`,
|
||||
wantBuilt: []string{"input", "chunk", "validator", "extract"},
|
||||
@@ -573,26 +411,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareRejectsInvalidComponentCheckpointFingerprint(t *testing.T) {
|
||||
failure := &constructionFailure{outputFingerprints: []CheckpointFingerprint{{Name: "identity"}}}
|
||||
registries, input := constructionRegistries(t, nil, failure)
|
||||
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = Prepare(resolved, registries, ModuleDependencies{})
|
||||
if err == nil || !strings.Contains(err.Error(), "checkpoint fingerprint") || !strings.Contains(err.Error(), "output:output") {
|
||||
t.Fatalf("Prepare() error = %v, want scoped checkpoint fingerprint error", err)
|
||||
}
|
||||
if len(input.requests) != 0 {
|
||||
t.Fatalf("input Parse calls = %d, want zero", len(input.requests))
|
||||
}
|
||||
}
|
||||
|
||||
type constructionFailure struct {
|
||||
requireExtractorLLM bool
|
||||
output error
|
||||
outputFingerprints []CheckpointFingerprint
|
||||
requireExtractorEngine bool
|
||||
output error
|
||||
}
|
||||
|
||||
func constructionProfile() PipelineProfile {
|
||||
@@ -664,8 +485,8 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
|
||||
extractSpec.ArtifactKind = "test/notes"
|
||||
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
|
||||
record("extract", &request)
|
||||
if failure.requireExtractorLLM && request.Dependencies.LLM == nil {
|
||||
return nil, errors.New("structured LLM client is required")
|
||||
if failure.requireExtractorEngine && request.Dependencies.Engine == nil {
|
||||
return nil, errors.New("native engine is required")
|
||||
}
|
||||
return &constructionExtractor{key: "extract", requests: &input.extractRequests}, nil
|
||||
}); err != nil {
|
||||
@@ -705,11 +526,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
|
||||
if failure.output != nil {
|
||||
return nil, failure.output
|
||||
}
|
||||
output := contracts.OutputEncoder(&typedTestOutput{key: "output"})
|
||||
if failure.outputFingerprints != nil {
|
||||
output = checkpointFingerprintOutput{OutputEncoder: output, fingerprints: failure.outputFingerprints}
|
||||
}
|
||||
return output, nil
|
||||
return &typedTestOutput{key: "output"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -719,17 +536,17 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
|
||||
type constructionInput struct {
|
||||
key string
|
||||
requests []contracts.ParseRequest
|
||||
extractRequests []contracts.TypedExtractionRequest
|
||||
extractRequests []contracts.TypedExtractionRequest[codecNotes]
|
||||
}
|
||||
|
||||
type constructionExtractor struct {
|
||||
key string
|
||||
requests *[]contracts.TypedExtractionRequest
|
||||
requests *[]contracts.TypedExtractionRequest[codecNotes]
|
||||
}
|
||||
|
||||
func (extractor *constructionExtractor) Key() string { return extractor.key }
|
||||
func (*constructionExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[codecNotes], error) {
|
||||
func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) {
|
||||
if extractor.requests != nil {
|
||||
*extractor.requests = append(*extractor.requests, request)
|
||||
}
|
||||
|
||||
@@ -126,148 +126,6 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) {
|
||||
correction := contracts.CorrectionProtocolSingleResponseV1
|
||||
catalog := newProfileCatalogWithOverrides(t,
|
||||
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
|
||||
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}},
|
||||
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||
)
|
||||
resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.ChunkCorrectionProtocol != correction {
|
||||
t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction)
|
||||
}
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction {
|
||||
t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction)
|
||||
}
|
||||
withoutCapability := cloneResolvedPipeline(resolved)
|
||||
withoutCapability.ChunkCorrectionProtocol = ""
|
||||
withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = ""
|
||||
withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = ""
|
||||
withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = ""
|
||||
withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability)
|
||||
if err != nil {
|
||||
t.Fatalf("resolvedPipelineDigest() error = %v", err)
|
||||
}
|
||||
if withoutCapabilityDigest == resolved.Digest {
|
||||
t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest)
|
||||
}
|
||||
|
||||
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.ChunkCorrectionProtocol != correction {
|
||||
t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction)
|
||||
}
|
||||
preparedLane := prepared.Steps[0].ArtifactLanes[0]
|
||||
if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction {
|
||||
t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareRequiresCorrectionCapabilityForValidatorRetries(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
protocol contracts.CorrectionProtocol
|
||||
retries int
|
||||
validators bool
|
||||
want string
|
||||
}{
|
||||
{name: "supported", protocol: contracts.CorrectionProtocolSingleResponseV1, retries: 1, validators: true},
|
||||
{name: "unsupported", retries: 1, validators: true, want: "does not declare correction protocol"},
|
||||
{name: "no validators", retries: 1},
|
||||
{name: "no retries", validators: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
catalog := newProfileCatalogWithOverrides(t,
|
||||
ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}},
|
||||
ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: test.protocol, Requires: []string{"source"}, Provides: []string{"chunk"}},
|
||||
ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||
ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||
ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||
)
|
||||
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) {
|
||||
return llmProfileTestChunkValidator{key: "chunk-validator"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := llmProfilePipeline()
|
||||
profile.Chunk.Retries = test.retries
|
||||
if test.validators {
|
||||
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "chunk-validator"}}}
|
||||
}
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
_, err = Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||
if test.want != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type llmProfileTestChunkValidator struct{ key string }
|
||||
|
||||
func (validator llmProfileTestChunkValidator) Name() string { return validator.key }
|
||||
|
||||
func (llmProfileTestChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (llmProfileTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
class contracts.ExecutionClass
|
||||
want string
|
||||
}{
|
||||
{name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"},
|
||||
{name: "LLM validator", class: contracts.ExecutionClassLLMBacked},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class})
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if test.want != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 {
|
||||
t.Fatalf("validator retries = %d, want 1", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
||||
resolved, err := ResolvePipeline(PipelineProfile{
|
||||
ID: "defaulted",
|
||||
@@ -552,148 +410,6 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) {
|
||||
pipelineAttempts := 2
|
||||
chunkAttempts := 1
|
||||
extractAttempts := 0
|
||||
validatorAttempts := 3
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = &pipelineAttempts
|
||||
profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.StructuredOutputRepairAttempts = &extractAttempts
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
encoded, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(resolved) error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) {
|
||||
t.Fatalf("resolved JSON = %s, want effective repair policy", encoded)
|
||||
}
|
||||
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
got *int
|
||||
want int
|
||||
}{
|
||||
{name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts},
|
||||
{name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts},
|
||||
{name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
{name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.got == nil || *test.got != test.want {
|
||||
t.Fatalf("repair attempts = %v, want %d", test.got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor")
|
||||
if extractChain == nil || len(extractChain.Validators) != 1 {
|
||||
t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain)
|
||||
}
|
||||
if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts {
|
||||
t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts)
|
||||
}
|
||||
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk")
|
||||
if chunkChain == nil || len(chunkChain.Validators) != 1 {
|
||||
t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain)
|
||||
}
|
||||
if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts {
|
||||
t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts)
|
||||
}
|
||||
|
||||
pipelineAttempts = 1
|
||||
chunkAttempts = 2
|
||||
extractAttempts = 3
|
||||
validatorAttempts = 0
|
||||
if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 {
|
||||
t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got)
|
||||
}
|
||||
if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 {
|
||||
t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got)
|
||||
}
|
||||
if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 {
|
||||
t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got)
|
||||
}
|
||||
if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 {
|
||||
t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*PipelineProfile)
|
||||
}{
|
||||
{name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }},
|
||||
{name: "validator", mutate: func(profile *PipelineProfile) {
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}}
|
||||
profile.Artifacts["events"] = lane
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
test.mutate(&profile)
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.StructuredOutputRepairAttempts = repairAttempts(2)
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved)
|
||||
}
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil {
|
||||
t.Fatalf("deterministic lane inherited repair attempts: %#v", lane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) {
|
||||
digests := make(map[string]string)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
attempts *int
|
||||
}{
|
||||
{name: "prompt owned", attempts: nil},
|
||||
{name: "disabled", attempts: repairAttempts(0)},
|
||||
{name: "configured", attempts: repairAttempts(1)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profile := llmProfilePipeline()
|
||||
profile.StructuredOutputRepairAttempts = test.attempts
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
digests[test.name] = resolved.Digest
|
||||
})
|
||||
}
|
||||
if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] {
|
||||
t.Fatalf("digests = %#v, want distinct effective repair policies", digests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
@@ -1811,7 +1527,6 @@ func TestResolvedPipelineDigestIncludesCompleteValidatorPolicy(t *testing.T) {
|
||||
value.ValidatorChains[0].Validators[0].Binding.Module = "other-validator"
|
||||
}},
|
||||
{name: "llm profile", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.LLMProfile = "fast" }},
|
||||
{name: "retries", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.Retries++ }},
|
||||
{name: "options", mutate: func(value *ResolvedPipeline) {
|
||||
value.ValidatorChains[0].Validators[0].Binding.Options["threshold"] = 0.75
|
||||
}},
|
||||
@@ -1877,7 +1592,6 @@ func validatorDigestFixture() ResolvedPipeline {
|
||||
Binding: ModuleBinding{
|
||||
Module: "semantic-validator",
|
||||
LLMProfile: "careful",
|
||||
Retries: 2,
|
||||
Options: map[string]any{"mode": "strict", "threshold": 0.5},
|
||||
References: ExternalReferenceMap(map[string]string{"rules": "rules.md", "examples": "examples.md"}),
|
||||
},
|
||||
@@ -2129,10 +1843,6 @@ func llmProfilePipeline() PipelineProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func repairAttempts(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
func llmProfileValues(profile string) map[string]string {
|
||||
return map[string]string{
|
||||
"input": profile,
|
||||
|
||||
@@ -143,17 +143,6 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
||||
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
|
||||
}
|
||||
|
||||
provenance := ReferenceProvenance(first)
|
||||
if len(provenance) != 2 {
|
||||
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
|
||||
}
|
||||
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
|
||||
}
|
||||
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(first)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
|
||||
@@ -163,7 +152,7 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
|
||||
func TestMaterializeReferencesStoresSetsForAllTargets(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
|
||||
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
|
||||
@@ -215,22 +204,6 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T)
|
||||
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
|
||||
}
|
||||
|
||||
provenance := ReferenceProvenance(materialized)
|
||||
if len(provenance) != 4 {
|
||||
t.Fatalf("ReferenceProvenance() = %#v, want four entries", provenance)
|
||||
}
|
||||
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
|
||||
}
|
||||
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
|
||||
}
|
||||
if provenance[2].Stage != string(StageMerge) || provenance[2].LaneID != "events" || provenance[2].SlotName != "merge_notes" || provenance[2].Digest != mergeItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[2] = %#v, want merge notes provenance", provenance[2])
|
||||
}
|
||||
if provenance[3].Stage != string(StageNormalize) || provenance[3].LaneID != "events" || provenance[3].SlotName != "normalization_notes" || provenance[3].Digest != normalizeItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesUsesLaneArtifactVariant(t *testing.T) {
|
||||
|
||||
@@ -1,478 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type capturedDebugRecorder struct {
|
||||
mu sync.Mutex
|
||||
json map[string][]byte
|
||||
bytes map[string][]byte
|
||||
failPath string
|
||||
}
|
||||
|
||||
func newCapturedDebugRecorder() *capturedDebugRecorder {
|
||||
return &capturedDebugRecorder{json: make(map[string][]byte), bytes: make(map[string][]byte)}
|
||||
}
|
||||
|
||||
func (*capturedDebugRecorder) Enabled() bool { return true }
|
||||
|
||||
func (r *capturedDebugRecorder) WriteJSON(name string, payload any) error {
|
||||
if name == r.failPath {
|
||||
return errors.New("debug recorder failure")
|
||||
}
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.json[name] = data
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *capturedDebugRecorder) WriteBytes(name string, data []byte) error {
|
||||
if name == r.failPath {
|
||||
return errors.New("debug recorder failure")
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.bytes[name] = append([]byte(nil), data...)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *capturedDebugRecorder) has(name string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
_, jsonOK := r.json[name]
|
||||
_, bytesOK := r.bytes[name]
|
||||
return jsonOK || bytesOK
|
||||
}
|
||||
|
||||
func (r *capturedDebugRecorder) names() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
names := make([]string, 0, len(r.json)+len(r.bytes))
|
||||
for name := range r.json {
|
||||
names = append(names, name)
|
||||
}
|
||||
for name := range r.bytes {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func (r *capturedDebugRecorder) envelope(t *testing.T, name string) debugTimedEnvelope {
|
||||
t.Helper()
|
||||
r.mu.Lock()
|
||||
data := append([]byte(nil), r.json[name]...)
|
||||
r.mu.Unlock()
|
||||
if len(data) == 0 {
|
||||
t.Fatalf("debug envelope %q was not written; names = %#v", name, r.names())
|
||||
}
|
||||
var envelope debugTimedEnvelope
|
||||
if err := json.Unmarshal(data, &envelope); err != nil {
|
||||
t.Fatalf("decode debug envelope %q: %v", name, err)
|
||||
}
|
||||
return envelope
|
||||
}
|
||||
|
||||
type attemptDebugLLM struct{}
|
||||
|
||||
func (attemptDebugLLM) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, _ any) (contracts.StructuredCompletionResponse, error) {
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: json.RawMessage(`{"accepted":true}`),
|
||||
Provider: "test",
|
||||
Model: "test-model",
|
||||
ProfileID: request.ProfileID,
|
||||
Debug: &contracts.LLMDebugMaterial{
|
||||
Prompt: &contracts.LLMDebugPrompt{PromptID: request.PromptID, Messages: []contracts.LLMDebugMessage{{Role: "user", Content: "test"}}},
|
||||
Response: &contracts.LLMDebugResponse{
|
||||
Content: `{"accepted":true}`,
|
||||
PromptID: request.PromptID,
|
||||
ModelName: "test-model",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline {
|
||||
return preparedAttemptDebugPipelineWithChunks(t, 1)
|
||||
}
|
||||
|
||||
func preparedAttemptDebugPipelineWithChunks(t *testing.T, chunkCount int) *PreparedPipeline {
|
||||
t.Helper()
|
||||
prepared := preparedConcurrentPipeline(t, chunkCount)
|
||||
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
|
||||
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
|
||||
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]
|
||||
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
|
||||
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
producerAttempts := 2
|
||||
validatorAttempts := 0
|
||||
lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts
|
||||
if len(lane.extractValidators.validators) == 0 {
|
||||
t.Fatal("extract validators are empty")
|
||||
}
|
||||
lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts
|
||||
|
||||
var observedProducer, observedValidator *int
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
observedProducer = request.StructuredOutputRepairAttempts
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
observedValidator = target.structuredOutputRepairAttempts
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts)
|
||||
}
|
||||
if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error {
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func TestRunnerWritesAttemptScopedMergeAndNormalizeDebug(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
debug := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
|
||||
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "merge"); err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("merge", "merge diagnostic")}}, nil
|
||||
}
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "normalize"); err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"normalized"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("normalize", "normalize diagnostic")}}, nil
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want one", len(output.NormalizeOutputs))
|
||||
}
|
||||
|
||||
wantPaths := []string{
|
||||
"merge/notes/input.json",
|
||||
"merge/notes/attempt-01.json",
|
||||
"merge/notes/attempt-01/prompt-0001.json",
|
||||
"merge/notes/attempt-01/response-0001.json",
|
||||
"merge/notes/attempt-01/response-content-0001.json",
|
||||
"merge/notes/output.json",
|
||||
"normalize/notes/input.json",
|
||||
"normalize/notes/attempt-01.json",
|
||||
"normalize/notes/attempt-01/prompt-0002.json",
|
||||
"normalize/notes/attempt-01/response-0002.json",
|
||||
"normalize/notes/attempt-01/response-content-0002.json",
|
||||
"normalize/notes/output.json",
|
||||
}
|
||||
for _, name := range wantPaths {
|
||||
if !debug.has(name) {
|
||||
t.Errorf("debug artifact %q missing; names = %#v", name, debug.names())
|
||||
}
|
||||
}
|
||||
for _, name := range debug.names() {
|
||||
if strings.HasPrefix(name, "unscoped-merge/") || strings.HasPrefix(name, "unscoped-normalize/") {
|
||||
t.Errorf("LLM call used fallback debug path %q", name)
|
||||
}
|
||||
}
|
||||
mergeEnvelope := debug.envelope(t, "merge/notes/attempt-01.json")
|
||||
normalizeEnvelope := debug.envelope(t, "normalize/notes/attempt-01.json")
|
||||
if len(mergeEnvelope.LLMCalls) != 1 || mergeEnvelope.LLMCalls[0].ResponsePath != "merge/notes/attempt-01/response-0001.json" {
|
||||
t.Fatalf("merge LLM calls = %#v, want attempt-scoped call", mergeEnvelope.LLMCalls)
|
||||
}
|
||||
if len(normalizeEnvelope.LLMCalls) != 1 || normalizeEnvelope.LLMCalls[0].ResponsePath != "normalize/notes/attempt-01/response-0002.json" {
|
||||
t.Fatalf("normalize LLM calls = %#v, want attempt-scoped call", normalizeEnvelope.LLMCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedDiagnosticsOnly(t *testing.T) {
|
||||
for _, stage := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
t.Run(string(stage), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
debug := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
attempts := 0
|
||||
operation := func(ctx context.Context) (erasedTypedResult, error) {
|
||||
attempts++
|
||||
if err := callAttemptDebugLLM(ctx, client, string(stage)); err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
scope := "accepted"
|
||||
if attempts == 1 {
|
||||
scope = "discarded"
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope))}, nil
|
||||
}
|
||||
validatorCalls := 0
|
||||
validator := preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
validatorCalls++
|
||||
return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}
|
||||
switch stage {
|
||||
case StageMerge:
|
||||
lane.resolved.Merge.Retries = 1
|
||||
lane.mergeValidators.validators = []preparedValidator{validator}
|
||||
lane.typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
return operation(ctx)
|
||||
}
|
||||
case StageNormalize:
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
lane.normalizeValidators.validators = []preparedValidator{validator}
|
||||
lane.typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return operation(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
laneID := lane.resolved.ID
|
||||
firstPath := fmt.Sprintf("%s/%s/attempt-01.json", stage, laneID)
|
||||
secondPath := fmt.Sprintf("%s/%s/attempt-02.json", stage, laneID)
|
||||
assertAttemptEnvelopeSequence(t, debug, fmt.Sprintf("%s/%s", stage, laneID), 1, 2)
|
||||
first := debug.envelope(t, firstPath)
|
||||
second := debug.envelope(t, secondPath)
|
||||
if len(first.LLMCalls) != 1 || len(second.LLMCalls) != 1 || first.LLMCalls[0].CallID == second.LLMCalls[0].CallID {
|
||||
t.Fatalf("retry LLM calls = first %#v, second %#v; want distinct calls", first.LLMCalls, second.LLMCalls)
|
||||
}
|
||||
if !strings.Contains(string(debug.json[firstPath]), "rejection") {
|
||||
t.Fatalf("first attempt envelope = %s, want rejection", debug.json[firstPath])
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" {
|
||||
t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*PreparedPipeline)
|
||||
path string
|
||||
wantError string
|
||||
wantBody string
|
||||
attemptError bool
|
||||
}{
|
||||
{
|
||||
name: "merge module error",
|
||||
configure: func(prepared *PreparedPipeline) {
|
||||
prepared.Steps[0].lanes[0].typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{}, errors.New("merge exploded")
|
||||
}
|
||||
},
|
||||
path: "merge/notes/attempt-01.json",
|
||||
wantError: "merge exploded",
|
||||
attemptError: true,
|
||||
},
|
||||
{
|
||||
name: "normalize validator error",
|
||||
configure: func(prepared *PreparedPipeline) {
|
||||
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureFailRun
|
||||
prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, errors.New("validator exploded")
|
||||
},
|
||||
}}
|
||||
},
|
||||
path: "normalize/notes/attempt-01.json",
|
||||
wantError: "validator exploded",
|
||||
wantBody: "output",
|
||||
},
|
||||
{
|
||||
name: "merge final rejection",
|
||||
configure: func(prepared *PreparedPipeline) {
|
||||
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
},
|
||||
path: "merge/notes/attempt-01.json",
|
||||
wantBody: "rejection",
|
||||
},
|
||||
{
|
||||
name: "normalize serialization error",
|
||||
configure: func(prepared *PreparedPipeline) {
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: "wrong artifact type"}, nil
|
||||
}
|
||||
},
|
||||
path: "normalize/notes/attempt-01.json",
|
||||
wantError: "serialize normalize candidate",
|
||||
attemptError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
debug := newCapturedDebugRecorder()
|
||||
tc.configure(prepared)
|
||||
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
envelope := debug.envelope(t, tc.path)
|
||||
if tc.wantError != "" {
|
||||
if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) {
|
||||
t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else if runErr != nil {
|
||||
t.Fatalf("Run() error = %v, want nil rejection outcome", runErr)
|
||||
}
|
||||
if tc.wantBody != "" && !strings.Contains(string(debug.json[tc.path]), tc.wantBody) {
|
||||
t.Fatalf("attempt envelope = %s, want %q", debug.json[tc.path], tc.wantBody)
|
||||
}
|
||||
if tc.name == "merge final rejection" && len(output.Rejected) != 1 {
|
||||
t.Fatalf("rejected outputs = %#v, want one", output.Rejected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerKeepsValidatorLLMCallsOutOfModuleAttempt(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
debug := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
|
||||
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "merge-module"); err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}}, nil
|
||||
}
|
||||
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "merge-validator"); err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
},
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
moduleEnvelope := debug.envelope(t, "merge/notes/attempt-01.json")
|
||||
validatorPath := "validate/merge/notes/typed%2Fmerge/01-llm-check-attempt-01.json"
|
||||
validatorEnvelope := debug.envelope(t, validatorPath)
|
||||
if len(moduleEnvelope.LLMCalls) != 1 || !strings.Contains(moduleEnvelope.LLMCalls[0].ResponsePath, "merge/notes/attempt-01/") {
|
||||
t.Fatalf("module LLM calls = %#v, want module call only", moduleEnvelope.LLMCalls)
|
||||
}
|
||||
if len(validatorEnvelope.LLMCalls) != 1 || !strings.Contains(validatorEnvelope.LLMCalls[0].ResponsePath, "validate/merge/notes/") {
|
||||
t.Fatalf("validator LLM calls = %#v, want validator call only", validatorEnvelope.LLMCalls)
|
||||
}
|
||||
if moduleEnvelope.LLMCalls[0].CallID == validatorEnvelope.LLMCalls[0].CallID {
|
||||
t.Fatalf("module and validator envelopes reference the same call: %#v", moduleEnvelope.LLMCalls)
|
||||
}
|
||||
}
|
||||
|
||||
type attemptReuseLoader struct {
|
||||
CheckpointLoader
|
||||
laneID string
|
||||
merge MergeCheckpoint
|
||||
normalize NormalizeCheckpoint
|
||||
}
|
||||
|
||||
func (l attemptReuseLoader) Enabled() bool { return true }
|
||||
|
||||
func (l attemptReuseLoader) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
if laneID == l.laneID {
|
||||
return l.merge, CheckpointDecision{Reused: true, Reason: "test reuse"}
|
||||
}
|
||||
return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||
}
|
||||
|
||||
func (l attemptReuseLoader) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
if laneID == l.laneID {
|
||||
return l.normalize, CheckpointDecision{Reused: true, Reason: "test reuse"}
|
||||
}
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||
}
|
||||
|
||||
func TestRunnerCheckpointReuseDoesNotSynthesizeModuleAttempts(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := prepared.Steps[0].lanes[0]
|
||||
merge, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Merge.Module, "source", codecNotes{Items: []string{"merged"}})
|
||||
if err != nil {
|
||||
t.Fatalf("checkpointArtifact(merge): %v", err)
|
||||
}
|
||||
normalize, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Normalize.Module, "source", codecNotes{Items: []string{"normalized"}})
|
||||
if err != nil {
|
||||
t.Fatalf("checkpointArtifact(normalize): %v", err)
|
||||
}
|
||||
loader := attemptReuseLoader{
|
||||
CheckpointLoader: NoopCheckpointLoader(),
|
||||
laneID: lane.resolved.ID,
|
||||
merge: MergeCheckpoint{Output: merge},
|
||||
normalize: NormalizeCheckpoint{Output: normalize},
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoint: loader}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
for _, name := range debug.names() {
|
||||
if (strings.HasPrefix(name, "merge/notes/attempt-") || strings.HasPrefix(name, "normalize/notes/attempt-")) && name != "" {
|
||||
t.Fatalf("checkpoint reuse synthesized module attempt artifact %q", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"merge/notes/input.json", "merge/notes/output.json", "normalize/notes/input.json", "normalize/notes/output.json"} {
|
||||
if !debug.has(name) {
|
||||
t.Errorf("checkpoint reuse missing stage-level artifact %q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerTreatsModuleAttemptDebugWriteFailureAsFrameworkError(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
debug := newCapturedDebugRecorder()
|
||||
debug.failPath = "merge/notes/attempt-01.json"
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err == nil || !strings.Contains(err.Error(), "write merge attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
|
||||
t.Fatalf("Run() error = %v, want merge attempt debug failure", err)
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,8 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -93,26 +91,6 @@ func matchingNotes(values []codecNotes, value string) int {
|
||||
return count
|
||||
}
|
||||
|
||||
type candidateCheckpointRecorder struct {
|
||||
CheckpointRecorder
|
||||
mergeSucceeded int
|
||||
normalizeSucceeded int
|
||||
mergeOutput CheckpointArtifact
|
||||
normalizeOutput CheckpointArtifact
|
||||
}
|
||||
|
||||
func (r *candidateCheckpointRecorder) MergeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error {
|
||||
r.mergeSucceeded++
|
||||
r.mergeOutput = cloneCheckpointArtifact(output)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *candidateCheckpointRecorder) NormalizeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error {
|
||||
r.normalizeSucceeded++
|
||||
r.normalizeOutput = cloneCheckpointArtifact(output)
|
||||
return nil
|
||||
}
|
||||
|
||||
func installObservedNotesCodec(t *testing.T, prepared *PreparedPipeline, codec *observedNotesCodec) {
|
||||
t.Helper()
|
||||
registry := NewArtifactCodecRegistry()
|
||||
@@ -130,7 +108,7 @@ func configureCandidateOperation(prepared *PreparedPipeline, target ModuleStage,
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
switch target {
|
||||
case StageExtract:
|
||||
lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: value}, nil
|
||||
}
|
||||
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
@@ -176,12 +154,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro
|
||||
func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
t.Run(string(target), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
if target == StageMerge {
|
||||
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
} else {
|
||||
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
}
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{}
|
||||
prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{}
|
||||
prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{}
|
||||
@@ -233,8 +206,7 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
@@ -251,23 +223,10 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) {
|
||||
}
|
||||
case StageNormalize:
|
||||
var normalized codecNotes
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) {
|
||||
if err := json.Unmarshal(output.normalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) {
|
||||
t.Fatalf("normalized output = %#v, %v; want %#v", normalized, err, candidate)
|
||||
}
|
||||
}
|
||||
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target)
|
||||
if target == StageExtract {
|
||||
attemptPath = "extract/notes/chunk-000001/attempt-01.json"
|
||||
}
|
||||
payload := debug.envelope(t, attemptPath).Payload.(map[string]any)
|
||||
content, err := base64.StdEncoding.DecodeString(payload["output"].(map[string]any)["content"].(map[string]any)["content_base64"].(string))
|
||||
var debugValue codecNotes
|
||||
if err == nil {
|
||||
err = json.Unmarshal(content, &debugValue)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(debugValue, candidate) {
|
||||
t.Fatalf("debug candidate = %#v, %v; want %#v", debugValue, err, candidate)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -284,7 +243,7 @@ func setCandidateValidators(prepared *PreparedPipeline, target ModuleStage, vali
|
||||
}
|
||||
|
||||
func TestRunnerReportsCandidateDecodeFailure(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
codec := &observedNotesCodec{candidateDecodeError: "decode-failure"}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
configureCandidateOperation(prepared, StageMerge, codecNotes{Items: []string{"decode-failure"}})
|
||||
@@ -298,131 +257,74 @@ func TestRunnerReportsCandidateDecodeFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerRejectsCandidatesBeforeFinalEncoding(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
t.Run(string(target), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
if target == StageMerge {
|
||||
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
} else {
|
||||
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
}
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
codec := &observedNotesCodec{}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}})
|
||||
setCandidateValidator(prepared, target, false)
|
||||
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want validator rejection", err)
|
||||
}
|
||||
if len(output.Rejected) != 1 || output.Rejected[0].Stage != string(target) || output.Rejected[0].ReasonCode != "candidate_rejected" {
|
||||
t.Fatalf("rejected outputs = %#v, want %s validator rejection", output.Rejected, target)
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err == nil || !strings.Contains(err.Error(), "candidate-check") || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("result=%#v error=%v, want rejection without publication", output, err)
|
||||
}
|
||||
if codec.candidateCalls("invalid") != 1 || codec.finalCalls("invalid") != 0 {
|
||||
t.Fatalf("invalid candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid"))
|
||||
}
|
||||
if target == StageMerge && recorder.mergeSucceeded != 0 {
|
||||
t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded)
|
||||
}
|
||||
if target == StageNormalize && recorder.normalizeSucceeded != 0 {
|
||||
t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded)
|
||||
t.Fatalf("rejected candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerFinalEncodesAcceptedCandidatesOnce(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
t.Run(string(target), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
codec := &observedNotesCodec{}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
value := "accepted-" + string(target)
|
||||
configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}})
|
||||
setCandidateValidator(prepared, target, true)
|
||||
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
loader := &extractResultLoader{CheckpointLoader: NoopCheckpointLoader(), decision: CheckpointDecision{Reason: "not found"}}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Checkpoint: loader})
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if codec.candidateCalls(value) != 1 || codec.finalCalls(value) != 1 {
|
||||
t.Fatalf("accepted candidate calls = candidate %d, final %d; want 1, 1", codec.candidateCalls(value), codec.finalCalls(value))
|
||||
}
|
||||
if target == StageMerge && recorder.mergeSucceeded != 1 {
|
||||
t.Fatalf("merge checkpoints = %d, want one", recorder.mergeSucceeded)
|
||||
}
|
||||
if target == StageNormalize && recorder.normalizeSucceeded != 1 {
|
||||
t.Fatalf("normalize checkpoints = %d, want one", recorder.normalizeSucceeded)
|
||||
}
|
||||
checkpoint := recorder.mergeOutput
|
||||
if target == StageNormalize {
|
||||
checkpoint = recorder.normalizeOutput
|
||||
}
|
||||
var stored codecNotes
|
||||
if err := json.Unmarshal(checkpoint.Artifact.Content, &stored); err != nil || !reflect.DeepEqual(stored, codecNotes{Items: []string{value}}) {
|
||||
t.Fatalf("checkpoint content = %s, %v; want accepted value %q", checkpoint.Artifact.Content, err, value)
|
||||
}
|
||||
gotStages := make([]string, len(output.CheckpointEvents))
|
||||
for i, event := range output.CheckpointEvents {
|
||||
gotStages[i] = event.Stage
|
||||
}
|
||||
wantStages := []string{"source", string(StageExtract), string(StageMerge), string(StageNormalize)}
|
||||
if !reflect.DeepEqual(gotStages, wantStages) {
|
||||
t.Fatalf("checkpoint event stages = %#v, want %#v", gotStages, wantStages)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRecordsCandidateAndFinalEncodingFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
target ModuleStage
|
||||
value string
|
||||
candidate bool
|
||||
wantError string
|
||||
}{
|
||||
{name: "merge candidate", target: StageMerge, value: "merge-candidate-failure", candidate: true, wantError: "serialize merge candidate"},
|
||||
{name: "normalize candidate", target: StageNormalize, value: "normalize-candidate-failure", candidate: true, wantError: "serialize normalize candidate"},
|
||||
{name: "merge final", target: StageMerge, value: "merge-final-failure", wantError: "serialize accepted merge output"},
|
||||
{name: "normalize final", target: StageNormalize, value: "normalize-final-failure", wantError: "serialize accepted normalize output"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
codec := &observedNotesCodec{}
|
||||
if tc.candidate {
|
||||
codec.candidateError = tc.value
|
||||
} else {
|
||||
codec.finalError = tc.value
|
||||
func TestRunnerReportsCandidateAndFinalEncodingFailures(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
for _, candidateFailure := range []bool{true, false} {
|
||||
name := string(target) + "/final"
|
||||
if candidateFailure {
|
||||
name = string(target) + "/candidate"
|
||||
}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
configureCandidateOperation(prepared, tc.target, codecNotes{Items: []string{tc.value}})
|
||||
setCandidateValidator(prepared, tc.target, true)
|
||||
recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, want %q", err, tc.wantError)
|
||||
}
|
||||
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", tc.target)
|
||||
envelope := debug.envelope(t, attemptPath)
|
||||
if !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.candidate && codec.finalCalls(tc.value) != 0 {
|
||||
t.Fatalf("final encode calls = %d, want none after candidate failure", codec.finalCalls(tc.value))
|
||||
}
|
||||
if tc.target == StageMerge && recorder.mergeSucceeded != 0 {
|
||||
t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded)
|
||||
}
|
||||
if tc.target == StageNormalize && recorder.normalizeSucceeded != 0 {
|
||||
t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded)
|
||||
}
|
||||
})
|
||||
t.Run(name, func(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
codec := &observedNotesCodec{}
|
||||
value := "encoding-failure"
|
||||
wantOperation := "encode"
|
||||
if candidateFailure {
|
||||
codec.candidateError = value
|
||||
wantOperation = "encode candidate"
|
||||
} else {
|
||||
codec.finalError = value
|
||||
}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}})
|
||||
setCandidateValidator(prepared, target, true)
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
var codecErr *ArtifactCodecOperationError
|
||||
if !errors.As(err, &codecErr) || codecErr.Operation != wantOperation || codecErr.Kind != "test/notes" || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("output=%#v error=%v codec error=%#v, want scoped %s failure without publication", output, err, codecErr, wantOperation)
|
||||
}
|
||||
if candidateFailure && codec.finalCalls(value) != 0 {
|
||||
t.Fatal("candidate failure reached final encoding")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Extract.Retries = 1
|
||||
|
||||
var mu sync.Mutex
|
||||
corrections := make(map[int]*contracts.SemanticCorrection)
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
chunkIndex := request.Chunk.Index
|
||||
if request.Correction != nil {
|
||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
mu.Lock()
|
||||
corrections[chunkIndex] = correction
|
||||
mu.Unlock()
|
||||
candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("corrected-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("corrected-%d", chunkIndex)}}, ModelCandidate: candidate}, nil
|
||||
}
|
||||
candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("initial-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("invalid-%d", chunkIndex)}}, ModelCandidate: candidate}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
value := target.value.(codecNotes)
|
||||
if value.Items[0] == fmt.Sprintf("invalid-%d", target.chunk.Index) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "incorrect_extract", Message: "candidate needs correction", CorrectionGuidance: fmt.Sprintf("correct chunk %d", target.chunk.Index)}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2, Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted extracts", output)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(corrections) != 2 {
|
||||
t.Fatalf("corrections = %#v, want one correction per chunk", corrections)
|
||||
}
|
||||
for index := 0; index < 2; index++ {
|
||||
correction := corrections[index]
|
||||
if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || !strings.Contains(correction.UserGuidance, fmt.Sprintf("correct chunk %d", index)) || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "incorrect_extract") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
|
||||
t.Fatalf("chunk %d correction = %#v, want its exact initial response and semantic replacement guidance only", index, correction)
|
||||
}
|
||||
}
|
||||
terminal := string(debug.json["extract/notes/chunk-000001/terminal.json"])
|
||||
if !strings.Contains(terminal, `"kind":"semantic_correction"`) || !strings.Contains(terminal, `"producer_attempt_count":2`) {
|
||||
t.Fatalf("extract terminal debug = %s, want correction provenance", terminal)
|
||||
}
|
||||
if strings.Contains(terminal, "initial-response-0") || strings.Contains(terminal, "corrected-response-0") || strings.Contains(terminal, "correct chunk 0") {
|
||||
t.Fatalf("extract terminal debug leaked correction content: %s", terminal)
|
||||
}
|
||||
manifest, err := json.Marshal(output.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(manifest), "initial-response-0") || strings.Contains(string(manifest), "corrected-response-0") || strings.Contains(string(manifest), "correct chunk 0") {
|
||||
t.Fatalf("manifest leaked correction content: %s", manifest)
|
||||
}
|
||||
}
|
||||
|
||||
func containsValidationSummary(summaries []artifacts.ValidationSummary, status string) bool {
|
||||
for _, summary := range summaries {
|
||||
if summary.Status == status {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestRunnerContinuesValidationIncompleteExtractWithoutCheckpoint(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
}
|
||||
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
|
||||
t.Fatalf("run output = %#v, want accepted incomplete extract", output)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") {
|
||||
t.Fatalf("manifest validation = %#v, want incomplete extract provenance", output.Manifest)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
|
||||
t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics)
|
||||
}
|
||||
if len(recorder.checkpoint.Outputs) != 0 || len(recorder.checkpoint.Rejected) != 0 {
|
||||
t.Fatalf("extract checkpoint = %#v, want no persisted incomplete output", recorder.checkpoint)
|
||||
}
|
||||
encoded := string(debug.json["extract/notes/output.json"])
|
||||
if !strings.Contains(encoded, `"validation_incomplete_chunks":[0]`) {
|
||||
t.Fatalf("extract output debug = %s, want incomplete chunk marker", encoded)
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,6 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,11 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type extractCaptureRecorder struct {
|
||||
CheckpointRecorder
|
||||
checkpoint ExtractCheckpoint
|
||||
}
|
||||
|
||||
type pipelineTypedMetadata map[string]any
|
||||
|
||||
type mutatingMetadataValidator struct {
|
||||
@@ -35,48 +28,14 @@ func (v *mutatingMetadataValidator) Validate(_ context.Context, request contract
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (r *extractCaptureRecorder) ExtractSucceeded(_ string, _ string, _ []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
|
||||
r.checkpoint = ExtractCheckpoint{
|
||||
Outputs: cloneCheckpointArtifacts(outputs),
|
||||
Rejected: cloneRejectedOutputs(rejected),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type extractResultLoader struct {
|
||||
CheckpointLoader
|
||||
checkpoint ExtractCheckpoint
|
||||
decision CheckpointDecision
|
||||
}
|
||||
|
||||
func (l *extractResultLoader) Enabled() bool { return true }
|
||||
|
||||
func (l *extractResultLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
return ExtractCheckpoint{
|
||||
Outputs: cloneCheckpointArtifacts(l.checkpoint.Outputs),
|
||||
Rejected: cloneRejectedOutputs(l.checkpoint.Rejected),
|
||||
}, l.decision
|
||||
}
|
||||
|
||||
func cloneCheckpointArtifacts(values []CheckpointArtifact) []CheckpointArtifact {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]CheckpointArtifact, len(values))
|
||||
for i := range values {
|
||||
cloned[i] = cloneCheckpointArtifact(values[i])
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
chunker := prepared.chunker.(*typedTestChunker)
|
||||
chunker.plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)}
|
||||
chunker.plan.Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)}
|
||||
|
||||
var gotRange, gotPlan string
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
gotRange = string(request.Chunk.Annotations["same"])
|
||||
gotPlan = string(request.Chunk.PlanAnnotations["same"])
|
||||
request.Chunk.Annotations["same"][0] = '['
|
||||
@@ -95,7 +54,7 @@ func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
input := prepared.input.(*typedTestInput)
|
||||
input.doc.Units[0].Metadata = map[string]any{
|
||||
"typed": pipelineTypedMetadata{"raw": json.RawMessage(`{"value":true}`)},
|
||||
@@ -114,7 +73,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes
|
||||
chunk: validator,
|
||||
}}
|
||||
var extractorSeen string
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
nested := request.Chunk.Units[0].Metadata["typed"].(pipelineTypedMetadata)
|
||||
extractorSeen = string(nested["raw"].(json.RawMessage))
|
||||
nested["raw"].(json.RawMessage)[0] = '['
|
||||
@@ -135,7 +94,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes
|
||||
}
|
||||
|
||||
func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
cyclic := make(map[string]any)
|
||||
cyclic["self"] = cyclic
|
||||
_, err := New().Run(context.Background(), RunInput{
|
||||
@@ -147,144 +106,3 @@ func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v, want contextual metadata clone failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
extractCalls := 0
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
extractCalls++
|
||||
return erasedTypedResult{
|
||||
Value: typedValueForLane(0, request.Chunk.Index),
|
||||
Diagnostics: []contracts.ProducerDiagnostic{{
|
||||
Disposition: contracts.DiagnosticDispositionObservation,
|
||||
Category: contracts.DiagnosticCategoryNormalization,
|
||||
ReasonCode: "accepted_extract_normalized",
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "extract", Message: "accepted extract"}},
|
||||
}},
|
||||
}, nil
|
||||
})
|
||||
|
||||
freshDebug := newCapturedDebugRecorder()
|
||||
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
freshLoader := &extractResultLoader{
|
||||
CheckpointLoader: NoopCheckpointLoader(),
|
||||
decision: CheckpointDecision{Reason: "extract checkpoint not found"},
|
||||
}
|
||||
fresh, err := New().Run(context.Background(), RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: []byte("input"),
|
||||
Checkpoints: recorder,
|
||||
Checkpoint: freshLoader,
|
||||
Debug: freshDebug,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("fresh Run() error = %v, want nil", err)
|
||||
}
|
||||
if extractCalls != 1 {
|
||||
t.Fatalf("fresh extract calls = %d, want 1", extractCalls)
|
||||
}
|
||||
assertExtractDecision(t, fresh.CheckpointEvents, "executed", "extract checkpoint not found")
|
||||
assertExtractDebugPaths(t, freshDebug, true)
|
||||
|
||||
reusedDebug := newCapturedDebugRecorder()
|
||||
reusedLoader := &extractResultLoader{
|
||||
CheckpointLoader: NoopCheckpointLoader(),
|
||||
checkpoint: recorder.checkpoint,
|
||||
decision: CheckpointDecision{Reused: true, Reason: "extract checkpoint matched"},
|
||||
}
|
||||
reused, err := New().Run(context.Background(), RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: []byte("input"),
|
||||
Checkpoint: reusedLoader,
|
||||
Debug: reusedDebug,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("reused Run() error = %v, want nil", err)
|
||||
}
|
||||
if extractCalls != 1 {
|
||||
t.Fatalf("extract calls after reuse = %d, want 1", extractCalls)
|
||||
}
|
||||
assertExtractDecision(t, reused.CheckpointEvents, "reused", "extract checkpoint matched")
|
||||
assertExtractDebugPaths(t, reusedDebug, false)
|
||||
if !reflect.DeepEqual(reused.NormalizeOutputs, fresh.NormalizeOutputs) {
|
||||
t.Fatalf("reused normalize outputs = %#v, want fresh outputs %#v", reused.NormalizeOutputs, fresh.NormalizeOutputs)
|
||||
}
|
||||
if !reflect.DeepEqual(reused.Diagnostics, fresh.Diagnostics) {
|
||||
t.Fatalf("reused diagnostics = %#v, want fresh diagnostics %#v", reused.Diagnostics, fresh.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPromotesOnlyAcceptedExtractRetryDiagnostics(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.Steps[0].lanes[0].resolved.Extract.Retries = 1
|
||||
attempts := 0
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
attempts++
|
||||
scope := "discarded"
|
||||
if attempts == 2 {
|
||||
scope = "accepted"
|
||||
}
|
||||
return erasedTypedResult{
|
||||
Value: typedValueForLane(0, request.Chunk.Index),
|
||||
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)},
|
||||
ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope)),
|
||||
}, nil
|
||||
})
|
||||
validatorCalls := 0
|
||||
prepared.Steps[0].lanes[0].extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
validatorCalls++
|
||||
return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract", CorrectionGuidance: "return an acceptable extract"}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("extract attempts = %d, want 2", attempts)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" {
|
||||
t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics)
|
||||
}
|
||||
assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1, 2)
|
||||
first := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json")
|
||||
if first.Error != "" || !strings.Contains(string(debug.json["extract/notes/chunk-000001/attempt-01.json"]), "rejection") {
|
||||
t.Fatalf("first extract attempt = %#v, want rejection without error", first)
|
||||
}
|
||||
name := "extract/notes/chunk-000001/attempt-02.json"
|
||||
if !debug.has(name) {
|
||||
t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names())
|
||||
}
|
||||
}
|
||||
|
||||
func assertExtractDecision(t *testing.T, events []CheckpointEvent, action CheckpointDecisionCategory, reason string) {
|
||||
t.Helper()
|
||||
for _, event := range events {
|
||||
if event.Stage == string(StageExtract) {
|
||||
if event.Action != action || event.Reason != reason {
|
||||
t.Fatalf("extract checkpoint event = %#v, want action %q reason %q", event, action, reason)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("extract checkpoint event missing from %#v", events)
|
||||
}
|
||||
|
||||
func assertExtractDebugPaths(t *testing.T, debug *capturedDebugRecorder, wantAttempt bool) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"extract/notes/input.json", "extract/notes/output.json"} {
|
||||
if !debug.has(name) {
|
||||
t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names())
|
||||
}
|
||||
}
|
||||
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
|
||||
if debug.has(attemptPath) != wantAttempt {
|
||||
t.Fatalf("attempt debug path present = %t, want %t; names = %#v", debug.has(attemptPath), wantAttempt, debug.names())
|
||||
}
|
||||
input := debug.envelope(t, "extract/notes/input.json")
|
||||
wantReuse := !wantAttempt
|
||||
if !strings.Contains(string(debug.json["extract/notes/input.json"]), `"reused":`+map[bool]string{true: "true", false: "false"}[wantReuse]) {
|
||||
t.Fatalf("extract input payload = %#v, want reused %t", input.Payload, wantReuse)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
)
|
||||
|
||||
func TestMergeLLMProfileManifestsDistinguishesEffectiveTargets(t *testing.T) {
|
||||
got := mergeLLMProfileManifests(
|
||||
[]artifacts.LLMProfileManifest{
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
|
||||
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
|
||||
},
|
||||
[]artifacts.LLMProfileManifest{
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
|
||||
},
|
||||
)
|
||||
want := []artifacts.LLMProfileManifest{
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
|
||||
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("merged profiles = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
retries int
|
||||
operation func(int) erasedTypedResult
|
||||
validator *preparedValidator
|
||||
wantCalls int
|
||||
wantItem string
|
||||
wantMessages []string
|
||||
wantRejected int
|
||||
wantDebug []string
|
||||
wantCheckpoint int
|
||||
}{
|
||||
{
|
||||
name: "accepts zero-retry fallback",
|
||||
retries: 0,
|
||||
operation: func(int) erasedTypedResult {
|
||||
return retryableNormalizeResult("fallback", "ordinary", "fallback-warning")
|
||||
},
|
||||
wantCalls: 1,
|
||||
wantItem: "fallback",
|
||||
wantMessages: []string{"ordinary", "fallback-warning"},
|
||||
wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`},
|
||||
wantCheckpoint: 1,
|
||||
},
|
||||
{
|
||||
name: "retries before accepting ordinary result",
|
||||
retries: 1,
|
||||
operation: func(attempt int) erasedTypedResult {
|
||||
if attempt == 1 {
|
||||
return retryableNormalizeResult("discarded", "discarded-ordinary", "discarded-fallback")
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"accepted"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted-warning")}}
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantItem: "accepted",
|
||||
wantMessages: []string{"accepted-warning"},
|
||||
wantDebug: []string{`"another_attempt":true`, `"fallback_accepted":false`},
|
||||
wantCheckpoint: 1,
|
||||
},
|
||||
{
|
||||
name: "accepts final fallback after exhaustion",
|
||||
retries: 1,
|
||||
operation: func(attempt int) erasedTypedResult {
|
||||
return retryableNormalizeResult(fmt.Sprintf("fallback-%d", attempt), fmt.Sprintf("ordinary-%d", attempt), fmt.Sprintf("fallback-warning-%d", attempt))
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantItem: "fallback-2",
|
||||
wantMessages: []string{"ordinary-2", "fallback-warning-2"},
|
||||
wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`},
|
||||
wantCheckpoint: 1,
|
||||
},
|
||||
{
|
||||
name: "keeps final fallback rejection terminal",
|
||||
retries: 1,
|
||||
operation: func(int) erasedTypedResult {
|
||||
return retryableNormalizeResult("rejected", "ordinary", "fallback-warning")
|
||||
},
|
||||
validator: &preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject-final-fallback"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback", CorrectionGuidance: "return an acceptable normalized artifact"}, nil
|
||||
},
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantRejected: 1,
|
||||
wantDebug: []string{`"fallback_accepted":true`, `"rejection"`},
|
||||
wantCheckpoint: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Normalize.Retries = tc.retries
|
||||
lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
if tc.validator != nil {
|
||||
lane.normalizeValidators.validators = []preparedValidator{*tc.validator}
|
||||
}
|
||||
calls := 0
|
||||
lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
return tc.operation(calls), nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoints: checkpoints})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != tc.wantCalls {
|
||||
t.Fatalf("normalize calls = %d, want %d", calls, tc.wantCalls)
|
||||
}
|
||||
if checkpoints.normalizeSucceeded != tc.wantCheckpoint {
|
||||
t.Fatalf("normalize checkpoints = %d, want %d", checkpoints.normalizeSucceeded, tc.wantCheckpoint)
|
||||
}
|
||||
if len(output.Rejected) != tc.wantRejected {
|
||||
t.Fatalf("rejected outputs = %#v, want %d", output.Rejected, tc.wantRejected)
|
||||
}
|
||||
var retryDebug strings.Builder
|
||||
for _, name := range debug.names() {
|
||||
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
|
||||
retryDebug.Write(debug.json[name])
|
||||
}
|
||||
}
|
||||
for _, fragment := range tc.wantDebug {
|
||||
if !strings.Contains(retryDebug.String(), fragment) {
|
||||
t.Fatalf("retry debug = %s, want %q", retryDebug.String(), fragment)
|
||||
}
|
||||
}
|
||||
if tc.wantRejected != 0 {
|
||||
if output.Rejected[0].AttemptCount != tc.wantCalls {
|
||||
t.Fatalf("rejection attempt count = %d, want %d", output.Rejected[0].AttemptCount, tc.wantCalls)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %#v, want one", output.NormalizeOutputs)
|
||||
}
|
||||
decoded, err := lane.typed.codec.decode(output.NormalizeOutputs[0].Artifact.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("decode normalized output: %v", err)
|
||||
}
|
||||
normalized, ok := decoded.(codecNotes)
|
||||
if !ok {
|
||||
t.Fatalf("decoded normalized output = %T, want codecNotes", decoded)
|
||||
}
|
||||
if got := firstNote(normalized); got != tc.wantItem {
|
||||
t.Fatalf("normalized item = %q, want %q", got, tc.wantItem)
|
||||
}
|
||||
var gotMessages []string
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
for _, sample := range group.Samples {
|
||||
gotMessages = append(gotMessages, sample.Message)
|
||||
}
|
||||
}
|
||||
if strings.Join(gotMessages, "|") != strings.Join(tc.wantMessages, "|") {
|
||||
t.Fatalf("durable diagnostics = %#v, want %#v", gotMessages, tc.wantMessages)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerForwardsModuleRequestedNormalizeCorrection(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
guidance = "Duplicate group 1 must use only supplied candidate numbers. Return one complete corrected response."
|
||||
)
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
var observed *contracts.SemanticCorrection
|
||||
calls := 0
|
||||
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
clone, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
observed = clone
|
||||
}
|
||||
if calls == 1 {
|
||||
return erasedTypedResult{
|
||||
Value: codecNotes{Items: []string{"safe fallback"}},
|
||||
ModelCandidate: attemptCandidate(t, defective),
|
||||
Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: "semantic_proposal_invalid",
|
||||
Message: "operator-facing proposal diagnostic",
|
||||
CorrectionGuidance: guidance,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"corrected"}}}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if calls != 2 || observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("normalize calls = %d correction = %#v", calls, observed)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted output", output)
|
||||
}
|
||||
var retryDebug strings.Builder
|
||||
for _, name := range debug.names() {
|
||||
if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") {
|
||||
retryDebug.Write(debug.json[name])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(retryDebug.String(), `"correction_available":true`) || strings.Contains(retryDebug.String(), guidance) || strings.Contains(retryDebug.String(), defective) {
|
||||
t.Fatalf("retry debug = %s, want safe correction metadata without content", retryDebug.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) {
|
||||
const (
|
||||
reasonSentinel = "reason-diagnostic-sentinel"
|
||||
messageSentinel = "message-diagnostic-sentinel"
|
||||
)
|
||||
reasonOverLimit := strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes-len(reasonSentinel)) + reasonSentinel + "x"
|
||||
messageOverLimit := strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
guidanceOverLimit := strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes-len(messageSentinel)) + messageSentinel + "x"
|
||||
tests := []struct {
|
||||
name string
|
||||
retry contracts.NormalizeRetry
|
||||
candidate *contracts.ModelCandidate
|
||||
wantError string
|
||||
hiddenValues []string
|
||||
}{
|
||||
{
|
||||
name: "accepts byte limits",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes),
|
||||
Message: strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "accepts correction guidance byte limit with candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes),
|
||||
},
|
||||
candidate: attemptCandidate(t, `{"duplicate_groups":[]}`),
|
||||
},
|
||||
{
|
||||
name: "rejects oversized reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonOverLimit,
|
||||
Message: messageSentinel,
|
||||
},
|
||||
wantError: "reason code exceeds maximum length",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects oversized message",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageOverLimit,
|
||||
},
|
||||
wantError: "message exceeds maximum length",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects invalid reason code UTF-8",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel + string([]byte{0xff}),
|
||||
Message: messageSentinel,
|
||||
},
|
||||
wantError: "reason code has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects invalid message UTF-8",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel + string([]byte{0xff}),
|
||||
},
|
||||
wantError: "message has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects oversized correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: guidanceOverLimit,
|
||||
},
|
||||
wantError: "correction guidance exceeds maximum length",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects invalid correction guidance UTF-8",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: messageSentinel + string([]byte{0xff}),
|
||||
},
|
||||
wantError: "correction guidance has invalid UTF-8",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank correction guidance",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: " \t\n ",
|
||||
},
|
||||
wantError: "correction guidance is blank",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects correction guidance without candidate",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: messageSentinel,
|
||||
CorrectionGuidance: "Return one complete corrected response.",
|
||||
},
|
||||
wantError: "requires a model candidate",
|
||||
hiddenValues: []string{reasonSentinel, messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank reason code",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: " \t\n ",
|
||||
Message: messageSentinel,
|
||||
},
|
||||
wantError: "reason code is blank",
|
||||
hiddenValues: []string{messageSentinel},
|
||||
},
|
||||
{
|
||||
name: "rejects blank message",
|
||||
retry: contracts.NormalizeRetry{
|
||||
ReasonCode: reasonSentinel,
|
||||
Message: " \t\n ",
|
||||
},
|
||||
wantError: "message is blank",
|
||||
hiddenValues: []string{reasonSentinel},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry, ModelCandidate: tc.candidate}, nil
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if tc.wantError == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want accepted retry fallback", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %#v, want one accepted fallback", output.NormalizeOutputs)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, want fixed %q error", err, tc.wantError)
|
||||
}
|
||||
var debugOutput strings.Builder
|
||||
for _, name := range debug.names() {
|
||||
debugOutput.Write(debug.json[name])
|
||||
}
|
||||
for _, value := range tc.hiddenValues {
|
||||
if strings.Contains(err.Error(), value) || strings.Contains(debugOutput.String(), value) {
|
||||
t.Fatalf("retry diagnostic leaked %q", value)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func retryableNormalizeResult(item, ordinary, fallback string) erasedTypedResult {
|
||||
return erasedTypedResult{
|
||||
Value: codecNotes{Items: []string{item}},
|
||||
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("attempt", ordinary)},
|
||||
Retry: &contracts.NormalizeRetry{
|
||||
ReasonCode: "retryable_normalization",
|
||||
Message: "safe fallback is available",
|
||||
FallbackDiagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("fallback", fallback)},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type diagnosticChunker struct {
|
||||
key string
|
||||
plan source.ChunkPlan
|
||||
calls int
|
||||
}
|
||||
|
||||
func (c *diagnosticChunker) Key() string { return c.key }
|
||||
func (*diagnosticChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (c *diagnosticChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
c.calls++
|
||||
return contracts.ChunkPlanResult{
|
||||
Plan: source.CloneChunkPlan(c.plan),
|
||||
Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", c.calls), "operation diagnostic")},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type chunkValidationFunc struct {
|
||||
name string
|
||||
validate func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
|
||||
}
|
||||
|
||||
func (v chunkValidationFunc) Name() string { return v.name }
|
||||
func (chunkValidationFunc) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v chunkValidationFunc) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return v.validate(ctx, request)
|
||||
}
|
||||
|
||||
func TestRunnerPromotesOnlyTerminalRejectionDiagnostics(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageChunk, StageExtract, StageMerge, StageNormalize} {
|
||||
t.Run(string(target), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
attempts := 0
|
||||
first := func() contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", attempts), "validator diagnostic")}}
|
||||
}
|
||||
reject := func() contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected", CorrectionGuidance: "return an acceptable candidate"}
|
||||
}
|
||||
debug := newCapturedDebugRecorder()
|
||||
recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
|
||||
switch target {
|
||||
case StageChunk:
|
||||
prepared.resolved.ChunkValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
chunker := prepared.chunker.(*typedTestChunker)
|
||||
prepared.chunker = &diagnosticChunker{key: prepared.resolved.Chunk.Module, plan: source.CloneChunkPlan(chunker.plan)}
|
||||
prepared.resolved.Chunk.Retries = 1
|
||||
prepared.chunkValidators.validators = []preparedValidator{
|
||||
{resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-approval", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return first(), nil
|
||||
}}},
|
||||
{resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-rejection", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return reject(), nil
|
||||
}}},
|
||||
}
|
||||
chunkerWithDiagnostics := prepared.chunker.(*diagnosticChunker)
|
||||
first = func() contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", chunkerWithDiagnostics.calls), "validator diagnostic")}}
|
||||
}
|
||||
case StageExtract:
|
||||
lane.resolved.Extract.Retries = 1
|
||||
lane.resolved.ExtractValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
attempts++
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
|
||||
})
|
||||
lane.extractValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
|
||||
case StageMerge:
|
||||
lane.resolved.Merge.Retries = 1
|
||||
lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
attempts++
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"merge"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
|
||||
}
|
||||
lane.mergeValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
|
||||
case StageNormalize:
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
attempts++
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"normalize"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil
|
||||
}
|
||||
lane.normalizeValidators.validators = rejectionDiagnosticTypedValidators(first, reject)
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
wantScopes := []string{"operation-2", "validator-2"}
|
||||
wantAttempts := 2
|
||||
if target == StageChunk {
|
||||
wantScopes = []string{"operation-1", "validator-1"}
|
||||
wantAttempts = 1
|
||||
}
|
||||
if got := rejectionDiagnosticScopes(output.Diagnostics.Groups); !reflect.DeepEqual(got, wantScopes) {
|
||||
t.Fatalf("published diagnostic scopes = %#v, want %#v", got, wantScopes)
|
||||
}
|
||||
if len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != wantAttempts {
|
||||
t.Fatalf("rejections = %#v, want terminal rejection after %d attempt(s)", output.Rejected, wantAttempts)
|
||||
}
|
||||
attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target)
|
||||
if target == StageChunk {
|
||||
attemptPath = "chunk/attempt-01.json"
|
||||
} else if target == StageExtract {
|
||||
attemptPath = "extract/notes/chunk-000001/attempt-01.json"
|
||||
}
|
||||
if !strings.Contains(string(debug.json[attemptPath]), "rejection") {
|
||||
t.Fatalf("first attempt debug = %s, want rejection", debug.json[attemptPath])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func rejectionDiagnosticTypedValidators(first func() contracts.ValidationResult, reject func() contracts.ValidationResult) []preparedValidator {
|
||||
return []preparedValidator{
|
||||
{resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return first(), nil
|
||||
}},
|
||||
{resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return reject(), nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func rejectionDiagnosticScopes(groups []contracts.DiagnosticGroup) []string {
|
||||
scopes := make([]string, len(groups))
|
||||
for index := range groups {
|
||||
scopes[index] = groups[index].Samples[0].Scope
|
||||
}
|
||||
return scopes
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type reuseLineageCheckpointSpy struct {
|
||||
CheckpointLoader
|
||||
CheckpointRecorder
|
||||
loads map[string]int
|
||||
writes map[string]int
|
||||
forbid map[string]struct{}
|
||||
}
|
||||
|
||||
func newReuseLineageCheckpointSpy() *reuseLineageCheckpointSpy {
|
||||
return &reuseLineageCheckpointSpy{
|
||||
CheckpointLoader: NoopCheckpointLoader(),
|
||||
CheckpointRecorder: NoopCheckpointRecorder(),
|
||||
loads: make(map[string]int),
|
||||
writes: make(map[string]int),
|
||||
forbid: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Enabled() bool { return true }
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) load(stage, laneID string) {
|
||||
key := stage + "/" + laneID
|
||||
if _, forbidden := s.forbid[key]; forbidden {
|
||||
panic("checkpoint lookup crossed validation-incomplete lineage: " + key)
|
||||
}
|
||||
s.loads[key]++
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) write(stage, action, laneID string) {
|
||||
s.writes[stage+"/"+action+"/"+laneID]++
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
s.load("extract", laneID)
|
||||
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
s.load("merge", laneID)
|
||||
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
s.load("normalize", laneID)
|
||||
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("extract", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ []CheckpointArtifact, _ []contracts.RejectedOutput) error {
|
||||
s.write("extract", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) ExtractFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("extract", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("merge", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error {
|
||||
s.write("merge", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
|
||||
s.write("merge", "rejected", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) MergeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("merge", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeRunning(laneID, _ string, _ []CheckpointFingerprint) error {
|
||||
s.write("normalize", "running", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error {
|
||||
s.write("normalize", "succeeded", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error {
|
||||
s.write("normalize", "rejected", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) NormalizeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error {
|
||||
s.write("normalize", "failed", laneID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *reuseLineageCheckpointSpy) writesFor(stage, laneID string) int {
|
||||
total := 0
|
||||
needle := stage + "/"
|
||||
suffix := "/" + laneID
|
||||
for key, count := range s.writes {
|
||||
if strings.HasPrefix(key, needle) && strings.HasSuffix(key, suffix) {
|
||||
total += count
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func unavailableTypedValidator(kind contracts.ArtifactKind) preparedValidator {
|
||||
return preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("typed/check"), Target: ValidatorTargetTyped, ArtifactKind: kind},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteExtractDisablesDownstreamCheckpointIO(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
|
||||
}
|
||||
if spy.loads["merge/notes"] != 0 || spy.loads["normalize/notes"] != 0 {
|
||||
t.Fatalf("downstream loads = %#v, want none", spy.loads)
|
||||
}
|
||||
if spy.writesFor("merge", "notes") != 0 || spy.writesFor("normalize", "notes") != 0 {
|
||||
t.Fatalf("downstream writes = %#v, want none", spy.writes)
|
||||
}
|
||||
if spy.writes["extract/succeeded/notes"] != 0 {
|
||||
t.Fatalf("extract writes = %#v, want no reusable success", spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteMergeDisablesNormalizeCheckpointIO(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.MergeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.mergeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs))
|
||||
}
|
||||
if spy.loads["merge/notes"] != 1 || spy.loads["normalize/notes"] != 0 {
|
||||
t.Fatalf("loads = %#v, want merge lookup only", spy.loads)
|
||||
}
|
||||
if spy.writes["merge/succeeded/notes"] != 0 || spy.writesFor("normalize", "notes") != 0 {
|
||||
t.Fatalf("writes = %#v, want no reusable merge or normalize state", spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationIncompleteNormalizeIsNotPublished(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
lane.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || spy.writes["normalize/succeeded/notes"] != 0 {
|
||||
t.Fatalf("output/writes = %d / %#v, want in-memory output without normalize publication", len(output.NormalizeOutputs), spy.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedReferenceFromIncompleteValidationDisablesDependentCheckpointIO(t *testing.T) {
|
||||
input, _, _ := handoffFixture(t, codecNotes{Items: []string{"producer"}})
|
||||
prepared := input.Prepared
|
||||
producer := &prepared.Steps[0].lanes[0]
|
||||
consumer := &prepared.Steps[1].lanes[0]
|
||||
producer.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
producer.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(producer.resolved.ArtifactKind)}
|
||||
referenceItems := 0
|
||||
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
referenceItems = len(request.References.Slots["producer-output"].Items)
|
||||
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
|
||||
}
|
||||
spy := newReuseLineageCheckpointSpy()
|
||||
for _, stage := range []string{"extract", "merge", "normalize"} {
|
||||
spy.forbid[stage+"/score"] = struct{}{}
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if referenceItems != 1 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("current-run handoff = items %d outputs %d, want one generated item and two outputs", referenceItems, len(output.NormalizeOutputs))
|
||||
}
|
||||
for _, stage := range []string{"extract", "merge", "normalize"} {
|
||||
if spy.writesFor(stage, "score") != 0 {
|
||||
t.Fatalf("dependent writes = %#v, want none for %s", spy.writes, stage)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunnerCorrectsRejectedMergeAndNormalizeCandidates(t *testing.T) {
|
||||
for _, target := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
t.Run(string(target), func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
var correction *contracts.SemanticCorrection
|
||||
validator := preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("correction-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
value := target.value.(codecNotes)
|
||||
if firstNote(value) == "invalid" {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "candidate needs correction", CorrectionGuidance: "produce the accepted value"}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
},
|
||||
}
|
||||
operation := func(observed *contracts.SemanticCorrection) (erasedTypedResult, error) {
|
||||
if observed != nil {
|
||||
cloned, err := contracts.CloneSemanticCorrection(observed)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
correction = cloned
|
||||
}
|
||||
item, response := "invalid", `{"items":["invalid"]}`
|
||||
if observed != nil {
|
||||
item, response = "accepted", `{"items":["accepted"]}`
|
||||
}
|
||||
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{item}}, ModelCandidate: candidate}, nil
|
||||
}
|
||||
switch target {
|
||||
case StageMerge:
|
||||
lane.resolved.Merge.Retries = 1
|
||||
lane.mergeValidators.validators = []preparedValidator{validator}
|
||||
lane.typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
return operation(request.Correction)
|
||||
}
|
||||
case StageNormalize:
|
||||
lane.resolved.Normalize.Retries = 1
|
||||
lane.normalizeValidators.validators = []preparedValidator{validator}
|
||||
lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return operation(request.Correction)
|
||||
}
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || !strings.Contains(correction.UserGuidance, "produce the accepted value") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "invalid_candidate") || strings.Contains(correction.UserGuidance, "candidate needs correction") {
|
||||
t.Fatalf("%s correction = %#v, want exact rejected response and semantic replacement guidance only", target, correction)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted output", output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRejectsDeterministicMergeCandidateWithoutCorrection(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.Merge.Retries = 1
|
||||
lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
calls := 0
|
||||
lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"invalid"}}}, nil
|
||||
}
|
||||
lane.mergeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
},
|
||||
}}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejection", err)
|
||||
}
|
||||
if calls != 1 || len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != 1 {
|
||||
t.Fatalf("merge calls = %d output = %#v, want one deterministic terminal rejection", calls, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerContinuesNormalizeAfterValidatorFailure(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
lane.normalizeValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("unavailable"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("validator unavailable")
|
||||
},
|
||||
}}
|
||||
checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: checkpoints})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want incomplete accepted output", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 || checkpoints.normalizeSucceeded != 0 {
|
||||
t.Fatalf("run output = %#v checkpoints = %#v, want non-checkpointed incomplete normalized output", output, checkpoints)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") {
|
||||
t.Fatalf("manifest validation = %#v, want incomplete normalization provenance", output.Manifest)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
|
||||
t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics)
|
||||
}
|
||||
}
|
||||
@@ -3,34 +3,39 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunnerUsesProvidedSessionWithoutSourceFallback(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
session string
|
||||
wantSet bool
|
||||
want string
|
||||
}{
|
||||
{name: "provided", session: " supplied-session ", wantSet: true, want: "supplied-session"},
|
||||
{name: "empty", wantSet: false},
|
||||
{name: "provided", session: " supplied-session ", want: "supplied-session"},
|
||||
{name: "empty"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
prepared, _ := preparedTerminalDebugPipeline(t)
|
||||
output, err := New().Run(context.Background(), RunInput{
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
var session string
|
||||
calls := 0
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
calls++
|
||||
session = request.SessionID
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
_, err := New().Run(context.Background(), RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: []byte("input"),
|
||||
SessionID: test.session,
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, found := output.Manifest.Metadata["session_id"]
|
||||
if found != test.wantSet {
|
||||
t.Fatalf("manifest session presence = %t, want %t; metadata = %#v", found, test.wantSet, output.Manifest.Metadata)
|
||||
}
|
||||
if found && value != test.want {
|
||||
t.Fatalf("manifest session = %#v, want %q", value, test.want)
|
||||
if calls != 1 || session != test.want {
|
||||
t.Fatalf("extract calls=%d session=%q, want one call and %q", calls, session, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,472 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type terminalChunker struct {
|
||||
key string
|
||||
plan source.ChunkPlan
|
||||
diagnostics []contracts.ProducerDiagnostic
|
||||
err error
|
||||
calls *int
|
||||
}
|
||||
|
||||
func (c terminalChunker) Key() string { return c.key }
|
||||
|
||||
func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
if c.calls != nil {
|
||||
(*c.calls)++
|
||||
}
|
||||
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: contracts.CloneProducerDiagnostics(c.diagnostics)}, c.err
|
||||
}
|
||||
|
||||
type terminalChunkValidator struct {
|
||||
result contracts.ValidationResult
|
||||
err error
|
||||
}
|
||||
|
||||
type retryingLLMChunkValidator struct {
|
||||
requests []contracts.ChunkValidationRequest
|
||||
calls int
|
||||
}
|
||||
|
||||
type observingChunkValidator struct {
|
||||
request contracts.ChunkValidationRequest
|
||||
}
|
||||
|
||||
type observingSerializedValidator struct {
|
||||
request contracts.SerializedValidationRequest
|
||||
}
|
||||
|
||||
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
||||
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
v.request = request
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (*observingSerializedValidator) Name() string { return "observing/serialized-validator" }
|
||||
func (*observingSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *observingSerializedValidator) Validate(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
||||
v.request = request
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
||||
|
||||
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return v.result, v.err
|
||||
}
|
||||
|
||||
func (*retryingLLMChunkValidator) Name() string { return "retrying/llm-chunk-validator" }
|
||||
|
||||
func (*retryingLLMChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (validator *retryingLLMChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
validator.calls++
|
||||
validator.requests = append(validator.requests, request)
|
||||
if validator.calls == 1 && len(request.Chunks) > 0 {
|
||||
request.Chunks[0].ID = "mutated"
|
||||
}
|
||||
if validator.calls == 1 {
|
||||
return contracts.ValidationResult{}, errors.New("temporary validator failure")
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) {
|
||||
t.Helper()
|
||||
marker := strings.TrimSuffix(prefix, "/") + "/attempt-"
|
||||
var got []int
|
||||
for _, name := range debug.names() {
|
||||
if !strings.HasPrefix(name, marker) || !strings.HasSuffix(name, ".json") {
|
||||
continue
|
||||
}
|
||||
remainder := strings.TrimSuffix(strings.TrimPrefix(name, marker), ".json")
|
||||
if strings.Contains(remainder, "/") {
|
||||
continue
|
||||
}
|
||||
value, err := strconv.Atoi(remainder)
|
||||
if err != nil {
|
||||
t.Fatalf("parse attempt index from %q: %v", name, err)
|
||||
}
|
||||
got = append(got, value)
|
||||
}
|
||||
sort.Ints(got)
|
||||
if !reflect.DeepEqual(got, attempts) {
|
||||
t.Fatalf("attempt envelopes under %q = %#v, want %#v; names = %#v", prefix, got, attempts, debug.names())
|
||||
}
|
||||
}
|
||||
|
||||
func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) {
|
||||
t.Helper()
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.resolved.ChunkValidationPolicy = ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: ValidatorFailureFailRun,
|
||||
}
|
||||
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: ValidatorFailureFailRun,
|
||||
}
|
||||
prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy = ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: ValidatorFailureFailRun,
|
||||
}
|
||||
prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy = ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: ValidatorFailureFailRun,
|
||||
}
|
||||
chunker, ok := prepared.chunker.(*typedTestChunker)
|
||||
if !ok {
|
||||
t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
|
||||
}
|
||||
return prepared, source.CloneChunkPlan(chunker.plan)
|
||||
}
|
||||
|
||||
func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
moduleError error
|
||||
validator terminalChunkValidator
|
||||
wantError string
|
||||
wantRejection bool
|
||||
attemptError bool
|
||||
}{
|
||||
{name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}},
|
||||
{name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed", attemptError: true},
|
||||
{name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected", CorrectionGuidance: "return an acceptable chunk plan"}}, wantRejection: true},
|
||||
{name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("chunk", "chunk diagnostic")}, err: tc.moduleError}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
|
||||
envelope := debug.envelope(t, "chunk/attempt-01.json")
|
||||
if tc.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if tc.wantRejection {
|
||||
if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "chunk_rejected" {
|
||||
t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
|
||||
}
|
||||
}
|
||||
if tc.name == "accepted" {
|
||||
encoded := string(debug.json["chunk/attempt-01.json"])
|
||||
if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) {
|
||||
t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetriesLLMValidatorsWithoutRegeneratingChunkCandidate(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
chunkCalls := 0
|
||||
prepared.resolved.Chunk.Retries = 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &chunkCalls}
|
||||
validator := &retryingLLMChunkValidator{}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: ModuleBinding{Module: validator.Name(), Retries: 1}, ExecutionClass: contracts.ExecutionClassLLMBacked, Target: ValidatorTargetChunk},
|
||||
chunk: validator,
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if chunkCalls != 1 || validator.calls != 2 {
|
||||
t.Fatalf("producer calls = %d validator calls = %d, want 1 and 2", chunkCalls, validator.calls)
|
||||
}
|
||||
if len(validator.requests) != 2 || len(validator.requests[0].Chunks) == 0 || len(validator.requests[1].Chunks) == 0 || validator.requests[1].Chunks[0].ID == "mutated" {
|
||||
t.Fatalf("validator requests = %#v, want independently owned immutable candidates", validator.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||
validator := &observingChunkValidator{}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
|
||||
chunk: validator,
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(validator.request.Chunks) != 1 {
|
||||
t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
|
||||
}
|
||||
chunk := validator.request.Chunks[0]
|
||||
if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
|
||||
t.Fatalf("materialized chunk identity = %#v", chunk)
|
||||
}
|
||||
if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
|
||||
t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
|
||||
}
|
||||
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
|
||||
t.Fatalf("materialized metadata = %#v", chunk.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerForwardsDetachedRepairAttemptsToChunkValidators(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||
chunkValidator := &observingChunkValidator{}
|
||||
serializedValidator := &observingSerializedValidator{}
|
||||
chunkAttempts := 2
|
||||
serializedAttempts := 3
|
||||
chunkBinding := Binding(chunkValidator.Name())
|
||||
chunkBinding.StructuredOutputRepairAttempts = &chunkAttempts
|
||||
serializedBinding := Binding(serializedValidator.Name())
|
||||
serializedBinding.StructuredOutputRepairAttempts = &serializedAttempts
|
||||
prepared.chunkValidators.validators = []preparedValidator{
|
||||
{resolved: ResolvedValidator{Binding: chunkBinding, Target: ValidatorTargetChunk}, chunk: chunkValidator},
|
||||
{resolved: ResolvedValidator{Binding: serializedBinding, Target: ValidatorTargetSerialized}, serialized: serializedValidator},
|
||||
}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if got := chunkValidator.request.StructuredOutputRepairAttempts; got == nil || *got != chunkAttempts || got == chunkBinding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("chunk validator repair attempts = %v, want detached value %d", got, chunkAttempts)
|
||||
}
|
||||
if got := serializedValidator.request.StructuredOutputRepairAttempts; got == nil || *got != serializedAttempts || got == serializedBinding.StructuredOutputRepairAttempts {
|
||||
t.Fatalf("serialized validator repair attempts = %v, want detached value %d", got, serializedAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
|
||||
prepared.resolved.Chunk.Retries = 1
|
||||
calls := 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err == nil || !strings.Contains(err.Error(), "valid JSON") {
|
||||
t.Fatalf("Run() error = %v, want malformed annotation error", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("plan calls = %d, want two attempts", calls)
|
||||
}
|
||||
assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2)
|
||||
}
|
||||
|
||||
func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
moduleError error
|
||||
validatorErr error
|
||||
reject bool
|
||||
candidateFail bool
|
||||
finalFail bool
|
||||
wantError string
|
||||
attemptError bool
|
||||
}{
|
||||
{name: "terminal rejection", reject: true},
|
||||
{name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed", attemptError: true},
|
||||
{name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"},
|
||||
{name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate", attemptError: true},
|
||||
{name: "final codec error", finalFail: true, wantError: "serialize accepted extract output", attemptError: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionRejectOutput,
|
||||
ValidatorFailure: ValidatorFailureFailRun,
|
||||
}
|
||||
value := "extract-terminal"
|
||||
codec := &observedNotesCodec{}
|
||||
if tc.candidateFail {
|
||||
codec.candidateError = value
|
||||
}
|
||||
if tc.finalFail {
|
||||
codec.finalError = value
|
||||
}
|
||||
installObservedNotesCodec(t, prepared, codec)
|
||||
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
if tc.moduleError != nil {
|
||||
return erasedTypedResult{}, tc.moduleError
|
||||
}
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{value}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("extract", "extract diagnostic")}}, nil
|
||||
})
|
||||
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected", CorrectionGuidance: "return an acceptable extract"}, tc.validatorErr
|
||||
},
|
||||
}}
|
||||
debug := newCapturedDebugRecorder()
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1)
|
||||
attemptPath := "extract/notes/chunk-000001/attempt-01.json"
|
||||
envelope := debug.envelope(t, attemptPath)
|
||||
if tc.wantError != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError)
|
||||
}
|
||||
if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) {
|
||||
t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError)
|
||||
}
|
||||
if !tc.attemptError && envelope.Error != "" {
|
||||
t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil rejection", err)
|
||||
}
|
||||
if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "extract_rejected" {
|
||||
t.Fatalf("extract rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
|
||||
t.Run("chunk", func(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
primary := errors.New("chunk operation failed")
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary}
|
||||
debug := newCapturedDebugRecorder()
|
||||
debug.failPath = "chunk/attempt-01.json"
|
||||
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
|
||||
t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipeline(t)
|
||||
primary := errors.New("extract operation failed")
|
||||
installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{}, primary
|
||||
})
|
||||
debug := newCapturedDebugRecorder()
|
||||
debug.failPath = "extract/notes/chunk-000001/attempt-01.json"
|
||||
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write extract attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
|
||||
t.Fatalf("Run() error = %v, want joined operation and debug errors", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
|
||||
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||
prepared.resolved.Chunk.Retries = 1
|
||||
calls := 0
|
||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||
prepared.chunkValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
|
||||
chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
|
||||
}}
|
||||
debug := newCapturedDebugRecorder()
|
||||
debug.failPath = "chunk/attempt-01.json"
|
||||
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||
if err == nil || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") {
|
||||
t.Fatalf("Run() error = %v, want terminal attempt debug failure", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("chunk calls = %d, want one attempt without retry", calls)
|
||||
}
|
||||
if debug.has("chunk/attempt-02.json") {
|
||||
t.Fatal("second chunk attempt envelope exists after non-retryable debug persistence failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) {
|
||||
prepared := preparedAttemptDebugPipelineWithChunks(t, 2)
|
||||
debug := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(attemptDebugLLM{}, debug)
|
||||
installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "extract-module"); err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{
|
||||
resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
|
||||
typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
if err := callAttemptDebugLLM(ctx, client, "extract-validator"); err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
},
|
||||
}}
|
||||
|
||||
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
validatorResponses := make(map[string]struct{})
|
||||
for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ {
|
||||
chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber)
|
||||
module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json")
|
||||
validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json")
|
||||
if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") {
|
||||
t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls)
|
||||
}
|
||||
if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") {
|
||||
t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls)
|
||||
}
|
||||
if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID {
|
||||
t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls)
|
||||
}
|
||||
validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{}
|
||||
}
|
||||
if len(validatorResponses) != 2 {
|
||||
t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses)
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,10 @@ func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
||||
invocation *contracts.CacheMode
|
||||
want contracts.ProducerControls
|
||||
}{
|
||||
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheAuto}},
|
||||
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModePermissive, 5, contracts.CacheRefresh}},
|
||||
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{contracts.ModeStrict, 0, contracts.CacheAuto}},
|
||||
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheBypass}},
|
||||
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheAuto}},
|
||||
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5, Cache: contracts.CacheRefresh}},
|
||||
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheAuto}},
|
||||
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -75,13 +75,13 @@ func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolved.ChunkControls != (contracts.ProducerControls{contracts.ModePermissive, 0, contracts.CacheBypass}) {
|
||||
if resolved.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
|
||||
t.Fatalf("chunk controls = %#v", resolved.ChunkControls)
|
||||
}
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
for _, got := range []contracts.ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
|
||||
if got != (contracts.ProducerControls{contracts.ModePermissive, 2, contracts.CacheBypass}) {
|
||||
if got != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}) {
|
||||
t.Fatalf("lane controls = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user