From c56c7d7ee9dc66df65a4a929ae81ea4ef184673f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 21 Nov 2026 23:20:17 +0000 Subject: [PATCH] Centralize candidate assessment and explicit producer fallback handling --- docs/roadmap/implementation.md | 15 + internal/framework/contracts/assessment.go | 71 +++ .../framework/contracts/assessment_test.go | 57 ++ .../pipeline/candidate_assessment.go | 195 +++++++ .../pipeline/candidate_assessment_test.go | 223 ++++++++ .../framework/pipeline/producer_attempts.go | 440 --------------- .../pipeline/producer_attempts_test.go | 530 ------------------ .../framework/pipeline/producer_outcome.go | 42 ++ .../pipeline/producer_outcome_test.go | 64 +++ internal/framework/pipeline/runner.go | 55 -- internal/framework/pipeline/runner_typed.go | 77 --- .../framework/pipeline/validation_executor.go | 227 -------- .../pipeline/validation_executor_test.go | 241 -------- 13 files changed, 667 insertions(+), 1570 deletions(-) create mode 100644 internal/framework/contracts/assessment.go create mode 100644 internal/framework/contracts/assessment_test.go create mode 100644 internal/framework/pipeline/candidate_assessment.go create mode 100644 internal/framework/pipeline/candidate_assessment_test.go delete mode 100644 internal/framework/pipeline/producer_attempts.go delete mode 100644 internal/framework/pipeline/producer_attempts_test.go create mode 100644 internal/framework/pipeline/producer_outcome.go create mode 100644 internal/framework/pipeline/producer_outcome_test.go delete mode 100644 internal/framework/pipeline/validation_executor.go delete mode 100644 internal/framework/pipeline/validation_executor_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 31da6921..cfdf3fbc 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -408,6 +408,21 @@ need not migrate every module caller. ## Stage 9: Centralize Candidate Assessment And Terminal Disposition +**Status:** Complete. Replaced retrying validator execution and the producer +attempt state machine with one ordered assessment path and strict/permissive +terminal handling. Typed and materialized-chunk assessors give each validator +detached candidate/context views and preserve rejection guidance separately from +failed/unavailable assessments and their native causes. Guidance aggregation has +no hard-coded correction assignment. Fallback construction runs once without +ordinary revalidation, never on caller cancellation, and cannot claim acceptance +or reuse eligibility. Added complete-chain, revised/unchanged candidate, isolation, +guidance-contract, unavailable-assessment, timeout, and cancellation coverage. +Contracts tests/vet, isolated terminal-mode tests, and `git diff --check` pass. +Full pipeline tests remain blocked by the recorded old ConfigKit YAML import; +production compilation also awaits migration of runner/debug/diagnostic consumers +of removed attempt/report types. No blocked test is claimed passing; runner and +native continuation integration remain assigned to Stages 10–11. + **Goal:** use one complete assessment path for initial and revised candidates. **Inspect/change:** `validation_executor.go`, `validation_policy.go`, diff --git a/internal/framework/contracts/assessment.go b/internal/framework/contracts/assessment.go new file mode 100644 index 00000000..ad329819 --- /dev/null +++ b/internal/framework/contracts/assessment.go @@ -0,0 +1,71 @@ +package contracts + +import ( + "errors" + "fmt" + "strings" +) + +// ValidationUnavailableError marks an assessment that could not be performed, +// rather than a detected defect in the candidate. Its cause remains inspectable. +type ValidationUnavailableError struct{ Cause error } + +func (e *ValidationUnavailableError) Error() string { + if e.Cause == nil { + return "validation unavailable" + } + return "validation unavailable: " + e.Cause.Error() +} + +func (e *ValidationUnavailableError) Unwrap() error { return e.Cause } + +// Accepted requires approval by every applicable validator. An empty configured +// chain has no additional acceptance criteria. +func (a CandidateAssessment) Accepted() bool { + for _, finding := range a.Validators { + if finding.Decision != AssessmentApproved { + return false + } + } + return true +} + +// Guidance returns only actionable rejections, in validator order, with exact +// duplicates removed. Domain correction templates own the surrounding assignment. +func (a CandidateAssessment) Guidance() ([]string, error) { + var guidance []string + seen := make(map[string]bool) + for _, finding := range a.Validators { + if finding.Decision != AssessmentRejected { + continue + } + text := finding.Result.CorrectionGuidance + if strings.TrimSpace(text) == "" { + return nil, fmt.Errorf("validator %q rejected output without correction guidance", finding.ValidatorName) + } + if !seen[text] { + seen[text] = true + guidance = append(guidance, text) + } + } + return guidance, nil +} + +// Err explains nonacceptance while preserving underlying operational causes. +// Neither this operator-facing text nor reason codes are correction instructions. +func (a CandidateAssessment) Err() error { + var failures []error + for _, finding := range a.Validators { + if finding.Decision == AssessmentApproved { + continue + } + if finding.Failure != nil { + failures = append(failures, fmt.Errorf("validator %q %s: %w", finding.ValidatorName, finding.Decision, finding.Failure)) + } else if finding.Result.Message != "" { + failures = append(failures, fmt.Errorf("validator %q %s: %s", finding.ValidatorName, finding.Decision, finding.Result.Message)) + } else { + failures = append(failures, fmt.Errorf("validator %q did not approve the candidate (%s)", finding.ValidatorName, finding.Decision)) + } + } + return errors.Join(failures...) +} diff --git a/internal/framework/contracts/assessment_test.go b/internal/framework/contracts/assessment_test.go new file mode 100644 index 00000000..8df5070b --- /dev/null +++ b/internal/framework/contracts/assessment_test.go @@ -0,0 +1,57 @@ +package contracts + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func TestCandidateAssessmentKeepsGuidanceSeparateAndOrdered(t *testing.T) { + unavailable := errors.New("provider unavailable") + assessment := CandidateAssessment{Validators: []ValidatorAssessment{ + {ValidatorName: "shape", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "shape-code", Message: "operator prose", CorrectionGuidance: "Return the required shape."}}, + {ValidatorName: "coverage", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "coverage-code", CorrectionGuidance: "Include all supported events."}}, + {ValidatorName: "duplicate", Decision: AssessmentRejected, Result: ValidationResult{CorrectionGuidance: "Return the required shape."}}, + {ValidatorName: "remote", Decision: AssessmentUnavailable, Failure: &ValidationUnavailableError{Cause: unavailable}}, + }} + guidance, err := assessment.Guidance() + if err != nil || !reflect.DeepEqual(guidance, []string{"Return the required shape.", "Include all supported events."}) { + t.Fatalf("guidance = %#v, %v", guidance, err) + } + if assessment.Accepted() || !errors.Is(assessment.Err(), unavailable) { + t.Fatalf("assessment = %#v, error = %v", assessment, assessment.Err()) + } + for _, diagnostic := range []string{"shape-code", "coverage-code", "operator prose", "provider unavailable"} { + if strings.Contains(strings.Join(guidance, "\n"), diagnostic) { + t.Fatalf("diagnostic became correction instructions: %q", diagnostic) + } + } +} + +func TestCandidateAssessmentRequiresEveryApproval(t *testing.T) { + if !(CandidateAssessment{}).Accepted() { + t.Fatal("empty validator chain rejected") + } + for _, decision := range []AssessmentDecision{AssessmentApproved, AssessmentRejected, AssessmentFailed, AssessmentUnavailable, ""} { + assessment := CandidateAssessment{Validators: []ValidatorAssessment{{Decision: AssessmentApproved}, {Decision: decision}}} + if got, want := assessment.Accepted(), decision == AssessmentApproved; got != want { + t.Fatalf("%s accepted = %t", decision, got) + } + } +} + +func TestCandidateAssessmentDoesNotInventMissingGuidance(t *testing.T) { + for _, text := range []string{"", " \t\n"} { + assessment := CandidateAssessment{Validators: []ValidatorAssessment{{ValidatorName: "validator", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "opaque", Message: "not instructions", CorrectionGuidance: text}}}} + if _, err := assessment.Guidance(); err == nil { + t.Fatal("missing actionable guidance accepted") + } + } + for _, decision := range []AssessmentDecision{AssessmentApproved, AssessmentFailed, AssessmentUnavailable} { + guidance, err := (CandidateAssessment{Validators: []ValidatorAssessment{{Decision: decision, Result: ValidationResult{Message: "not instructions"}}}}).Guidance() + if err != nil || len(guidance) != 0 { + t.Fatalf("nonrejection guidance = %#v, %v", guidance, err) + } + } +} diff --git a/internal/framework/pipeline/candidate_assessment.go b/internal/framework/pipeline/candidate_assessment.go new file mode 100644 index 00000000..702d9be5 --- /dev/null +++ b/internal/framework/pipeline/candidate_assessment.go @@ -0,0 +1,195 @@ +package pipeline + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +// executeValidationChain invokes every applicable validator once, in configured +// order. Only caller cancellation or a framework failure interrupts the chain. +func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) { + assessment := contracts.CandidateAssessment{} + if ctx == nil { + return assessment, errors.New("validator execution context must not be nil") + } + if invoke == nil { + return assessment, errors.New("validator invocation must not be nil") + } + if err := ctx.Err(); err != nil { + return assessment, err + } + for index, validator := range chain.validators { + if err := ctx.Err(); err != nil { + return assessment, err + } + validator.position = index + 1 + result, err := invoke(ctx, validator) + if canceled := ctx.Err(); canceled != nil { + return assessment, canceled + } + var frameworkError *assessmentFrameworkError + if errors.As(err, &frameworkError) { + return assessment, err + } + finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result} + finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics) + if err != nil { + finding.Decision, finding.Failure = contracts.AssessmentFailed, err + var unavailable *contracts.ValidationUnavailableError + if errors.As(err, &unavailable) { + finding.Decision = contracts.AssessmentUnavailable + } + } else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil { + finding.Decision = contracts.AssessmentFailed + finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr) + } else { + finding.Decision = contracts.AssessmentRejected + if result.Approved { + finding.Decision = contracts.AssessmentApproved + } + } + assessment.Validators = append(assessment.Validators, finding) + } + if err := ctx.Err(); err != nil { + return assessment, err + } + return assessment, nil +} + +type assessmentFrameworkError struct{ cause error } + +func (e *assessmentFrameworkError) Error() string { return e.cause.Error() } +func (e *assessmentFrameworkError) Unwrap() error { return e.cause } + +// typedCandidateAssessor binds one operation's complete chain. Each invocation +// captures the new candidate once and decodes a detached view for each validator. +func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] { + return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) { + if ctx == nil { + return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil") + } + if err := ctx.Err(); err != nil { + return contracts.CandidateAssessment{}, err + } + content, err := codec.encodeCandidate(value) + if err != nil { + return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err) + } + return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + request, err := isolatedValidationTarget(target) + if err != nil { + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err} + } + request.llmProfile = validator.resolved.Binding.LLMProfile + switch validator.resolved.Target { + case ValidatorTargetTyped: + request.value, err = codec.decodeCandidate(append([]byte(nil), content...)) + if err != nil { + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)} + } + if reflect.TypeOf(request.value) != codec.valueType { + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))} + } + return validator.typedValidate(ctx, validator.typed, request) + case ValidatorTargetSerialized: + return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{ + Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey, + Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput, + SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile, + Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks, + Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), + }) + default: + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)} + } + }) + } +} + +func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) { + request := target + request.sourceInput = target.sourceInput.Clone() + request.references = CloneReferenceSet(target.references) + var err error + request.metadata, err = source.CloneMetadata(target.metadata) + if err != nil { + return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err) + } + request.source, err = cloneAssessmentSource(target.source) + if err != nil { + return typedValidationTarget{}, err + } + request.chunk, err = cloneSourceChunkPtr(target.chunk) + if err != nil { + return typedValidationTarget{}, err + } + request.chunks, err = cloneSourceChunks(target.chunks) + return request, err +} + +func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) { + if doc == nil { + return nil, nil + } + copy := *doc + var err error + copy.Metadata, err = source.CloneMetadata(doc.Metadata) + if err != nil { + return nil, fmt.Errorf("clone validation source metadata: %w", err) + } + copy.Units, err = cloneSourceUnits(doc.Units) + if err != nil { + return nil, err + } + return ©, nil +} + +// chunkCandidateAssessor assesses materialized chunks, not raw model plans. +// The runner's plan callback owns materialization before this boundary. +func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] { + return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) { + if ctx == nil { + return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil") + } + if err := ctx.Err(); err != nil { + return contracts.CandidateAssessment{}, err + } + content, err := json.Marshal(chunks) + if err != nil { + return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err) + } + target := target + target.chunks = chunks + schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)} + return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + request, err := isolatedValidationTarget(target) + if err != nil { + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err} + } + request.llmProfile = validator.resolved.Binding.LLMProfile + switch validator.resolved.Target { + case ValidatorTargetChunk: + return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{ + ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID, + SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references, + LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks, + }) + case ValidatorTargetSerialized: + return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{ + Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID, + SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references, + LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks, + Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...), + }) + default: + return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)} + } + }) + } +} diff --git a/internal/framework/pipeline/candidate_assessment_test.go b/internal/framework/pipeline/candidate_assessment_test.go new file mode 100644 index 00000000..6e4bbaa0 --- /dev/null +++ b/internal/framework/pipeline/candidate_assessment_test.go @@ -0,0 +1,223 @@ +package pipeline + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) { + chain := preparedValidatorChain{} + names := []string{"first", "second", "unavailable", "failed", "timeout"} + for _, name := range names { + chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}}) + } + failure := errors.New("validator failed") + var called []string + assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + name := validator.resolved.Binding.Module + called = append(called, name) + switch name { + case "unavailable": + return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure} + case "failed": + return contracts.ValidationResult{}, failure + case "timeout": + return contracts.ValidationResult{}, context.DeadlineExceeded + default: + return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil + } + }) + if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() { + t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err) + } + var decisions []contracts.AssessmentDecision + for _, finding := range assessment.Validators { + decisions = append(decisions, finding.Decision) + } + want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed} + if !reflect.DeepEqual(decisions, want) { + t.Fatalf("decisions = %#v", decisions) + } + guidance, err := assessment.Guidance() + if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) { + t.Fatalf("guidance = %#v, %v", guidance, err) + } + if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) { + t.Fatalf("assessment lost underlying causes: %v", assessment.Err()) + } +} + +func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}} + assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { + calls++ + cancel() + return contracts.ValidationResult{Approved: true}, nil + }) + if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 { + t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls) + } +} + +func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) { + calls := 0 + chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}} + assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { + calls++ + return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil + }) + guidance, guidanceErr := assessment.Guidance() + if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil { + t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr) + } + for _, finding := range assessment.Validators { + if finding.Decision != contracts.AssessmentFailed { + t.Fatalf("invalid rejection = %#v", finding) + } + } +} + +type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) + +func (notesAssessmentValidator) Name() string { return "notes" } +func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { + return v(ctx, request) +} + +func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator { + t.Helper() + registry := NewValidatorRegistry() + if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil { + t.Fatal(err) + } + entry, _ := registry.typedEntry(name, "test/notes") + built, err := entry.builder(BuildRequest{}) + if err != nil { + t.Fatal(err) + } + return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate} +} + +func assessmentNotesCodec(t *testing.T) artifactCodecEntry { + t.Helper() + registry := NewArtifactCodecRegistry() + if err := RegisterArtifactCodec(registry, notesCodec()); err != nil { + t.Fatal(err) + } + entry, _, err := registry.entry("test/notes") + if err != nil { + t.Fatal(err) + } + return entry +} + +func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) { + candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}} + doc := typedTestDocument() + doc.Metadata = map[string]any{"label": "original"} + target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}} + first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { + request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed" + request.Source.Units[0].Text = "changed" + request.Source.Metadata["label"] = "changed" + request.Metadata["label"] = "changed" + return contracts.ValidationResult{Approved: true}, nil + }) + second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { + if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" { + t.Fatalf("validator received a mutated view: %#v", request) + } + return contracts.ValidationResult{Approved: true}, nil + }) + assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}}) + assessment, err := assess(context.Background(), candidate) + if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" { + t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate) + } +} + +func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) { + calls := []string{} + makeValidator := func(name, accepted string) preparedValidator { + return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { + value := request.Value.Items[0] + calls = append(calls, name+":"+value) + if value == accepted { + return contracts.ValidationResult{Approved: true}, nil + } + return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil + }) + } + assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}}) + for _, text := range []string{"one", "two", "two"} { + assessment, err := assess(context.Background(), codecNotes{Items: []string{text}}) + if err != nil || assessment.Accepted() { + t.Fatalf("assessment for %s = %#v, %v", text, assessment, err) + } + want := contracts.AssessmentRejected + if text == "one" { + want = contracts.AssessmentApproved + } + if assessment.Validators[0].Decision != want { + t.Fatalf("earlier approval survived revised candidate: %#v", assessment) + } + } + if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) { + t.Fatalf("assessments = %#v", calls) + } +} + +type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) + +func (chunkAssessmentValidator) Name() string { return "chunk assessment" } +func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { + return v(ctx, request) +} + +func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) { + chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}} + originalJSON, err := json.Marshal(chunks) + if err != nil { + t.Fatal(err) + } + calls := 0 + mutator := preparedValidator{ + resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk}, + chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { + calls++ + request.Chunks[0].Content[0] = 'x' + request.Chunks[0].Metadata["label"] = "changed" + return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil + }), + } + observer := preparedValidator{ + resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized}, + serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) { + calls++ + if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) { + t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks) + } + return contracts.ValidationResult{Approved: true}, nil + }), + } + assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks) + if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" { + t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks) + } +} diff --git a/internal/framework/pipeline/producer_attempts.go b/internal/framework/pipeline/producer_attempts.go deleted file mode 100644 index 4bb03131..00000000 --- a/internal/framework/pipeline/producer_attempts.go +++ /dev/null @@ -1,440 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "fmt" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -// producerAttemptKind records why a producer invocation followed the prior -// one. It is deliberately independent of any artifact family. -type producerAttemptKind string - -const ( - producerAttemptInitial producerAttemptKind = "initial" - producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry" - producerAttemptStructuralRetry producerAttemptKind = "structural_retry" - producerAttemptModuleRetry producerAttemptKind = "module_requested_retry" - producerAttemptSemanticRetry producerAttemptKind = "semantic_correction" -) - -type producerAttemptOutcome string - -const ( - producerAttemptAccepted producerAttemptOutcome = "accepted" - producerAttemptRejected producerAttemptOutcome = "rejected" - producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted" - producerAttemptRetried producerAttemptOutcome = "retried" - producerAttemptFailed producerAttemptOutcome = "failed" -) - -type producerTerminalAction string - -const ( - producerTerminalAccepted producerTerminalAction = "accepted" - producerTerminalRejected producerTerminalAction = "reject_output" - producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue" - producerTerminalFailed producerTerminalAction = "fail_run" -) - -// producerAttemptRequest contains only attempt-local control material. The -// producer reconstructs its ordinary request from its own durable inputs. -type producerAttemptRequest struct { - Number int - Kind producerAttemptKind - Correction *contracts.SemanticCorrection -} - -// producerRetryDirective asks for another producer invocation while retaining -// the current value as a safe fallback if its shared budget is exhausted. -// Artifact-specific adapters are responsible for validating and populating it. -type producerRetryDirective struct { - CorrectionGuidance string - FallbackDiagnostics []contracts.ProducerDiagnostic -} - -func (directive *producerRetryDirective) clone() *producerRetryDirective { - if directive == nil { - return nil - } - return &producerRetryDirective{ - CorrectionGuidance: directive.CorrectionGuidance, - FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics), - } -} - -// producerAttemptOutput is intentionally artifact-neutral. Value remains -// opaque to the state machine; Candidate is the attempt-local response that -// may support semantic correction. -type producerAttemptOutput struct { - Value any - Candidate *contracts.ModelCandidate - Diagnostics []contracts.ProducerDiagnostic - Retry *producerRetryDirective -} - -func (output producerAttemptOutput) clone() (producerAttemptOutput, error) { - candidate, err := contracts.CloneModelCandidate(output.Candidate) - if err != nil { - return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err) - } - return producerAttemptOutput{ - Value: output.Value, - Candidate: candidate, - Diagnostics: contracts.CloneProducerDiagnostics(output.Diagnostics), - Retry: output.Retry.clone(), - }, nil -} - -type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) -type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error) - -type producerAttemptConfig struct { - Retries int - Policy ValidationPolicy - AllowStructuralRetry bool -} - -// producerAttemptProvenance is ordered by producer invocation. It retains the -// settled validation report for later debug and durable-provenance adapters. -type producerAttemptProvenance struct { - Number int - Kind producerAttemptKind - Outcome producerAttemptOutcome - Validation validationReport -} - -func (provenance producerAttemptProvenance) clone() producerAttemptProvenance { - provenance.Validation = cloneValidationReport(provenance.Validation) - return provenance -} - -// producerAttemptTerminal describes the state-machine decision without -// materializing an artifact or persisting stage-specific diagnostics. -type producerAttemptTerminal struct { - Action producerTerminalAction - Value any - Diagnostics []contracts.ProducerDiagnostic - Rejection *contracts.RejectedOutput - Validation validationReport - ValidationIncomplete bool - Provenance []producerAttemptProvenance -} - -func (terminal producerAttemptTerminal) clone() producerAttemptTerminal { - terminal.Diagnostics = contracts.CloneProducerDiagnostics(terminal.Diagnostics) - if terminal.Rejection != nil { - rejection := *terminal.Rejection - rejection.Validation = cloneValidationSummaryPtr(rejection.Validation) - terminal.Rejection = &rejection - } - terminal.Validation = cloneValidationReport(terminal.Validation) - terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance) - return terminal -} - -func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance { - if len(provenance) == 0 { - return nil - } - cloned := make([]producerAttemptProvenance, len(provenance)) - for index, item := range provenance { - cloned[index] = item.clone() - } - return cloned -} - -func cloneValidationReport(report validationReport) validationReport { - return validationReport{records: report.Records()} -} - -func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) { - if ctx == nil { - return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil") - } - if config.Retries < 0 { - return producerAttemptTerminal{}, errors.New("producer retries must not be negative") - } - if produce == nil { - return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil") - } - if validate == nil { - return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil") - } - - attemptLimit := config.Retries + 1 - provenance := make([]producerAttemptProvenance, 0, attemptLimit) - kind := producerAttemptInitial - var correction *contracts.SemanticCorrection - - for number := 1; number <= attemptLimit; number++ { - if err := ctx.Err(); err != nil { - return failedProducerAttempt(provenance), err - } - requestCorrection, err := contracts.CloneSemanticCorrection(correction) - if err != nil { - return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err) - } - output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection}) - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - if isImmediateProducerFailure(err) { - return failedProducerAttempt(provenance), err - } - if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry { - if number < attemptLimit { - kind, correction = producerAttemptStructuralRetry, nil - continue - } - return applyStructuralTerminalPolicy(config.Policy, provenance, number, err) - } - if number < attemptLimit { - kind, correction = producerAttemptOperationalRetry, nil - continue - } - return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err) - } - if err := validateProducerAttemptDiagnostics(output); err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), err - } - - output, err = output.clone() - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), err - } - if output.Retry != nil && number < attemptLimit { - correction, err = moduleRetryCorrection(output) - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), err - } - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried}) - kind = producerAttemptModuleRetry - continue - } - if output.Retry != nil && output.Retry.CorrectionGuidance != "" { - if _, err := moduleRetryCorrection(output); err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), err - } - } - if output.Retry != nil { - output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...) - } - correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate) - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err) - } - - report, err := validate(ctx, output) - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed}) - return failedProducerAttempt(provenance), err - } - report = cloneValidationReport(report) - if err := ctx.Err(); err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report}) - return failedProducerAttempt(provenance), err - } - - if rejection := report.FirstRejection(); rejection != nil { - if correctionCandidate != nil { - if err := correctionCandidate.Validate(); err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report}) - return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err) - } - } - if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 { - correctionRequest, guidanceErr := report.CorrectionRequest() - if guidanceErr != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report}) - return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr) - } - correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest) - if err != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report}) - return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err) - } - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report}) - kind = producerAttemptSemanticRetry - continue - } - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report}) - return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection) - } - - if incomplete := firstIncompleteValidation(report); incomplete != nil { - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report}) - if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue { - return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil - } - return failedProducerAttempt(provenance), validatorFailureError(*incomplete) - } - - provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report}) - return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil - } - - return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically") -} - -func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) { - if output.Retry == nil || output.Retry.CorrectionGuidance == "" { - return nil, nil - } - if output.Candidate == nil { - return nil, errors.New("module retry correction guidance requires a model candidate") - } - if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 { - return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1) - } - correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance) - if err != nil { - return nil, fmt.Errorf("construct module retry semantic correction: %w", err) - } - return correction, nil -} - -func validateProducerAttemptDiagnostics(output producerAttemptOutput) error { - if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil { - return fmt.Errorf("producer returned invalid diagnostics: %w", err) - } - if output.Retry != nil { - if err := contracts.ValidateProducerDiagnostics(output.Retry.FallbackDiagnostics); err != nil { - return fmt.Errorf("producer returned invalid retry fallback diagnostics: %w", err) - } - } - return nil -} - -func isImmediateProducerFailure(err error) bool { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return true - } - var debugErr *attemptDebugPersistenceError - return errors.As(err, &debugErr) -} - -func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) { - switch policy.ProducerStructuralFailure { - case ProducerStructuralFailureRejectOutput: - return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil - case ProducerStructuralFailureFailRun: - return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err) - default: - return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure) - } -} - -func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) { - rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath} - switch policy.SemanticRejection { - case SemanticRejectionRejectOutput: - return producerAttemptTerminal{Action: producerTerminalRejected, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil - case SemanticRejectionFailRun: - return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message) - default: - return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection) - } -} - -func firstIncompleteValidation(report validationReport) *validationRecord { - for _, record := range report.records { - if record.outcome == validationFailed || record.outcome == validationSkipped { - clone := record.clone() - return &clone - } - } - return nil -} - -func cloneProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic) []contracts.ProducerDiagnostic { - return contracts.CloneProducerDiagnostics(diagnostics) -} - -// incompleteValidationDiagnostics reports every applicable validator that -// could not complete under warn_continue. It uses fixed text so provider -// errors and arbitrary validator prose cannot cross this boundary. -func incompleteValidationDiagnostics(report validationReport) []validationDiagnosticRecord { - diagnostics := make([]validationDiagnosticRecord, 0) - for _, record := range report.records { - if record.outcome != validationFailed && record.outcome != validationSkipped { - continue - } - diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: contracts.ProducerDiagnostic{ - Disposition: contracts.DiagnosticDispositionWarning, - Category: contracts.DiagnosticCategoryValidationIncomplete, - ReasonCode: "validator_execution_incomplete", - OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{ - Scope: record.validatorName, - Message: "Validator execution did not complete within its configured budget.", - }}, - }}) - } - return diagnostics -} - -// validationSummary projects a terminal state-machine result into the durable -// bounded representation used by manifests, rejections, and receipts. -func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary { - summary := artifacts.ValidationSummary{ - Stage: string(stage), - StepID: stepID, - LaneID: laneID, - ModuleKey: moduleKey, - ChunkID: chunkID, - ChunkIndex: chunkIndex, - ProducerAttemptCount: len(terminal.Provenance), - TerminalAction: string(terminal.Action), - } - switch { - case terminal.Action == producerTerminalRejected: - summary.Status = "rejected" - case terminal.Action == producerTerminalFailed: - summary.Status = "incomplete" - case terminal.ValidationIncomplete: - summary.Status = "incomplete" - default: - summary.Status = "complete" - } - seenValidators := make(map[string]struct{}) - seenReasons := make(map[string]struct{}) - seenIncomplete := make(map[string]struct{}) - for _, record := range terminal.Validation.records { - if record.reasonCode != "" { - if _, exists := seenReasons[record.reasonCode]; !exists { - seenReasons[record.reasonCode] = struct{}{} - summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode) - } - } - if record.outcome == validationRejected { - if _, exists := seenValidators[record.validatorName]; !exists { - seenValidators[record.validatorName] = struct{}{} - summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName) - } - } - if record.outcome == validationFailed || record.outcome == validationSkipped { - if _, exists := seenIncomplete[record.validatorName]; !exists { - seenIncomplete[record.validatorName] = struct{}{} - summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName) - } - } - } - if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" { - if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists { - summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode) - } - } - return summary -} - -func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal { - return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)} -} diff --git a/internal/framework/pipeline/producer_attempts_test.go b/internal/framework/pipeline/producer_attempts_test.go deleted file mode 100644 index e854e1cf..00000000 --- a/internal/framework/pipeline/producer_attempts_test.go +++ /dev/null @@ -1,530 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) { - candidate := attemptCandidate(t, "first response") - producerCalls := 0 - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - producerCalls++ - switch request.Number { - case 1: - return producerAttemptOutput{}, errors.New("temporary producer failure") - case 2: - return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput - case 3: - return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil - default: - t.Fatalf("unexpected producer attempt %d", request.Number) - return producerAttemptOutput{}, nil - } - }, approveAttempt) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" { - t.Fatalf("terminal = %#v, want accepted value", terminal) - } - if producerCalls != 3 { - t.Fatalf("producer calls = %d, want 3", producerCalls) - } - if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) { - t.Fatalf("attempt kinds = %v", got) - } -} - -func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) { - first := attemptCandidate(t, "first response") - second := attemptCandidate(t, "second response") - var corrections []*contracts.SemanticCorrection - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - if request.Correction != nil { - corrections = append(corrections, request.Correction) - } - switch request.Number { - case 1: - return producerAttemptOutput{Value: "one", Candidate: first}, nil - case 2: - return producerAttemptOutput{Value: "two", Candidate: second}, nil - case 3: - return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil - default: - t.Fatalf("unexpected producer attempt %d", request.Number) - return producerAttemptOutput{}, nil - } - }, func(_ context.Context, output producerAttemptOutput) (validationReport, error) { - if output.Value == "three" { - return validationReport{}, nil - } - return rejectedAttemptReport("semantic_defect"), nil - }) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted { - t.Fatalf("terminal action = %q, want accepted", terminal.Action) - } - if len(corrections) != 2 { - t.Fatalf("correction count = %d, want 2", len(corrections)) - } - if corrections[0] == corrections[1] { - t.Fatal("semantic corrections reused the same pointer") - } - if got := string(corrections[0].AssistantResponse); got != "first response" { - t.Fatalf("first correction response = %q", got) - } - if got := string(corrections[1].AssistantResponse); got != "second response" { - t.Fatalf("second correction response = %q", got) - } - if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) { - t.Fatalf("attempt kinds = %v", got) - } -} - -func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) { - tests := []struct { - name string - config producerAttemptConfig - produce producerAttemptProducer - validate producerAttemptValidator - wantAction producerTerminalAction - wantError bool - wantValue any - }{ - { - name: "structural reject output", - config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput - }, - validate: approveAttempt, - wantAction: producerTerminalRejected, - }, - { - name: "structural fail run", - config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput - }, - validate: approveAttempt, - wantAction: producerTerminalFailed, - wantError: true, - }, - { - name: "operational failure after retry exhaustion", - config: producerAttemptConfig{Policy: DefaultValidationPolicy()}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, errors.New("producer unavailable") - }, - validate: approveAttempt, - wantAction: producerTerminalFailed, - wantError: true, - }, - { - name: "semantic reject output", - config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil - }, - validate: func(context.Context, producerAttemptOutput) (validationReport, error) { - return rejectedAttemptReport("bad"), nil - }, - wantAction: producerTerminalRejected, - }, - { - name: "semantic fail run", - config: producerAttemptConfig{Policy: DefaultValidationPolicy()}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil - }, - validate: func(context.Context, producerAttemptOutput) (validationReport, error) { - return rejectedAttemptReport("bad"), nil - }, - wantAction: producerTerminalFailed, - wantError: true, - }, - { - name: "validator failure warns and continues", - config: producerAttemptConfig{Policy: DefaultValidationPolicy()}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{Value: "kept"}, nil - }, - validate: func(context.Context, producerAttemptOutput) (validationReport, error) { - return failedAttemptReport(), nil - }, - wantAction: producerTerminalIncompleteAccepted, - wantValue: "kept", - }, - { - name: "validator failure fails run", - config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}}, - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, nil - }, - validate: func(context.Context, producerAttemptOutput) (validationReport, error) { - return failedAttemptReport(), nil - }, - wantAction: producerTerminalFailed, - wantError: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate) - if (err != nil) != test.wantError { - t.Fatalf("error = %v, want error %t", err, test.wantError) - } - if terminal.Action != test.wantAction { - t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction) - } - if terminal.Value != test.wantValue { - t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue) - } - if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) { - t.Fatalf("rejected terminal = %#v, want rejection without value", terminal) - } - }) - } -} - -func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) { - calls := 0 - policy := DefaultValidationPolicy() - policy.SemanticRejection = SemanticRejectionRejectOutput - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - calls++ - return producerAttemptOutput{Value: "deterministic"}, nil - }, func(context.Context, producerAttemptOutput) (validationReport, error) { - return rejectedAttemptReport("deterministic_rejection"), nil - }) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 { - t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls) - } -} - -func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) { - t.Run("retry", func(t *testing.T) { - calls := 0 - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - calls++ - if request.Correction != nil { - t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction) - } - if request.Number == 1 { - return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil - } - return producerAttemptOutput{Value: "replacement"}, nil - }, approveAttempt) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 { - t.Fatalf("terminal = %#v, calls = %d", terminal, calls) - } - if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) { - t.Fatalf("attempt kinds = %v", got) - } - }) - - t.Run("feedback retry", func(t *testing.T) { - const ( - defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` - guidance = "Use only candidate numbers from the supplied candidate list. Return one complete corrected response." - ) - var observed *contracts.SemanticCorrection - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - if request.Number == 1 { - return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil - } - observed = request.Correction - return producerAttemptOutput{Value: "corrected"}, nil - }, approveAttempt) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" { - t.Fatalf("terminal = %#v, want corrected accepted value", terminal) - } - if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance { - t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed) - } - if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) { - t.Fatalf("attempt kinds = %v", got) - } - }) - - t.Run("feedback-free retry clears prior correction", func(t *testing.T) { - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - switch request.Number { - case 1: - return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil - case 2: - if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" { - t.Fatalf("second attempt correction = %#v", request.Correction) - } - return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil - case 3: - if request.Correction != nil { - t.Fatalf("third attempt retained stale correction %#v", request.Correction) - } - return producerAttemptOutput{Value: "accepted"}, nil - default: - t.Fatalf("unexpected producer attempt %d", request.Number) - return producerAttemptOutput{}, nil - } - }, approveAttempt) - if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" { - t.Fatalf("terminal = %#v, error = %v", terminal, err) - } - }) - - t.Run("fallback", func(t *testing.T) { - fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}} - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{fallbackDiagnostic}}}, nil - }, approveAttempt) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Diagnostics, []contracts.ProducerDiagnostic{fallbackDiagnostic}) { - t.Fatalf("terminal = %#v", terminal) - } - }) -} - -func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) { - producerCalls := 0 - validatorCalls := 0 - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - producerCalls++ - return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil - }, func(context.Context, producerAttemptOutput) (validationReport, error) { - validatorCalls++ - return validationReport{}, nil - }) - if err == nil || !strings.Contains(err.Error(), "requires a model candidate") { - t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err) - } - if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 { - t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls) - } -} - -func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) { - tests := []struct { - name string - output producerAttemptOutput - want string - }{ - { - name: "producer diagnostics", - output: producerAttemptOutput{Diagnostics: []contracts.ProducerDiagnostic{{}}}, - want: "producer returned invalid diagnostics", - }, - { - name: "retry fallback diagnostics", - output: producerAttemptOutput{Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{{}}}}, - want: "producer returned invalid retry fallback diagnostics", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - producerCalls := 0 - validatorCalls := 0 - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - producerCalls++ - return test.output, nil - }, func(context.Context, producerAttemptOutput) (validationReport, error) { - validatorCalls++ - return validationReport{}, nil - }) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("runProducerAttempts() error = %v, want %q", err, test.want) - } - if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 { - t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d; want immediate framework failure", terminal, producerCalls, validatorCalls) - } - }) - } -} - -func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) { - firstDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded warning")} - secondDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted warning")} - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - if request.Number == 1 { - return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Diagnostics: firstDiagnostics}, nil - } - return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Diagnostics: secondDiagnostics}, nil - }, func(_ context.Context, output producerAttemptOutput) (validationReport, error) { - if output.Value == "first" { - report := rejectedAttemptReport("defect") - report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")}) - return report, nil - } - return validationReport{}, nil - }) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalAccepted { - t.Fatalf("terminal action = %q, want accepted", terminal.Action) - } - if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) { - t.Fatalf("attempt kinds = %v", got) - } - if got := terminal.Diagnostics; !reflect.DeepEqual(got, secondDiagnostics) { - t.Fatalf("terminal diagnostics = %#v, want %#v", got, secondDiagnostics) - } - if len(terminal.Provenance[0].Validation.records) != 2 { - t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records) - } -} - -func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) { - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - if request.Number == 1 { - return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil - } - return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil - }, func(_ context.Context, output producerAttemptOutput) (validationReport, error) { - if output.Value == "first" { - return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil - } - return validationReport{}, nil - }) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3) - if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) { - t.Fatalf("summary = %#v", summary) - } - if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 { - t.Fatalf("corrected success summary retained prior findings: %#v", summary) - } -} - -func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) { - report := validationReport{records: []validationRecord{ - {validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")}, - {validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"}, - {validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")}, - }} - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{Value: "candidate"}, nil - }, func(context.Context, producerAttemptOutput) (validationReport, error) { - return report, nil - }) - if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) - } - if terminal.Action != producerTerminalIncompleteAccepted { - t.Fatalf("terminal action = %q", terminal.Action) - } - groups, groupErr := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil) - if groupErr != nil { - t.Fatalf("terminalDiagnosticGroups() error = %v", groupErr) - } - if len(groups) != 3 { - t.Fatalf("diagnostic groups = %#v, want every failed and skipped validator", groups) - } - for index, validator := range []string{"first", "second", "third"} { - if group := groups[index]; group.Origin.ValidatorKey != validator || group.Disposition != contracts.DiagnosticDispositionWarning || group.Category != contracts.DiagnosticCategoryValidationIncomplete || group.ReasonCode != "validator_execution_incomplete" { - t.Fatalf("diagnostic group %d = %#v, want incomplete warning for %q", index, group, validator) - } - } - summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0) - if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) { - t.Fatalf("summary = %#v", summary) - } -} - -func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) { - tests := []struct { - name string - context context.Context - produce producerAttemptProducer - }{ - { - name: "cancelled context", - context: cancelledAttemptContext(), - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, nil - }, - }, - { - name: "debug persistence failure", - context: context.Background(), - produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) { - return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")} - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - calls := 0 - producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - calls++ - return test.produce(ctx, request) - } - terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt) - if err == nil || terminal.Action != producerTerminalFailed { - t.Fatalf("terminal = %#v, error = %v", terminal, err) - } - if test.name == "cancelled context" && calls != 0 { - t.Fatalf("cancelled producer calls = %d, want 0", calls) - } - if test.name == "debug persistence failure" && calls != 1 { - t.Fatalf("debug failure producer calls = %d, want 1", calls) - } - }) - } -} - -func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate { - t.Helper() - candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1) - if err != nil { - t.Fatalf("NewModelCandidate() error = %v", err) - } - return candidate -} - -func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) { - return validationReport{}, nil -} - -func rejectedAttemptReport(reason string) validationReport { - return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}} -} - -func failedAttemptReport() validationReport { - return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}} -} - -func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind { - kinds := make([]producerAttemptKind, len(provenance)) - for index, item := range provenance { - kinds[index] = item.Kind - } - return kinds -} - -func cancelledAttemptContext() context.Context { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - return ctx -} diff --git a/internal/framework/pipeline/producer_outcome.go b/internal/framework/pipeline/producer_outcome.go new file mode 100644 index 00000000..07ecd73b --- /dev/null +++ b/internal/framework/pipeline/producer_outcome.go @@ -0,0 +1,42 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +// finishProducerFailure applies the resolved terminal mode without retrying +// production or running ordinary validators against a supported fallback. +func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) { + var zero T + if ctx == nil { + return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil") + } + if err := ctx.Err(); err != nil { + return zero, contracts.ProducerOutcome{}, err + } + if cause == nil { + cause = errors.New("producer did not yield an accepted result") + } + switch mode { + case contracts.ModeStrict: + return zero, contracts.ProducerOutcome{}, cause + case contracts.ModePermissive: + if fallback == nil { + return zero, contracts.ProducerOutcome{}, fmt.Errorf("producer has no supported fallback: %w", cause) + } + value, err := fallback(ctx) + if canceled := ctx.Err(); canceled != nil { + return zero, contracts.ProducerOutcome{}, canceled + } + if err != nil { + return zero, contracts.ProducerOutcome{}, errors.Join(cause, fmt.Errorf("construct producer fallback: %w", err)) + } + return value, contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: cause.Error(), ReuseEligible: false}, nil + default: + return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode) + } +} diff --git a/internal/framework/pipeline/producer_outcome_test.go b/internal/framework/pipeline/producer_outcome_test.go new file mode 100644 index 00000000..419febb7 --- /dev/null +++ b/internal/framework/pipeline/producer_outcome_test.go @@ -0,0 +1,64 @@ +package pipeline + +import ( + "context" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) { + failure := errors.New("candidate not accepted") + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + calls := 0 + value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil }) + if mode == contracts.ModeStrict { + if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" { + t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls) + } + } else if err != nil || calls != 1 || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || outcome.FallbackReason == "" { + t.Fatalf("permissive = %q %#v %v, calls %d", value, outcome, err, calls) + } + } +} + +func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) { + original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback") + _, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction }) + if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" { + t.Fatalf("outcome = %#v, error = %v", outcome, err) + } + if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) { + t.Fatalf("unsupported fallback = %v", err) + } +} + +func TestProducerCancellationNeverDeliversFallback(t *testing.T) { + for _, cancelDuringFallback := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + if !cancelDuringFallback { + cancel() + } + value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) { + calls++ + cancel() + return "not delivered", nil + }) + cancel() + if !errors.Is(err, context.Canceled) || value != "" || outcome.Disposition != "" { + t.Fatalf("cancellation = %q %#v %v", value, outcome, err) + } + if !cancelDuringFallback && calls != 0 { + t.Fatal("fallback started after cancellation") + } + } +} + +func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) { + value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil }) + if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback { + t.Fatalf("local timeout = %q %#v %v", value, outcome, err) + } +} diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 1343cbc6..a5c6e54c 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -492,61 +492,6 @@ func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (ret return last, nil } -func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) { - content, err := json.Marshal(chunks) - if err != nil { - return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err) - } - schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)} - report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) { - binding := item.resolved.Binding - started := time.Now().UTC() - attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt) - validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath) - var result contracts.ValidationResult - requestMetadata, cloneErr := cloneMetadata(metadata) - if cloneErr != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr)) - } - requestChunks, cloneErr := cloneSourceChunks(chunks) - if cloneErr != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr)) - } - switch item.resolved.Target { - case ValidatorTargetChunk: - result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks}) - case ValidatorTargetSerialized: - result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)}) - default: - return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)) - } - if err == nil { - err = contracts.ValidateValidationResult(result) - } - debugContent := debugContentEnvelope(content, "application/json", nil, nil) - debugContent.ContentDigest = debugContentDigest(content) - debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)} - if err != nil { - debugCall.Error = err.Error() - } - if err != nil { - validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err) - if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil { - return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))) - } - return validationInvocation{}, validationErr - } - if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)) - } - return validationInvocation{result: result}, nil - }) - if err != nil { - return report, err - } - return report, nil -} - func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain { for _, chain := range chains { if chain.Stage != stage { diff --git a/internal/framework/pipeline/runner_typed.go b/internal/framework/pipeline/runner_typed.go index f790809f..dd26d599 100644 --- a/internal/framework/pipeline/runner_typed.go +++ b/internal/framework/pipeline/runner_typed.go @@ -583,83 +583,6 @@ func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, m return nil } -func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) { - if len(chain.validators) > 0 && target.candidate == nil { - candidate, err := validationCandidateArtifact(codec, target) - if err != nil { - return validationReport{}, fmt.Errorf("serialize %s candidate for validation: %w", target.stage, err) - } - target.candidate = &candidate - } - report, err := executeValidationChain(ctx, chain, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) { - binding := item.resolved.Binding - var result contracts.ValidationResult - var err error - started := time.Now().UTC() - pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)} - if target.chunk != nil { - pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1)) - } - pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)) - attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt) - validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath) - requestTarget := target - requestTarget.sourceInput = target.sourceInput.Clone() - requestTarget.references = CloneReferenceSet(target.references) - requestTarget.metadata, err = cloneMetadata(target.metadata) - if err != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation metadata: %w", err)) - } - requestTarget.chunk, err = cloneSourceChunkPtr(target.chunk) - if err != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunk: %w", err)) - } - requestTarget.chunks, err = cloneSourceChunks(target.chunks) - if err != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunks: %w", err)) - } - switch item.resolved.Target { - case ValidatorTargetTyped: - candidateValue, decodeErr := decodeTypedValidationCandidate(codec, *target.candidate) - if decodeErr != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("decode %s candidate for typed validator %q: %w", target.stage, binding.Module, decodeErr)) - } - requestTarget.value = candidateValue - requestTarget.llmProfile = binding.LLMProfile - requestTarget.structuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts) - result, err = item.typedValidate(validatorCtx, item.typed, requestTarget) - case ValidatorTargetSerialized: - artifact := cloneCheckpointArtifact(*target.candidate) - result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)}) - default: - return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)) - } - if err == nil { - err = contracts.ValidateValidationResult(result) - } - artifact := cloneCheckpointArtifact(*target.candidate) - debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(artifact), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)} - if err != nil { - debugCall.Error = err.Error() - } - if err != nil { - validationErr := fmt.Errorf("validate typed %s output with validator %q: %w", target.stage, binding.Module, err) - if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil { - return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr))) - } - return validationInvocation{}, validationErr - } - if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil { - return validationInvocation{}, fatalValidationError(fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr)) - } - return validationInvocation{result: result}, nil - }) - if err != nil { - return report, err - } - return report, nil -} - func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput { rejection := report.FirstRejection() if rejection == nil { diff --git a/internal/framework/pipeline/validation_executor.go b/internal/framework/pipeline/validation_executor.go deleted file mode 100644 index bb1f8d02..00000000 --- a/internal/framework/pipeline/validation_executor.go +++ /dev/null @@ -1,227 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "fmt" - "strings" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type validationOutcome string - -const ( - validationApproved validationOutcome = "approved" - validationRejected validationOutcome = "rejected" - validationFailed validationOutcome = "failed" - validationSkipped validationOutcome = "skipped" -) - -const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n" - -// validationRecord captures the settled result of one configured validator. -// Its fields remain private so reports cannot expose mutable diagnostic storage. -type validationRecord struct { - validatorName string - outcome validationOutcome - attemptCount int - reasonCode string - message string - diagnosticPath string - diagnostics []contracts.ProducerDiagnostic - correctionGuidance string - failure error -} - -func (record validationRecord) clone() validationRecord { - record.diagnostics = contracts.CloneProducerDiagnostics(record.diagnostics) - return record -} - -// validationReport is the immutable, ordered result of one validator chain. -type validationReport struct { - records []validationRecord -} - -func (report validationReport) Records() []validationRecord { - records := make([]validationRecord, len(report.records)) - for index, record := range report.records { - records[index] = record.clone() - } - return records -} - -func (report validationReport) Diagnostics() []validationDiagnosticRecord { - var diagnostics []validationDiagnosticRecord - for _, record := range report.records { - if record.outcome != validationApproved && record.outcome != validationRejected { - continue - } - for _, diagnostic := range record.diagnostics { - diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: cloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0]}) - } - } - return diagnostics -} - -type validationDiagnosticRecord struct { - validatorName string - diagnostic contracts.ProducerDiagnostic -} - -func (report validationReport) FirstRejection() *validationRecord { - for _, record := range report.records { - if record.outcome == validationRejected { - clone := record.clone() - return &clone - } - } - return nil -} - -func (report validationReport) FirstFailure() *validationRecord { - for _, record := range report.records { - if record.outcome == validationFailed { - clone := record.clone() - return &clone - } - } - return nil -} - -func (report validationReport) CorrectionRequest() (string, error) { - seen := make(map[string]struct{}) - parts := make([]string, 0, len(report.records)) - for _, record := range report.records { - if record.outcome != validationRejected { - continue - } - guidance := strings.TrimSpace(record.correctionGuidance) - if guidance == "" { - return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName) - } - if _, exists := seen[guidance]; exists { - continue - } - seen[guidance] = struct{}{} - parts = append(parts, guidance) - } - if len(parts) == 0 { - return "", errors.New("validation report contains no correction guidance") - } - var builder strings.Builder - builder.WriteString(correctionRequestIntroduction) - for index, guidance := range parts { - fmt.Fprintf(&builder, "%d. %s", index+1, guidance) - if index+1 < len(parts) { - builder.WriteByte('\n') - } - } - request := builder.String() - if len(request) > contracts.MaxCorrectionGuidanceBytes { - return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes) - } - return request, nil -} - -type validationInvocation struct { - result contracts.ValidationResult - skipped bool - reason string - message string -} - -func skippedValidation(reason, message string) validationInvocation { - return validationInvocation{skipped: true, reason: strings.TrimSpace(reason), message: strings.TrimSpace(message)} -} - -type validationInvoker func(context.Context, preparedValidator, int) (validationInvocation, error) - -type validationFrameworkError struct{ err error } - -func (err validationFrameworkError) Error() string { return err.err.Error() } - -func (err validationFrameworkError) Unwrap() error { return err.err } - -func fatalValidationError(err error) error { - if err == nil { - return nil - } - return validationFrameworkError{err: err} -} - -func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke validationInvoker) (validationReport, error) { - if ctx == nil { - return validationReport{}, errors.New("validator execution context must not be nil") - } - if invoke == nil { - return validationReport{}, errors.New("validator invocation must not be nil") - } - report := validationReport{records: make([]validationRecord, 0, len(chain.validators))} - for index, validator := range chain.validators { - validator.position = index + 1 - if err := ctx.Err(); err != nil { - return validationReport{}, err - } - binding := validator.resolved.Binding - attemptLimit := 1 - if validator.resolved.ExecutionClass == contracts.ExecutionClassLLMBacked { - attemptLimit += binding.Retries - } - for attempt := 1; attempt <= attemptLimit; attempt++ { - if err := ctx.Err(); err != nil { - return validationReport{}, err - } - invocation, err := invoke(ctx, validator, attempt) - if err != nil { - var frameworkErr validationFrameworkError - if errors.As(err, &frameworkErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return validationReport{}, err - } - if attempt == attemptLimit { - report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator failed after retry exhaustion", failure: err}) - } - continue - } - if invocation.skipped { - report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationSkipped, attemptCount: attempt, reasonCode: invocation.reason, message: invocation.message}) - break - } - if err := contracts.ValidateProducerDiagnostics(invocation.result.Diagnostics); err != nil { - return validationReport{}, fatalValidationError(fmt.Errorf("validator %q returned invalid diagnostics: %w", binding.Module, err)) - } - if err := contracts.ValidateValidationResult(invocation.result); err != nil { - if attempt == attemptLimit { - report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator returned an invalid result", failure: err}) - } - continue - } - if invocation.result.Approved { - report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), diagnosticPath: invocation.result.DiagnosticArtifactPath}) - break - } - message := invocation.result.Message - if message == "" { - message = "output rejected" - } - report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), correctionGuidance: invocation.result.CorrectionGuidance}) - break - } - } - return report, nil -} - -func validatorFailureError(record validationRecord) error { - if record.failure != nil { - return record.failure - } - return fmt.Errorf("validator %q failed after %d attempt(s)", record.validatorName, record.attemptCount) -} - -func validatorAttemptPath(base string, attempt int) string { - if attempt == 1 { - return base - } - return fmt.Sprintf("%s-validator-%02d", base, attempt) -} diff --git a/internal/framework/pipeline/validation_executor_test.go b/internal/framework/pipeline/validation_executor_test.go deleted file mode 100644 index 5a2a3fc5..00000000 --- a/internal/framework/pipeline/validation_executor_test.go +++ /dev/null @@ -1,241 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) { - tests := []struct { - name string - chain preparedValidatorChain - invoke validationInvoker - want []validationOutcome - wantCalls []string - wantGuidance []string - wantDiagnostics []string - }{ - { - name: "all approved", - chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)), - invoke: validationSequence(map[string][]validationStep{ - "shape": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("shape")}}}}, - "refs": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("refs")}}}}, - }), - want: []validationOutcome{validationApproved, validationApproved}, - wantCalls: []string{"shape:1", "refs:1"}, - wantDiagnostics: []string{"shape", "refs"}, - }, - { - name: "multiple rejections deduplicate guidance", - chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)), - invoke: validationSequence(map[string][]validationStep{ - "shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}}, - "refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}}, - "coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}}, - }), - want: []validationOutcome{validationRejected, validationRejected, validationRejected}, - wantCalls: []string{"shape:1", "refs:1", "coverage:1"}, - wantGuidance: []string{"repair shape", "repair references"}, - }, - { - name: "rejection and failure both settle", - chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)), - invoke: validationSequence(map[string][]validationStep{ - "shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}}, - "remote": {{err: errors.New("offline")}}, - }), - want: []validationOutcome{validationRejected, validationFailed}, - wantCalls: []string{"shape:1", "remote:1"}, - wantGuidance: []string{"repair shape"}, - }, - { - name: "failure only has no correction guidance", - chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)), - invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}), - want: []validationOutcome{validationFailed}, - wantCalls: []string{"remote:1"}, - }, - { - name: "skipped is retained without guidance", - chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)), - invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}), - want: []validationOutcome{validationSkipped}, - wantCalls: []string{"optional:1"}, - }, - { - name: "LLM failure retries then succeeds", - chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)), - invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}), - want: []validationOutcome{validationApproved}, - wantCalls: []string{"remote:1", "remote:2", "remote:3"}, - }, - { - name: "LLM failure retry exhaustion records once", - chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)), - invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}), - want: []validationOutcome{validationFailed}, - wantCalls: []string{"remote:1", "remote:2"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - calls := []string(nil) - invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) { - calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt))) - return test.invoke(ctx, validator, attempt) - } - report, err := executeValidationChain(context.Background(), test.chain, invoke) - if err != nil { - t.Fatalf("executeValidationChain() error = %v", err) - } - records := report.Records() - outcomes := make([]validationOutcome, len(records)) - for index, record := range records { - outcomes[index] = record.outcome - } - if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) { - t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls) - } - if len(test.wantGuidance) > 0 { - guidance, err := report.CorrectionRequest() - if err != nil { - t.Fatalf("CorrectionRequest() error = %v", err) - } - if !strings.Contains(guidance, "complete corrected replacement") { - t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance) - } - for _, item := range test.wantGuidance { - if strings.Count(guidance, item) != 1 { - t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item) - } - } - } - diagnostics := report.Diagnostics() - var diagnosticCodes []string - for _, record := range diagnostics { - diagnosticCodes = append(diagnosticCodes, record.diagnostic.ReasonCode) - } - if !reflect.DeepEqual(diagnosticCodes, test.wantDiagnostics) { - t.Fatalf("Diagnostics() = %#v, want codes %#v", diagnostics, test.wantDiagnostics) - } - }) - } -} - -func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) { - candidate := []byte(`{"events":["one"]}`) - chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)) - var received [][]byte - report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) { - requestCandidate := append([]byte(nil), candidate...) - received = append(received, requestCandidate) - requestCandidate[0] = 'x' - if attempt == 1 { - return validationInvocation{}, errors.New("retry") - } - return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil - }) - if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 { - t.Fatalf("report = %#v, error = %v", report, err) - } - if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) { - t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received) - } -} - -func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) { - guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes) - report := validationReport{records: make([]validationRecord, 17)} - for index := range report.records { - unique := []byte(guidance) - unique[0] = byte('a' + index) - report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)} - } - report.records[0].diagnostics = []contracts.ProducerDiagnostic{validationDiagnostic("diagnostic")} - if _, err := report.CorrectionRequest(); err == nil { - t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error") - } - records := report.Records() - records[0].diagnostics[0].ReasonCode = "changed" - if report.Records()[0].diagnostics[0].ReasonCode != "diagnostic" { - t.Fatal("Records() exposed mutable diagnostic storage") - } -} - -func validationDiagnostic(reasonCode string) contracts.ProducerDiagnostic { - return contracts.ProducerDiagnostic{ - Disposition: contracts.DiagnosticDispositionObservation, - Category: contracts.DiagnosticCategoryNormalization, - ReasonCode: reasonCode, - OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{Scope: "validator", Message: reasonCode}}, - } -} - -func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) { - chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1)) - framework := errors.New("debug persistence failed") - if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) { - return validationInvocation{}, fatalValidationError(framework) - }); !errors.Is(err, framework) { - t.Fatalf("framework error = %v, want %v", err, framework) - } - canceled, cancel := context.WithCancel(context.Background()) - cancel() - if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) { - return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil - }); !errors.Is(err, context.Canceled) { - t.Fatalf("cancellation error = %v, want context canceled", err) - } -} - -func TestExecuteValidationChainRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) { - chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 2)) - calls := 0 - _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) { - calls++ - return validationInvocation{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{}}}}, nil - }) - var frameworkErr validationFrameworkError - if !errors.As(err, &frameworkErr) || calls != 1 || !strings.Contains(err.Error(), "validator \"validator\" returned invalid diagnostics") { - t.Fatalf("error = %v calls = %d, want one fatal invalid-diagnostics result", err, calls) - } -} - -type validationStep struct { - result contracts.ValidationResult - invocation validationInvocation - err error -} - -func validationSequence(steps map[string][]validationStep) validationInvoker { - positions := make(map[string]int) - return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) { - name := validator.resolved.Binding.Module - index := positions[name] - positions[name]++ - if index >= len(steps[name]) { - return validationInvocation{}, errors.New("unexpected validator invocation") - } - step := steps[name][index] - if step.invocation.skipped { - return step.invocation, step.err - } - return validationInvocation{result: step.result}, step.err - } -} - -func validationChain(validators ...preparedValidator) preparedValidatorChain { - return preparedValidatorChain{validators: validators} -} - -func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator { - return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}} -}