Centralize candidate assessment and explicit producer fallback handling

This commit is contained in:
2026-11-21 23:20:17 +00:00
parent 797320b4da
commit c56c7d7ee9
13 changed files with 667 additions and 1570 deletions

View File

@@ -0,0 +1,195 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// executeValidationChain invokes every applicable validator once, in configured
// order. Only caller cancellation or a framework failure interrupts the chain.
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) {
assessment := contracts.CandidateAssessment{}
if ctx == nil {
return assessment, errors.New("validator execution context must not be nil")
}
if invoke == nil {
return assessment, errors.New("validator invocation must not be nil")
}
if err := ctx.Err(); err != nil {
return assessment, err
}
for index, validator := range chain.validators {
if err := ctx.Err(); err != nil {
return assessment, err
}
validator.position = index + 1
result, err := invoke(ctx, validator)
if canceled := ctx.Err(); canceled != nil {
return assessment, canceled
}
var frameworkError *assessmentFrameworkError
if errors.As(err, &frameworkError) {
return assessment, err
}
finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result}
finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics)
if err != nil {
finding.Decision, finding.Failure = contracts.AssessmentFailed, err
var unavailable *contracts.ValidationUnavailableError
if errors.As(err, &unavailable) {
finding.Decision = contracts.AssessmentUnavailable
}
} else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil {
finding.Decision = contracts.AssessmentFailed
finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr)
} else {
finding.Decision = contracts.AssessmentRejected
if result.Approved {
finding.Decision = contracts.AssessmentApproved
}
}
assessment.Validators = append(assessment.Validators, finding)
}
if err := ctx.Err(); err != nil {
return assessment, err
}
return assessment, nil
}
type assessmentFrameworkError struct{ cause error }
func (e *assessmentFrameworkError) Error() string { return e.cause.Error() }
func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
// typedCandidateAssessor binds one operation's complete chain. Each invocation
// captures the new candidate once and decodes a detached view for each validator.
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.CandidateAssessment{}, err
}
content, err := codec.encodeCandidate(value)
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
switch validator.resolved.Target {
case ValidatorTargetTyped:
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)}
}
if reflect.TypeOf(request.value) != codec.valueType {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))}
}
return validator.typedValidate(ctx, validator.typed, request)
case ValidatorTargetSerialized:
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey,
Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput,
SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile,
Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks,
Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...),
})
default:
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)}
}
})
}
}
func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) {
request := target
request.sourceInput = target.sourceInput.Clone()
request.references = CloneReferenceSet(target.references)
var err error
request.metadata, err = source.CloneMetadata(target.metadata)
if err != nil {
return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err)
}
request.source, err = cloneAssessmentSource(target.source)
if err != nil {
return typedValidationTarget{}, err
}
request.chunk, err = cloneSourceChunkPtr(target.chunk)
if err != nil {
return typedValidationTarget{}, err
}
request.chunks, err = cloneSourceChunks(target.chunks)
return request, err
}
func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) {
if doc == nil {
return nil, nil
}
copy := *doc
var err error
copy.Metadata, err = source.CloneMetadata(doc.Metadata)
if err != nil {
return nil, fmt.Errorf("clone validation source metadata: %w", err)
}
copy.Units, err = cloneSourceUnits(doc.Units)
if err != nil {
return nil, err
}
return &copy, nil
}
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
// The runner's plan callback owns materialization before this boundary.
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.CandidateAssessment{}, err
}
content, err := json.Marshal(chunks)
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err)
}
target := target
target.chunks = chunks
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
switch validator.resolved.Target {
case ValidatorTargetChunk:
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
})
case ValidatorTargetSerialized:
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...),
})
default:
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)}
}
})
}
}

View File

@@ -0,0 +1,223 @@
package pipeline
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) {
chain := preparedValidatorChain{}
names := []string{"first", "second", "unavailable", "failed", "timeout"}
for _, name := range names {
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}})
}
failure := errors.New("validator failed")
var called []string
assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
name := validator.resolved.Binding.Module
called = append(called, name)
switch name {
case "unavailable":
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure}
case "failed":
return contracts.ValidationResult{}, failure
case "timeout":
return contracts.ValidationResult{}, context.DeadlineExceeded
default:
return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil
}
})
if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() {
t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err)
}
var decisions []contracts.AssessmentDecision
for _, finding := range assessment.Validators {
decisions = append(decisions, finding.Decision)
}
want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed}
if !reflect.DeepEqual(decisions, want) {
t.Fatalf("decisions = %#v", decisions)
}
guidance, err := assessment.Guidance()
if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) {
t.Fatalf("guidance = %#v, %v", guidance, err)
}
if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) {
t.Fatalf("assessment lost underlying causes: %v", assessment.Err())
}
}
func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls := 0
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
calls++
cancel()
return contracts.ValidationResult{Approved: true}, nil
})
if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 {
t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls)
}
}
func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) {
calls := 0
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
calls++
return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil
})
guidance, guidanceErr := assessment.Guidance()
if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil {
t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr)
}
for _, finding := range assessment.Validators {
if finding.Decision != contracts.AssessmentFailed {
t.Fatalf("invalid rejection = %#v", finding)
}
}
}
type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error)
func (notesAssessmentValidator) Name() string { return "notes" }
func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
return v(ctx, request)
}
func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator {
t.Helper()
registry := NewValidatorRegistry()
if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil {
t.Fatal(err)
}
entry, _ := registry.typedEntry(name, "test/notes")
built, err := entry.builder(BuildRequest{})
if err != nil {
t.Fatal(err)
}
return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate}
}
func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
t.Helper()
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatal(err)
}
entry, _, err := registry.entry("test/notes")
if err != nil {
t.Fatal(err)
}
return entry
}
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
doc := typedTestDocument()
doc.Metadata = map[string]any{"label": "original"}
target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}}
first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed"
request.Source.Units[0].Text = "changed"
request.Source.Metadata["label"] = "changed"
request.Metadata["label"] = "changed"
return contracts.ValidationResult{Approved: true}, nil
})
second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" {
t.Fatalf("validator received a mutated view: %#v", request)
}
return contracts.ValidationResult{Approved: true}, nil
})
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}})
assessment, err := assess(context.Background(), candidate)
if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" {
t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate)
}
}
func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) {
calls := []string{}
makeValidator := func(name, accepted string) preparedValidator {
return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
value := request.Value.Items[0]
calls = append(calls, name+":"+value)
if value == accepted {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil
})
}
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}})
for _, text := range []string{"one", "two", "two"} {
assessment, err := assess(context.Background(), codecNotes{Items: []string{text}})
if err != nil || assessment.Accepted() {
t.Fatalf("assessment for %s = %#v, %v", text, assessment, err)
}
want := contracts.AssessmentRejected
if text == "one" {
want = contracts.AssessmentApproved
}
if assessment.Validators[0].Decision != want {
t.Fatalf("earlier approval survived revised candidate: %#v", assessment)
}
}
if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) {
t.Fatalf("assessments = %#v", calls)
}
}
type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
func (chunkAssessmentValidator) Name() string { return "chunk assessment" }
func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return v(ctx, request)
}
func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) {
chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}}
originalJSON, err := json.Marshal(chunks)
if err != nil {
t.Fatal(err)
}
calls := 0
mutator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk},
chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
calls++
request.Chunks[0].Content[0] = 'x'
request.Chunks[0].Metadata["label"] = "changed"
return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil
}),
}
observer := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized},
serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
calls++
if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) {
t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks)
}
return contracts.ValidationResult{Approved: true}, nil
}),
}
assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks)
if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" {
t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks)
}
}

View File

@@ -1,440 +0,0 @@
package pipeline
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// producerAttemptKind records why a producer invocation followed the prior
// one. It is deliberately independent of any artifact family.
type producerAttemptKind string
const (
producerAttemptInitial producerAttemptKind = "initial"
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
)
type producerAttemptOutcome string
const (
producerAttemptAccepted producerAttemptOutcome = "accepted"
producerAttemptRejected producerAttemptOutcome = "rejected"
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
producerAttemptRetried producerAttemptOutcome = "retried"
producerAttemptFailed producerAttemptOutcome = "failed"
)
type producerTerminalAction string
const (
producerTerminalAccepted producerTerminalAction = "accepted"
producerTerminalRejected producerTerminalAction = "reject_output"
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
producerTerminalFailed producerTerminalAction = "fail_run"
)
// producerAttemptRequest contains only attempt-local control material. The
// producer reconstructs its ordinary request from its own durable inputs.
type producerAttemptRequest struct {
Number int
Kind producerAttemptKind
Correction *contracts.SemanticCorrection
}
// producerRetryDirective asks for another producer invocation while retaining
// the current value as a safe fallback if its shared budget is exhausted.
// Artifact-specific adapters are responsible for validating and populating it.
type producerRetryDirective struct {
CorrectionGuidance string
FallbackDiagnostics []contracts.ProducerDiagnostic
}
func (directive *producerRetryDirective) clone() *producerRetryDirective {
if directive == nil {
return nil
}
return &producerRetryDirective{
CorrectionGuidance: directive.CorrectionGuidance,
FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics),
}
}
// producerAttemptOutput is intentionally artifact-neutral. Value remains
// opaque to the state machine; Candidate is the attempt-local response that
// may support semantic correction.
type producerAttemptOutput struct {
Value any
Candidate *contracts.ModelCandidate
Diagnostics []contracts.ProducerDiagnostic
Retry *producerRetryDirective
}
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
candidate, err := contracts.CloneModelCandidate(output.Candidate)
if err != nil {
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
}
return producerAttemptOutput{
Value: output.Value,
Candidate: candidate,
Diagnostics: contracts.CloneProducerDiagnostics(output.Diagnostics),
Retry: output.Retry.clone(),
}, nil
}
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
type producerAttemptConfig struct {
Retries int
Policy ValidationPolicy
AllowStructuralRetry bool
}
// producerAttemptProvenance is ordered by producer invocation. It retains the
// settled validation report for later debug and durable-provenance adapters.
type producerAttemptProvenance struct {
Number int
Kind producerAttemptKind
Outcome producerAttemptOutcome
Validation validationReport
}
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
provenance.Validation = cloneValidationReport(provenance.Validation)
return provenance
}
// producerAttemptTerminal describes the state-machine decision without
// materializing an artifact or persisting stage-specific diagnostics.
type producerAttemptTerminal struct {
Action producerTerminalAction
Value any
Diagnostics []contracts.ProducerDiagnostic
Rejection *contracts.RejectedOutput
Validation validationReport
ValidationIncomplete bool
Provenance []producerAttemptProvenance
}
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
terminal.Diagnostics = contracts.CloneProducerDiagnostics(terminal.Diagnostics)
if terminal.Rejection != nil {
rejection := *terminal.Rejection
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
terminal.Rejection = &rejection
}
terminal.Validation = cloneValidationReport(terminal.Validation)
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
return terminal
}
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
if len(provenance) == 0 {
return nil
}
cloned := make([]producerAttemptProvenance, len(provenance))
for index, item := range provenance {
cloned[index] = item.clone()
}
return cloned
}
func cloneValidationReport(report validationReport) validationReport {
return validationReport{records: report.Records()}
}
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
if ctx == nil {
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
}
if config.Retries < 0 {
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
}
if produce == nil {
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
}
if validate == nil {
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
}
attemptLimit := config.Retries + 1
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
kind := producerAttemptInitial
var correction *contracts.SemanticCorrection
for number := 1; number <= attemptLimit; number++ {
if err := ctx.Err(); err != nil {
return failedProducerAttempt(provenance), err
}
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
if err != nil {
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
}
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
if isImmediateProducerFailure(err) {
return failedProducerAttempt(provenance), err
}
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
if number < attemptLimit {
kind, correction = producerAttemptStructuralRetry, nil
continue
}
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
}
if number < attemptLimit {
kind, correction = producerAttemptOperationalRetry, nil
continue
}
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
}
if err := validateProducerAttemptDiagnostics(output); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
output, err = output.clone()
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
if output.Retry != nil && number < attemptLimit {
correction, err = moduleRetryCorrection(output)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
kind = producerAttemptModuleRetry
continue
}
if output.Retry != nil && output.Retry.CorrectionGuidance != "" {
if _, err := moduleRetryCorrection(output); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
}
if output.Retry != nil {
output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...)
}
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
}
report, err := validate(ctx, output)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
return failedProducerAttempt(provenance), err
}
report = cloneValidationReport(report)
if err := ctx.Err(); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), err
}
if rejection := report.FirstRejection(); rejection != nil {
if correctionCandidate != nil {
if err := correctionCandidate.Validate(); err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
}
}
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
correctionRequest, guidanceErr := report.CorrectionRequest()
if guidanceErr != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
}
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
if err != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
kind = producerAttemptSemanticRetry
continue
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
}
if incomplete := firstIncompleteValidation(report); incomplete != nil {
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
}
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
}
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
}
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
}
func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) {
if output.Retry == nil || output.Retry.CorrectionGuidance == "" {
return nil, nil
}
if output.Candidate == nil {
return nil, errors.New("module retry correction guidance requires a model candidate")
}
if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 {
return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1)
}
correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance)
if err != nil {
return nil, fmt.Errorf("construct module retry semantic correction: %w", err)
}
return correction, nil
}
func validateProducerAttemptDiagnostics(output producerAttemptOutput) error {
if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil {
return fmt.Errorf("producer returned invalid diagnostics: %w", err)
}
if output.Retry != nil {
if err := contracts.ValidateProducerDiagnostics(output.Retry.FallbackDiagnostics); err != nil {
return fmt.Errorf("producer returned invalid retry fallback diagnostics: %w", err)
}
}
return nil
}
func isImmediateProducerFailure(err error) bool {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
var debugErr *attemptDebugPersistenceError
return errors.As(err, &debugErr)
}
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
switch policy.ProducerStructuralFailure {
case ProducerStructuralFailureRejectOutput:
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
case ProducerStructuralFailureFailRun:
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
default:
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
}
}
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
switch policy.SemanticRejection {
case SemanticRejectionRejectOutput:
return producerAttemptTerminal{Action: producerTerminalRejected, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
case SemanticRejectionFailRun:
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
default:
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
}
}
func firstIncompleteValidation(report validationReport) *validationRecord {
for _, record := range report.records {
if record.outcome == validationFailed || record.outcome == validationSkipped {
clone := record.clone()
return &clone
}
}
return nil
}
func cloneProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic) []contracts.ProducerDiagnostic {
return contracts.CloneProducerDiagnostics(diagnostics)
}
// incompleteValidationDiagnostics reports every applicable validator that
// could not complete under warn_continue. It uses fixed text so provider
// errors and arbitrary validator prose cannot cross this boundary.
func incompleteValidationDiagnostics(report validationReport) []validationDiagnosticRecord {
diagnostics := make([]validationDiagnosticRecord, 0)
for _, record := range report.records {
if record.outcome != validationFailed && record.outcome != validationSkipped {
continue
}
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryValidationIncomplete,
ReasonCode: "validator_execution_incomplete",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{
Scope: record.validatorName,
Message: "Validator execution did not complete within its configured budget.",
}},
}})
}
return diagnostics
}
// validationSummary projects a terminal state-machine result into the durable
// bounded representation used by manifests, rejections, and receipts.
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
summary := artifacts.ValidationSummary{
Stage: string(stage),
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
ChunkID: chunkID,
ChunkIndex: chunkIndex,
ProducerAttemptCount: len(terminal.Provenance),
TerminalAction: string(terminal.Action),
}
switch {
case terminal.Action == producerTerminalRejected:
summary.Status = "rejected"
case terminal.Action == producerTerminalFailed:
summary.Status = "incomplete"
case terminal.ValidationIncomplete:
summary.Status = "incomplete"
default:
summary.Status = "complete"
}
seenValidators := make(map[string]struct{})
seenReasons := make(map[string]struct{})
seenIncomplete := make(map[string]struct{})
for _, record := range terminal.Validation.records {
if record.reasonCode != "" {
if _, exists := seenReasons[record.reasonCode]; !exists {
seenReasons[record.reasonCode] = struct{}{}
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
}
}
if record.outcome == validationRejected {
if _, exists := seenValidators[record.validatorName]; !exists {
seenValidators[record.validatorName] = struct{}{}
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
}
}
if record.outcome == validationFailed || record.outcome == validationSkipped {
if _, exists := seenIncomplete[record.validatorName]; !exists {
seenIncomplete[record.validatorName] = struct{}{}
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
}
}
}
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
}
}
return summary
}
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
}

View File

@@ -1,530 +0,0 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) {
candidate := attemptCandidate(t, "first response")
producerCalls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
producerCalls++
switch request.Number {
case 1:
return producerAttemptOutput{}, errors.New("temporary producer failure")
case 2:
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
case 3:
return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil
default:
t.Fatalf("unexpected producer attempt %d", request.Number)
return producerAttemptOutput{}, nil
}
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
t.Fatalf("terminal = %#v, want accepted value", terminal)
}
if producerCalls != 3 {
t.Fatalf("producer calls = %d, want 3", producerCalls)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
}
func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) {
first := attemptCandidate(t, "first response")
second := attemptCandidate(t, "second response")
var corrections []*contracts.SemanticCorrection
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Correction != nil {
corrections = append(corrections, request.Correction)
}
switch request.Number {
case 1:
return producerAttemptOutput{Value: "one", Candidate: first}, nil
case 2:
return producerAttemptOutput{Value: "two", Candidate: second}, nil
case 3:
return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil
default:
t.Fatalf("unexpected producer attempt %d", request.Number)
return producerAttemptOutput{}, nil
}
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "three" {
return validationReport{}, nil
}
return rejectedAttemptReport("semantic_defect"), nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted {
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
}
if len(corrections) != 2 {
t.Fatalf("correction count = %d, want 2", len(corrections))
}
if corrections[0] == corrections[1] {
t.Fatal("semantic corrections reused the same pointer")
}
if got := string(corrections[0].AssistantResponse); got != "first response" {
t.Fatalf("first correction response = %q", got)
}
if got := string(corrections[1].AssistantResponse); got != "second response" {
t.Fatalf("second correction response = %q", got)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
}
func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) {
tests := []struct {
name string
config producerAttemptConfig
produce producerAttemptProducer
validate producerAttemptValidator
wantAction producerTerminalAction
wantError bool
wantValue any
}{
{
name: "structural reject output",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
},
validate: approveAttempt,
wantAction: producerTerminalRejected,
},
{
name: "structural fail run",
config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
},
validate: approveAttempt,
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "operational failure after retry exhaustion",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, errors.New("producer unavailable")
},
validate: approveAttempt,
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "semantic reject output",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("bad"), nil
},
wantAction: producerTerminalRejected,
},
{
name: "semantic fail run",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("bad"), nil
},
wantAction: producerTerminalFailed,
wantError: true,
},
{
name: "validator failure warns and continues",
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "kept"}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return failedAttemptReport(), nil
},
wantAction: producerTerminalIncompleteAccepted,
wantValue: "kept",
},
{
name: "validator failure fails run",
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}},
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, nil
},
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
return failedAttemptReport(), nil
},
wantAction: producerTerminalFailed,
wantError: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate)
if (err != nil) != test.wantError {
t.Fatalf("error = %v, want error %t", err, test.wantError)
}
if terminal.Action != test.wantAction {
t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction)
}
if terminal.Value != test.wantValue {
t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue)
}
if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) {
t.Fatalf("rejected terminal = %#v, want rejection without value", terminal)
}
})
}
}
func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) {
calls := 0
policy := DefaultValidationPolicy()
policy.SemanticRejection = SemanticRejectionRejectOutput
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
calls++
return producerAttemptOutput{Value: "deterministic"}, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
return rejectedAttemptReport("deterministic_rejection"), nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 {
t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls)
}
}
func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
t.Run("retry", func(t *testing.T) {
calls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
calls++
if request.Correction != nil {
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
}
if request.Number == 1 {
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
}
return producerAttemptOutput{Value: "replacement"}, nil
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 {
t.Fatalf("terminal = %#v, calls = %d", terminal, calls)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
})
t.Run("feedback retry", func(t *testing.T) {
const (
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
guidance = "Use only candidate numbers from the supplied candidate list. Return one complete corrected response."
)
var observed *contracts.SemanticCorrection
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
}
observed = request.Correction
return producerAttemptOutput{Value: "corrected"}, nil
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
}
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
})
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
switch request.Number {
case 1:
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
case 2:
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
t.Fatalf("second attempt correction = %#v", request.Correction)
}
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
case 3:
if request.Correction != nil {
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
}
return producerAttemptOutput{Value: "accepted"}, nil
default:
t.Fatalf("unexpected producer attempt %d", request.Number)
return producerAttemptOutput{}, nil
}
}, approveAttempt)
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
t.Fatalf("terminal = %#v, error = %v", terminal, err)
}
})
t.Run("fallback", func(t *testing.T) {
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{fallbackDiagnostic}}}, nil
}, approveAttempt)
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Diagnostics, []contracts.ProducerDiagnostic{fallbackDiagnostic}) {
t.Fatalf("terminal = %#v", terminal)
}
})
}
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
producerCalls := 0
validatorCalls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
producerCalls++
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
validatorCalls++
return validationReport{}, nil
})
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
}
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
}
}
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
tests := []struct {
name string
output producerAttemptOutput
want string
}{
{
name: "producer diagnostics",
output: producerAttemptOutput{Diagnostics: []contracts.ProducerDiagnostic{{}}},
want: "producer returned invalid diagnostics",
},
{
name: "retry fallback diagnostics",
output: producerAttemptOutput{Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{{}}}},
want: "producer returned invalid retry fallback diagnostics",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
producerCalls := 0
validatorCalls := 0
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
producerCalls++
return test.output, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
validatorCalls++
return validationReport{}, nil
})
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("runProducerAttempts() error = %v, want %q", err, test.want)
}
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d; want immediate framework failure", terminal, producerCalls, validatorCalls)
}
})
}
}
func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) {
firstDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded warning")}
secondDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted warning")}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Diagnostics: firstDiagnostics}, nil
}
return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Diagnostics: secondDiagnostics}, nil
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "first" {
report := rejectedAttemptReport("defect")
report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")})
return report, nil
}
return validationReport{}, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalAccepted {
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
}
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) {
t.Fatalf("attempt kinds = %v", got)
}
if got := terminal.Diagnostics; !reflect.DeepEqual(got, secondDiagnostics) {
t.Fatalf("terminal diagnostics = %#v, want %#v", got, secondDiagnostics)
}
if len(terminal.Provenance[0].Validation.records) != 2 {
t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records)
}
}
func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) {
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
if request.Number == 1 {
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil
}
return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
if output.Value == "first" {
return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil
}
return validationReport{}, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3)
if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) {
t.Fatalf("summary = %#v", summary)
}
if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 {
t.Fatalf("corrected success summary retained prior findings: %#v", summary)
}
}
func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) {
report := validationReport{records: []validationRecord{
{validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")},
{validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"},
{validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")},
}}
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{Value: "candidate"}, nil
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
return report, nil
})
if err != nil {
t.Fatalf("runProducerAttempts() error = %v", err)
}
if terminal.Action != producerTerminalIncompleteAccepted {
t.Fatalf("terminal action = %q", terminal.Action)
}
groups, groupErr := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil)
if groupErr != nil {
t.Fatalf("terminalDiagnosticGroups() error = %v", groupErr)
}
if len(groups) != 3 {
t.Fatalf("diagnostic groups = %#v, want every failed and skipped validator", groups)
}
for index, validator := range []string{"first", "second", "third"} {
if group := groups[index]; group.Origin.ValidatorKey != validator || group.Disposition != contracts.DiagnosticDispositionWarning || group.Category != contracts.DiagnosticCategoryValidationIncomplete || group.ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostic group %d = %#v, want incomplete warning for %q", index, group, validator)
}
}
summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0)
if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) {
t.Fatalf("summary = %#v", summary)
}
}
func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) {
tests := []struct {
name string
context context.Context
produce producerAttemptProducer
}{
{
name: "cancelled context",
context: cancelledAttemptContext(),
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, nil
},
},
{
name: "debug persistence failure",
context: context.Background(),
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
calls := 0
producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
calls++
return test.produce(ctx, request)
}
terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt)
if err == nil || terminal.Action != producerTerminalFailed {
t.Fatalf("terminal = %#v, error = %v", terminal, err)
}
if test.name == "cancelled context" && calls != 0 {
t.Fatalf("cancelled producer calls = %d, want 0", calls)
}
if test.name == "debug persistence failure" && calls != 1 {
t.Fatalf("debug failure producer calls = %d, want 1", calls)
}
})
}
}
func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate {
t.Helper()
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
t.Fatalf("NewModelCandidate() error = %v", err)
}
return candidate
}
func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) {
return validationReport{}, nil
}
func rejectedAttemptReport(reason string) validationReport {
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}}
}
func failedAttemptReport() validationReport {
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}}
}
func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind {
kinds := make([]producerAttemptKind, len(provenance))
for index, item := range provenance {
kinds[index] = item.Kind
}
return kinds
}
func cancelledAttemptContext() context.Context {
ctx, cancel := context.WithCancel(context.Background())
cancel()
return ctx
}

View File

@@ -0,0 +1,42 @@
package pipeline
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// finishProducerFailure applies the resolved terminal mode without retrying
// production or running ordinary validators against a supported fallback.
func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
var zero T
if ctx == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
}
if err := ctx.Err(); err != nil {
return zero, contracts.ProducerOutcome{}, err
}
if cause == nil {
cause = errors.New("producer did not yield an accepted result")
}
switch mode {
case contracts.ModeStrict:
return zero, contracts.ProducerOutcome{}, cause
case contracts.ModePermissive:
if fallback == nil {
return zero, contracts.ProducerOutcome{}, fmt.Errorf("producer has no supported fallback: %w", cause)
}
value, err := fallback(ctx)
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
if err != nil {
return zero, contracts.ProducerOutcome{}, errors.Join(cause, fmt.Errorf("construct producer fallback: %w", err))
}
return value, contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: cause.Error(), ReuseEligible: false}, nil
default:
return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode)
}
}

View File

@@ -0,0 +1,64 @@
package pipeline
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) {
failure := errors.New("candidate not accepted")
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
calls := 0
value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil })
if mode == contracts.ModeStrict {
if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" {
t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls)
}
} else if err != nil || calls != 1 || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || outcome.FallbackReason == "" {
t.Fatalf("permissive = %q %#v %v, calls %d", value, outcome, err, calls)
}
}
}
func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) {
original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback")
_, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction })
if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" {
t.Fatalf("outcome = %#v, error = %v", outcome, err)
}
if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) {
t.Fatalf("unsupported fallback = %v", err)
}
}
func TestProducerCancellationNeverDeliversFallback(t *testing.T) {
for _, cancelDuringFallback := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
calls := 0
if !cancelDuringFallback {
cancel()
}
value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) {
calls++
cancel()
return "not delivered", nil
})
cancel()
if !errors.Is(err, context.Canceled) || value != "" || outcome.Disposition != "" {
t.Fatalf("cancellation = %q %#v %v", value, outcome, err)
}
if !cancelDuringFallback && calls != 0 {
t.Fatal("fallback started after cancellation")
}
}
}
func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) {
value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil })
if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback {
t.Fatalf("local timeout = %q %#v %v", value, outcome, err)
}
}

View File

@@ -492,61 +492,6 @@ func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (ret
return last, nil
}
func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
content, err := json.Marshal(chunks)
if err != nil {
return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err)
}
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
binding := item.resolved.Binding
started := time.Now().UTC()
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
var result contracts.ValidationResult
requestMetadata, cloneErr := cloneMetadata(metadata)
if cloneErr != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
}
requestChunks, cloneErr := cloneSourceChunks(chunks)
if cloneErr != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
}
switch item.resolved.Target {
case ValidatorTargetChunk:
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
case ValidatorTargetSerialized:
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
default:
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
}
if err == nil {
err = contracts.ValidateValidationResult(result)
}
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
debugContent.ContentDigest = debugContentDigest(content)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil {
debugCall.Error = err.Error()
}
if err != nil {
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
}
return validationInvocation{}, validationErr
}
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
}
return validationInvocation{result: result}, nil
})
if err != nil {
return report, err
}
return report, nil
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains {
if chain.Stage != stage {

View File

@@ -583,83 +583,6 @@ func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, m
return nil
}
func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
if len(chain.validators) > 0 && target.candidate == nil {
candidate, err := validationCandidateArtifact(codec, target)
if err != nil {
return validationReport{}, fmt.Errorf("serialize %s candidate for validation: %w", target.stage, err)
}
target.candidate = &candidate
}
report, err := executeValidationChain(ctx, chain, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
binding := item.resolved.Binding
var result contracts.ValidationResult
var err error
started := time.Now().UTC()
pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)}
if target.chunk != nil {
pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1))
}
pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt))
attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt)
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
requestTarget := target
requestTarget.sourceInput = target.sourceInput.Clone()
requestTarget.references = CloneReferenceSet(target.references)
requestTarget.metadata, err = cloneMetadata(target.metadata)
if err != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation metadata: %w", err))
}
requestTarget.chunk, err = cloneSourceChunkPtr(target.chunk)
if err != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunk: %w", err))
}
requestTarget.chunks, err = cloneSourceChunks(target.chunks)
if err != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunks: %w", err))
}
switch item.resolved.Target {
case ValidatorTargetTyped:
candidateValue, decodeErr := decodeTypedValidationCandidate(codec, *target.candidate)
if decodeErr != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("decode %s candidate for typed validator %q: %w", target.stage, binding.Module, decodeErr))
}
requestTarget.value = candidateValue
requestTarget.llmProfile = binding.LLMProfile
requestTarget.structuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
result, err = item.typedValidate(validatorCtx, item.typed, requestTarget)
case ValidatorTargetSerialized:
artifact := cloneCheckpointArtifact(*target.candidate)
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
default:
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module))
}
if err == nil {
err = contracts.ValidateValidationResult(result)
}
artifact := cloneCheckpointArtifact(*target.candidate)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(artifact), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil {
debugCall.Error = err.Error()
}
if err != nil {
validationErr := fmt.Errorf("validate typed %s output with validator %q: %w", target.stage, binding.Module, err)
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr)))
}
return validationInvocation{}, validationErr
}
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
return validationInvocation{}, fatalValidationError(fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr))
}
return validationInvocation{result: result}, nil
})
if err != nil {
return report, err
}
return report, nil
}
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
rejection := report.FirstRejection()
if rejection == nil {

View File

@@ -1,227 +0,0 @@
package pipeline
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type validationOutcome string
const (
validationApproved validationOutcome = "approved"
validationRejected validationOutcome = "rejected"
validationFailed validationOutcome = "failed"
validationSkipped validationOutcome = "skipped"
)
const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"
// validationRecord captures the settled result of one configured validator.
// Its fields remain private so reports cannot expose mutable diagnostic storage.
type validationRecord struct {
validatorName string
outcome validationOutcome
attemptCount int
reasonCode string
message string
diagnosticPath string
diagnostics []contracts.ProducerDiagnostic
correctionGuidance string
failure error
}
func (record validationRecord) clone() validationRecord {
record.diagnostics = contracts.CloneProducerDiagnostics(record.diagnostics)
return record
}
// validationReport is the immutable, ordered result of one validator chain.
type validationReport struct {
records []validationRecord
}
func (report validationReport) Records() []validationRecord {
records := make([]validationRecord, len(report.records))
for index, record := range report.records {
records[index] = record.clone()
}
return records
}
func (report validationReport) Diagnostics() []validationDiagnosticRecord {
var diagnostics []validationDiagnosticRecord
for _, record := range report.records {
if record.outcome != validationApproved && record.outcome != validationRejected {
continue
}
for _, diagnostic := range record.diagnostics {
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: cloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0]})
}
}
return diagnostics
}
type validationDiagnosticRecord struct {
validatorName string
diagnostic contracts.ProducerDiagnostic
}
func (report validationReport) FirstRejection() *validationRecord {
for _, record := range report.records {
if record.outcome == validationRejected {
clone := record.clone()
return &clone
}
}
return nil
}
func (report validationReport) FirstFailure() *validationRecord {
for _, record := range report.records {
if record.outcome == validationFailed {
clone := record.clone()
return &clone
}
}
return nil
}
func (report validationReport) CorrectionRequest() (string, error) {
seen := make(map[string]struct{})
parts := make([]string, 0, len(report.records))
for _, record := range report.records {
if record.outcome != validationRejected {
continue
}
guidance := strings.TrimSpace(record.correctionGuidance)
if guidance == "" {
return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
}
if _, exists := seen[guidance]; exists {
continue
}
seen[guidance] = struct{}{}
parts = append(parts, guidance)
}
if len(parts) == 0 {
return "", errors.New("validation report contains no correction guidance")
}
var builder strings.Builder
builder.WriteString(correctionRequestIntroduction)
for index, guidance := range parts {
fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
if index+1 < len(parts) {
builder.WriteByte('\n')
}
}
request := builder.String()
if len(request) > contracts.MaxCorrectionGuidanceBytes {
return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
}
return request, nil
}
type validationInvocation struct {
result contracts.ValidationResult
skipped bool
reason string
message string
}
func skippedValidation(reason, message string) validationInvocation {
return validationInvocation{skipped: true, reason: strings.TrimSpace(reason), message: strings.TrimSpace(message)}
}
type validationInvoker func(context.Context, preparedValidator, int) (validationInvocation, error)
type validationFrameworkError struct{ err error }
func (err validationFrameworkError) Error() string { return err.err.Error() }
func (err validationFrameworkError) Unwrap() error { return err.err }
func fatalValidationError(err error) error {
if err == nil {
return nil
}
return validationFrameworkError{err: err}
}
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke validationInvoker) (validationReport, error) {
if ctx == nil {
return validationReport{}, errors.New("validator execution context must not be nil")
}
if invoke == nil {
return validationReport{}, errors.New("validator invocation must not be nil")
}
report := validationReport{records: make([]validationRecord, 0, len(chain.validators))}
for index, validator := range chain.validators {
validator.position = index + 1
if err := ctx.Err(); err != nil {
return validationReport{}, err
}
binding := validator.resolved.Binding
attemptLimit := 1
if validator.resolved.ExecutionClass == contracts.ExecutionClassLLMBacked {
attemptLimit += binding.Retries
}
for attempt := 1; attempt <= attemptLimit; attempt++ {
if err := ctx.Err(); err != nil {
return validationReport{}, err
}
invocation, err := invoke(ctx, validator, attempt)
if err != nil {
var frameworkErr validationFrameworkError
if errors.As(err, &frameworkErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return validationReport{}, err
}
if attempt == attemptLimit {
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator failed after retry exhaustion", failure: err})
}
continue
}
if invocation.skipped {
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationSkipped, attemptCount: attempt, reasonCode: invocation.reason, message: invocation.message})
break
}
if err := contracts.ValidateProducerDiagnostics(invocation.result.Diagnostics); err != nil {
return validationReport{}, fatalValidationError(fmt.Errorf("validator %q returned invalid diagnostics: %w", binding.Module, err))
}
if err := contracts.ValidateValidationResult(invocation.result); err != nil {
if attempt == attemptLimit {
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator returned an invalid result", failure: err})
}
continue
}
if invocation.result.Approved {
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), diagnosticPath: invocation.result.DiagnosticArtifactPath})
break
}
message := invocation.result.Message
if message == "" {
message = "output rejected"
}
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), correctionGuidance: invocation.result.CorrectionGuidance})
break
}
}
return report, nil
}
func validatorFailureError(record validationRecord) error {
if record.failure != nil {
return record.failure
}
return fmt.Errorf("validator %q failed after %d attempt(s)", record.validatorName, record.attemptCount)
}
func validatorAttemptPath(base string, attempt int) string {
if attempt == 1 {
return base
}
return fmt.Sprintf("%s-validator-%02d", base, attempt)
}

View File

@@ -1,241 +0,0 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
tests := []struct {
name string
chain preparedValidatorChain
invoke validationInvoker
want []validationOutcome
wantCalls []string
wantGuidance []string
wantDiagnostics []string
}{
{
name: "all approved",
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)),
invoke: validationSequence(map[string][]validationStep{
"shape": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("shape")}}}},
"refs": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("refs")}}}},
}),
want: []validationOutcome{validationApproved, validationApproved},
wantCalls: []string{"shape:1", "refs:1"},
wantDiagnostics: []string{"shape", "refs"},
},
{
name: "multiple rejections deduplicate guidance",
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)),
invoke: validationSequence(map[string][]validationStep{
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}},
"refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}},
"coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}},
}),
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
wantGuidance: []string{"repair shape", "repair references"},
},
{
name: "rejection and failure both settle",
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
invoke: validationSequence(map[string][]validationStep{
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}},
"remote": {{err: errors.New("offline")}},
}),
want: []validationOutcome{validationRejected, validationFailed},
wantCalls: []string{"shape:1", "remote:1"},
wantGuidance: []string{"repair shape"},
},
{
name: "failure only has no correction guidance",
chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}),
want: []validationOutcome{validationFailed},
wantCalls: []string{"remote:1"},
},
{
name: "skipped is retained without guidance",
chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)),
invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}),
want: []validationOutcome{validationSkipped},
wantCalls: []string{"optional:1"},
},
{
name: "LLM failure retries then succeeds",
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)),
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}),
want: []validationOutcome{validationApproved},
wantCalls: []string{"remote:1", "remote:2", "remote:3"},
},
{
name: "LLM failure retry exhaustion records once",
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)),
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}),
want: []validationOutcome{validationFailed},
wantCalls: []string{"remote:1", "remote:2"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
calls := []string(nil)
invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) {
calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt)))
return test.invoke(ctx, validator, attempt)
}
report, err := executeValidationChain(context.Background(), test.chain, invoke)
if err != nil {
t.Fatalf("executeValidationChain() error = %v", err)
}
records := report.Records()
outcomes := make([]validationOutcome, len(records))
for index, record := range records {
outcomes[index] = record.outcome
}
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
}
if len(test.wantGuidance) > 0 {
guidance, err := report.CorrectionRequest()
if err != nil {
t.Fatalf("CorrectionRequest() error = %v", err)
}
if !strings.Contains(guidance, "complete corrected replacement") {
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
}
for _, item := range test.wantGuidance {
if strings.Count(guidance, item) != 1 {
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
}
}
}
diagnostics := report.Diagnostics()
var diagnosticCodes []string
for _, record := range diagnostics {
diagnosticCodes = append(diagnosticCodes, record.diagnostic.ReasonCode)
}
if !reflect.DeepEqual(diagnosticCodes, test.wantDiagnostics) {
t.Fatalf("Diagnostics() = %#v, want codes %#v", diagnostics, test.wantDiagnostics)
}
})
}
}
func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
candidate := []byte(`{"events":["one"]}`)
chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1))
var received [][]byte
report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) {
requestCandidate := append([]byte(nil), candidate...)
received = append(received, requestCandidate)
requestCandidate[0] = 'x'
if attempt == 1 {
return validationInvocation{}, errors.New("retry")
}
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
})
if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 {
t.Fatalf("report = %#v, error = %v", report, err)
}
if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) {
t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received)
}
}
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
report := validationReport{records: make([]validationRecord, 17)}
for index := range report.records {
unique := []byte(guidance)
unique[0] = byte('a' + index)
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
}
report.records[0].diagnostics = []contracts.ProducerDiagnostic{validationDiagnostic("diagnostic")}
if _, err := report.CorrectionRequest(); err == nil {
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
}
records := report.Records()
records[0].diagnostics[0].ReasonCode = "changed"
if report.Records()[0].diagnostics[0].ReasonCode != "diagnostic" {
t.Fatal("Records() exposed mutable diagnostic storage")
}
}
func validationDiagnostic(reasonCode string) contracts.ProducerDiagnostic {
return contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionObservation,
Category: contracts.DiagnosticCategoryNormalization,
ReasonCode: reasonCode,
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "validator", Message: reasonCode}},
}
}
func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) {
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1))
framework := errors.New("debug persistence failed")
if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
return validationInvocation{}, fatalValidationError(framework)
}); !errors.Is(err, framework) {
t.Fatalf("framework error = %v, want %v", err, framework)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
}); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation error = %v, want context canceled", err)
}
}
func TestExecuteValidationChainRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 2))
calls := 0
_, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
calls++
return validationInvocation{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{}}}}, nil
})
var frameworkErr validationFrameworkError
if !errors.As(err, &frameworkErr) || calls != 1 || !strings.Contains(err.Error(), "validator \"validator\" returned invalid diagnostics") {
t.Fatalf("error = %v calls = %d, want one fatal invalid-diagnostics result", err, calls)
}
}
type validationStep struct {
result contracts.ValidationResult
invocation validationInvocation
err error
}
func validationSequence(steps map[string][]validationStep) validationInvoker {
positions := make(map[string]int)
return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) {
name := validator.resolved.Binding.Module
index := positions[name]
positions[name]++
if index >= len(steps[name]) {
return validationInvocation{}, errors.New("unexpected validator invocation")
}
step := steps[name][index]
if step.invocation.skipped {
return step.invocation, step.err
}
return validationInvocation{result: step.result}, step.err
}
}
func validationChain(validators ...preparedValidator) preparedValidatorChain {
return preparedValidatorChain{validators: validators}
}
func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator {
return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}}
}