Centralize candidate assessment and explicit producer fallback handling
This commit is contained in:
195
internal/framework/pipeline/candidate_assessment.go
Normal file
195
internal/framework/pipeline/candidate_assessment.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// executeValidationChain invokes every applicable validator once, in configured
|
||||
// order. Only caller cancellation or a framework failure interrupts the chain.
|
||||
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) {
|
||||
assessment := contracts.CandidateAssessment{}
|
||||
if ctx == nil {
|
||||
return assessment, errors.New("validator execution context must not be nil")
|
||||
}
|
||||
if invoke == nil {
|
||||
return assessment, errors.New("validator invocation must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return assessment, err
|
||||
}
|
||||
for index, validator := range chain.validators {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return assessment, err
|
||||
}
|
||||
validator.position = index + 1
|
||||
result, err := invoke(ctx, validator)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return assessment, canceled
|
||||
}
|
||||
var frameworkError *assessmentFrameworkError
|
||||
if errors.As(err, &frameworkError) {
|
||||
return assessment, err
|
||||
}
|
||||
finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result}
|
||||
finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics)
|
||||
if err != nil {
|
||||
finding.Decision, finding.Failure = contracts.AssessmentFailed, err
|
||||
var unavailable *contracts.ValidationUnavailableError
|
||||
if errors.As(err, &unavailable) {
|
||||
finding.Decision = contracts.AssessmentUnavailable
|
||||
}
|
||||
} else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil {
|
||||
finding.Decision = contracts.AssessmentFailed
|
||||
finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr)
|
||||
} else {
|
||||
finding.Decision = contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
finding.Decision = contracts.AssessmentApproved
|
||||
}
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, finding)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return assessment, err
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
|
||||
type assessmentFrameworkError struct{ cause error }
|
||||
|
||||
func (e *assessmentFrameworkError) Error() string { return e.cause.Error() }
|
||||
func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
|
||||
|
||||
// typedCandidateAssessor binds one operation's complete chain. Each invocation
|
||||
// captures the new candidate once and decodes a detached view for each validator.
|
||||
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
|
||||
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
||||
if ctx == nil {
|
||||
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
content, err := codec.encodeCandidate(value)
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
|
||||
}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
request, err := isolatedValidationTarget(target)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||
}
|
||||
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||
switch validator.resolved.Target {
|
||||
case ValidatorTargetTyped:
|
||||
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)}
|
||||
}
|
||||
if reflect.TypeOf(request.value) != codec.valueType {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))}
|
||||
}
|
||||
return validator.typedValidate(ctx, validator.typed, request)
|
||||
case ValidatorTargetSerialized:
|
||||
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
||||
Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey,
|
||||
Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput,
|
||||
SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile,
|
||||
Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks,
|
||||
Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...),
|
||||
})
|
||||
default:
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) {
|
||||
request := target
|
||||
request.sourceInput = target.sourceInput.Clone()
|
||||
request.references = CloneReferenceSet(target.references)
|
||||
var err error
|
||||
request.metadata, err = source.CloneMetadata(target.metadata)
|
||||
if err != nil {
|
||||
return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err)
|
||||
}
|
||||
request.source, err = cloneAssessmentSource(target.source)
|
||||
if err != nil {
|
||||
return typedValidationTarget{}, err
|
||||
}
|
||||
request.chunk, err = cloneSourceChunkPtr(target.chunk)
|
||||
if err != nil {
|
||||
return typedValidationTarget{}, err
|
||||
}
|
||||
request.chunks, err = cloneSourceChunks(target.chunks)
|
||||
return request, err
|
||||
}
|
||||
|
||||
func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) {
|
||||
if doc == nil {
|
||||
return nil, nil
|
||||
}
|
||||
copy := *doc
|
||||
var err error
|
||||
copy.Metadata, err = source.CloneMetadata(doc.Metadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clone validation source metadata: %w", err)
|
||||
}
|
||||
copy.Units, err = cloneSourceUnits(doc.Units)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ©, nil
|
||||
}
|
||||
|
||||
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
|
||||
// The runner's plan callback owns materialization before this boundary.
|
||||
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
|
||||
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
|
||||
if ctx == nil {
|
||||
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
content, err := json.Marshal(chunks)
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err)
|
||||
}
|
||||
target := target
|
||||
target.chunks = chunks
|
||||
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
request, err := isolatedValidationTarget(target)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||
}
|
||||
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||
switch validator.resolved.Target {
|
||||
case ValidatorTargetChunk:
|
||||
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
|
||||
ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
||||
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
||||
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
||||
})
|
||||
case ValidatorTargetSerialized:
|
||||
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
||||
Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
||||
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
||||
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
||||
Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...),
|
||||
})
|
||||
default:
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
223
internal/framework/pipeline/candidate_assessment_test.go
Normal file
223
internal/framework/pipeline/candidate_assessment_test.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) {
|
||||
chain := preparedValidatorChain{}
|
||||
names := []string{"first", "second", "unavailable", "failed", "timeout"}
|
||||
for _, name := range names {
|
||||
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}})
|
||||
}
|
||||
failure := errors.New("validator failed")
|
||||
var called []string
|
||||
assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
name := validator.resolved.Binding.Module
|
||||
called = append(called, name)
|
||||
switch name {
|
||||
case "unavailable":
|
||||
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure}
|
||||
case "failed":
|
||||
return contracts.ValidationResult{}, failure
|
||||
case "timeout":
|
||||
return contracts.ValidationResult{}, context.DeadlineExceeded
|
||||
default:
|
||||
return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil
|
||||
}
|
||||
})
|
||||
if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() {
|
||||
t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err)
|
||||
}
|
||||
var decisions []contracts.AssessmentDecision
|
||||
for _, finding := range assessment.Validators {
|
||||
decisions = append(decisions, finding.Decision)
|
||||
}
|
||||
want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed}
|
||||
if !reflect.DeepEqual(decisions, want) {
|
||||
t.Fatalf("decisions = %#v", decisions)
|
||||
}
|
||||
guidance, err := assessment.Guidance()
|
||||
if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) {
|
||||
t.Fatalf("guidance = %#v, %v", guidance, err)
|
||||
}
|
||||
if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) {
|
||||
t.Fatalf("assessment lost underlying causes: %v", assessment.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
||||
assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 {
|
||||
t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) {
|
||||
calls := 0
|
||||
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
||||
assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
||||
calls++
|
||||
return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil
|
||||
})
|
||||
guidance, guidanceErr := assessment.Guidance()
|
||||
if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil {
|
||||
t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr)
|
||||
}
|
||||
for _, finding := range assessment.Validators {
|
||||
if finding.Decision != contracts.AssessmentFailed {
|
||||
t.Fatalf("invalid rejection = %#v", finding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error)
|
||||
|
||||
func (notesAssessmentValidator) Name() string { return "notes" }
|
||||
func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||
return v(ctx, request)
|
||||
}
|
||||
|
||||
func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator {
|
||||
t.Helper()
|
||||
registry := NewValidatorRegistry()
|
||||
if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, _ := registry.typedEntry(name, "test/notes")
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate}
|
||||
}
|
||||
|
||||
func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
|
||||
t.Helper()
|
||||
registry := NewArtifactCodecRegistry()
|
||||
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, _, err := registry.entry("test/notes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
|
||||
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
|
||||
doc := typedTestDocument()
|
||||
doc.Metadata = map[string]any{"label": "original"}
|
||||
target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}}
|
||||
first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||
request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed"
|
||||
request.Source.Units[0].Text = "changed"
|
||||
request.Source.Metadata["label"] = "changed"
|
||||
request.Metadata["label"] = "changed"
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})
|
||||
second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||
if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" {
|
||||
t.Fatalf("validator received a mutated view: %#v", request)
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})
|
||||
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}})
|
||||
assessment, err := assess(context.Background(), candidate)
|
||||
if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" {
|
||||
t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) {
|
||||
calls := []string{}
|
||||
makeValidator := func(name, accepted string) preparedValidator {
|
||||
return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||
value := request.Value.Items[0]
|
||||
calls = append(calls, name+":"+value)
|
||||
if value == accepted {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil
|
||||
})
|
||||
}
|
||||
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}})
|
||||
for _, text := range []string{"one", "two", "two"} {
|
||||
assessment, err := assess(context.Background(), codecNotes{Items: []string{text}})
|
||||
if err != nil || assessment.Accepted() {
|
||||
t.Fatalf("assessment for %s = %#v, %v", text, assessment, err)
|
||||
}
|
||||
want := contracts.AssessmentRejected
|
||||
if text == "one" {
|
||||
want = contracts.AssessmentApproved
|
||||
}
|
||||
if assessment.Validators[0].Decision != want {
|
||||
t.Fatalf("earlier approval survived revised candidate: %#v", assessment)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) {
|
||||
t.Fatalf("assessments = %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
|
||||
|
||||
func (chunkAssessmentValidator) Name() string { return "chunk assessment" }
|
||||
func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
return v(ctx, request)
|
||||
}
|
||||
|
||||
func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) {
|
||||
chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}}
|
||||
originalJSON, err := json.Marshal(chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
mutator := preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk},
|
||||
chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
calls++
|
||||
request.Chunks[0].Content[0] = 'x'
|
||||
request.Chunks[0].Metadata["label"] = "changed"
|
||||
return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil
|
||||
}),
|
||||
}
|
||||
observer := preparedValidator{
|
||||
resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized},
|
||||
serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
||||
calls++
|
||||
if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) {
|
||||
t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks)
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}),
|
||||
}
|
||||
assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks)
|
||||
if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" {
|
||||
t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks)
|
||||
}
|
||||
}
|
||||
@@ -1,440 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// producerAttemptKind records why a producer invocation followed the prior
|
||||
// one. It is deliberately independent of any artifact family.
|
||||
type producerAttemptKind string
|
||||
|
||||
const (
|
||||
producerAttemptInitial producerAttemptKind = "initial"
|
||||
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
|
||||
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
|
||||
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
|
||||
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
|
||||
)
|
||||
|
||||
type producerAttemptOutcome string
|
||||
|
||||
const (
|
||||
producerAttemptAccepted producerAttemptOutcome = "accepted"
|
||||
producerAttemptRejected producerAttemptOutcome = "rejected"
|
||||
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
|
||||
producerAttemptRetried producerAttemptOutcome = "retried"
|
||||
producerAttemptFailed producerAttemptOutcome = "failed"
|
||||
)
|
||||
|
||||
type producerTerminalAction string
|
||||
|
||||
const (
|
||||
producerTerminalAccepted producerTerminalAction = "accepted"
|
||||
producerTerminalRejected producerTerminalAction = "reject_output"
|
||||
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
|
||||
producerTerminalFailed producerTerminalAction = "fail_run"
|
||||
)
|
||||
|
||||
// producerAttemptRequest contains only attempt-local control material. The
|
||||
// producer reconstructs its ordinary request from its own durable inputs.
|
||||
type producerAttemptRequest struct {
|
||||
Number int
|
||||
Kind producerAttemptKind
|
||||
Correction *contracts.SemanticCorrection
|
||||
}
|
||||
|
||||
// producerRetryDirective asks for another producer invocation while retaining
|
||||
// the current value as a safe fallback if its shared budget is exhausted.
|
||||
// Artifact-specific adapters are responsible for validating and populating it.
|
||||
type producerRetryDirective struct {
|
||||
CorrectionGuidance string
|
||||
FallbackDiagnostics []contracts.ProducerDiagnostic
|
||||
}
|
||||
|
||||
func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
||||
if directive == nil {
|
||||
return nil
|
||||
}
|
||||
return &producerRetryDirective{
|
||||
CorrectionGuidance: directive.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
||||
// opaque to the state machine; Candidate is the attempt-local response that
|
||||
// may support semantic correction.
|
||||
type producerAttemptOutput struct {
|
||||
Value any
|
||||
Candidate *contracts.ModelCandidate
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
Retry *producerRetryDirective
|
||||
}
|
||||
|
||||
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
|
||||
candidate, err := contracts.CloneModelCandidate(output.Candidate)
|
||||
if err != nil {
|
||||
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
|
||||
}
|
||||
return producerAttemptOutput{
|
||||
Value: output.Value,
|
||||
Candidate: candidate,
|
||||
Diagnostics: contracts.CloneProducerDiagnostics(output.Diagnostics),
|
||||
Retry: output.Retry.clone(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
|
||||
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
|
||||
|
||||
type producerAttemptConfig struct {
|
||||
Retries int
|
||||
Policy ValidationPolicy
|
||||
AllowStructuralRetry bool
|
||||
}
|
||||
|
||||
// producerAttemptProvenance is ordered by producer invocation. It retains the
|
||||
// settled validation report for later debug and durable-provenance adapters.
|
||||
type producerAttemptProvenance struct {
|
||||
Number int
|
||||
Kind producerAttemptKind
|
||||
Outcome producerAttemptOutcome
|
||||
Validation validationReport
|
||||
}
|
||||
|
||||
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
|
||||
provenance.Validation = cloneValidationReport(provenance.Validation)
|
||||
return provenance
|
||||
}
|
||||
|
||||
// producerAttemptTerminal describes the state-machine decision without
|
||||
// materializing an artifact or persisting stage-specific diagnostics.
|
||||
type producerAttemptTerminal struct {
|
||||
Action producerTerminalAction
|
||||
Value any
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
Rejection *contracts.RejectedOutput
|
||||
Validation validationReport
|
||||
ValidationIncomplete bool
|
||||
Provenance []producerAttemptProvenance
|
||||
}
|
||||
|
||||
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
|
||||
terminal.Diagnostics = contracts.CloneProducerDiagnostics(terminal.Diagnostics)
|
||||
if terminal.Rejection != nil {
|
||||
rejection := *terminal.Rejection
|
||||
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
|
||||
terminal.Rejection = &rejection
|
||||
}
|
||||
terminal.Validation = cloneValidationReport(terminal.Validation)
|
||||
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
|
||||
return terminal
|
||||
}
|
||||
|
||||
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
|
||||
if len(provenance) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make([]producerAttemptProvenance, len(provenance))
|
||||
for index, item := range provenance {
|
||||
cloned[index] = item.clone()
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func cloneValidationReport(report validationReport) validationReport {
|
||||
return validationReport{records: report.Records()}
|
||||
}
|
||||
|
||||
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
|
||||
if ctx == nil {
|
||||
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
|
||||
}
|
||||
if config.Retries < 0 {
|
||||
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
|
||||
}
|
||||
if produce == nil {
|
||||
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
|
||||
}
|
||||
if validate == nil {
|
||||
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
|
||||
}
|
||||
|
||||
attemptLimit := config.Retries + 1
|
||||
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
|
||||
kind := producerAttemptInitial
|
||||
var correction *contracts.SemanticCorrection
|
||||
|
||||
for number := 1; number <= attemptLimit; number++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
|
||||
if err != nil {
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
|
||||
}
|
||||
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
if isImmediateProducerFailure(err) {
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
|
||||
if number < attemptLimit {
|
||||
kind, correction = producerAttemptStructuralRetry, nil
|
||||
continue
|
||||
}
|
||||
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
|
||||
}
|
||||
if number < attemptLimit {
|
||||
kind, correction = producerAttemptOperationalRetry, nil
|
||||
continue
|
||||
}
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
|
||||
}
|
||||
if err := validateProducerAttemptDiagnostics(output); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
|
||||
output, err = output.clone()
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
if output.Retry != nil && number < attemptLimit {
|
||||
correction, err = moduleRetryCorrection(output)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
||||
kind = producerAttemptModuleRetry
|
||||
continue
|
||||
}
|
||||
if output.Retry != nil && output.Retry.CorrectionGuidance != "" {
|
||||
if _, err := moduleRetryCorrection(output); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
}
|
||||
if output.Retry != nil {
|
||||
output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...)
|
||||
}
|
||||
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
|
||||
}
|
||||
|
||||
report, err := validate(ctx, output)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
report = cloneValidationReport(report)
|
||||
if err := ctx.Err(); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), err
|
||||
}
|
||||
|
||||
if rejection := report.FirstRejection(); rejection != nil {
|
||||
if correctionCandidate != nil {
|
||||
if err := correctionCandidate.Validate(); err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
|
||||
}
|
||||
}
|
||||
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
||||
correctionRequest, guidanceErr := report.CorrectionRequest()
|
||||
if guidanceErr != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
|
||||
}
|
||||
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
|
||||
if err != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
|
||||
}
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
|
||||
kind = producerAttemptSemanticRetry
|
||||
continue
|
||||
}
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
|
||||
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
|
||||
}
|
||||
|
||||
if incomplete := firstIncompleteValidation(report); incomplete != nil {
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
|
||||
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
|
||||
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||
}
|
||||
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
|
||||
}
|
||||
|
||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
|
||||
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||
}
|
||||
|
||||
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
||||
}
|
||||
|
||||
func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) {
|
||||
if output.Retry == nil || output.Retry.CorrectionGuidance == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if output.Candidate == nil {
|
||||
return nil, errors.New("module retry correction guidance requires a model candidate")
|
||||
}
|
||||
if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||
return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1)
|
||||
}
|
||||
correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct module retry semantic correction: %w", err)
|
||||
}
|
||||
return correction, nil
|
||||
}
|
||||
|
||||
func validateProducerAttemptDiagnostics(output producerAttemptOutput) error {
|
||||
if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil {
|
||||
return fmt.Errorf("producer returned invalid diagnostics: %w", err)
|
||||
}
|
||||
if output.Retry != nil {
|
||||
if err := contracts.ValidateProducerDiagnostics(output.Retry.FallbackDiagnostics); err != nil {
|
||||
return fmt.Errorf("producer returned invalid retry fallback diagnostics: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isImmediateProducerFailure(err error) bool {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var debugErr *attemptDebugPersistenceError
|
||||
return errors.As(err, &debugErr)
|
||||
}
|
||||
|
||||
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
|
||||
switch policy.ProducerStructuralFailure {
|
||||
case ProducerStructuralFailureRejectOutput:
|
||||
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||
case ProducerStructuralFailureFailRun:
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
|
||||
default:
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
|
||||
}
|
||||
}
|
||||
|
||||
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
|
||||
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
|
||||
switch policy.SemanticRejection {
|
||||
case SemanticRejectionRejectOutput:
|
||||
return producerAttemptTerminal{Action: producerTerminalRejected, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
||||
case SemanticRejectionFailRun:
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
|
||||
default:
|
||||
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
|
||||
}
|
||||
}
|
||||
|
||||
func firstIncompleteValidation(report validationReport) *validationRecord {
|
||||
for _, record := range report.records {
|
||||
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
||||
clone := record.clone()
|
||||
return &clone
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic) []contracts.ProducerDiagnostic {
|
||||
return contracts.CloneProducerDiagnostics(diagnostics)
|
||||
}
|
||||
|
||||
// incompleteValidationDiagnostics reports every applicable validator that
|
||||
// could not complete under warn_continue. It uses fixed text so provider
|
||||
// errors and arbitrary validator prose cannot cross this boundary.
|
||||
func incompleteValidationDiagnostics(report validationReport) []validationDiagnosticRecord {
|
||||
diagnostics := make([]validationDiagnosticRecord, 0)
|
||||
for _, record := range report.records {
|
||||
if record.outcome != validationFailed && record.outcome != validationSkipped {
|
||||
continue
|
||||
}
|
||||
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: contracts.ProducerDiagnostic{
|
||||
Disposition: contracts.DiagnosticDispositionWarning,
|
||||
Category: contracts.DiagnosticCategoryValidationIncomplete,
|
||||
ReasonCode: "validator_execution_incomplete",
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{
|
||||
Scope: record.validatorName,
|
||||
Message: "Validator execution did not complete within its configured budget.",
|
||||
}},
|
||||
}})
|
||||
}
|
||||
return diagnostics
|
||||
}
|
||||
|
||||
// validationSummary projects a terminal state-machine result into the durable
|
||||
// bounded representation used by manifests, rejections, and receipts.
|
||||
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
|
||||
summary := artifacts.ValidationSummary{
|
||||
Stage: string(stage),
|
||||
StepID: stepID,
|
||||
LaneID: laneID,
|
||||
ModuleKey: moduleKey,
|
||||
ChunkID: chunkID,
|
||||
ChunkIndex: chunkIndex,
|
||||
ProducerAttemptCount: len(terminal.Provenance),
|
||||
TerminalAction: string(terminal.Action),
|
||||
}
|
||||
switch {
|
||||
case terminal.Action == producerTerminalRejected:
|
||||
summary.Status = "rejected"
|
||||
case terminal.Action == producerTerminalFailed:
|
||||
summary.Status = "incomplete"
|
||||
case terminal.ValidationIncomplete:
|
||||
summary.Status = "incomplete"
|
||||
default:
|
||||
summary.Status = "complete"
|
||||
}
|
||||
seenValidators := make(map[string]struct{})
|
||||
seenReasons := make(map[string]struct{})
|
||||
seenIncomplete := make(map[string]struct{})
|
||||
for _, record := range terminal.Validation.records {
|
||||
if record.reasonCode != "" {
|
||||
if _, exists := seenReasons[record.reasonCode]; !exists {
|
||||
seenReasons[record.reasonCode] = struct{}{}
|
||||
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
|
||||
}
|
||||
}
|
||||
if record.outcome == validationRejected {
|
||||
if _, exists := seenValidators[record.validatorName]; !exists {
|
||||
seenValidators[record.validatorName] = struct{}{}
|
||||
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
|
||||
}
|
||||
}
|
||||
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
||||
if _, exists := seenIncomplete[record.validatorName]; !exists {
|
||||
seenIncomplete[record.validatorName] = struct{}{}
|
||||
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
|
||||
}
|
||||
}
|
||||
}
|
||||
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
|
||||
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
|
||||
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
|
||||
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
|
||||
}
|
||||
@@ -1,530 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) {
|
||||
candidate := attemptCandidate(t, "first response")
|
||||
producerCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{}, errors.New("temporary producer failure")
|
||||
case 2:
|
||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||
case 3:
|
||||
return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||
t.Fatalf("terminal = %#v, want accepted value", terminal)
|
||||
}
|
||||
if producerCalls != 3 {
|
||||
t.Fatalf("producer calls = %d, want 3", producerCalls)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) {
|
||||
first := attemptCandidate(t, "first response")
|
||||
second := attemptCandidate(t, "second response")
|
||||
var corrections []*contracts.SemanticCorrection
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Correction != nil {
|
||||
corrections = append(corrections, request.Correction)
|
||||
}
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{Value: "one", Candidate: first}, nil
|
||||
case 2:
|
||||
return producerAttemptOutput{Value: "two", Candidate: second}, nil
|
||||
case 3:
|
||||
return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||
if output.Value == "three" {
|
||||
return validationReport{}, nil
|
||||
}
|
||||
return rejectedAttemptReport("semantic_defect"), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted {
|
||||
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
||||
}
|
||||
if len(corrections) != 2 {
|
||||
t.Fatalf("correction count = %d, want 2", len(corrections))
|
||||
}
|
||||
if corrections[0] == corrections[1] {
|
||||
t.Fatal("semantic corrections reused the same pointer")
|
||||
}
|
||||
if got := string(corrections[0].AssistantResponse); got != "first response" {
|
||||
t.Fatalf("first correction response = %q", got)
|
||||
}
|
||||
if got := string(corrections[1].AssistantResponse); got != "second response" {
|
||||
t.Fatalf("second correction response = %q", got)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config producerAttemptConfig
|
||||
produce producerAttemptProducer
|
||||
validate producerAttemptValidator
|
||||
wantAction producerTerminalAction
|
||||
wantError bool
|
||||
wantValue any
|
||||
}{
|
||||
{
|
||||
name: "structural reject output",
|
||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||
},
|
||||
validate: approveAttempt,
|
||||
wantAction: producerTerminalRejected,
|
||||
},
|
||||
{
|
||||
name: "structural fail run",
|
||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
||||
},
|
||||
validate: approveAttempt,
|
||||
wantAction: producerTerminalFailed,
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "operational failure after retry exhaustion",
|
||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, errors.New("producer unavailable")
|
||||
},
|
||||
validate: approveAttempt,
|
||||
wantAction: producerTerminalFailed,
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "semantic reject output",
|
||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil
|
||||
},
|
||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return rejectedAttemptReport("bad"), nil
|
||||
},
|
||||
wantAction: producerTerminalRejected,
|
||||
},
|
||||
{
|
||||
name: "semantic fail run",
|
||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil
|
||||
},
|
||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return rejectedAttemptReport("bad"), nil
|
||||
},
|
||||
wantAction: producerTerminalFailed,
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "validator failure warns and continues",
|
||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{Value: "kept"}, nil
|
||||
},
|
||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return failedAttemptReport(), nil
|
||||
},
|
||||
wantAction: producerTerminalIncompleteAccepted,
|
||||
wantValue: "kept",
|
||||
},
|
||||
{
|
||||
name: "validator failure fails run",
|
||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}},
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, nil
|
||||
},
|
||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return failedAttemptReport(), nil
|
||||
},
|
||||
wantAction: producerTerminalFailed,
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate)
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("error = %v, want error %t", err, test.wantError)
|
||||
}
|
||||
if terminal.Action != test.wantAction {
|
||||
t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction)
|
||||
}
|
||||
if terminal.Value != test.wantValue {
|
||||
t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue)
|
||||
}
|
||||
if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) {
|
||||
t.Fatalf("rejected terminal = %#v, want rejection without value", terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) {
|
||||
calls := 0
|
||||
policy := DefaultValidationPolicy()
|
||||
policy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
return producerAttemptOutput{Value: "deterministic"}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return rejectedAttemptReport("deterministic_rejection"), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 {
|
||||
t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
||||
t.Run("retry", func(t *testing.T) {
|
||||
calls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
|
||||
}
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
||||
}
|
||||
return producerAttemptOutput{Value: "replacement"}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 {
|
||||
t.Fatalf("terminal = %#v, calls = %d", terminal, calls)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback retry", func(t *testing.T) {
|
||||
const (
|
||||
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
||||
guidance = "Use only candidate numbers from the supplied candidate list. Return one complete corrected response."
|
||||
)
|
||||
var observed *contracts.SemanticCorrection
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
|
||||
}
|
||||
observed = request.Correction
|
||||
return producerAttemptOutput{Value: "corrected"}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
|
||||
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
|
||||
}
|
||||
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
||||
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
switch request.Number {
|
||||
case 1:
|
||||
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
|
||||
case 2:
|
||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
|
||||
t.Fatalf("second attempt correction = %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
|
||||
case 3:
|
||||
if request.Correction != nil {
|
||||
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
|
||||
}
|
||||
return producerAttemptOutput{Value: "accepted"}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
||||
return producerAttemptOutput{}, nil
|
||||
}
|
||||
}, approveAttempt)
|
||||
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{fallbackDiagnostic}}}, nil
|
||||
}, approveAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Diagnostics, []contracts.ProducerDiagnostic{fallbackDiagnostic}) {
|
||||
t.Fatalf("terminal = %#v", terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
|
||||
producerCalls := 0
|
||||
validatorCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
validatorCalls++
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
|
||||
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
output producerAttemptOutput
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "producer diagnostics",
|
||||
output: producerAttemptOutput{Diagnostics: []contracts.ProducerDiagnostic{{}}},
|
||||
want: "producer returned invalid diagnostics",
|
||||
},
|
||||
{
|
||||
name: "retry fallback diagnostics",
|
||||
output: producerAttemptOutput{Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{{}}}},
|
||||
want: "producer returned invalid retry fallback diagnostics",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
producerCalls := 0
|
||||
validatorCalls := 0
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
producerCalls++
|
||||
return test.output, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
validatorCalls++
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("runProducerAttempts() error = %v, want %q", err, test.want)
|
||||
}
|
||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d; want immediate framework failure", terminal, producerCalls, validatorCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) {
|
||||
firstDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded warning")}
|
||||
secondDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted warning")}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Diagnostics: firstDiagnostics}, nil
|
||||
}
|
||||
return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Diagnostics: secondDiagnostics}, nil
|
||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||
if output.Value == "first" {
|
||||
report := rejectedAttemptReport("defect")
|
||||
report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")})
|
||||
return report, nil
|
||||
}
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalAccepted {
|
||||
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
||||
}
|
||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) {
|
||||
t.Fatalf("attempt kinds = %v", got)
|
||||
}
|
||||
if got := terminal.Diagnostics; !reflect.DeepEqual(got, secondDiagnostics) {
|
||||
t.Fatalf("terminal diagnostics = %#v, want %#v", got, secondDiagnostics)
|
||||
}
|
||||
if len(terminal.Provenance[0].Validation.records) != 2 {
|
||||
t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) {
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
if request.Number == 1 {
|
||||
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil
|
||||
}
|
||||
return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil
|
||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
||||
if output.Value == "first" {
|
||||
return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil
|
||||
}
|
||||
return validationReport{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3)
|
||||
if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 {
|
||||
t.Fatalf("corrected success summary retained prior findings: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) {
|
||||
report := validationReport{records: []validationRecord{
|
||||
{validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")},
|
||||
{validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"},
|
||||
{validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")},
|
||||
}}
|
||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{Value: "candidate"}, nil
|
||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return report, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
||||
}
|
||||
if terminal.Action != producerTerminalIncompleteAccepted {
|
||||
t.Fatalf("terminal action = %q", terminal.Action)
|
||||
}
|
||||
groups, groupErr := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil)
|
||||
if groupErr != nil {
|
||||
t.Fatalf("terminalDiagnosticGroups() error = %v", groupErr)
|
||||
}
|
||||
if len(groups) != 3 {
|
||||
t.Fatalf("diagnostic groups = %#v, want every failed and skipped validator", groups)
|
||||
}
|
||||
for index, validator := range []string{"first", "second", "third"} {
|
||||
if group := groups[index]; group.Origin.ValidatorKey != validator || group.Disposition != contracts.DiagnosticDispositionWarning || group.Category != contracts.DiagnosticCategoryValidationIncomplete || group.ReasonCode != "validator_execution_incomplete" {
|
||||
t.Fatalf("diagnostic group %d = %#v, want incomplete warning for %q", index, group, validator)
|
||||
}
|
||||
}
|
||||
summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0)
|
||||
if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
context context.Context
|
||||
produce producerAttemptProducer
|
||||
}{
|
||||
{
|
||||
name: "cancelled context",
|
||||
context: cancelledAttemptContext(),
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "debug persistence failure",
|
||||
context: context.Background(),
|
||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
||||
calls++
|
||||
return test.produce(ctx, request)
|
||||
}
|
||||
terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt)
|
||||
if err == nil || terminal.Action != producerTerminalFailed {
|
||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
||||
}
|
||||
if test.name == "cancelled context" && calls != 0 {
|
||||
t.Fatalf("cancelled producer calls = %d, want 0", calls)
|
||||
}
|
||||
if test.name == "debug persistence failure" && calls != 1 {
|
||||
t.Fatalf("debug failure producer calls = %d, want 1", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate {
|
||||
t.Helper()
|
||||
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewModelCandidate() error = %v", err)
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
|
||||
func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) {
|
||||
return validationReport{}, nil
|
||||
}
|
||||
|
||||
func rejectedAttemptReport(reason string) validationReport {
|
||||
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}}
|
||||
}
|
||||
|
||||
func failedAttemptReport() validationReport {
|
||||
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}}
|
||||
}
|
||||
|
||||
func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind {
|
||||
kinds := make([]producerAttemptKind, len(provenance))
|
||||
for index, item := range provenance {
|
||||
kinds[index] = item.Kind
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func cancelledAttemptContext() context.Context {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
return ctx
|
||||
}
|
||||
42
internal/framework/pipeline/producer_outcome.go
Normal file
42
internal/framework/pipeline/producer_outcome.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
// finishProducerFailure applies the resolved terminal mode without retrying
|
||||
// production or running ordinary validators against a supported fallback.
|
||||
func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
|
||||
var zero T
|
||||
if ctx == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if cause == nil {
|
||||
cause = errors.New("producer did not yield an accepted result")
|
||||
}
|
||||
switch mode {
|
||||
case contracts.ModeStrict:
|
||||
return zero, contracts.ProducerOutcome{}, cause
|
||||
case contracts.ModePermissive:
|
||||
if fallback == nil {
|
||||
return zero, contracts.ProducerOutcome{}, fmt.Errorf("producer has no supported fallback: %w", cause)
|
||||
}
|
||||
value, err := fallback(ctx)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
if err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.Join(cause, fmt.Errorf("construct producer fallback: %w", err))
|
||||
}
|
||||
return value, contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: cause.Error(), ReuseEligible: false}, nil
|
||||
default:
|
||||
return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode)
|
||||
}
|
||||
}
|
||||
64
internal/framework/pipeline/producer_outcome_test.go
Normal file
64
internal/framework/pipeline/producer_outcome_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) {
|
||||
failure := errors.New("candidate not accepted")
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
calls := 0
|
||||
value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil })
|
||||
if mode == contracts.ModeStrict {
|
||||
if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" {
|
||||
t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls)
|
||||
}
|
||||
} else if err != nil || calls != 1 || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || outcome.FallbackReason == "" {
|
||||
t.Fatalf("permissive = %q %#v %v, calls %d", value, outcome, err, calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) {
|
||||
original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback")
|
||||
_, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction })
|
||||
if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" {
|
||||
t.Fatalf("outcome = %#v, error = %v", outcome, err)
|
||||
}
|
||||
if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) {
|
||||
t.Fatalf("unsupported fallback = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerCancellationNeverDeliversFallback(t *testing.T) {
|
||||
for _, cancelDuringFallback := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := 0
|
||||
if !cancelDuringFallback {
|
||||
cancel()
|
||||
}
|
||||
value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return "not delivered", nil
|
||||
})
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || value != "" || outcome.Disposition != "" {
|
||||
t.Fatalf("cancellation = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
if !cancelDuringFallback && calls != 0 {
|
||||
t.Fatal("fallback started after cancellation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) {
|
||||
value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil })
|
||||
if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback {
|
||||
t.Fatalf("local timeout = %q %#v %v", value, outcome, err)
|
||||
}
|
||||
}
|
||||
@@ -492,61 +492,6 @@ func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (ret
|
||||
return last, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
|
||||
content, err := json.Marshal(chunks)
|
||||
if err != nil {
|
||||
return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err)
|
||||
}
|
||||
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
||||
report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
|
||||
binding := item.resolved.Binding
|
||||
started := time.Now().UTC()
|
||||
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
|
||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
||||
var result contracts.ValidationResult
|
||||
requestMetadata, cloneErr := cloneMetadata(metadata)
|
||||
if cloneErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
|
||||
}
|
||||
requestChunks, cloneErr := cloneSourceChunks(chunks)
|
||||
if cloneErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
|
||||
}
|
||||
switch item.resolved.Target {
|
||||
case ValidatorTargetChunk:
|
||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
|
||||
case ValidatorTargetSerialized:
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
||||
default:
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
|
||||
}
|
||||
if err == nil {
|
||||
err = contracts.ValidateValidationResult(result)
|
||||
}
|
||||
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
|
||||
debugContent.ContentDigest = debugContentDigest(content)
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
|
||||
if err != nil {
|
||||
debugCall.Error = err.Error()
|
||||
}
|
||||
if err != nil {
|
||||
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
|
||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
|
||||
}
|
||||
return validationInvocation{}, validationErr
|
||||
}
|
||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
|
||||
}
|
||||
return validationInvocation{result: result}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||
for _, chain := range chains {
|
||||
if chain.Stage != stage {
|
||||
|
||||
@@ -583,83 +583,6 @@ func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
|
||||
if len(chain.validators) > 0 && target.candidate == nil {
|
||||
candidate, err := validationCandidateArtifact(codec, target)
|
||||
if err != nil {
|
||||
return validationReport{}, fmt.Errorf("serialize %s candidate for validation: %w", target.stage, err)
|
||||
}
|
||||
target.candidate = &candidate
|
||||
}
|
||||
report, err := executeValidationChain(ctx, chain, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
|
||||
binding := item.resolved.Binding
|
||||
var result contracts.ValidationResult
|
||||
var err error
|
||||
started := time.Now().UTC()
|
||||
pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)}
|
||||
if target.chunk != nil {
|
||||
pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1))
|
||||
}
|
||||
pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt))
|
||||
attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt)
|
||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
||||
requestTarget := target
|
||||
requestTarget.sourceInput = target.sourceInput.Clone()
|
||||
requestTarget.references = CloneReferenceSet(target.references)
|
||||
requestTarget.metadata, err = cloneMetadata(target.metadata)
|
||||
if err != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation metadata: %w", err))
|
||||
}
|
||||
requestTarget.chunk, err = cloneSourceChunkPtr(target.chunk)
|
||||
if err != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunk: %w", err))
|
||||
}
|
||||
requestTarget.chunks, err = cloneSourceChunks(target.chunks)
|
||||
if err != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunks: %w", err))
|
||||
}
|
||||
switch item.resolved.Target {
|
||||
case ValidatorTargetTyped:
|
||||
candidateValue, decodeErr := decodeTypedValidationCandidate(codec, *target.candidate)
|
||||
if decodeErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("decode %s candidate for typed validator %q: %w", target.stage, binding.Module, decodeErr))
|
||||
}
|
||||
requestTarget.value = candidateValue
|
||||
requestTarget.llmProfile = binding.LLMProfile
|
||||
requestTarget.structuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||
result, err = item.typedValidate(validatorCtx, item.typed, requestTarget)
|
||||
case ValidatorTargetSerialized:
|
||||
artifact := cloneCheckpointArtifact(*target.candidate)
|
||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
|
||||
default:
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module))
|
||||
}
|
||||
if err == nil {
|
||||
err = contracts.ValidateValidationResult(result)
|
||||
}
|
||||
artifact := cloneCheckpointArtifact(*target.candidate)
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(artifact), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
|
||||
if err != nil {
|
||||
debugCall.Error = err.Error()
|
||||
}
|
||||
if err != nil {
|
||||
validationErr := fmt.Errorf("validate typed %s output with validator %q: %w", target.stage, binding.Module, err)
|
||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr)))
|
||||
}
|
||||
return validationInvocation{}, validationErr
|
||||
}
|
||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr))
|
||||
}
|
||||
return validationInvocation{result: result}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
|
||||
rejection := report.FirstRejection()
|
||||
if rejection == nil {
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type validationOutcome string
|
||||
|
||||
const (
|
||||
validationApproved validationOutcome = "approved"
|
||||
validationRejected validationOutcome = "rejected"
|
||||
validationFailed validationOutcome = "failed"
|
||||
validationSkipped validationOutcome = "skipped"
|
||||
)
|
||||
|
||||
const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"
|
||||
|
||||
// validationRecord captures the settled result of one configured validator.
|
||||
// Its fields remain private so reports cannot expose mutable diagnostic storage.
|
||||
type validationRecord struct {
|
||||
validatorName string
|
||||
outcome validationOutcome
|
||||
attemptCount int
|
||||
reasonCode string
|
||||
message string
|
||||
diagnosticPath string
|
||||
diagnostics []contracts.ProducerDiagnostic
|
||||
correctionGuidance string
|
||||
failure error
|
||||
}
|
||||
|
||||
func (record validationRecord) clone() validationRecord {
|
||||
record.diagnostics = contracts.CloneProducerDiagnostics(record.diagnostics)
|
||||
return record
|
||||
}
|
||||
|
||||
// validationReport is the immutable, ordered result of one validator chain.
|
||||
type validationReport struct {
|
||||
records []validationRecord
|
||||
}
|
||||
|
||||
func (report validationReport) Records() []validationRecord {
|
||||
records := make([]validationRecord, len(report.records))
|
||||
for index, record := range report.records {
|
||||
records[index] = record.clone()
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func (report validationReport) Diagnostics() []validationDiagnosticRecord {
|
||||
var diagnostics []validationDiagnosticRecord
|
||||
for _, record := range report.records {
|
||||
if record.outcome != validationApproved && record.outcome != validationRejected {
|
||||
continue
|
||||
}
|
||||
for _, diagnostic := range record.diagnostics {
|
||||
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: cloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0]})
|
||||
}
|
||||
}
|
||||
return diagnostics
|
||||
}
|
||||
|
||||
type validationDiagnosticRecord struct {
|
||||
validatorName string
|
||||
diagnostic contracts.ProducerDiagnostic
|
||||
}
|
||||
|
||||
func (report validationReport) FirstRejection() *validationRecord {
|
||||
for _, record := range report.records {
|
||||
if record.outcome == validationRejected {
|
||||
clone := record.clone()
|
||||
return &clone
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (report validationReport) FirstFailure() *validationRecord {
|
||||
for _, record := range report.records {
|
||||
if record.outcome == validationFailed {
|
||||
clone := record.clone()
|
||||
return &clone
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (report validationReport) CorrectionRequest() (string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
parts := make([]string, 0, len(report.records))
|
||||
for _, record := range report.records {
|
||||
if record.outcome != validationRejected {
|
||||
continue
|
||||
}
|
||||
guidance := strings.TrimSpace(record.correctionGuidance)
|
||||
if guidance == "" {
|
||||
return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
|
||||
}
|
||||
if _, exists := seen[guidance]; exists {
|
||||
continue
|
||||
}
|
||||
seen[guidance] = struct{}{}
|
||||
parts = append(parts, guidance)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", errors.New("validation report contains no correction guidance")
|
||||
}
|
||||
var builder strings.Builder
|
||||
builder.WriteString(correctionRequestIntroduction)
|
||||
for index, guidance := range parts {
|
||||
fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
|
||||
if index+1 < len(parts) {
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
request := builder.String()
|
||||
if len(request) > contracts.MaxCorrectionGuidanceBytes {
|
||||
return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type validationInvocation struct {
|
||||
result contracts.ValidationResult
|
||||
skipped bool
|
||||
reason string
|
||||
message string
|
||||
}
|
||||
|
||||
func skippedValidation(reason, message string) validationInvocation {
|
||||
return validationInvocation{skipped: true, reason: strings.TrimSpace(reason), message: strings.TrimSpace(message)}
|
||||
}
|
||||
|
||||
type validationInvoker func(context.Context, preparedValidator, int) (validationInvocation, error)
|
||||
|
||||
type validationFrameworkError struct{ err error }
|
||||
|
||||
func (err validationFrameworkError) Error() string { return err.err.Error() }
|
||||
|
||||
func (err validationFrameworkError) Unwrap() error { return err.err }
|
||||
|
||||
func fatalValidationError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return validationFrameworkError{err: err}
|
||||
}
|
||||
|
||||
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke validationInvoker) (validationReport, error) {
|
||||
if ctx == nil {
|
||||
return validationReport{}, errors.New("validator execution context must not be nil")
|
||||
}
|
||||
if invoke == nil {
|
||||
return validationReport{}, errors.New("validator invocation must not be nil")
|
||||
}
|
||||
report := validationReport{records: make([]validationRecord, 0, len(chain.validators))}
|
||||
for index, validator := range chain.validators {
|
||||
validator.position = index + 1
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validationReport{}, err
|
||||
}
|
||||
binding := validator.resolved.Binding
|
||||
attemptLimit := 1
|
||||
if validator.resolved.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
attemptLimit += binding.Retries
|
||||
}
|
||||
for attempt := 1; attempt <= attemptLimit; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validationReport{}, err
|
||||
}
|
||||
invocation, err := invoke(ctx, validator, attempt)
|
||||
if err != nil {
|
||||
var frameworkErr validationFrameworkError
|
||||
if errors.As(err, &frameworkErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return validationReport{}, err
|
||||
}
|
||||
if attempt == attemptLimit {
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator failed after retry exhaustion", failure: err})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if invocation.skipped {
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationSkipped, attemptCount: attempt, reasonCode: invocation.reason, message: invocation.message})
|
||||
break
|
||||
}
|
||||
if err := contracts.ValidateProducerDiagnostics(invocation.result.Diagnostics); err != nil {
|
||||
return validationReport{}, fatalValidationError(fmt.Errorf("validator %q returned invalid diagnostics: %w", binding.Module, err))
|
||||
}
|
||||
if err := contracts.ValidateValidationResult(invocation.result); err != nil {
|
||||
if attempt == attemptLimit {
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator returned an invalid result", failure: err})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if invocation.result.Approved {
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), diagnosticPath: invocation.result.DiagnosticArtifactPath})
|
||||
break
|
||||
}
|
||||
message := invocation.result.Message
|
||||
if message == "" {
|
||||
message = "output rejected"
|
||||
}
|
||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), correctionGuidance: invocation.result.CorrectionGuidance})
|
||||
break
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func validatorFailureError(record validationRecord) error {
|
||||
if record.failure != nil {
|
||||
return record.failure
|
||||
}
|
||||
return fmt.Errorf("validator %q failed after %d attempt(s)", record.validatorName, record.attemptCount)
|
||||
}
|
||||
|
||||
func validatorAttemptPath(base string, attempt int) string {
|
||||
if attempt == 1 {
|
||||
return base
|
||||
}
|
||||
return fmt.Sprintf("%s-validator-%02d", base, attempt)
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chain preparedValidatorChain
|
||||
invoke validationInvoker
|
||||
want []validationOutcome
|
||||
wantCalls []string
|
||||
wantGuidance []string
|
||||
wantDiagnostics []string
|
||||
}{
|
||||
{
|
||||
name: "all approved",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("shape")}}}},
|
||||
"refs": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("refs")}}}},
|
||||
}),
|
||||
want: []validationOutcome{validationApproved, validationApproved},
|
||||
wantCalls: []string{"shape:1", "refs:1"},
|
||||
wantDiagnostics: []string{"shape", "refs"},
|
||||
},
|
||||
{
|
||||
name: "multiple rejections deduplicate guidance",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}},
|
||||
"refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}},
|
||||
"coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}},
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
|
||||
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
|
||||
wantGuidance: []string{"repair shape", "repair references"},
|
||||
},
|
||||
{
|
||||
name: "rejection and failure both settle",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}},
|
||||
"remote": {{err: errors.New("offline")}},
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationFailed},
|
||||
wantCalls: []string{"shape:1", "remote:1"},
|
||||
wantGuidance: []string{"repair shape"},
|
||||
},
|
||||
{
|
||||
name: "failure only has no correction guidance",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}),
|
||||
want: []validationOutcome{validationFailed},
|
||||
wantCalls: []string{"remote:1"},
|
||||
},
|
||||
{
|
||||
name: "skipped is retained without guidance",
|
||||
chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}),
|
||||
want: []validationOutcome{validationSkipped},
|
||||
wantCalls: []string{"optional:1"},
|
||||
},
|
||||
{
|
||||
name: "LLM failure retries then succeeds",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}),
|
||||
want: []validationOutcome{validationApproved},
|
||||
wantCalls: []string{"remote:1", "remote:2", "remote:3"},
|
||||
},
|
||||
{
|
||||
name: "LLM failure retry exhaustion records once",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}),
|
||||
want: []validationOutcome{validationFailed},
|
||||
wantCalls: []string{"remote:1", "remote:2"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := []string(nil)
|
||||
invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) {
|
||||
calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt)))
|
||||
return test.invoke(ctx, validator, attempt)
|
||||
}
|
||||
report, err := executeValidationChain(context.Background(), test.chain, invoke)
|
||||
if err != nil {
|
||||
t.Fatalf("executeValidationChain() error = %v", err)
|
||||
}
|
||||
records := report.Records()
|
||||
outcomes := make([]validationOutcome, len(records))
|
||||
for index, record := range records {
|
||||
outcomes[index] = record.outcome
|
||||
}
|
||||
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
|
||||
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
|
||||
}
|
||||
if len(test.wantGuidance) > 0 {
|
||||
guidance, err := report.CorrectionRequest()
|
||||
if err != nil {
|
||||
t.Fatalf("CorrectionRequest() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(guidance, "complete corrected replacement") {
|
||||
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
|
||||
}
|
||||
for _, item := range test.wantGuidance {
|
||||
if strings.Count(guidance, item) != 1 {
|
||||
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
diagnostics := report.Diagnostics()
|
||||
var diagnosticCodes []string
|
||||
for _, record := range diagnostics {
|
||||
diagnosticCodes = append(diagnosticCodes, record.diagnostic.ReasonCode)
|
||||
}
|
||||
if !reflect.DeepEqual(diagnosticCodes, test.wantDiagnostics) {
|
||||
t.Fatalf("Diagnostics() = %#v, want codes %#v", diagnostics, test.wantDiagnostics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
|
||||
candidate := []byte(`{"events":["one"]}`)
|
||||
chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1))
|
||||
var received [][]byte
|
||||
report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) {
|
||||
requestCandidate := append([]byte(nil), candidate...)
|
||||
received = append(received, requestCandidate)
|
||||
requestCandidate[0] = 'x'
|
||||
if attempt == 1 {
|
||||
return validationInvocation{}, errors.New("retry")
|
||||
}
|
||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
||||
})
|
||||
if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 {
|
||||
t.Fatalf("report = %#v, error = %v", report, err)
|
||||
}
|
||||
if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) {
|
||||
t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
|
||||
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
|
||||
report := validationReport{records: make([]validationRecord, 17)}
|
||||
for index := range report.records {
|
||||
unique := []byte(guidance)
|
||||
unique[0] = byte('a' + index)
|
||||
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
|
||||
}
|
||||
report.records[0].diagnostics = []contracts.ProducerDiagnostic{validationDiagnostic("diagnostic")}
|
||||
if _, err := report.CorrectionRequest(); err == nil {
|
||||
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
|
||||
}
|
||||
records := report.Records()
|
||||
records[0].diagnostics[0].ReasonCode = "changed"
|
||||
if report.Records()[0].diagnostics[0].ReasonCode != "diagnostic" {
|
||||
t.Fatal("Records() exposed mutable diagnostic storage")
|
||||
}
|
||||
}
|
||||
|
||||
func validationDiagnostic(reasonCode string) contracts.ProducerDiagnostic {
|
||||
return contracts.ProducerDiagnostic{
|
||||
Disposition: contracts.DiagnosticDispositionObservation,
|
||||
Category: contracts.DiagnosticCategoryNormalization,
|
||||
ReasonCode: reasonCode,
|
||||
OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: "validator", Message: reasonCode}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) {
|
||||
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1))
|
||||
framework := errors.New("debug persistence failed")
|
||||
if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
||||
return validationInvocation{}, fatalValidationError(framework)
|
||||
}); !errors.Is(err, framework) {
|
||||
t.Fatalf("framework error = %v, want %v", err, framework)
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationChainRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
||||
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 2))
|
||||
calls := 0
|
||||
_, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
||||
calls++
|
||||
return validationInvocation{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{}}}}, nil
|
||||
})
|
||||
var frameworkErr validationFrameworkError
|
||||
if !errors.As(err, &frameworkErr) || calls != 1 || !strings.Contains(err.Error(), "validator \"validator\" returned invalid diagnostics") {
|
||||
t.Fatalf("error = %v calls = %d, want one fatal invalid-diagnostics result", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
type validationStep struct {
|
||||
result contracts.ValidationResult
|
||||
invocation validationInvocation
|
||||
err error
|
||||
}
|
||||
|
||||
func validationSequence(steps map[string][]validationStep) validationInvoker {
|
||||
positions := make(map[string]int)
|
||||
return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) {
|
||||
name := validator.resolved.Binding.Module
|
||||
index := positions[name]
|
||||
positions[name]++
|
||||
if index >= len(steps[name]) {
|
||||
return validationInvocation{}, errors.New("unexpected validator invocation")
|
||||
}
|
||||
step := steps[name][index]
|
||||
if step.invocation.skipped {
|
||||
return step.invocation, step.err
|
||||
}
|
||||
return validationInvocation{result: step.result}, step.err
|
||||
}
|
||||
}
|
||||
|
||||
func validationChain(validators ...preparedValidator) preparedValidatorChain {
|
||||
return preparedValidatorChain{validators: validators}
|
||||
}
|
||||
|
||||
func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator {
|
||||
return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}}
|
||||
}
|
||||
Reference in New Issue
Block a user