Centralize candidate assessment and explicit producer fallback handling
This commit is contained in:
@@ -408,6 +408,21 @@ need not migrate every module caller.
|
|||||||
|
|
||||||
## Stage 9: Centralize Candidate Assessment And Terminal Disposition
|
## Stage 9: Centralize Candidate Assessment And Terminal Disposition
|
||||||
|
|
||||||
|
**Status:** Complete. Replaced retrying validator execution and the producer
|
||||||
|
attempt state machine with one ordered assessment path and strict/permissive
|
||||||
|
terminal handling. Typed and materialized-chunk assessors give each validator
|
||||||
|
detached candidate/context views and preserve rejection guidance separately from
|
||||||
|
failed/unavailable assessments and their native causes. Guidance aggregation has
|
||||||
|
no hard-coded correction assignment. Fallback construction runs once without
|
||||||
|
ordinary revalidation, never on caller cancellation, and cannot claim acceptance
|
||||||
|
or reuse eligibility. Added complete-chain, revised/unchanged candidate, isolation,
|
||||||
|
guidance-contract, unavailable-assessment, timeout, and cancellation coverage.
|
||||||
|
Contracts tests/vet, isolated terminal-mode tests, and `git diff --check` pass.
|
||||||
|
Full pipeline tests remain blocked by the recorded old ConfigKit YAML import;
|
||||||
|
production compilation also awaits migration of runner/debug/diagnostic consumers
|
||||||
|
of removed attempt/report types. No blocked test is claimed passing; runner and
|
||||||
|
native continuation integration remain assigned to Stages 10–11.
|
||||||
|
|
||||||
**Goal:** use one complete assessment path for initial and revised candidates.
|
**Goal:** use one complete assessment path for initial and revised candidates.
|
||||||
|
|
||||||
**Inspect/change:** `validation_executor.go`, `validation_policy.go`,
|
**Inspect/change:** `validation_executor.go`, `validation_policy.go`,
|
||||||
|
|||||||
71
internal/framework/contracts/assessment.go
Normal file
71
internal/framework/contracts/assessment.go
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidationUnavailableError marks an assessment that could not be performed,
|
||||||
|
// rather than a detected defect in the candidate. Its cause remains inspectable.
|
||||||
|
type ValidationUnavailableError struct{ Cause error }
|
||||||
|
|
||||||
|
func (e *ValidationUnavailableError) Error() string {
|
||||||
|
if e.Cause == nil {
|
||||||
|
return "validation unavailable"
|
||||||
|
}
|
||||||
|
return "validation unavailable: " + e.Cause.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ValidationUnavailableError) Unwrap() error { return e.Cause }
|
||||||
|
|
||||||
|
// Accepted requires approval by every applicable validator. An empty configured
|
||||||
|
// chain has no additional acceptance criteria.
|
||||||
|
func (a CandidateAssessment) Accepted() bool {
|
||||||
|
for _, finding := range a.Validators {
|
||||||
|
if finding.Decision != AssessmentApproved {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guidance returns only actionable rejections, in validator order, with exact
|
||||||
|
// duplicates removed. Domain correction templates own the surrounding assignment.
|
||||||
|
func (a CandidateAssessment) Guidance() ([]string, error) {
|
||||||
|
var guidance []string
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for _, finding := range a.Validators {
|
||||||
|
if finding.Decision != AssessmentRejected {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
text := finding.Result.CorrectionGuidance
|
||||||
|
if strings.TrimSpace(text) == "" {
|
||||||
|
return nil, fmt.Errorf("validator %q rejected output without correction guidance", finding.ValidatorName)
|
||||||
|
}
|
||||||
|
if !seen[text] {
|
||||||
|
seen[text] = true
|
||||||
|
guidance = append(guidance, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return guidance, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Err explains nonacceptance while preserving underlying operational causes.
|
||||||
|
// Neither this operator-facing text nor reason codes are correction instructions.
|
||||||
|
func (a CandidateAssessment) Err() error {
|
||||||
|
var failures []error
|
||||||
|
for _, finding := range a.Validators {
|
||||||
|
if finding.Decision == AssessmentApproved {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if finding.Failure != nil {
|
||||||
|
failures = append(failures, fmt.Errorf("validator %q %s: %w", finding.ValidatorName, finding.Decision, finding.Failure))
|
||||||
|
} else if finding.Result.Message != "" {
|
||||||
|
failures = append(failures, fmt.Errorf("validator %q %s: %s", finding.ValidatorName, finding.Decision, finding.Result.Message))
|
||||||
|
} else {
|
||||||
|
failures = append(failures, fmt.Errorf("validator %q did not approve the candidate (%s)", finding.ValidatorName, finding.Decision))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(failures...)
|
||||||
|
}
|
||||||
57
internal/framework/contracts/assessment_test.go
Normal file
57
internal/framework/contracts/assessment_test.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCandidateAssessmentKeepsGuidanceSeparateAndOrdered(t *testing.T) {
|
||||||
|
unavailable := errors.New("provider unavailable")
|
||||||
|
assessment := CandidateAssessment{Validators: []ValidatorAssessment{
|
||||||
|
{ValidatorName: "shape", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "shape-code", Message: "operator prose", CorrectionGuidance: "Return the required shape."}},
|
||||||
|
{ValidatorName: "coverage", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "coverage-code", CorrectionGuidance: "Include all supported events."}},
|
||||||
|
{ValidatorName: "duplicate", Decision: AssessmentRejected, Result: ValidationResult{CorrectionGuidance: "Return the required shape."}},
|
||||||
|
{ValidatorName: "remote", Decision: AssessmentUnavailable, Failure: &ValidationUnavailableError{Cause: unavailable}},
|
||||||
|
}}
|
||||||
|
guidance, err := assessment.Guidance()
|
||||||
|
if err != nil || !reflect.DeepEqual(guidance, []string{"Return the required shape.", "Include all supported events."}) {
|
||||||
|
t.Fatalf("guidance = %#v, %v", guidance, err)
|
||||||
|
}
|
||||||
|
if assessment.Accepted() || !errors.Is(assessment.Err(), unavailable) {
|
||||||
|
t.Fatalf("assessment = %#v, error = %v", assessment, assessment.Err())
|
||||||
|
}
|
||||||
|
for _, diagnostic := range []string{"shape-code", "coverage-code", "operator prose", "provider unavailable"} {
|
||||||
|
if strings.Contains(strings.Join(guidance, "\n"), diagnostic) {
|
||||||
|
t.Fatalf("diagnostic became correction instructions: %q", diagnostic)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateAssessmentRequiresEveryApproval(t *testing.T) {
|
||||||
|
if !(CandidateAssessment{}).Accepted() {
|
||||||
|
t.Fatal("empty validator chain rejected")
|
||||||
|
}
|
||||||
|
for _, decision := range []AssessmentDecision{AssessmentApproved, AssessmentRejected, AssessmentFailed, AssessmentUnavailable, ""} {
|
||||||
|
assessment := CandidateAssessment{Validators: []ValidatorAssessment{{Decision: AssessmentApproved}, {Decision: decision}}}
|
||||||
|
if got, want := assessment.Accepted(), decision == AssessmentApproved; got != want {
|
||||||
|
t.Fatalf("%s accepted = %t", decision, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateAssessmentDoesNotInventMissingGuidance(t *testing.T) {
|
||||||
|
for _, text := range []string{"", " \t\n"} {
|
||||||
|
assessment := CandidateAssessment{Validators: []ValidatorAssessment{{ValidatorName: "validator", Decision: AssessmentRejected, Result: ValidationResult{ReasonCode: "opaque", Message: "not instructions", CorrectionGuidance: text}}}}
|
||||||
|
if _, err := assessment.Guidance(); err == nil {
|
||||||
|
t.Fatal("missing actionable guidance accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, decision := range []AssessmentDecision{AssessmentApproved, AssessmentFailed, AssessmentUnavailable} {
|
||||||
|
guidance, err := (CandidateAssessment{Validators: []ValidatorAssessment{{Decision: decision, Result: ValidationResult{Message: "not instructions"}}}}).Guidance()
|
||||||
|
if err != nil || len(guidance) != 0 {
|
||||||
|
t.Fatalf("nonrejection guidance = %#v, %v", guidance, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
195
internal/framework/pipeline/candidate_assessment.go
Normal file
195
internal/framework/pipeline/candidate_assessment.go
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// executeValidationChain invokes every applicable validator once, in configured
|
||||||
|
// order. Only caller cancellation or a framework failure interrupts the chain.
|
||||||
|
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) {
|
||||||
|
assessment := contracts.CandidateAssessment{}
|
||||||
|
if ctx == nil {
|
||||||
|
return assessment, errors.New("validator execution context must not be nil")
|
||||||
|
}
|
||||||
|
if invoke == nil {
|
||||||
|
return assessment, errors.New("validator invocation must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return assessment, err
|
||||||
|
}
|
||||||
|
for index, validator := range chain.validators {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return assessment, err
|
||||||
|
}
|
||||||
|
validator.position = index + 1
|
||||||
|
result, err := invoke(ctx, validator)
|
||||||
|
if canceled := ctx.Err(); canceled != nil {
|
||||||
|
return assessment, canceled
|
||||||
|
}
|
||||||
|
var frameworkError *assessmentFrameworkError
|
||||||
|
if errors.As(err, &frameworkError) {
|
||||||
|
return assessment, err
|
||||||
|
}
|
||||||
|
finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result}
|
||||||
|
finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics)
|
||||||
|
if err != nil {
|
||||||
|
finding.Decision, finding.Failure = contracts.AssessmentFailed, err
|
||||||
|
var unavailable *contracts.ValidationUnavailableError
|
||||||
|
if errors.As(err, &unavailable) {
|
||||||
|
finding.Decision = contracts.AssessmentUnavailable
|
||||||
|
}
|
||||||
|
} else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil {
|
||||||
|
finding.Decision = contracts.AssessmentFailed
|
||||||
|
finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr)
|
||||||
|
} else {
|
||||||
|
finding.Decision = contracts.AssessmentRejected
|
||||||
|
if result.Approved {
|
||||||
|
finding.Decision = contracts.AssessmentApproved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assessment.Validators = append(assessment.Validators, finding)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return assessment, err
|
||||||
|
}
|
||||||
|
return assessment, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type assessmentFrameworkError struct{ cause error }
|
||||||
|
|
||||||
|
func (e *assessmentFrameworkError) Error() string { return e.cause.Error() }
|
||||||
|
func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
|
||||||
|
|
||||||
|
// typedCandidateAssessor binds one operation's complete chain. Each invocation
|
||||||
|
// captures the new candidate once and decodes a detached view for each validator.
|
||||||
|
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
|
||||||
|
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.CandidateAssessment{}, err
|
||||||
|
}
|
||||||
|
content, err := codec.encodeCandidate(value)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
|
||||||
|
}
|
||||||
|
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
request, err := isolatedValidationTarget(target)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||||
|
}
|
||||||
|
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||||
|
switch validator.resolved.Target {
|
||||||
|
case ValidatorTargetTyped:
|
||||||
|
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)}
|
||||||
|
}
|
||||||
|
if reflect.TypeOf(request.value) != codec.valueType {
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))}
|
||||||
|
}
|
||||||
|
return validator.typedValidate(ctx, validator.typed, request)
|
||||||
|
case ValidatorTargetSerialized:
|
||||||
|
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
||||||
|
Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey,
|
||||||
|
Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput,
|
||||||
|
SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile,
|
||||||
|
Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks,
|
||||||
|
Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...),
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) {
|
||||||
|
request := target
|
||||||
|
request.sourceInput = target.sourceInput.Clone()
|
||||||
|
request.references = CloneReferenceSet(target.references)
|
||||||
|
var err error
|
||||||
|
request.metadata, err = source.CloneMetadata(target.metadata)
|
||||||
|
if err != nil {
|
||||||
|
return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err)
|
||||||
|
}
|
||||||
|
request.source, err = cloneAssessmentSource(target.source)
|
||||||
|
if err != nil {
|
||||||
|
return typedValidationTarget{}, err
|
||||||
|
}
|
||||||
|
request.chunk, err = cloneSourceChunkPtr(target.chunk)
|
||||||
|
if err != nil {
|
||||||
|
return typedValidationTarget{}, err
|
||||||
|
}
|
||||||
|
request.chunks, err = cloneSourceChunks(target.chunks)
|
||||||
|
return request, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) {
|
||||||
|
if doc == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
copy := *doc
|
||||||
|
var err error
|
||||||
|
copy.Metadata, err = source.CloneMetadata(doc.Metadata)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clone validation source metadata: %w", err)
|
||||||
|
}
|
||||||
|
copy.Units, err = cloneSourceUnits(doc.Units)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ©, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
|
||||||
|
// The runner's plan callback owns materialization before this boundary.
|
||||||
|
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
|
||||||
|
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.CandidateAssessment{}, err
|
||||||
|
}
|
||||||
|
content, err := json.Marshal(chunks)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err)
|
||||||
|
}
|
||||||
|
target := target
|
||||||
|
target.chunks = chunks
|
||||||
|
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
||||||
|
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
request, err := isolatedValidationTarget(target)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||||
|
}
|
||||||
|
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||||
|
switch validator.resolved.Target {
|
||||||
|
case ValidatorTargetChunk:
|
||||||
|
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
|
||||||
|
ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
||||||
|
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
||||||
|
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
||||||
|
})
|
||||||
|
case ValidatorTargetSerialized:
|
||||||
|
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
||||||
|
Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
||||||
|
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
||||||
|
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
||||||
|
Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...),
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
223
internal/framework/pipeline/candidate_assessment_test.go
Normal file
223
internal/framework/pipeline/candidate_assessment_test.go
Normal file
@@ -0,0 +1,223 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) {
|
||||||
|
chain := preparedValidatorChain{}
|
||||||
|
names := []string{"first", "second", "unavailable", "failed", "timeout"}
|
||||||
|
for _, name := range names {
|
||||||
|
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}})
|
||||||
|
}
|
||||||
|
failure := errors.New("validator failed")
|
||||||
|
var called []string
|
||||||
|
assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
name := validator.resolved.Binding.Module
|
||||||
|
called = append(called, name)
|
||||||
|
switch name {
|
||||||
|
case "unavailable":
|
||||||
|
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure}
|
||||||
|
case "failed":
|
||||||
|
return contracts.ValidationResult{}, failure
|
||||||
|
case "timeout":
|
||||||
|
return contracts.ValidationResult{}, context.DeadlineExceeded
|
||||||
|
default:
|
||||||
|
return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() {
|
||||||
|
t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err)
|
||||||
|
}
|
||||||
|
var decisions []contracts.AssessmentDecision
|
||||||
|
for _, finding := range assessment.Validators {
|
||||||
|
decisions = append(decisions, finding.Decision)
|
||||||
|
}
|
||||||
|
want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed}
|
||||||
|
if !reflect.DeepEqual(decisions, want) {
|
||||||
|
t.Fatalf("decisions = %#v", decisions)
|
||||||
|
}
|
||||||
|
guidance, err := assessment.Guidance()
|
||||||
|
if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) {
|
||||||
|
t.Fatalf("guidance = %#v, %v", guidance, err)
|
||||||
|
}
|
||||||
|
if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("assessment lost underlying causes: %v", assessment.Err())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
||||||
|
assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
calls++
|
||||||
|
cancel()
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 {
|
||||||
|
t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
||||||
|
assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
||||||
|
calls++
|
||||||
|
return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil
|
||||||
|
})
|
||||||
|
guidance, guidanceErr := assessment.Guidance()
|
||||||
|
if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil {
|
||||||
|
t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr)
|
||||||
|
}
|
||||||
|
for _, finding := range assessment.Validators {
|
||||||
|
if finding.Decision != contracts.AssessmentFailed {
|
||||||
|
t.Fatalf("invalid rejection = %#v", finding)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error)
|
||||||
|
|
||||||
|
func (notesAssessmentValidator) Name() string { return "notes" }
|
||||||
|
func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||||
|
return v(ctx, request)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator {
|
||||||
|
t.Helper()
|
||||||
|
registry := NewValidatorRegistry()
|
||||||
|
if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry, _ := registry.typedEntry(name, "test/notes")
|
||||||
|
built, err := entry.builder(BuildRequest{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
|
||||||
|
t.Helper()
|
||||||
|
registry := NewArtifactCodecRegistry()
|
||||||
|
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry, _, err := registry.entry("test/notes")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return entry
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
|
||||||
|
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
|
||||||
|
doc := typedTestDocument()
|
||||||
|
doc.Metadata = map[string]any{"label": "original"}
|
||||||
|
target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}}
|
||||||
|
first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||||
|
request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed"
|
||||||
|
request.Source.Units[0].Text = "changed"
|
||||||
|
request.Source.Metadata["label"] = "changed"
|
||||||
|
request.Metadata["label"] = "changed"
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
})
|
||||||
|
second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||||
|
if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" {
|
||||||
|
t.Fatalf("validator received a mutated view: %#v", request)
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
})
|
||||||
|
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}})
|
||||||
|
assessment, err := assess(context.Background(), candidate)
|
||||||
|
if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" {
|
||||||
|
t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) {
|
||||||
|
calls := []string{}
|
||||||
|
makeValidator := func(name, accepted string) preparedValidator {
|
||||||
|
return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||||
|
value := request.Value.Items[0]
|
||||||
|
calls = append(calls, name+":"+value)
|
||||||
|
if value == accepted {
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}})
|
||||||
|
for _, text := range []string{"one", "two", "two"} {
|
||||||
|
assessment, err := assess(context.Background(), codecNotes{Items: []string{text}})
|
||||||
|
if err != nil || assessment.Accepted() {
|
||||||
|
t.Fatalf("assessment for %s = %#v, %v", text, assessment, err)
|
||||||
|
}
|
||||||
|
want := contracts.AssessmentRejected
|
||||||
|
if text == "one" {
|
||||||
|
want = contracts.AssessmentApproved
|
||||||
|
}
|
||||||
|
if assessment.Validators[0].Decision != want {
|
||||||
|
t.Fatalf("earlier approval survived revised candidate: %#v", assessment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) {
|
||||||
|
t.Fatalf("assessments = %#v", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
|
||||||
|
|
||||||
|
func (chunkAssessmentValidator) Name() string { return "chunk assessment" }
|
||||||
|
func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
return v(ctx, request)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) {
|
||||||
|
chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}}
|
||||||
|
originalJSON, err := json.Marshal(chunks)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
mutator := preparedValidator{
|
||||||
|
resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk},
|
||||||
|
chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
calls++
|
||||||
|
request.Chunks[0].Content[0] = 'x'
|
||||||
|
request.Chunks[0].Metadata["label"] = "changed"
|
||||||
|
return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
observer := preparedValidator{
|
||||||
|
resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized},
|
||||||
|
serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
calls++
|
||||||
|
if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) {
|
||||||
|
t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks)
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks)
|
||||||
|
if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" {
|
||||||
|
t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,440 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// producerAttemptKind records why a producer invocation followed the prior
|
|
||||||
// one. It is deliberately independent of any artifact family.
|
|
||||||
type producerAttemptKind string
|
|
||||||
|
|
||||||
const (
|
|
||||||
producerAttemptInitial producerAttemptKind = "initial"
|
|
||||||
producerAttemptOperationalRetry producerAttemptKind = "operational_error_retry"
|
|
||||||
producerAttemptStructuralRetry producerAttemptKind = "structural_retry"
|
|
||||||
producerAttemptModuleRetry producerAttemptKind = "module_requested_retry"
|
|
||||||
producerAttemptSemanticRetry producerAttemptKind = "semantic_correction"
|
|
||||||
)
|
|
||||||
|
|
||||||
type producerAttemptOutcome string
|
|
||||||
|
|
||||||
const (
|
|
||||||
producerAttemptAccepted producerAttemptOutcome = "accepted"
|
|
||||||
producerAttemptRejected producerAttemptOutcome = "rejected"
|
|
||||||
producerAttemptIncompleteAccepted producerAttemptOutcome = "incomplete_accepted"
|
|
||||||
producerAttemptRetried producerAttemptOutcome = "retried"
|
|
||||||
producerAttemptFailed producerAttemptOutcome = "failed"
|
|
||||||
)
|
|
||||||
|
|
||||||
type producerTerminalAction string
|
|
||||||
|
|
||||||
const (
|
|
||||||
producerTerminalAccepted producerTerminalAction = "accepted"
|
|
||||||
producerTerminalRejected producerTerminalAction = "reject_output"
|
|
||||||
producerTerminalIncompleteAccepted producerTerminalAction = "warn_continue"
|
|
||||||
producerTerminalFailed producerTerminalAction = "fail_run"
|
|
||||||
)
|
|
||||||
|
|
||||||
// producerAttemptRequest contains only attempt-local control material. The
|
|
||||||
// producer reconstructs its ordinary request from its own durable inputs.
|
|
||||||
type producerAttemptRequest struct {
|
|
||||||
Number int
|
|
||||||
Kind producerAttemptKind
|
|
||||||
Correction *contracts.SemanticCorrection
|
|
||||||
}
|
|
||||||
|
|
||||||
// producerRetryDirective asks for another producer invocation while retaining
|
|
||||||
// the current value as a safe fallback if its shared budget is exhausted.
|
|
||||||
// Artifact-specific adapters are responsible for validating and populating it.
|
|
||||||
type producerRetryDirective struct {
|
|
||||||
CorrectionGuidance string
|
|
||||||
FallbackDiagnostics []contracts.ProducerDiagnostic
|
|
||||||
}
|
|
||||||
|
|
||||||
func (directive *producerRetryDirective) clone() *producerRetryDirective {
|
|
||||||
if directive == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &producerRetryDirective{
|
|
||||||
CorrectionGuidance: directive.CorrectionGuidance,
|
|
||||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(directive.FallbackDiagnostics),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// producerAttemptOutput is intentionally artifact-neutral. Value remains
|
|
||||||
// opaque to the state machine; Candidate is the attempt-local response that
|
|
||||||
// may support semantic correction.
|
|
||||||
type producerAttemptOutput struct {
|
|
||||||
Value any
|
|
||||||
Candidate *contracts.ModelCandidate
|
|
||||||
Diagnostics []contracts.ProducerDiagnostic
|
|
||||||
Retry *producerRetryDirective
|
|
||||||
}
|
|
||||||
|
|
||||||
func (output producerAttemptOutput) clone() (producerAttemptOutput, error) {
|
|
||||||
candidate, err := contracts.CloneModelCandidate(output.Candidate)
|
|
||||||
if err != nil {
|
|
||||||
return producerAttemptOutput{}, fmt.Errorf("clone model candidate: %w", err)
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{
|
|
||||||
Value: output.Value,
|
|
||||||
Candidate: candidate,
|
|
||||||
Diagnostics: contracts.CloneProducerDiagnostics(output.Diagnostics),
|
|
||||||
Retry: output.Retry.clone(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type producerAttemptProducer func(context.Context, producerAttemptRequest) (producerAttemptOutput, error)
|
|
||||||
type producerAttemptValidator func(context.Context, producerAttemptOutput) (validationReport, error)
|
|
||||||
|
|
||||||
type producerAttemptConfig struct {
|
|
||||||
Retries int
|
|
||||||
Policy ValidationPolicy
|
|
||||||
AllowStructuralRetry bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// producerAttemptProvenance is ordered by producer invocation. It retains the
|
|
||||||
// settled validation report for later debug and durable-provenance adapters.
|
|
||||||
type producerAttemptProvenance struct {
|
|
||||||
Number int
|
|
||||||
Kind producerAttemptKind
|
|
||||||
Outcome producerAttemptOutcome
|
|
||||||
Validation validationReport
|
|
||||||
}
|
|
||||||
|
|
||||||
func (provenance producerAttemptProvenance) clone() producerAttemptProvenance {
|
|
||||||
provenance.Validation = cloneValidationReport(provenance.Validation)
|
|
||||||
return provenance
|
|
||||||
}
|
|
||||||
|
|
||||||
// producerAttemptTerminal describes the state-machine decision without
|
|
||||||
// materializing an artifact or persisting stage-specific diagnostics.
|
|
||||||
type producerAttemptTerminal struct {
|
|
||||||
Action producerTerminalAction
|
|
||||||
Value any
|
|
||||||
Diagnostics []contracts.ProducerDiagnostic
|
|
||||||
Rejection *contracts.RejectedOutput
|
|
||||||
Validation validationReport
|
|
||||||
ValidationIncomplete bool
|
|
||||||
Provenance []producerAttemptProvenance
|
|
||||||
}
|
|
||||||
|
|
||||||
func (terminal producerAttemptTerminal) clone() producerAttemptTerminal {
|
|
||||||
terminal.Diagnostics = contracts.CloneProducerDiagnostics(terminal.Diagnostics)
|
|
||||||
if terminal.Rejection != nil {
|
|
||||||
rejection := *terminal.Rejection
|
|
||||||
rejection.Validation = cloneValidationSummaryPtr(rejection.Validation)
|
|
||||||
terminal.Rejection = &rejection
|
|
||||||
}
|
|
||||||
terminal.Validation = cloneValidationReport(terminal.Validation)
|
|
||||||
terminal.Provenance = cloneProducerAttemptProvenance(terminal.Provenance)
|
|
||||||
return terminal
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneProducerAttemptProvenance(provenance []producerAttemptProvenance) []producerAttemptProvenance {
|
|
||||||
if len(provenance) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := make([]producerAttemptProvenance, len(provenance))
|
|
||||||
for index, item := range provenance {
|
|
||||||
cloned[index] = item.clone()
|
|
||||||
}
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneValidationReport(report validationReport) validationReport {
|
|
||||||
return validationReport{records: report.Records()}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runProducerAttempts(ctx context.Context, config producerAttemptConfig, produce producerAttemptProducer, validate producerAttemptValidator) (producerAttemptTerminal, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return producerAttemptTerminal{}, errors.New("producer attempt context must not be nil")
|
|
||||||
}
|
|
||||||
if config.Retries < 0 {
|
|
||||||
return producerAttemptTerminal{}, errors.New("producer retries must not be negative")
|
|
||||||
}
|
|
||||||
if produce == nil {
|
|
||||||
return producerAttemptTerminal{}, errors.New("producer attempt closure must not be nil")
|
|
||||||
}
|
|
||||||
if validate == nil {
|
|
||||||
return producerAttemptTerminal{}, errors.New("producer validation closure must not be nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
attemptLimit := config.Retries + 1
|
|
||||||
provenance := make([]producerAttemptProvenance, 0, attemptLimit)
|
|
||||||
kind := producerAttemptInitial
|
|
||||||
var correction *contracts.SemanticCorrection
|
|
||||||
|
|
||||||
for number := 1; number <= attemptLimit; number++ {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
requestCorrection, err := contracts.CloneSemanticCorrection(correction)
|
|
||||||
if err != nil {
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("clone semantic correction: %w", err)
|
|
||||||
}
|
|
||||||
output, err := produce(ctx, producerAttemptRequest{Number: number, Kind: kind, Correction: requestCorrection})
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
if isImmediateProducerFailure(err) {
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
if errors.Is(err, contracts.ErrInvalidStructuredOutput) && config.AllowStructuralRetry {
|
|
||||||
if number < attemptLimit {
|
|
||||||
kind, correction = producerAttemptStructuralRetry, nil
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return applyStructuralTerminalPolicy(config.Policy, provenance, number, err)
|
|
||||||
}
|
|
||||||
if number < attemptLimit {
|
|
||||||
kind, correction = producerAttemptOperationalRetry, nil
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("producer failed after %d attempt(s): %w", number, err)
|
|
||||||
}
|
|
||||||
if err := validateProducerAttemptDiagnostics(output); err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
|
|
||||||
output, err = output.clone()
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
if output.Retry != nil && number < attemptLimit {
|
|
||||||
correction, err = moduleRetryCorrection(output)
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried})
|
|
||||||
kind = producerAttemptModuleRetry
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if output.Retry != nil && output.Retry.CorrectionGuidance != "" {
|
|
||||||
if _, err := moduleRetryCorrection(output); err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if output.Retry != nil {
|
|
||||||
output.Diagnostics = append(output.Diagnostics, contracts.CloneProducerDiagnostics(output.Retry.FallbackDiagnostics)...)
|
|
||||||
}
|
|
||||||
correctionCandidate, err := contracts.CloneModelCandidate(output.Candidate)
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("clone correction candidate: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
report, err := validate(ctx, output)
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
report = cloneValidationReport(report)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
||||||
return failedProducerAttempt(provenance), err
|
|
||||||
}
|
|
||||||
|
|
||||||
if rejection := report.FirstRejection(); rejection != nil {
|
|
||||||
if correctionCandidate != nil {
|
|
||||||
if err := correctionCandidate.Validate(); err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("validate producer model candidate: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if number < attemptLimit && correctionCandidate != nil && correctionCandidate.Protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
|
||||||
correctionRequest, guidanceErr := report.CorrectionRequest()
|
|
||||||
if guidanceErr != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction request: %w", guidanceErr)
|
|
||||||
}
|
|
||||||
correction, err = contracts.NewSemanticCorrection(correctionCandidate.Response, correctionRequest)
|
|
||||||
if err != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptFailed, Validation: report})
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("construct semantic correction: %w", err)
|
|
||||||
}
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRetried, Validation: report})
|
|
||||||
kind = producerAttemptSemanticRetry
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptRejected, Validation: report})
|
|
||||||
return applySemanticTerminalPolicy(config.Policy, provenance, number, output, report, *rejection)
|
|
||||||
}
|
|
||||||
|
|
||||||
if incomplete := firstIncompleteValidation(report); incomplete != nil {
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptIncompleteAccepted, Validation: report})
|
|
||||||
if config.Policy.ValidatorFailure == ValidatorFailureWarnContinue {
|
|
||||||
return producerAttemptTerminal{Action: producerTerminalIncompleteAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, ValidationIncomplete: true, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
||||||
}
|
|
||||||
return failedProducerAttempt(provenance), validatorFailureError(*incomplete)
|
|
||||||
}
|
|
||||||
|
|
||||||
provenance = append(provenance, producerAttemptProvenance{Number: number, Kind: kind, Outcome: producerAttemptAccepted, Validation: report})
|
|
||||||
return producerAttemptTerminal{Action: producerTerminalAccepted, Value: output.Value, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Validation: report, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return failedProducerAttempt(provenance), errors.New("producer attempt budget was not exhausted deterministically")
|
|
||||||
}
|
|
||||||
|
|
||||||
func moduleRetryCorrection(output producerAttemptOutput) (*contracts.SemanticCorrection, error) {
|
|
||||||
if output.Retry == nil || output.Retry.CorrectionGuidance == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if output.Candidate == nil {
|
|
||||||
return nil, errors.New("module retry correction guidance requires a model candidate")
|
|
||||||
}
|
|
||||||
if output.Candidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 {
|
|
||||||
return nil, fmt.Errorf("module retry correction guidance requires protocol %q", contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
}
|
|
||||||
correction, err := contracts.NewSemanticCorrection(output.Candidate.Response, output.Retry.CorrectionGuidance)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("construct module retry semantic correction: %w", err)
|
|
||||||
}
|
|
||||||
return correction, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateProducerAttemptDiagnostics(output producerAttemptOutput) error {
|
|
||||||
if err := contracts.ValidateProducerDiagnostics(output.Diagnostics); err != nil {
|
|
||||||
return fmt.Errorf("producer returned invalid diagnostics: %w", err)
|
|
||||||
}
|
|
||||||
if output.Retry != nil {
|
|
||||||
if err := contracts.ValidateProducerDiagnostics(output.Retry.FallbackDiagnostics); err != nil {
|
|
||||||
return fmt.Errorf("producer returned invalid retry fallback diagnostics: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isImmediateProducerFailure(err error) bool {
|
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
var debugErr *attemptDebugPersistenceError
|
|
||||||
return errors.As(err, &debugErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
func applyStructuralTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, err error) (producerAttemptTerminal, error) {
|
|
||||||
switch policy.ProducerStructuralFailure {
|
|
||||||
case ProducerStructuralFailureRejectOutput:
|
|
||||||
return producerAttemptTerminal{Action: producerTerminalRejected, Rejection: &contracts.RejectedOutput{ReasonCode: "invalid_structured_output", Message: "producer returned invalid structured output", AttemptCount: number}, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
||||||
case ProducerStructuralFailureFailRun:
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("producer returned invalid structured output after %d attempt(s): %w", number, err)
|
|
||||||
default:
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("unknown producer structural-failure action %q", policy.ProducerStructuralFailure)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func applySemanticTerminalPolicy(policy ValidationPolicy, provenance []producerAttemptProvenance, number int, output producerAttemptOutput, report validationReport, rejection validationRecord) (producerAttemptTerminal, error) {
|
|
||||||
rejected := contracts.RejectedOutput{ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: number, DiagnosticArtifactPath: rejection.diagnosticPath}
|
|
||||||
switch policy.SemanticRejection {
|
|
||||||
case SemanticRejectionRejectOutput:
|
|
||||||
return producerAttemptTerminal{Action: producerTerminalRejected, Diagnostics: cloneProducerDiagnostics(output.Diagnostics), Rejection: &rejected, Validation: report, ValidationIncomplete: firstIncompleteValidation(report) != nil, Provenance: cloneProducerAttemptProvenance(provenance)}, nil
|
|
||||||
case SemanticRejectionFailRun:
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("producer candidate rejected after %d attempt(s): %s", number, rejection.message)
|
|
||||||
default:
|
|
||||||
return failedProducerAttempt(provenance), fmt.Errorf("unknown semantic-rejection action %q", policy.SemanticRejection)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func firstIncompleteValidation(report validationReport) *validationRecord {
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
|
||||||
clone := record.clone()
|
|
||||||
return &clone
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic) []contracts.ProducerDiagnostic {
|
|
||||||
return contracts.CloneProducerDiagnostics(diagnostics)
|
|
||||||
}
|
|
||||||
|
|
||||||
// incompleteValidationDiagnostics reports every applicable validator that
|
|
||||||
// could not complete under warn_continue. It uses fixed text so provider
|
|
||||||
// errors and arbitrary validator prose cannot cross this boundary.
|
|
||||||
func incompleteValidationDiagnostics(report validationReport) []validationDiagnosticRecord {
|
|
||||||
diagnostics := make([]validationDiagnosticRecord, 0)
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome != validationFailed && record.outcome != validationSkipped {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: contracts.ProducerDiagnostic{
|
|
||||||
Disposition: contracts.DiagnosticDispositionWarning,
|
|
||||||
Category: contracts.DiagnosticCategoryValidationIncomplete,
|
|
||||||
ReasonCode: "validator_execution_incomplete",
|
|
||||||
OccurrenceCount: 1,
|
|
||||||
Samples: []contracts.DiagnosticSample{{
|
|
||||||
Scope: record.validatorName,
|
|
||||||
Message: "Validator execution did not complete within its configured budget.",
|
|
||||||
}},
|
|
||||||
}})
|
|
||||||
}
|
|
||||||
return diagnostics
|
|
||||||
}
|
|
||||||
|
|
||||||
// validationSummary projects a terminal state-machine result into the durable
|
|
||||||
// bounded representation used by manifests, rejections, and receipts.
|
|
||||||
func validationSummary(terminal producerAttemptTerminal, stage ModuleStage, stepID, laneID, moduleKey, chunkID string, chunkIndex int) artifacts.ValidationSummary {
|
|
||||||
summary := artifacts.ValidationSummary{
|
|
||||||
Stage: string(stage),
|
|
||||||
StepID: stepID,
|
|
||||||
LaneID: laneID,
|
|
||||||
ModuleKey: moduleKey,
|
|
||||||
ChunkID: chunkID,
|
|
||||||
ChunkIndex: chunkIndex,
|
|
||||||
ProducerAttemptCount: len(terminal.Provenance),
|
|
||||||
TerminalAction: string(terminal.Action),
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case terminal.Action == producerTerminalRejected:
|
|
||||||
summary.Status = "rejected"
|
|
||||||
case terminal.Action == producerTerminalFailed:
|
|
||||||
summary.Status = "incomplete"
|
|
||||||
case terminal.ValidationIncomplete:
|
|
||||||
summary.Status = "incomplete"
|
|
||||||
default:
|
|
||||||
summary.Status = "complete"
|
|
||||||
}
|
|
||||||
seenValidators := make(map[string]struct{})
|
|
||||||
seenReasons := make(map[string]struct{})
|
|
||||||
seenIncomplete := make(map[string]struct{})
|
|
||||||
for _, record := range terminal.Validation.records {
|
|
||||||
if record.reasonCode != "" {
|
|
||||||
if _, exists := seenReasons[record.reasonCode]; !exists {
|
|
||||||
seenReasons[record.reasonCode] = struct{}{}
|
|
||||||
summary.ReasonCodes = append(summary.ReasonCodes, record.reasonCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if record.outcome == validationRejected {
|
|
||||||
if _, exists := seenValidators[record.validatorName]; !exists {
|
|
||||||
seenValidators[record.validatorName] = struct{}{}
|
|
||||||
summary.RejectingValidators = append(summary.RejectingValidators, record.validatorName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if record.outcome == validationFailed || record.outcome == validationSkipped {
|
|
||||||
if _, exists := seenIncomplete[record.validatorName]; !exists {
|
|
||||||
seenIncomplete[record.validatorName] = struct{}{}
|
|
||||||
summary.IncompleteValidators = append(summary.IncompleteValidators, record.validatorName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if terminal.Rejection != nil && terminal.Rejection.ReasonCode != "" {
|
|
||||||
if _, exists := seenReasons[terminal.Rejection.ReasonCode]; !exists {
|
|
||||||
summary.ReasonCodes = append(summary.ReasonCodes, terminal.Rejection.ReasonCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return summary
|
|
||||||
}
|
|
||||||
|
|
||||||
func failedProducerAttempt(provenance []producerAttemptProvenance) producerAttemptTerminal {
|
|
||||||
return producerAttemptTerminal{Action: producerTerminalFailed, Provenance: cloneProducerAttemptProvenance(provenance)}
|
|
||||||
}
|
|
||||||
@@ -1,530 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsUsesOneSharedBudget(t *testing.T) {
|
|
||||||
candidate := attemptCandidate(t, "first response")
|
|
||||||
producerCalls := 0
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy(), AllowStructuralRetry: true}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
producerCalls++
|
|
||||||
switch request.Number {
|
|
||||||
case 1:
|
|
||||||
return producerAttemptOutput{}, errors.New("temporary producer failure")
|
|
||||||
case 2:
|
|
||||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
|
||||||
case 3:
|
|
||||||
return producerAttemptOutput{Value: "accepted", Candidate: candidate}, nil
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
|
||||||
return producerAttemptOutput{}, nil
|
|
||||||
}
|
|
||||||
}, approveAttempt)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
|
||||||
t.Fatalf("terminal = %#v, want accepted value", terminal)
|
|
||||||
}
|
|
||||||
if producerCalls != 3 {
|
|
||||||
t.Fatalf("producer calls = %d, want 3", producerCalls)
|
|
||||||
}
|
|
||||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptOperationalRetry, producerAttemptStructuralRetry}) {
|
|
||||||
t.Fatalf("attempt kinds = %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsReplacesSemanticCorrectionWithLatestResponse(t *testing.T) {
|
|
||||||
first := attemptCandidate(t, "first response")
|
|
||||||
second := attemptCandidate(t, "second response")
|
|
||||||
var corrections []*contracts.SemanticCorrection
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
if request.Correction != nil {
|
|
||||||
corrections = append(corrections, request.Correction)
|
|
||||||
}
|
|
||||||
switch request.Number {
|
|
||||||
case 1:
|
|
||||||
return producerAttemptOutput{Value: "one", Candidate: first}, nil
|
|
||||||
case 2:
|
|
||||||
return producerAttemptOutput{Value: "two", Candidate: second}, nil
|
|
||||||
case 3:
|
|
||||||
return producerAttemptOutput{Value: "three", Candidate: attemptCandidate(t, "third response")}, nil
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
|
||||||
return producerAttemptOutput{}, nil
|
|
||||||
}
|
|
||||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
|
||||||
if output.Value == "three" {
|
|
||||||
return validationReport{}, nil
|
|
||||||
}
|
|
||||||
return rejectedAttemptReport("semantic_defect"), nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted {
|
|
||||||
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
|
||||||
}
|
|
||||||
if len(corrections) != 2 {
|
|
||||||
t.Fatalf("correction count = %d, want 2", len(corrections))
|
|
||||||
}
|
|
||||||
if corrections[0] == corrections[1] {
|
|
||||||
t.Fatal("semantic corrections reused the same pointer")
|
|
||||||
}
|
|
||||||
if got := string(corrections[0].AssistantResponse); got != "first response" {
|
|
||||||
t.Fatalf("first correction response = %q", got)
|
|
||||||
}
|
|
||||||
if got := string(corrections[1].AssistantResponse); got != "second response" {
|
|
||||||
t.Fatalf("second correction response = %q", got)
|
|
||||||
}
|
|
||||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry, producerAttemptSemanticRetry}) {
|
|
||||||
t.Fatalf("attempt kinds = %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsAppliesTerminalPolicies(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
config producerAttemptConfig
|
|
||||||
produce producerAttemptProducer
|
|
||||||
validate producerAttemptValidator
|
|
||||||
wantAction producerTerminalAction
|
|
||||||
wantError bool
|
|
||||||
wantValue any
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "structural reject output",
|
|
||||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureWarnContinue}, AllowStructuralRetry: true},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
|
||||||
},
|
|
||||||
validate: approveAttempt,
|
|
||||||
wantAction: producerTerminalRejected,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "structural fail run",
|
|
||||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy(), AllowStructuralRetry: true},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, contracts.ErrInvalidStructuredOutput
|
|
||||||
},
|
|
||||||
validate: approveAttempt,
|
|
||||||
wantAction: producerTerminalFailed,
|
|
||||||
wantError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "operational failure after retry exhaustion",
|
|
||||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, errors.New("producer unavailable")
|
|
||||||
},
|
|
||||||
validate: approveAttempt,
|
|
||||||
wantAction: producerTerminalFailed,
|
|
||||||
wantError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "semantic reject output",
|
|
||||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{Value: "discard", Candidate: attemptCandidate(t, "response")}, nil
|
|
||||||
},
|
|
||||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return rejectedAttemptReport("bad"), nil
|
|
||||||
},
|
|
||||||
wantAction: producerTerminalRejected,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "semantic fail run",
|
|
||||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{Candidate: attemptCandidate(t, "response")}, nil
|
|
||||||
},
|
|
||||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return rejectedAttemptReport("bad"), nil
|
|
||||||
},
|
|
||||||
wantAction: producerTerminalFailed,
|
|
||||||
wantError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "validator failure warns and continues",
|
|
||||||
config: producerAttemptConfig{Policy: DefaultValidationPolicy()},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{Value: "kept"}, nil
|
|
||||||
},
|
|
||||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return failedAttemptReport(), nil
|
|
||||||
},
|
|
||||||
wantAction: producerTerminalIncompleteAccepted,
|
|
||||||
wantValue: "kept",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "validator failure fails run",
|
|
||||||
config: producerAttemptConfig{Policy: ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureFailRun, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}},
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, nil
|
|
||||||
},
|
|
||||||
validate: func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return failedAttemptReport(), nil
|
|
||||||
},
|
|
||||||
wantAction: producerTerminalFailed,
|
|
||||||
wantError: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), test.config, test.produce, test.validate)
|
|
||||||
if (err != nil) != test.wantError {
|
|
||||||
t.Fatalf("error = %v, want error %t", err, test.wantError)
|
|
||||||
}
|
|
||||||
if terminal.Action != test.wantAction {
|
|
||||||
t.Fatalf("terminal action = %q, want %q", terminal.Action, test.wantAction)
|
|
||||||
}
|
|
||||||
if terminal.Value != test.wantValue {
|
|
||||||
t.Fatalf("terminal value = %#v, want %#v", terminal.Value, test.wantValue)
|
|
||||||
}
|
|
||||||
if test.wantAction == producerTerminalRejected && (terminal.Rejection == nil || terminal.Value != nil) {
|
|
||||||
t.Fatalf("rejected terminal = %#v, want rejection without value", terminal)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsDoesNotRetryUncorrectableRejection(t *testing.T) {
|
|
||||||
calls := 0
|
|
||||||
policy := DefaultValidationPolicy()
|
|
||||||
policy.SemanticRejection = SemanticRejectionRejectOutput
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: policy}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
calls++
|
|
||||||
return producerAttemptOutput{Value: "deterministic"}, nil
|
|
||||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return rejectedAttemptReport("deterministic_rejection"), nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if calls != 1 || terminal.Action != producerTerminalRejected || len(terminal.Provenance) != 1 {
|
|
||||||
t.Fatalf("terminal = %#v, calls = %d; want immediate rejection", terminal, calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsUsesModuleRetryBudgetAndFallback(t *testing.T) {
|
|
||||||
t.Run("retry", func(t *testing.T) {
|
|
||||||
calls := 0
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
calls++
|
|
||||||
if request.Correction != nil {
|
|
||||||
t.Fatalf("feedback-free module retry correction = %#v, want nil", request.Correction)
|
|
||||||
}
|
|
||||||
if request.Number == 1 {
|
|
||||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{}}, nil
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{Value: "replacement"}, nil
|
|
||||||
}, approveAttempt)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "replacement" || calls != 2 {
|
|
||||||
t.Fatalf("terminal = %#v, calls = %d", terminal, calls)
|
|
||||||
}
|
|
||||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
|
||||||
t.Fatalf("attempt kinds = %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("feedback retry", func(t *testing.T) {
|
|
||||||
const (
|
|
||||||
defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
|
|
||||||
guidance = "Use only candidate numbers from the supplied candidate list. Return one complete corrected response."
|
|
||||||
)
|
|
||||||
var observed *contracts.SemanticCorrection
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
if request.Number == 1 {
|
|
||||||
return producerAttemptOutput{Value: "safe fallback", Candidate: attemptCandidate(t, defective), Retry: &producerRetryDirective{CorrectionGuidance: guidance}}, nil
|
|
||||||
}
|
|
||||||
observed = request.Correction
|
|
||||||
return producerAttemptOutput{Value: "corrected"}, nil
|
|
||||||
}, approveAttempt)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "corrected" {
|
|
||||||
t.Fatalf("terminal = %#v, want corrected accepted value", terminal)
|
|
||||||
}
|
|
||||||
if observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance {
|
|
||||||
t.Fatalf("module retry correction = %#v, want exact latest response and guidance", observed)
|
|
||||||
}
|
|
||||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptModuleRetry}) {
|
|
||||||
t.Fatalf("attempt kinds = %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("feedback-free retry clears prior correction", func(t *testing.T) {
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 2, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
switch request.Number {
|
|
||||||
case 1:
|
|
||||||
return producerAttemptOutput{Value: "first fallback", Candidate: attemptCandidate(t, "first defective response"), Retry: &producerRetryDirective{CorrectionGuidance: "Correct the first response."}}, nil
|
|
||||||
case 2:
|
|
||||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != "first defective response" {
|
|
||||||
t.Fatalf("second attempt correction = %#v", request.Correction)
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{Value: "second fallback", Retry: &producerRetryDirective{}}, nil
|
|
||||||
case 3:
|
|
||||||
if request.Correction != nil {
|
|
||||||
t.Fatalf("third attempt retained stale correction %#v", request.Correction)
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{Value: "accepted"}, nil
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected producer attempt %d", request.Number)
|
|
||||||
return producerAttemptOutput{}, nil
|
|
||||||
}
|
|
||||||
}, approveAttempt)
|
|
||||||
if err != nil || terminal.Action != producerTerminalAccepted || terminal.Value != "accepted" {
|
|
||||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("fallback", func(t *testing.T) {
|
|
||||||
fallbackDiagnostic := contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "fallback", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "fallback", Message: "fallback warning"}}}
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{Value: "fallback", Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{fallbackDiagnostic}}}, nil
|
|
||||||
}, approveAttempt)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted || terminal.Value != "fallback" || !reflect.DeepEqual(terminal.Diagnostics, []contracts.ProducerDiagnostic{fallbackDiagnostic}) {
|
|
||||||
t.Fatalf("terminal = %#v", terminal)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsRejectsModuleCorrectionWithoutModelCandidate(t *testing.T) {
|
|
||||||
producerCalls := 0
|
|
||||||
validatorCalls := 0
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
producerCalls++
|
|
||||||
return producerAttemptOutput{Value: "safe", Retry: &producerRetryDirective{CorrectionGuidance: "Return a complete corrected response."}}, nil
|
|
||||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
validatorCalls++
|
|
||||||
return validationReport{}, nil
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "requires a model candidate") {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v, want model-candidate contract failure", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
|
||||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d", terminal, producerCalls, validatorCalls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
output producerAttemptOutput
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "producer diagnostics",
|
|
||||||
output: producerAttemptOutput{Diagnostics: []contracts.ProducerDiagnostic{{}}},
|
|
||||||
want: "producer returned invalid diagnostics",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "retry fallback diagnostics",
|
|
||||||
output: producerAttemptOutput{Retry: &producerRetryDirective{FallbackDiagnostics: []contracts.ProducerDiagnostic{{}}}},
|
|
||||||
want: "producer returned invalid retry fallback diagnostics",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
producerCalls := 0
|
|
||||||
validatorCalls := 0
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
producerCalls++
|
|
||||||
return test.output, nil
|
|
||||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
validatorCalls++
|
|
||||||
return validationReport{}, nil
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v, want %q", err, test.want)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalFailed || producerCalls != 1 || validatorCalls != 0 {
|
|
||||||
t.Fatalf("terminal = %#v, producer calls = %d, validator calls = %d; want immediate framework failure", terminal, producerCalls, validatorCalls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsRejectionWinsOverValidatorFailure(t *testing.T) {
|
|
||||||
firstDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded warning")}
|
|
||||||
secondDiagnostics := []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted warning")}
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
if request.Number == 1 {
|
|
||||||
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "defective"), Diagnostics: firstDiagnostics}, nil
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{Value: "second", Candidate: attemptCandidate(t, "corrected"), Diagnostics: secondDiagnostics}, nil
|
|
||||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
|
||||||
if output.Value == "first" {
|
|
||||||
report := rejectedAttemptReport("defect")
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: "unavailable", outcome: validationFailed, failure: errors.New("validator unavailable")})
|
|
||||||
return report, nil
|
|
||||||
}
|
|
||||||
return validationReport{}, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalAccepted {
|
|
||||||
t.Fatalf("terminal action = %q, want accepted", terminal.Action)
|
|
||||||
}
|
|
||||||
if got := attemptKinds(terminal.Provenance); !reflect.DeepEqual(got, []producerAttemptKind{producerAttemptInitial, producerAttemptSemanticRetry}) {
|
|
||||||
t.Fatalf("attempt kinds = %v", got)
|
|
||||||
}
|
|
||||||
if got := terminal.Diagnostics; !reflect.DeepEqual(got, secondDiagnostics) {
|
|
||||||
t.Fatalf("terminal diagnostics = %#v, want %#v", got, secondDiagnostics)
|
|
||||||
}
|
|
||||||
if len(terminal.Provenance[0].Validation.records) != 2 {
|
|
||||||
t.Fatalf("first validation records = %#v, want rejection and failure", terminal.Provenance[0].Validation.records)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidationSummaryIsBoundedAndCorrectedSuccessIsQuiet(t *testing.T) {
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
if request.Number == 1 {
|
|
||||||
return producerAttemptOutput{Value: "first", Candidate: attemptCandidate(t, "sensitive defective response")}, nil
|
|
||||||
}
|
|
||||||
return producerAttemptOutput{Value: "corrected", Candidate: attemptCandidate(t, "sensitive corrected response")}, nil
|
|
||||||
}, func(_ context.Context, output producerAttemptOutput) (validationReport, error) {
|
|
||||||
if output.Value == "first" {
|
|
||||||
return validationReport{records: []validationRecord{{validatorName: "first", outcome: validationRejected, reasonCode: "needs_fix", message: "sensitive diagnostic", correctionGuidance: "sensitive guidance"}}}, nil
|
|
||||||
}
|
|
||||||
return validationReport{}, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
summary := validationSummary(terminal, StageExtract, "step", "lane", "module", "chunk", 3)
|
|
||||||
if summary.Status != "complete" || summary.ProducerAttemptCount != 2 || summary.TerminalAction != string(producerTerminalAccepted) {
|
|
||||||
t.Fatalf("summary = %#v", summary)
|
|
||||||
}
|
|
||||||
if len(summary.ReasonCodes) != 0 || len(summary.RejectingValidators) != 0 || len(summary.IncompleteValidators) != 0 {
|
|
||||||
t.Fatalf("corrected success summary retained prior findings: %#v", summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWarnContinueRecordsOneWarningForEachExhaustedValidator(t *testing.T) {
|
|
||||||
report := validationReport{records: []validationRecord{
|
|
||||||
{validatorName: "first", outcome: validationFailed, attemptCount: 2, failure: errors.New("provider error with sensitive details")},
|
|
||||||
{validatorName: "second", outcome: validationSkipped, attemptCount: 1, reasonCode: "missing_prerequisite", message: "sensitive skipped detail"},
|
|
||||||
{validatorName: "third", outcome: validationFailed, attemptCount: 1, failure: errors.New("other provider error")},
|
|
||||||
}}
|
|
||||||
terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Policy: DefaultValidationPolicy()}, func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{Value: "candidate"}, nil
|
|
||||||
}, func(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return report, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runProducerAttempts() error = %v", err)
|
|
||||||
}
|
|
||||||
if terminal.Action != producerTerminalIncompleteAccepted {
|
|
||||||
t.Fatalf("terminal action = %q", terminal.Action)
|
|
||||||
}
|
|
||||||
groups, groupErr := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil)
|
|
||||||
if groupErr != nil {
|
|
||||||
t.Fatalf("terminalDiagnosticGroups() error = %v", groupErr)
|
|
||||||
}
|
|
||||||
if len(groups) != 3 {
|
|
||||||
t.Fatalf("diagnostic groups = %#v, want every failed and skipped validator", groups)
|
|
||||||
}
|
|
||||||
for index, validator := range []string{"first", "second", "third"} {
|
|
||||||
if group := groups[index]; group.Origin.ValidatorKey != validator || group.Disposition != contracts.DiagnosticDispositionWarning || group.Category != contracts.DiagnosticCategoryValidationIncomplete || group.ReasonCode != "validator_execution_incomplete" {
|
|
||||||
t.Fatalf("diagnostic group %d = %#v, want incomplete warning for %q", index, group, validator)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
summary := validationSummary(terminal, StageNormalize, "step", "lane", "module", "", 0)
|
|
||||||
if summary.Status != "incomplete" || !reflect.DeepEqual(summary.IncompleteValidators, []string{"first", "second", "third"}) || !reflect.DeepEqual(summary.ReasonCodes, []string{"missing_prerequisite"}) {
|
|
||||||
t.Fatalf("summary = %#v", summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunProducerAttemptsStopsForCancellationAndDebugFailure(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
context context.Context
|
|
||||||
produce producerAttemptProducer
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "cancelled context",
|
|
||||||
context: cancelledAttemptContext(),
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, nil
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "debug persistence failure",
|
|
||||||
context: context.Background(),
|
|
||||||
produce: func(context.Context, producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
return producerAttemptOutput{}, &attemptDebugPersistenceError{label: "attempt", err: errors.New("write debug")}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
calls := 0
|
|
||||||
producer := func(ctx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
|
|
||||||
calls++
|
|
||||||
return test.produce(ctx, request)
|
|
||||||
}
|
|
||||||
terminal, err := runProducerAttempts(test.context, producerAttemptConfig{Retries: 3, Policy: DefaultValidationPolicy()}, producer, approveAttempt)
|
|
||||||
if err == nil || terminal.Action != producerTerminalFailed {
|
|
||||||
t.Fatalf("terminal = %#v, error = %v", terminal, err)
|
|
||||||
}
|
|
||||||
if test.name == "cancelled context" && calls != 0 {
|
|
||||||
t.Fatalf("cancelled producer calls = %d, want 0", calls)
|
|
||||||
}
|
|
||||||
if test.name == "debug persistence failure" && calls != 1 {
|
|
||||||
t.Fatalf("debug failure producer calls = %d, want 1", calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func attemptCandidate(t *testing.T, response string) *contracts.ModelCandidate {
|
|
||||||
t.Helper()
|
|
||||||
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewModelCandidate() error = %v", err)
|
|
||||||
}
|
|
||||||
return candidate
|
|
||||||
}
|
|
||||||
|
|
||||||
func approveAttempt(context.Context, producerAttemptOutput) (validationReport, error) {
|
|
||||||
return validationReport{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func rejectedAttemptReport(reason string) validationReport {
|
|
||||||
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: reason, message: "candidate rejected", correctionGuidance: "fix the defect"}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func failedAttemptReport() validationReport {
|
|
||||||
return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationFailed, attemptCount: 1, failure: errors.New("validator failure")}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func attemptKinds(provenance []producerAttemptProvenance) []producerAttemptKind {
|
|
||||||
kinds := make([]producerAttemptKind, len(provenance))
|
|
||||||
for index, item := range provenance {
|
|
||||||
kinds[index] = item.Kind
|
|
||||||
}
|
|
||||||
return kinds
|
|
||||||
}
|
|
||||||
|
|
||||||
func cancelledAttemptContext() context.Context {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
return ctx
|
|
||||||
}
|
|
||||||
42
internal/framework/pipeline/producer_outcome.go
Normal file
42
internal/framework/pipeline/producer_outcome.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// finishProducerFailure applies the resolved terminal mode without retrying
|
||||||
|
// production or running ordinary validators against a supported fallback.
|
||||||
|
func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) {
|
||||||
|
var zero T
|
||||||
|
if ctx == nil {
|
||||||
|
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return zero, contracts.ProducerOutcome{}, err
|
||||||
|
}
|
||||||
|
if cause == nil {
|
||||||
|
cause = errors.New("producer did not yield an accepted result")
|
||||||
|
}
|
||||||
|
switch mode {
|
||||||
|
case contracts.ModeStrict:
|
||||||
|
return zero, contracts.ProducerOutcome{}, cause
|
||||||
|
case contracts.ModePermissive:
|
||||||
|
if fallback == nil {
|
||||||
|
return zero, contracts.ProducerOutcome{}, fmt.Errorf("producer has no supported fallback: %w", cause)
|
||||||
|
}
|
||||||
|
value, err := fallback(ctx)
|
||||||
|
if canceled := ctx.Err(); canceled != nil {
|
||||||
|
return zero, contracts.ProducerOutcome{}, canceled
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return zero, contracts.ProducerOutcome{}, errors.Join(cause, fmt.Errorf("construct producer fallback: %w", err))
|
||||||
|
}
|
||||||
|
return value, contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: cause.Error(), ReuseEligible: false}, nil
|
||||||
|
default:
|
||||||
|
return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
64
internal/framework/pipeline/producer_outcome_test.go
Normal file
64
internal/framework/pipeline/producer_outcome_test.go
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) {
|
||||||
|
failure := errors.New("candidate not accepted")
|
||||||
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||||
|
calls := 0
|
||||||
|
value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil })
|
||||||
|
if mode == contracts.ModeStrict {
|
||||||
|
if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" {
|
||||||
|
t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls)
|
||||||
|
}
|
||||||
|
} else if err != nil || calls != 1 || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || outcome.FallbackReason == "" {
|
||||||
|
t.Fatalf("permissive = %q %#v %v, calls %d", value, outcome, err, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) {
|
||||||
|
original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback")
|
||||||
|
_, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction })
|
||||||
|
if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" {
|
||||||
|
t.Fatalf("outcome = %#v, error = %v", outcome, err)
|
||||||
|
}
|
||||||
|
if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) {
|
||||||
|
t.Fatalf("unsupported fallback = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProducerCancellationNeverDeliversFallback(t *testing.T) {
|
||||||
|
for _, cancelDuringFallback := range []bool{false, true} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
calls := 0
|
||||||
|
if !cancelDuringFallback {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) {
|
||||||
|
calls++
|
||||||
|
cancel()
|
||||||
|
return "not delivered", nil
|
||||||
|
})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || value != "" || outcome.Disposition != "" {
|
||||||
|
t.Fatalf("cancellation = %q %#v %v", value, outcome, err)
|
||||||
|
}
|
||||||
|
if !cancelDuringFallback && calls != 0 {
|
||||||
|
t.Fatal("fallback started after cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) {
|
||||||
|
value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil })
|
||||||
|
if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback {
|
||||||
|
t.Fatalf("local timeout = %q %#v %v", value, outcome, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -492,61 +492,6 @@ func runSimpleRetry(ctx context.Context, retries int, run func(attempt int) (ret
|
|||||||
return last, nil
|
return last, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateChunkReport(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
|
|
||||||
content, err := json.Marshal(chunks)
|
|
||||||
if err != nil {
|
|
||||||
return validationReport{}, fmt.Errorf("encode canonical chunks for validation: %w", err)
|
|
||||||
}
|
|
||||||
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
|
||||||
report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
|
|
||||||
binding := item.resolved.Binding
|
|
||||||
started := time.Now().UTC()
|
|
||||||
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
|
|
||||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
|
||||||
var result contracts.ValidationResult
|
|
||||||
requestMetadata, cloneErr := cloneMetadata(metadata)
|
|
||||||
if cloneErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
|
|
||||||
}
|
|
||||||
requestChunks, cloneErr := cloneSourceChunks(chunks)
|
|
||||||
if cloneErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
|
|
||||||
}
|
|
||||||
switch item.resolved.Target {
|
|
||||||
case ValidatorTargetChunk:
|
|
||||||
result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks})
|
|
||||||
case ValidatorTargetSerialized:
|
|
||||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
|
|
||||||
default:
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = contracts.ValidateValidationResult(result)
|
|
||||||
}
|
|
||||||
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
|
|
||||||
debugContent.ContentDigest = debugContentDigest(content)
|
|
||||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
|
|
||||||
if err != nil {
|
|
||||||
debugCall.Error = err.Error()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
|
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
|
|
||||||
}
|
|
||||||
return validationInvocation{}, validationErr
|
|
||||||
}
|
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
|
|
||||||
}
|
|
||||||
return validationInvocation{result: result}, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return report, err
|
|
||||||
}
|
|
||||||
return report, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||||
for _, chain := range chains {
|
for _, chain := range chains {
|
||||||
if chain.Stage != stage {
|
if chain.Stage != stage {
|
||||||
|
|||||||
@@ -583,83 +583,6 @@ func setTypedLaneManifestMetadata(output *RunOutput, laneID string, extractor, m
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateTypedReport(ctx context.Context, codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain, attempt int, debug DebugRecorder) (validationReport, error) {
|
|
||||||
if len(chain.validators) > 0 && target.candidate == nil {
|
|
||||||
candidate, err := validationCandidateArtifact(codec, target)
|
|
||||||
if err != nil {
|
|
||||||
return validationReport{}, fmt.Errorf("serialize %s candidate for validation: %w", target.stage, err)
|
|
||||||
}
|
|
||||||
target.candidate = &candidate
|
|
||||||
}
|
|
||||||
report, err := executeValidationChain(ctx, chain, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
|
|
||||||
binding := item.resolved.Binding
|
|
||||||
var result contracts.ValidationResult
|
|
||||||
var err error
|
|
||||||
started := time.Now().UTC()
|
|
||||||
pathParts := []string{"validate", fileio.EncodePathComponent(string(target.stage)), fileio.EncodePathComponent(target.laneID), fileio.EncodePathComponent(target.moduleKey)}
|
|
||||||
if target.chunk != nil {
|
|
||||||
pathParts = append(pathParts, fmt.Sprintf("chunk-%06d", target.chunk.Index+1))
|
|
||||||
}
|
|
||||||
pathParts = append(pathParts, fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt))
|
|
||||||
attemptPath := validatorAttemptPath(path.Join(pathParts...), validatorAttempt)
|
|
||||||
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
|
|
||||||
requestTarget := target
|
|
||||||
requestTarget.sourceInput = target.sourceInput.Clone()
|
|
||||||
requestTarget.references = CloneReferenceSet(target.references)
|
|
||||||
requestTarget.metadata, err = cloneMetadata(target.metadata)
|
|
||||||
if err != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation metadata: %w", err))
|
|
||||||
}
|
|
||||||
requestTarget.chunk, err = cloneSourceChunkPtr(target.chunk)
|
|
||||||
if err != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunk: %w", err))
|
|
||||||
}
|
|
||||||
requestTarget.chunks, err = cloneSourceChunks(target.chunks)
|
|
||||||
if err != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone typed validation chunks: %w", err))
|
|
||||||
}
|
|
||||||
switch item.resolved.Target {
|
|
||||||
case ValidatorTargetTyped:
|
|
||||||
candidateValue, decodeErr := decodeTypedValidationCandidate(codec, *target.candidate)
|
|
||||||
if decodeErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("decode %s candidate for typed validator %q: %w", target.stage, binding.Module, decodeErr))
|
|
||||||
}
|
|
||||||
requestTarget.value = candidateValue
|
|
||||||
requestTarget.llmProfile = binding.LLMProfile
|
|
||||||
requestTarget.structuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
|
||||||
result, err = item.typedValidate(validatorCtx, item.typed, requestTarget)
|
|
||||||
case ValidatorTargetSerialized:
|
|
||||||
artifact := cloneCheckpointArtifact(*target.candidate)
|
|
||||||
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: requestTarget.sourceInput, SessionID: target.sessionID, References: requestTarget.references, LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestTarget.metadata, Chunk: requestTarget.chunk, Chunks: requestTarget.chunks, Schema: contracts.CloneArtifactSchema(artifact.Artifact.Schema), MediaType: artifact.Artifact.MediaType, Content: append([]byte(nil), artifact.Artifact.Content...)})
|
|
||||||
default:
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module))
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = contracts.ValidateValidationResult(result)
|
|
||||||
}
|
|
||||||
artifact := cloneCheckpointArtifact(*target.candidate)
|
|
||||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(artifact), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
|
|
||||||
if err != nil {
|
|
||||||
debugCall.Error = err.Error()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
validationErr := fmt.Errorf("validate typed %s output with validator %q: %w", target.stage, binding.Module, err)
|
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr)))
|
|
||||||
}
|
|
||||||
return validationInvocation{}, validationErr
|
|
||||||
}
|
|
||||||
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(target.stage), StepID: target.stepID, LaneID: target.laneID, ModuleKey: target.moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
|
|
||||||
return validationInvocation{}, fatalValidationError(fmt.Errorf("write typed validator attempt debug artifact: %w", debugErr))
|
|
||||||
}
|
|
||||||
return validationInvocation{result: result}, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return report, err
|
|
||||||
}
|
|
||||||
return report, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
|
func typedRejection(report validationReport, target typedValidationTarget, attempt int) *contracts.RejectedOutput {
|
||||||
rejection := report.FirstRejection()
|
rejection := report.FirstRejection()
|
||||||
if rejection == nil {
|
if rejection == nil {
|
||||||
|
|||||||
@@ -1,227 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type validationOutcome string
|
|
||||||
|
|
||||||
const (
|
|
||||||
validationApproved validationOutcome = "approved"
|
|
||||||
validationRejected validationOutcome = "rejected"
|
|
||||||
validationFailed validationOutcome = "failed"
|
|
||||||
validationSkipped validationOutcome = "skipped"
|
|
||||||
)
|
|
||||||
|
|
||||||
const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"
|
|
||||||
|
|
||||||
// validationRecord captures the settled result of one configured validator.
|
|
||||||
// Its fields remain private so reports cannot expose mutable diagnostic storage.
|
|
||||||
type validationRecord struct {
|
|
||||||
validatorName string
|
|
||||||
outcome validationOutcome
|
|
||||||
attemptCount int
|
|
||||||
reasonCode string
|
|
||||||
message string
|
|
||||||
diagnosticPath string
|
|
||||||
diagnostics []contracts.ProducerDiagnostic
|
|
||||||
correctionGuidance string
|
|
||||||
failure error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (record validationRecord) clone() validationRecord {
|
|
||||||
record.diagnostics = contracts.CloneProducerDiagnostics(record.diagnostics)
|
|
||||||
return record
|
|
||||||
}
|
|
||||||
|
|
||||||
// validationReport is the immutable, ordered result of one validator chain.
|
|
||||||
type validationReport struct {
|
|
||||||
records []validationRecord
|
|
||||||
}
|
|
||||||
|
|
||||||
func (report validationReport) Records() []validationRecord {
|
|
||||||
records := make([]validationRecord, len(report.records))
|
|
||||||
for index, record := range report.records {
|
|
||||||
records[index] = record.clone()
|
|
||||||
}
|
|
||||||
return records
|
|
||||||
}
|
|
||||||
|
|
||||||
func (report validationReport) Diagnostics() []validationDiagnosticRecord {
|
|
||||||
var diagnostics []validationDiagnosticRecord
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome != validationApproved && record.outcome != validationRejected {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, diagnostic := range record.diagnostics {
|
|
||||||
diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: cloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0]})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return diagnostics
|
|
||||||
}
|
|
||||||
|
|
||||||
type validationDiagnosticRecord struct {
|
|
||||||
validatorName string
|
|
||||||
diagnostic contracts.ProducerDiagnostic
|
|
||||||
}
|
|
||||||
|
|
||||||
func (report validationReport) FirstRejection() *validationRecord {
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome == validationRejected {
|
|
||||||
clone := record.clone()
|
|
||||||
return &clone
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (report validationReport) FirstFailure() *validationRecord {
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome == validationFailed {
|
|
||||||
clone := record.clone()
|
|
||||||
return &clone
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (report validationReport) CorrectionRequest() (string, error) {
|
|
||||||
seen := make(map[string]struct{})
|
|
||||||
parts := make([]string, 0, len(report.records))
|
|
||||||
for _, record := range report.records {
|
|
||||||
if record.outcome != validationRejected {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
guidance := strings.TrimSpace(record.correctionGuidance)
|
|
||||||
if guidance == "" {
|
|
||||||
return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
|
|
||||||
}
|
|
||||||
if _, exists := seen[guidance]; exists {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[guidance] = struct{}{}
|
|
||||||
parts = append(parts, guidance)
|
|
||||||
}
|
|
||||||
if len(parts) == 0 {
|
|
||||||
return "", errors.New("validation report contains no correction guidance")
|
|
||||||
}
|
|
||||||
var builder strings.Builder
|
|
||||||
builder.WriteString(correctionRequestIntroduction)
|
|
||||||
for index, guidance := range parts {
|
|
||||||
fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
|
|
||||||
if index+1 < len(parts) {
|
|
||||||
builder.WriteByte('\n')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
request := builder.String()
|
|
||||||
if len(request) > contracts.MaxCorrectionGuidanceBytes {
|
|
||||||
return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
|
|
||||||
}
|
|
||||||
return request, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type validationInvocation struct {
|
|
||||||
result contracts.ValidationResult
|
|
||||||
skipped bool
|
|
||||||
reason string
|
|
||||||
message string
|
|
||||||
}
|
|
||||||
|
|
||||||
func skippedValidation(reason, message string) validationInvocation {
|
|
||||||
return validationInvocation{skipped: true, reason: strings.TrimSpace(reason), message: strings.TrimSpace(message)}
|
|
||||||
}
|
|
||||||
|
|
||||||
type validationInvoker func(context.Context, preparedValidator, int) (validationInvocation, error)
|
|
||||||
|
|
||||||
type validationFrameworkError struct{ err error }
|
|
||||||
|
|
||||||
func (err validationFrameworkError) Error() string { return err.err.Error() }
|
|
||||||
|
|
||||||
func (err validationFrameworkError) Unwrap() error { return err.err }
|
|
||||||
|
|
||||||
func fatalValidationError(err error) error {
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return validationFrameworkError{err: err}
|
|
||||||
}
|
|
||||||
|
|
||||||
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke validationInvoker) (validationReport, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return validationReport{}, errors.New("validator execution context must not be nil")
|
|
||||||
}
|
|
||||||
if invoke == nil {
|
|
||||||
return validationReport{}, errors.New("validator invocation must not be nil")
|
|
||||||
}
|
|
||||||
report := validationReport{records: make([]validationRecord, 0, len(chain.validators))}
|
|
||||||
for index, validator := range chain.validators {
|
|
||||||
validator.position = index + 1
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return validationReport{}, err
|
|
||||||
}
|
|
||||||
binding := validator.resolved.Binding
|
|
||||||
attemptLimit := 1
|
|
||||||
if validator.resolved.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
|
||||||
attemptLimit += binding.Retries
|
|
||||||
}
|
|
||||||
for attempt := 1; attempt <= attemptLimit; attempt++ {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return validationReport{}, err
|
|
||||||
}
|
|
||||||
invocation, err := invoke(ctx, validator, attempt)
|
|
||||||
if err != nil {
|
|
||||||
var frameworkErr validationFrameworkError
|
|
||||||
if errors.As(err, &frameworkErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return validationReport{}, err
|
|
||||||
}
|
|
||||||
if attempt == attemptLimit {
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator failed after retry exhaustion", failure: err})
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if invocation.skipped {
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationSkipped, attemptCount: attempt, reasonCode: invocation.reason, message: invocation.message})
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err := contracts.ValidateProducerDiagnostics(invocation.result.Diagnostics); err != nil {
|
|
||||||
return validationReport{}, fatalValidationError(fmt.Errorf("validator %q returned invalid diagnostics: %w", binding.Module, err))
|
|
||||||
}
|
|
||||||
if err := contracts.ValidateValidationResult(invocation.result); err != nil {
|
|
||||||
if attempt == attemptLimit {
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator returned an invalid result", failure: err})
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if invocation.result.Approved {
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), diagnosticPath: invocation.result.DiagnosticArtifactPath})
|
|
||||||
break
|
|
||||||
}
|
|
||||||
message := invocation.result.Message
|
|
||||||
if message == "" {
|
|
||||||
message = "output rejected"
|
|
||||||
}
|
|
||||||
report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), correctionGuidance: invocation.result.CorrectionGuidance})
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return report, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validatorFailureError(record validationRecord) error {
|
|
||||||
if record.failure != nil {
|
|
||||||
return record.failure
|
|
||||||
}
|
|
||||||
return fmt.Errorf("validator %q failed after %d attempt(s)", record.validatorName, record.attemptCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validatorAttemptPath(base string, attempt int) string {
|
|
||||||
if attempt == 1 {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s-validator-%02d", base, attempt)
|
|
||||||
}
|
|
||||||
@@ -1,241 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
chain preparedValidatorChain
|
|
||||||
invoke validationInvoker
|
|
||||||
want []validationOutcome
|
|
||||||
wantCalls []string
|
|
||||||
wantGuidance []string
|
|
||||||
wantDiagnostics []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "all approved",
|
|
||||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{
|
|
||||||
"shape": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("shape")}}}},
|
|
||||||
"refs": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("refs")}}}},
|
|
||||||
}),
|
|
||||||
want: []validationOutcome{validationApproved, validationApproved},
|
|
||||||
wantCalls: []string{"shape:1", "refs:1"},
|
|
||||||
wantDiagnostics: []string{"shape", "refs"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple rejections deduplicate guidance",
|
|
||||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{
|
|
||||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}},
|
|
||||||
"refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}},
|
|
||||||
"coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}},
|
|
||||||
}),
|
|
||||||
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
|
|
||||||
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
|
|
||||||
wantGuidance: []string{"repair shape", "repair references"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "rejection and failure both settle",
|
|
||||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{
|
|
||||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}},
|
|
||||||
"remote": {{err: errors.New("offline")}},
|
|
||||||
}),
|
|
||||||
want: []validationOutcome{validationRejected, validationFailed},
|
|
||||||
wantCalls: []string{"shape:1", "remote:1"},
|
|
||||||
wantGuidance: []string{"repair shape"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "failure only has no correction guidance",
|
|
||||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}),
|
|
||||||
want: []validationOutcome{validationFailed},
|
|
||||||
wantCalls: []string{"remote:1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "skipped is retained without guidance",
|
|
||||||
chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}),
|
|
||||||
want: []validationOutcome{validationSkipped},
|
|
||||||
wantCalls: []string{"optional:1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "LLM failure retries then succeeds",
|
|
||||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}),
|
|
||||||
want: []validationOutcome{validationApproved},
|
|
||||||
wantCalls: []string{"remote:1", "remote:2", "remote:3"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "LLM failure retry exhaustion records once",
|
|
||||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)),
|
|
||||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}),
|
|
||||||
want: []validationOutcome{validationFailed},
|
|
||||||
wantCalls: []string{"remote:1", "remote:2"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
calls := []string(nil)
|
|
||||||
invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) {
|
|
||||||
calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt)))
|
|
||||||
return test.invoke(ctx, validator, attempt)
|
|
||||||
}
|
|
||||||
report, err := executeValidationChain(context.Background(), test.chain, invoke)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("executeValidationChain() error = %v", err)
|
|
||||||
}
|
|
||||||
records := report.Records()
|
|
||||||
outcomes := make([]validationOutcome, len(records))
|
|
||||||
for index, record := range records {
|
|
||||||
outcomes[index] = record.outcome
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
|
|
||||||
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
|
|
||||||
}
|
|
||||||
if len(test.wantGuidance) > 0 {
|
|
||||||
guidance, err := report.CorrectionRequest()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("CorrectionRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(guidance, "complete corrected replacement") {
|
|
||||||
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
|
|
||||||
}
|
|
||||||
for _, item := range test.wantGuidance {
|
|
||||||
if strings.Count(guidance, item) != 1 {
|
|
||||||
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
diagnostics := report.Diagnostics()
|
|
||||||
var diagnosticCodes []string
|
|
||||||
for _, record := range diagnostics {
|
|
||||||
diagnosticCodes = append(diagnosticCodes, record.diagnostic.ReasonCode)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(diagnosticCodes, test.wantDiagnostics) {
|
|
||||||
t.Fatalf("Diagnostics() = %#v, want codes %#v", diagnostics, test.wantDiagnostics)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
|
|
||||||
candidate := []byte(`{"events":["one"]}`)
|
|
||||||
chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1))
|
|
||||||
var received [][]byte
|
|
||||||
report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) {
|
|
||||||
requestCandidate := append([]byte(nil), candidate...)
|
|
||||||
received = append(received, requestCandidate)
|
|
||||||
requestCandidate[0] = 'x'
|
|
||||||
if attempt == 1 {
|
|
||||||
return validationInvocation{}, errors.New("retry")
|
|
||||||
}
|
|
||||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
|
||||||
})
|
|
||||||
if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 {
|
|
||||||
t.Fatalf("report = %#v, error = %v", report, err)
|
|
||||||
}
|
|
||||||
if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) {
|
|
||||||
t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
|
|
||||||
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
|
|
||||||
report := validationReport{records: make([]validationRecord, 17)}
|
|
||||||
for index := range report.records {
|
|
||||||
unique := []byte(guidance)
|
|
||||||
unique[0] = byte('a' + index)
|
|
||||||
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
|
|
||||||
}
|
|
||||||
report.records[0].diagnostics = []contracts.ProducerDiagnostic{validationDiagnostic("diagnostic")}
|
|
||||||
if _, err := report.CorrectionRequest(); err == nil {
|
|
||||||
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
|
|
||||||
}
|
|
||||||
records := report.Records()
|
|
||||||
records[0].diagnostics[0].ReasonCode = "changed"
|
|
||||||
if report.Records()[0].diagnostics[0].ReasonCode != "diagnostic" {
|
|
||||||
t.Fatal("Records() exposed mutable diagnostic storage")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validationDiagnostic(reasonCode string) contracts.ProducerDiagnostic {
|
|
||||||
return contracts.ProducerDiagnostic{
|
|
||||||
Disposition: contracts.DiagnosticDispositionObservation,
|
|
||||||
Category: contracts.DiagnosticCategoryNormalization,
|
|
||||||
ReasonCode: reasonCode,
|
|
||||||
OccurrenceCount: 1,
|
|
||||||
Samples: []contracts.DiagnosticSample{{Scope: "validator", Message: reasonCode}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) {
|
|
||||||
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1))
|
|
||||||
framework := errors.New("debug persistence failed")
|
|
||||||
if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
||||||
return validationInvocation{}, fatalValidationError(framework)
|
|
||||||
}); !errors.Is(err, framework) {
|
|
||||||
t.Fatalf("framework error = %v, want %v", err, framework)
|
|
||||||
}
|
|
||||||
canceled, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
||||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
|
||||||
}); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("cancellation error = %v, want context canceled", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteValidationChainRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
|
||||||
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 2))
|
|
||||||
calls := 0
|
|
||||||
_, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
||||||
calls++
|
|
||||||
return validationInvocation{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{}}}}, nil
|
|
||||||
})
|
|
||||||
var frameworkErr validationFrameworkError
|
|
||||||
if !errors.As(err, &frameworkErr) || calls != 1 || !strings.Contains(err.Error(), "validator \"validator\" returned invalid diagnostics") {
|
|
||||||
t.Fatalf("error = %v calls = %d, want one fatal invalid-diagnostics result", err, calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type validationStep struct {
|
|
||||||
result contracts.ValidationResult
|
|
||||||
invocation validationInvocation
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func validationSequence(steps map[string][]validationStep) validationInvoker {
|
|
||||||
positions := make(map[string]int)
|
|
||||||
return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) {
|
|
||||||
name := validator.resolved.Binding.Module
|
|
||||||
index := positions[name]
|
|
||||||
positions[name]++
|
|
||||||
if index >= len(steps[name]) {
|
|
||||||
return validationInvocation{}, errors.New("unexpected validator invocation")
|
|
||||||
}
|
|
||||||
step := steps[name][index]
|
|
||||||
if step.invocation.skipped {
|
|
||||||
return step.invocation, step.err
|
|
||||||
}
|
|
||||||
return validationInvocation{result: step.result}, step.err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validationChain(validators ...preparedValidator) preparedValidatorChain {
|
|
||||||
return preparedValidatorChain{validators: validators}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator {
|
|
||||||
return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user