Files
notarius/internal/framework/pipeline/candidate_assessment.go

196 lines
8.6 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// executeValidationChain invokes every applicable validator once, in configured
// order. Only caller cancellation or a framework failure interrupts the chain.
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) {
assessment := contracts.CandidateAssessment{}
if ctx == nil {
return assessment, errors.New("validator execution context must not be nil")
}
if invoke == nil {
return assessment, errors.New("validator invocation must not be nil")
}
if err := ctx.Err(); err != nil {
return assessment, err
}
for index, validator := range chain.validators {
if err := ctx.Err(); err != nil {
return assessment, err
}
validator.position = index + 1
result, err := invoke(ctx, validator)
if canceled := ctx.Err(); canceled != nil {
return assessment, canceled
}
var frameworkError *assessmentFrameworkError
if errors.As(err, &frameworkError) {
return assessment, err
}
finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result}
finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics)
if err != nil {
finding.Decision, finding.Failure = contracts.AssessmentFailed, err
var unavailable *contracts.ValidationUnavailableError
if errors.As(err, &unavailable) {
finding.Decision = contracts.AssessmentUnavailable
}
} else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil {
finding.Decision = contracts.AssessmentFailed
finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr)
} else {
finding.Decision = contracts.AssessmentRejected
if result.Approved {
finding.Decision = contracts.AssessmentApproved
}
}
assessment.Validators = append(assessment.Validators, finding)
}
if err := ctx.Err(); err != nil {
return assessment, err
}
return assessment, nil
}
type assessmentFrameworkError struct{ cause error }
func (e *assessmentFrameworkError) Error() string { return e.cause.Error() }
func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
// typedCandidateAssessor binds one operation's complete chain. Each invocation
// captures the new candidate once and decodes a detached view for each validator.
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.CandidateAssessment{}, err
}
content, err := codec.encodeCandidate(value)
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
switch validator.resolved.Target {
case ValidatorTargetTyped:
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)}
}
if reflect.TypeOf(request.value) != codec.valueType {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))}
}
return validator.typedValidate(ctx, validator.typed, request)
case ValidatorTargetSerialized:
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey,
Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput,
SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile,
Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks,
Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...),
})
default:
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)}
}
})
}
}
func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) {
request := target
request.sourceInput = target.sourceInput.Clone()
request.references = CloneReferenceSet(target.references)
var err error
request.metadata, err = source.CloneMetadata(target.metadata)
if err != nil {
return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err)
}
request.source, err = cloneAssessmentSource(target.source)
if err != nil {
return typedValidationTarget{}, err
}
request.chunk, err = cloneSourceChunkPtr(target.chunk)
if err != nil {
return typedValidationTarget{}, err
}
request.chunks, err = cloneSourceChunks(target.chunks)
return request, err
}
func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) {
if doc == nil {
return nil, nil
}
copy := *doc
var err error
copy.Metadata, err = source.CloneMetadata(doc.Metadata)
if err != nil {
return nil, fmt.Errorf("clone validation source metadata: %w", err)
}
copy.Units, err = cloneSourceUnits(doc.Units)
if err != nil {
return nil, err
}
return &copy, nil
}
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
// The runner's plan callback owns materialization before this boundary.
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.CandidateAssessment{}, err
}
content, err := json.Marshal(chunks)
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err)
}
target := target
target.chunks = chunks
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
switch validator.resolved.Target {
case ValidatorTargetChunk:
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
})
case ValidatorTargetSerialized:
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...),
})
default:
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)}
}
})
}
}