237 lines
11 KiB
Go
237 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strconv"
|
|
"sync/atomic"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
// executeValidationChain invokes every applicable validator once, in configured
|
|
// order. Only caller cancellation or a framework failure interrupts the chain.
|
|
func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke func(context.Context, preparedValidator) (contracts.ValidationResult, error)) (contracts.CandidateAssessment, error) {
|
|
assessment := contracts.CandidateAssessment{}
|
|
if ctx == nil {
|
|
return assessment, errors.New("validator execution context must not be nil")
|
|
}
|
|
if invoke == nil {
|
|
return assessment, errors.New("validator invocation must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return assessment, err
|
|
}
|
|
for index, validator := range chain.validators {
|
|
if err := ctx.Err(); err != nil {
|
|
return assessment, err
|
|
}
|
|
validator.position = index + 1
|
|
result, err := invoke(ctx, validator)
|
|
var captureErr *contracts.CaptureError
|
|
if errors.As(err, &captureErr) {
|
|
return assessment, errors.Join(ctx.Err(), err)
|
|
}
|
|
if canceled := ctx.Err(); canceled != nil {
|
|
return assessment, canceled
|
|
}
|
|
var frameworkError *assessmentFrameworkError
|
|
if errors.As(err, &frameworkError) {
|
|
return assessment, err
|
|
}
|
|
finding := contracts.ValidatorAssessment{ValidatorName: validator.resolved.Binding.Module, Result: result}
|
|
finding.Result.Diagnostics = contracts.CloneProducerDiagnostics(result.Diagnostics)
|
|
if err != nil {
|
|
finding.Decision, finding.Failure = contracts.AssessmentFailed, err
|
|
var unavailable *contracts.ValidationUnavailableError
|
|
if errors.As(err, &unavailable) {
|
|
finding.Decision = contracts.AssessmentUnavailable
|
|
}
|
|
} else if contractErr := contracts.ValidateValidationResult(result); contractErr != nil {
|
|
finding.Decision = contracts.AssessmentFailed
|
|
finding.Failure = fmt.Errorf("invalid validation result: %w", contractErr)
|
|
} else {
|
|
finding.Decision = contracts.AssessmentRejected
|
|
if result.Approved {
|
|
finding.Decision = contracts.AssessmentApproved
|
|
}
|
|
}
|
|
assessment.Validators = append(assessment.Validators, finding)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return assessment, err
|
|
}
|
|
return assessment, nil
|
|
}
|
|
|
|
type assessmentFrameworkError struct{ cause error }
|
|
|
|
func (e *assessmentFrameworkError) Error() string { return e.cause.Error() }
|
|
func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
|
|
|
|
// typedCandidateAssessor binds one operation's complete chain. Each invocation
|
|
// captures the new candidate once and decodes a detached view for each validator.
|
|
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
|
|
var candidateNumber atomic.Uint64
|
|
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
|
if ctx == nil {
|
|
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
content, err := codec.encodeCandidate(value)
|
|
if err != nil {
|
|
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
|
|
}
|
|
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
|
|
if capture != nil {
|
|
if err := capture.WriteBytes("candidate.json", content); err != nil {
|
|
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
|
|
}
|
|
}
|
|
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
|
|
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
|
|
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
|
|
request, err := isolatedValidationTarget(target)
|
|
if err != nil {
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
|
}
|
|
request.llmProfile = validator.resolved.Binding.LLMProfile
|
|
request.execution.Capture = validatorCapture
|
|
switch validator.resolved.Target {
|
|
case ValidatorTargetTyped:
|
|
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
|
|
if err != nil {
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("decode candidate for validator %q: %w", validator.resolved.Binding.Module, err)}
|
|
}
|
|
if reflect.TypeOf(request.value) != codec.valueType {
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: newArtifactCodecTypeError("decode candidate", codec.spec.Kind, codec.valueType, reflect.TypeOf(request.value))}
|
|
}
|
|
return validator.typedValidate(ctx, validator.typed, request)
|
|
case ValidatorTargetSerialized:
|
|
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
|
Stage: string(request.stage), LaneID: request.laneID, ModuleKey: request.moduleKey,
|
|
Source: request.source, SourceID: request.sourceID, SourceInput: request.sourceInput,
|
|
SessionID: request.sessionID, References: request.references, LLMProfile: request.llmProfile,
|
|
Execution: request.execution, Metadata: request.metadata, Chunk: request.chunk, Chunks: request.chunks,
|
|
Schema: contracts.CloneArtifactSchema(codec.spec.Schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...),
|
|
})
|
|
default:
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with typed artifact validation", validator.resolved.Binding.Module)}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func isolatedValidationTarget(target typedValidationTarget) (typedValidationTarget, error) {
|
|
request := target
|
|
request.sourceInput = target.sourceInput.Clone()
|
|
request.references = CloneReferenceSet(target.references)
|
|
var err error
|
|
request.metadata, err = source.CloneMetadata(target.metadata)
|
|
if err != nil {
|
|
return typedValidationTarget{}, fmt.Errorf("clone validation metadata: %w", err)
|
|
}
|
|
request.source, err = cloneAssessmentSource(target.source)
|
|
if err != nil {
|
|
return typedValidationTarget{}, err
|
|
}
|
|
request.chunk, err = cloneSourceChunkPtr(target.chunk)
|
|
if err != nil {
|
|
return typedValidationTarget{}, err
|
|
}
|
|
request.chunks, err = cloneSourceChunks(target.chunks)
|
|
return request, err
|
|
}
|
|
|
|
func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, error) {
|
|
if doc == nil {
|
|
return nil, nil
|
|
}
|
|
copy := *doc
|
|
var err error
|
|
copy.Metadata, err = source.CloneMetadata(doc.Metadata)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("clone validation source metadata: %w", err)
|
|
}
|
|
copy.Units, err = cloneSourceUnits(doc.Units)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ©, nil
|
|
}
|
|
|
|
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
|
|
// The runner's plan callback owns materialization before this boundary.
|
|
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
|
|
var candidateNumber atomic.Uint64
|
|
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
|
|
if ctx == nil {
|
|
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
content, err := json.Marshal(chunks)
|
|
if err != nil {
|
|
return contracts.CandidateAssessment{}, fmt.Errorf("encode chunks for validation: %w", err)
|
|
}
|
|
target := target
|
|
target.chunks = chunks
|
|
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
|
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
|
|
if capture != nil {
|
|
if err := capture.WriteBytes("candidate.json", content); err != nil {
|
|
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
|
|
}
|
|
}
|
|
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
|
|
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
|
|
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
|
|
request, err := isolatedValidationTarget(target)
|
|
if err != nil {
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
|
}
|
|
request.llmProfile = validator.resolved.Binding.LLMProfile
|
|
request.execution.Capture = validatorCapture
|
|
switch validator.resolved.Target {
|
|
case ValidatorTargetChunk:
|
|
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
|
|
ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
|
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
|
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
|
})
|
|
case ValidatorTargetSerialized:
|
|
return validator.serialized.Validate(ctx, contracts.SerializedValidationRequest{
|
|
Stage: string(StageChunk), ModuleKey: request.moduleKey, Source: request.source, SourceID: request.sourceID,
|
|
SourceInput: request.sourceInput, SessionID: request.sessionID, References: request.references,
|
|
LLMProfile: request.llmProfile, Execution: request.execution, Metadata: request.metadata, Chunks: request.chunks,
|
|
Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...),
|
|
})
|
|
default:
|
|
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: fmt.Errorf("validator %q is incompatible with chunk validation", validator.resolved.Binding.Module)}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func captureValidationFinding(writer contracts.EvidenceWriter, finding contracts.ValidationResult, failure error) error {
|
|
if writer == nil {
|
|
return failure
|
|
}
|
|
errorText := ""
|
|
if failure != nil {
|
|
errorText = failure.Error()
|
|
}
|
|
if err := writer.WriteJSON("finding.json", map[string]any{"result": finding, "correction_guidance": finding.CorrectionGuidance, "error": errorText}); err != nil {
|
|
return errors.Join(failure, &contracts.CaptureError{Cause: err})
|
|
}
|
|
return failure
|
|
}
|