Files
notarius/internal/framework/pipeline/candidate_assessment_test.go

285 lines
13 KiB
Go

package pipeline
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) {
chain := preparedValidatorChain{}
names := []string{"first", "second", "unavailable", "failed", "timeout"}
for _, name := range names {
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}})
}
failure := errors.New("validator failed")
var called []string
assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
name := validator.resolved.Binding.Module
called = append(called, name)
switch name {
case "unavailable":
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure}
case "failed":
return contracts.ValidationResult{}, failure
case "timeout":
return contracts.ValidationResult{}, context.DeadlineExceeded
default:
return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil
}
})
if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() {
t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err)
}
var decisions []contracts.AssessmentDecision
for _, finding := range assessment.Validators {
decisions = append(decisions, finding.Decision)
}
want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed}
if !reflect.DeepEqual(decisions, want) {
t.Fatalf("decisions = %#v", decisions)
}
guidance, err := assessment.Guidance()
if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) {
t.Fatalf("guidance = %#v, %v", guidance, err)
}
if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) {
t.Fatalf("assessment lost underlying causes: %v", assessment.Err())
}
}
func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls := 0
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
calls++
cancel()
return contracts.ValidationResult{Approved: true}, nil
})
if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 {
t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls)
}
}
func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) {
calls := 0
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
calls++
return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil
})
guidance, guidanceErr := assessment.Guidance()
if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil {
t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr)
}
for _, finding := range assessment.Validators {
if finding.Decision != contracts.AssessmentFailed {
t.Fatalf("invalid rejection = %#v", finding)
}
}
}
type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error)
func (notesAssessmentValidator) Name() string { return "notes" }
func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
return v(ctx, request)
}
func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator {
t.Helper()
registry := NewValidatorRegistry()
if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil {
t.Fatal(err)
}
entry, _ := registry.typedEntry(name, "test/notes")
built, err := entry.builder(BuildRequest{})
if err != nil {
t.Fatal(err)
}
return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate}
}
func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
t.Helper()
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatal(err)
}
entry, _, err := registry.entry("test/notes")
if err != nil {
t.Fatal(err)
}
return entry
}
func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) {
evidence := &nativeEvidence{}
content := strings.Repeat("sk-ordinary content ", 1000)
candidate := codecNotes{Items: []string{content}}
validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
if request.Execution.Capture == nil {
t.Fatal("validator has no scoped capture")
}
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil
})
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}})
for range 2 {
assessment, err := assess(context.Background(), candidate)
if err != nil || assessment.Accepted() {
t.Fatalf("assessment=%#v %v", assessment, err)
}
}
if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 {
t.Fatal("revised candidates or findings were overwritten")
}
for name, data := range evidence.files {
if strings.HasSuffix(name, "/candidate.json") {
decoded, err := assessmentNotesCodec(t).decodeCandidate(data)
if err != nil || !reflect.DeepEqual(decoded, candidate) {
t.Fatalf("codec candidate changed: %#v, %v", decoded, err)
}
}
if strings.HasSuffix(name, "/finding.json") {
var finding struct {
Guidance string `json:"correction_guidance"`
}
if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content {
t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err)
}
}
}
}
func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) {
primary := errors.New("capture failed")
called := 0
_, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
called++
return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary}
})
if called != 1 || !errors.Is(err, primary) {
t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err)
}
}
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
doc := typedTestDocument()
doc.Metadata = map[string]any{"label": "original"}
target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}}
first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed"
request.Source.Units[0].Text = "changed"
request.Source.Metadata["label"] = "changed"
request.Metadata["label"] = "changed"
return contracts.ValidationResult{Approved: true}, nil
})
second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" {
t.Fatalf("validator received a mutated view: %#v", request)
}
return contracts.ValidationResult{Approved: true}, nil
})
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}})
assessment, err := assess(context.Background(), candidate)
if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" {
t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate)
}
}
func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) {
calls := []string{}
makeValidator := func(name, accepted string) preparedValidator {
return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
value := request.Value.Items[0]
calls = append(calls, name+":"+value)
if value == accepted {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil
})
}
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}})
for _, text := range []string{"one", "two", "two"} {
assessment, err := assess(context.Background(), codecNotes{Items: []string{text}})
if err != nil || assessment.Accepted() {
t.Fatalf("assessment for %s = %#v, %v", text, assessment, err)
}
want := contracts.AssessmentRejected
if text == "one" {
want = contracts.AssessmentApproved
}
if assessment.Validators[0].Decision != want {
t.Fatalf("earlier approval survived revised candidate: %#v", assessment)
}
}
if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) {
t.Fatalf("assessments = %#v", calls)
}
}
type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
func (chunkAssessmentValidator) Name() string { return "chunk assessment" }
func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return v(ctx, request)
}
func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) {
chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}}
originalJSON, err := json.Marshal(chunks)
if err != nil {
t.Fatal(err)
}
calls := 0
mutator := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk},
chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
calls++
request.Chunks[0].Content[0] = 'x'
request.Chunks[0].Metadata["label"] = "changed"
return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil
}),
}
observer := preparedValidator{
resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized},
serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
calls++
if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) {
t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks)
}
return contracts.ValidationResult{Approved: true}, nil
}),
}
assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks)
if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" {
t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks)
}
}
type serializedValidationFunc func(context.Context, contracts.SerializedValidationRequest) (contracts.ValidationResult, error)
func (serializedValidationFunc) Name() string { return "candidate-serialized" }
func (serializedValidationFunc) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validate serializedValidationFunc) Validate(ctx context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
return validate(ctx, request)
}