224 lines
11 KiB
Go
224 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) {
|
|
chain := preparedValidatorChain{}
|
|
names := []string{"first", "second", "unavailable", "failed", "timeout"}
|
|
for _, name := range names {
|
|
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}})
|
|
}
|
|
failure := errors.New("validator failed")
|
|
var called []string
|
|
assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
|
name := validator.resolved.Binding.Module
|
|
called = append(called, name)
|
|
switch name {
|
|
case "unavailable":
|
|
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure}
|
|
case "failed":
|
|
return contracts.ValidationResult{}, failure
|
|
case "timeout":
|
|
return contracts.ValidationResult{}, context.DeadlineExceeded
|
|
default:
|
|
return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil
|
|
}
|
|
})
|
|
if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() {
|
|
t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err)
|
|
}
|
|
var decisions []contracts.AssessmentDecision
|
|
for _, finding := range assessment.Validators {
|
|
decisions = append(decisions, finding.Decision)
|
|
}
|
|
want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed}
|
|
if !reflect.DeepEqual(decisions, want) {
|
|
t.Fatalf("decisions = %#v", decisions)
|
|
}
|
|
guidance, err := assessment.Guidance()
|
|
if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) {
|
|
t.Fatalf("guidance = %#v, %v", guidance, err)
|
|
}
|
|
if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) {
|
|
t.Fatalf("assessment lost underlying causes: %v", assessment.Err())
|
|
}
|
|
}
|
|
|
|
func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
calls := 0
|
|
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
|
assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
|
calls++
|
|
cancel()
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
})
|
|
if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 {
|
|
t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls)
|
|
}
|
|
}
|
|
|
|
func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) {
|
|
calls := 0
|
|
chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}}
|
|
assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
|
calls++
|
|
return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil
|
|
})
|
|
guidance, guidanceErr := assessment.Guidance()
|
|
if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil {
|
|
t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr)
|
|
}
|
|
for _, finding := range assessment.Validators {
|
|
if finding.Decision != contracts.AssessmentFailed {
|
|
t.Fatalf("invalid rejection = %#v", finding)
|
|
}
|
|
}
|
|
}
|
|
|
|
type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error)
|
|
|
|
func (notesAssessmentValidator) Name() string { return "notes" }
|
|
func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
|
return contracts.ExecutionClassDeterministic
|
|
}
|
|
func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
return v(ctx, request)
|
|
}
|
|
|
|
func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator {
|
|
t.Helper()
|
|
registry := NewValidatorRegistry()
|
|
if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entry, _ := registry.typedEntry(name, "test/notes")
|
|
built, err := entry.builder(BuildRequest{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate}
|
|
}
|
|
|
|
func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
|
|
t.Helper()
|
|
registry := NewArtifactCodecRegistry()
|
|
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entry, _, err := registry.entry("test/notes")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return entry
|
|
}
|
|
|
|
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
|
|
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
|
|
doc := typedTestDocument()
|
|
doc.Metadata = map[string]any{"label": "original"}
|
|
target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}}
|
|
first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed"
|
|
request.Source.Units[0].Text = "changed"
|
|
request.Source.Metadata["label"] = "changed"
|
|
request.Metadata["label"] = "changed"
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
})
|
|
second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" {
|
|
t.Fatalf("validator received a mutated view: %#v", request)
|
|
}
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
})
|
|
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}})
|
|
assessment, err := assess(context.Background(), candidate)
|
|
if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" {
|
|
t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate)
|
|
}
|
|
}
|
|
|
|
func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) {
|
|
calls := []string{}
|
|
makeValidator := func(name, accepted string) preparedValidator {
|
|
return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
value := request.Value.Items[0]
|
|
calls = append(calls, name+":"+value)
|
|
if value == accepted {
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
}
|
|
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil
|
|
})
|
|
}
|
|
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}})
|
|
for _, text := range []string{"one", "two", "two"} {
|
|
assessment, err := assess(context.Background(), codecNotes{Items: []string{text}})
|
|
if err != nil || assessment.Accepted() {
|
|
t.Fatalf("assessment for %s = %#v, %v", text, assessment, err)
|
|
}
|
|
want := contracts.AssessmentRejected
|
|
if text == "one" {
|
|
want = contracts.AssessmentApproved
|
|
}
|
|
if assessment.Validators[0].Decision != want {
|
|
t.Fatalf("earlier approval survived revised candidate: %#v", assessment)
|
|
}
|
|
}
|
|
if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) {
|
|
t.Fatalf("assessments = %#v", calls)
|
|
}
|
|
}
|
|
|
|
type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error)
|
|
|
|
func (chunkAssessmentValidator) Name() string { return "chunk assessment" }
|
|
func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass {
|
|
return contracts.ExecutionClassDeterministic
|
|
}
|
|
func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
|
return v(ctx, request)
|
|
}
|
|
|
|
func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) {
|
|
chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}}
|
|
originalJSON, err := json.Marshal(chunks)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := 0
|
|
mutator := preparedValidator{
|
|
resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk},
|
|
chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
|
calls++
|
|
request.Chunks[0].Content[0] = 'x'
|
|
request.Chunks[0].Metadata["label"] = "changed"
|
|
return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil
|
|
}),
|
|
}
|
|
observer := preparedValidator{
|
|
resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized},
|
|
serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) {
|
|
calls++
|
|
if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) {
|
|
t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks)
|
|
}
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
}),
|
|
}
|
|
assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks)
|
|
if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" {
|
|
t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks)
|
|
}
|
|
}
|