package pipeline import ( "bytes" "context" "encoding/json" "errors" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestCandidateAssessmentCompletesTheChainWithoutRetries(t *testing.T) { chain := preparedValidatorChain{} names := []string{"first", "second", "unavailable", "failed", "timeout"} for _, name := range names { chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), ExecutionClass: contracts.ExecutionClassLLMBacked}}) } failure := errors.New("validator failed") var called []string assessment, err := executeValidationChain(context.Background(), chain, func(_ context.Context, validator preparedValidator) (contracts.ValidationResult, error) { name := validator.resolved.Binding.Module called = append(called, name) switch name { case "unavailable": return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: failure} case "failed": return contracts.ValidationResult{}, failure case "timeout": return contracts.ValidationResult{}, context.DeadlineExceeded default: return contracts.ValidationResult{ReasonCode: name, CorrectionGuidance: "Correct " + name}, nil } }) if err != nil || !reflect.DeepEqual(called, names) || assessment.Accepted() { t.Fatalf("assessment = %#v, calls %#v, error %v", assessment, called, err) } var decisions []contracts.AssessmentDecision for _, finding := range assessment.Validators { decisions = append(decisions, finding.Decision) } want := []contracts.AssessmentDecision{contracts.AssessmentRejected, contracts.AssessmentRejected, contracts.AssessmentUnavailable, contracts.AssessmentFailed, contracts.AssessmentFailed} if !reflect.DeepEqual(decisions, want) { t.Fatalf("decisions = %#v", decisions) } guidance, err := assessment.Guidance() if err != nil || !reflect.DeepEqual(guidance, []string{"Correct first", "Correct second"}) { t.Fatalf("guidance = %#v, %v", guidance, err) } if !errors.Is(assessment.Err(), failure) || !errors.Is(assessment.Err(), context.DeadlineExceeded) { t.Fatalf("assessment lost underlying causes: %v", assessment.Err()) } } func TestCandidateAssessmentStopsForCallerCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() calls := 0 chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}} assessment, err := executeValidationChain(ctx, chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { calls++ cancel() return contracts.ValidationResult{Approved: true}, nil }) if !errors.Is(err, context.Canceled) || calls != 1 || len(assessment.Validators) != 0 { t.Fatalf("cancellation = %#v %v, calls %d", assessment, err, calls) } } func TestCandidateAssessmentInvalidGuidanceIsFailureNotCorrection(t *testing.T) { calls := 0 chain := preparedValidatorChain{validators: []preparedValidator{{}, {}}} assessment, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { calls++ return contracts.ValidationResult{ReasonCode: "opaque", Message: "not correction guidance"}, nil }) guidance, guidanceErr := assessment.Guidance() if err != nil || guidanceErr != nil || calls != 2 || len(guidance) != 0 || assessment.Accepted() || assessment.Err() == nil { t.Fatalf("invalid guidance = %#v %v, guidance %#v %v", assessment, err, guidance, guidanceErr) } for _, finding := range assessment.Validators { if finding.Decision != contracts.AssessmentFailed { t.Fatalf("invalid rejection = %#v", finding) } } } type notesAssessmentValidator func(context.Context, contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) func (notesAssessmentValidator) Name() string { return "notes" } func (notesAssessmentValidator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } func (v notesAssessmentValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { return v(ctx, request) } func assessmentNotesValidator(t *testing.T, name string, validate notesAssessmentValidator) preparedValidator { t.Helper() registry := NewValidatorRegistry() if err := RegisterTypedValidator(registry, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return validate, nil }); err != nil { t.Fatal(err) } entry, _ := registry.typedEntry(name, "test/notes") built, err := entry.builder(BuildRequest{}) if err != nil { t.Fatal(err) } return preparedValidator{resolved: ResolvedValidator{Binding: Binding(name), Target: ValidatorTargetTyped}, typed: built, typedValidate: entry.validate} } func assessmentNotesCodec(t *testing.T) artifactCodecEntry { t.Helper() registry := NewArtifactCodecRegistry() if err := RegisterArtifactCodec(registry, notesCodec()); err != nil { t.Fatal(err) } entry, _, err := registry.entry("test/notes") if err != nil { t.Fatal(err) } return entry } func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) { evidence := &nativeEvidence{} content := strings.Repeat("sk-ordinary content ", 1000) candidate := codecNotes{Items: []string{content}} validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { if request.Execution.Capture == nil { t.Fatal("validator has no scoped capture") } return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil }) assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}}) for range 2 { assessment, err := assess(context.Background(), candidate) if err != nil || assessment.Accepted() { t.Fatalf("assessment=%#v %v", assessment, err) } } if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 { t.Fatal("revised candidates or findings were overwritten") } for name, data := range evidence.files { if strings.HasSuffix(name, "/candidate.json") { decoded, err := assessmentNotesCodec(t).decodeCandidate(data) if err != nil || !reflect.DeepEqual(decoded, candidate) { t.Fatalf("codec candidate changed: %#v, %v", decoded, err) } } if strings.HasSuffix(name, "/finding.json") { var finding struct { Guidance string `json:"correction_guidance"` } if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content { t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err) } } } } func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) { primary := errors.New("capture failed") called := 0 _, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { called++ return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary} }) if called != 1 || !errors.Is(err, primary) { t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err) } } func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) { candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}} doc := typedTestDocument() doc.Metadata = map[string]any{"label": "original"} target := typedValidationTarget{stage: StageExtract, source: doc, sourceID: doc.ID, metadata: map[string]any{"label": "original"}} first := assessmentNotesValidator(t, "mutator", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { request.Value.Items[0], request.Value.Labels["label"], request.Value.Details.Name = "changed", "changed", "changed" request.Source.Units[0].Text = "changed" request.Source.Metadata["label"] = "changed" request.Metadata["label"] = "changed" return contracts.ValidationResult{Approved: true}, nil }) second := assessmentNotesValidator(t, "observer", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { if !reflect.DeepEqual(request.Value, candidate) || request.Source.Units[0].Text != doc.Units[0].Text || request.Source.Metadata["label"] != "original" || request.Metadata["label"] != "original" { t.Fatalf("validator received a mutated view: %#v", request) } return contracts.ValidationResult{Approved: true}, nil }) assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), target, preparedValidatorChain{validators: []preparedValidator{first, second}}) assessment, err := assess(context.Background(), candidate) if err != nil || !assessment.Accepted() || candidate.Items[0] != "original" || candidate.Details.Name != "original" { t.Fatalf("assessment = %#v, error %v, candidate %#v", assessment, err, candidate) } } func TestTypedAssessmentRechecksEarlierApprovalsAndUnchangedRevisions(t *testing.T) { calls := []string{} makeValidator := func(name, accepted string) preparedValidator { return assessmentNotesValidator(t, name, func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { value := request.Value.Items[0] calls = append(calls, name+":"+value) if value == accepted { return contracts.ValidationResult{Approved: true}, nil } return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Satisfy " + name}, nil }) } assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{}, preparedValidatorChain{validators: []preparedValidator{makeValidator("first", "one"), makeValidator("second", "two")}}) for _, text := range []string{"one", "two", "two"} { assessment, err := assess(context.Background(), codecNotes{Items: []string{text}}) if err != nil || assessment.Accepted() { t.Fatalf("assessment for %s = %#v, %v", text, assessment, err) } want := contracts.AssessmentRejected if text == "one" { want = contracts.AssessmentApproved } if assessment.Validators[0].Decision != want { t.Fatalf("earlier approval survived revised candidate: %#v", assessment) } } if !reflect.DeepEqual(calls, []string{"first:one", "second:one", "first:two", "second:two", "first:two", "second:two"}) { t.Fatalf("assessments = %#v", calls) } } type chunkAssessmentValidator func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) func (chunkAssessmentValidator) Name() string { return "chunk assessment" } func (chunkAssessmentValidator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } func (v chunkAssessmentValidator) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { return v(ctx, request) } func TestChunkAssessmentIsolatesTypedAndSerializedViewsAfterRejection(t *testing.T) { chunks := []source.Chunk{{ID: "chunk", Content: []byte("original"), Metadata: map[string]any{"label": "original"}}} originalJSON, err := json.Marshal(chunks) if err != nil { t.Fatal(err) } calls := 0 mutator := preparedValidator{ resolved: ResolvedValidator{Binding: Binding("chunk mutator"), Target: ValidatorTargetChunk}, chunk: chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { calls++ request.Chunks[0].Content[0] = 'x' request.Chunks[0].Metadata["label"] = "changed" return contracts.ValidationResult{ReasonCode: "reject", CorrectionGuidance: "Correct the partition."}, nil }), } observer := preparedValidator{ resolved: ResolvedValidator{Binding: Binding("serialized observer"), Target: ValidatorTargetSerialized}, serialized: serializedValidationFunc(func(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) { calls++ if !bytes.Equal(request.Content, originalJSON) || !reflect.DeepEqual(request.Chunks, chunks) { t.Fatalf("chunk validator changed a later view: %s %#v", request.Content, request.Chunks) } return contracts.ValidationResult{Approved: true}, nil }), } assessment, err := chunkCandidateAssessor(typedValidationTarget{stage: StageChunk}, preparedValidatorChain{validators: []preparedValidator{mutator, observer}})(context.Background(), chunks) if err != nil || calls != 2 || assessment.Accepted() || string(chunks[0].Content) != "original" || chunks[0].Metadata["label"] != "original" { t.Fatalf("assessment = %#v %v, calls %d, chunks %#v", assessment, err, calls, chunks) } } type serializedValidationFunc func(context.Context, contracts.SerializedValidationRequest) (contracts.ValidationResult, error) func (serializedValidationFunc) Name() string { return "candidate-serialized" } func (serializedValidationFunc) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } func (validate serializedValidationFunc) Validate(ctx context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) { return validate(ctx, request) }