audita/internal/framework/runner/feedback_test.go

package runner

import (
	"context"
	"fmt"
	"os"
	"testing"

	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
	"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
	validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)

func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
	candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
		return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
	}
	var seen []int
	semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
		var decisions []validators.Decision
		for _, p := range req.CandidateProposal {
			seen = append(seen, p.ProposalIndex)
			decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"})
		}
		return validators.Result{Decisions: decisions}, nil
	}}, validatormetadata.ExecutionClassLLMBacked)
	input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
	result, err := validateSectionCandidates(context.Background(), input)
	if err != nil {
		t.Fatal(err)
	}
	reasons := map[int]map[string]bool{}
	for _, d := range result.rejected {
		if reasons[d.ProposalIndex] == nil {
			reasons[d.ProposalIndex] = map[string]bool{}
		}
		reasons[d.ProposalIndex][d.ReasonCode] = true
	}
	if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
		t.Fatalf("findings = %+v", reasons)
	}
	if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
		t.Fatalf("approved=%v semantic input=%v", result.approved, seen)
	}
}

func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
	candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
	for _, budget := range []int{0, 1} {
		t.Run(fmt.Sprint(budget), func(t *testing.T) {
			var later int
			first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
				var decisions []validators.Decision
				for _, p := range req.CandidateProposal {
					decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
				}
				return validators.Result{Decisions: decisions}, nil
			}}
			second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
				later = len(req.CandidateProposal)
				var decisions []validators.Decision
				for _, p := range req.CandidateProposal {
					decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
				}
				return validators.Result{Decisions: decisions}, nil
			}}
			input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
			result, err := validateSectionCandidates(context.Background(), input)
			if err != nil {
				t.Fatal(err)
			}
			if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget {
				t.Fatalf("result=%+v later=%d", result, later)
			}
		})
	}
}

func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
	rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
	findings := repairFindings(2, rejected)
	if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
		t.Fatalf("findings: %+v", findings)
	}
}

func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
		{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
		{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
	}}
	validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
	if err != nil {
		t.Fatal(err)
	}
	dir := t.TempDir()
	attempt := 1
	var paths []string
	for _, index := range []int{2, 3} {
		candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
		input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
		result, err := validateSectionCandidates(context.Background(), input)
		if err != nil {
			t.Fatal(err)
		}
		if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
			t.Fatalf("decisions: %+v", result.decisions)
		}
		paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
	}
	if paths[0] == paths[1] {
		t.Fatalf("colliding paths: %v", paths)
	}
	for _, path := range paths {
		if _, err := os.Stat(path); err != nil {
			t.Fatal(err)
		}
	}
}

func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
	validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
	if err != nil {
		t.Fatal(err)
	}
	candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
	input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
	result, err := validateSectionCandidates(context.Background(), input)
	if err != nil {
		t.Fatal(err)
	}
	if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
		t.Fatalf("result: %+v", result)
	}
}