Retain processing evidence only for selected outputs
This commit is contained in:
94
internal/framework/runner/evidence_test.go
Normal file
94
internal/framework/runner/evidence_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
)
|
||||
|
||||
func evidenceFixture(size int) (*Runner, RunInput) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 1
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
|
||||
for i := range transcript.Segments {
|
||||
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
|
||||
}
|
||||
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
module := outcomeModule{
|
||||
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
|
||||
category := req.Section.Index % 4
|
||||
if category == 0 {
|
||||
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
|
||||
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
|
||||
}
|
||||
operation.Succeeded = true
|
||||
original := "word"
|
||||
if category == 1 {
|
||||
original = "absent"
|
||||
}
|
||||
if category == 2 {
|
||||
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
|
||||
}
|
||||
return contracts.ProposalResult{
|
||||
Operations: []outcomes.Operation{operation},
|
||||
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
|
||||
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
|
||||
r, input := evidenceFixture(40)
|
||||
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
|
||||
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
|
||||
var baseline *schema.Transcript
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
selection EvidenceSelection
|
||||
units int
|
||||
candidates int
|
||||
}{
|
||||
{name: "summary"},
|
||||
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
|
||||
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
|
||||
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
input.Evidence = &tc.selection
|
||||
out, err := r.Run(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Totals == nil || *out.Totals != want {
|
||||
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
|
||||
}
|
||||
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
|
||||
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
|
||||
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
|
||||
}
|
||||
if baseline == nil {
|
||||
baseline = out.FinalTranscript
|
||||
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
|
||||
t.Fatal("output selection changed final corrections")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user