95 lines
3.9 KiB
Go
95 lines
3.9 KiB
Go
|
|
package runner
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"reflect"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||
|
|
)
|
||
|
|
|
||
|
|
func evidenceFixture(size int) (*Runner, RunInput) {
|
||
|
|
cfg := runnerTestConfig()
|
||
|
|
cfg.Chunking.MaxSectionTokens = 1
|
||
|
|
cfg.Chunking.MinSectionTokens = 0
|
||
|
|
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
|
||
|
|
for i := range transcript.Segments {
|
||
|
|
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
|
||
|
|
}
|
||
|
|
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
|
||
|
|
var decisions []validators.Decision
|
||
|
|
for _, candidate := range req.CandidateProposal {
|
||
|
|
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
||
|
|
}
|
||
|
|
return validators.Result{Decisions: decisions}, nil
|
||
|
|
}}
|
||
|
|
module := outcomeModule{
|
||
|
|
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
|
||
|
|
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||
|
|
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
|
||
|
|
category := req.Section.Index % 4
|
||
|
|
if category == 0 {
|
||
|
|
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
|
||
|
|
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
|
||
|
|
}
|
||
|
|
operation.Succeeded = true
|
||
|
|
original := "word"
|
||
|
|
if category == 1 {
|
||
|
|
original = "absent"
|
||
|
|
}
|
||
|
|
if category == 2 {
|
||
|
|
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
|
||
|
|
}
|
||
|
|
return contracts.ProposalResult{
|
||
|
|
Operations: []outcomes.Operation{operation},
|
||
|
|
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
|
||
|
|
}, nil
|
||
|
|
},
|
||
|
|
}
|
||
|
|
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
|
||
|
|
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
|
||
|
|
r, input := evidenceFixture(40)
|
||
|
|
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
|
||
|
|
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
|
||
|
|
var baseline *schema.Transcript
|
||
|
|
for _, tc := range []struct {
|
||
|
|
name string
|
||
|
|
selection EvidenceSelection
|
||
|
|
units int
|
||
|
|
candidates int
|
||
|
|
}{
|
||
|
|
{name: "summary"},
|
||
|
|
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
|
||
|
|
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
|
||
|
|
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
|
||
|
|
} {
|
||
|
|
t.Run(tc.name, func(t *testing.T) {
|
||
|
|
input.Evidence = &tc.selection
|
||
|
|
out, err := r.Run(context.Background(), input)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if out.Totals == nil || *out.Totals != want {
|
||
|
|
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
|
||
|
|
}
|
||
|
|
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
|
||
|
|
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
|
||
|
|
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
|
||
|
|
}
|
||
|
|
if baseline == nil {
|
||
|
|
baseline = out.FinalTranscript
|
||
|
|
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
|
||
|
|
t.Fatal("output selection changed final corrections")
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|