eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
internal/framework/runner/runner_test.go
View File

@@ -11,8 +11,8 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
@@ -717,10 +717,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
}
func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" {
t.Fatalf("unexpected instance names: %+v", specs)
}
@@ -920,10 +917,13 @@ func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
@@ -955,10 +955,9 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -978,10 +977,9 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1007,10 +1005,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
@@ -1058,7 +1055,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
@@ -1083,10 +1080,9 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
@@ -1106,10 +1102,9 @@ func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
@@ -1139,10 +1134,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
@@ -1161,13 +1155,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1177,8 +1166,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1227,10 +1215,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Config: req.Config,
Glossary: req.Glossary,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
DiagnosticsDir: req.DiagnosticsDir,
@@ -1281,8 +1266,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ProposalLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
ProposalDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1315,13 +1299,11 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
Glossary: &schema.Glossary{},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1349,13 +1331,11 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Glossary = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)