eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/framework/runner/cancellation_test.go
View File

@@ -10,7 +10,6 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
@@ -21,7 +20,8 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type contextValidator struct {
@@ -47,23 +47,20 @@ func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := "version: 2\nbackends:\n shared:\n endpoint: http://provider.invalid/v1\n concurrency: 1\n queue_capacity: 1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 0\n"
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil {
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
transport := &cancelTransport{entered: make(chan struct{}, 3)}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: transport})
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +72,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
started <- struct{}{}
}
}
client, err := runtime.NewRoleClient("validation", hooks)
client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks)
if err != nil {
t.Fatal(err)
}
@@ -101,7 +98,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
var out map[string]any
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
PromptID: prompts.PromptIDValidatorEditorialReview, PromptVersion: prompts.VersionV1,
PromptID: prompts.PromptIDValidatorEditorialReview,
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
}, &out)
@@ -120,7 +117,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
done := make(chan pipelineOutcome, 1)
go func() {
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Config: &loaded.Config, Sections: sections})
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections})
done <- pipelineOutcome{result: result, err: err}
}()
select {

92
internal/framework/runner/runner_test.go
View File

@@ -11,8 +11,8 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
@@ -717,10 +717,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
}
func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" {
t.Fatalf("unexpected instance names: %+v", specs)
}
@@ -920,10 +917,13 @@ func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
@@ -955,10 +955,9 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -978,10 +977,9 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1007,10 +1005,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
@@ -1058,7 +1055,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
@@ -1083,10 +1080,9 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
@@ -1106,10 +1102,9 @@ func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
@@ -1139,10 +1134,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
@@ -1161,13 +1155,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1177,8 +1166,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1227,10 +1215,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Config: req.Config,
Glossary: req.Glossary,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
DiagnosticsDir: req.DiagnosticsDir,
@@ -1281,8 +1266,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ProposalLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
ProposalDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1315,13 +1299,11 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
Glossary: &schema.Glossary{},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1349,13 +1331,11 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Glossary = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)