Files
audita/internal/framework/runner/cancellation_test.go

259 lines
11 KiB
Go
Raw Normal View History

package runner
import (
"context"
"errors"
"net/http"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type contextValidator struct {
validate func(context.Context, validators.Request) (validators.Result, error)
}
func (v contextValidator) Name() string { return "cancel-test" }
func (v contextValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
return v.validate(ctx, req)
}
type cancelTransport struct {
entered chan struct{}
calls atomic.Int32
}
func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
t.calls.Add(1)
t.entered <- struct{}{}
<-req.Context().Done()
return nil, req.Context().Err()
}
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
transport := &cancelTransport{entered: make(chan struct{}, 3)}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
collector := llm.NewObservationCollector()
hooks := collector.Hooks()
started := make(chan struct{}, 2)
hooks.OnEvent = func(event pk.ExecutionEvent) {
if event.Kind == pk.ExecutionStarted {
started <- struct{}{}
}
}
client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks)
if err != nil {
t.Fatal(err)
}
origin := errors.New("originating validator failure")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
validator := contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
if *req.SectionIndex == 0 {
// Hold the failure until one call is active and both peer executions started.
for i := 0; i < 2; i++ {
select {
case <-started:
case <-ctx.Done():
return validators.Result{}, ctx.Err()
}
}
select {
case <-transport.entered:
case <-ctx.Done():
return validators.Result{}, ctx.Err()
}
return validators.Result{}, origin
}
var out map[string]any
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
PromptID: prompts.PromptIDValidatorEditorialReview,
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
}, &out)
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
}}
sections := []chunking.Section{}
for i := 0; i < 3; i++ {
sections = append(sections, chunking.Section{Index: i, Segments: []schema.Segment{{ID: 1, Text: "hello"}}})
}
type pipelineOutcome struct {
result modulePipelineResult
err error
}
done := make(chan pipelineOutcome, 1)
go func() {
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections})
done <- pipelineOutcome{result: result, err: err}
}()
select {
case outcome := <-done:
if !errors.Is(outcome.err, origin) {
t.Fatalf("lost originating failure: %v", outcome.err)
}
if len(outcome.result.WorkUnits) != 3 {
t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits)
}
var failure *ExecutionFailure
if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() ||
outcome.result.WorkUnits[0].Failure != failure {
t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits)
}
for _, peer := range outcome.result.WorkUnits[1:] {
if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure {
t.Fatalf("lost peer interruption: %+v", peer)
}
}
if len(outcome.result.Approved) != 0 {
t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved)
}
case <-time.After(2 * time.Second):
t.Fatal("validator failure did not cancel and join peers")
}
if transport.calls.Load() != 1 {
t.Fatalf("pending call dispatched: %d", transport.calls.Load())
}
snapshot := collector.Snapshot()
if len(snapshot.Calls) != 2 {
t.Fatalf("partial observations missing: %+v", snapshot)
}
for _, call := range snapshot.Calls {
if call.Finished.IsZero() || call.Failure != structuredoutput.FailureRuntime {
t.Fatalf("unfinished/cause lost: %+v", call)
}
}
if got := llm.Summarize(snapshot); got.ProviderAttempts != 1 {
t.Fatalf("attempt count: %+v", got)
}
}
func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
out, err := New(fakeFactory{}).Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}},
})
if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" {
t.Fatalf("canceled entry: result=%+v error=%v", out, err)
}
}
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
origin := errors.New("earlier proposal failed")
laterValidationStarted := make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
close(laterValidationStarted)
<-ctx.Done()
return validators.Result{}, ctx.Err()
}},
}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index == 0 {
select {
case <-laterValidationStarted:
return contracts.ProposalResult{}, origin
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
SegmentReferences: []int{1},
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
}, nil
},
}
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure {
t.Fatalf("lost originating proposal failure: %+v", failure)
}
candidate := out.Candidates[0]
later := out.WorkUnits[1]
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
candidate.SectionIndex == nil || *candidate.SectionIndex != later.Section.Index ||
out.FinalTranscript.Segments[1].Text != "other text" {
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
}
}
func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil
}
}
last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
cancel()
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")},
"second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}},
}})
out, err := r.Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}},
})
var failure *ExecutionFailure
if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 ||
len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed ||
len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 ||
!out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure)
}
}