259 lines
12 KiB
Go
259 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
|
ck "go.maximumdirect.net/configkit"
|
|
pk "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type contextValidator struct {
|
|
validate func(context.Context, validators.Request) (validators.Result, error)
|
|
}
|
|
|
|
func (v contextValidator) Name() string { return "cancel-test" }
|
|
func (v contextValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
|
|
return v.validate(ctx, req)
|
|
}
|
|
|
|
type cancelTransport struct {
|
|
entered chan struct{}
|
|
calls atomic.Int32
|
|
}
|
|
|
|
func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
t.calls.Add(1)
|
|
t.entered <- struct{}{}
|
|
<-req.Context().Done()
|
|
return nil, req.Context().Err()
|
|
}
|
|
|
|
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
|
dir := t.TempDir()
|
|
main := filepath.Join(dir, "config.yml")
|
|
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n"
|
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
transport := &cancelTransport{entered: make(chan struct{}, 3)}
|
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
collector := llm.NewObservationCollector()
|
|
hooks := collector.Hooks()
|
|
started := make(chan struct{}, 2)
|
|
hooks.OnEvent = func(event pk.ExecutionEvent) {
|
|
if event.Kind == pk.ExecutionStarted {
|
|
started <- struct{}{}
|
|
}
|
|
}
|
|
client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
origin := errors.New("originating validator failure")
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
validator := contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
|
|
if *req.SectionIndex == 0 {
|
|
// Hold the failure until one call is active and both peer executions started.
|
|
for i := 0; i < 2; i++ {
|
|
select {
|
|
case <-started:
|
|
case <-ctx.Done():
|
|
return validators.Result{}, ctx.Err()
|
|
}
|
|
}
|
|
select {
|
|
case <-transport.entered:
|
|
case <-ctx.Done():
|
|
return validators.Result{}, ctx.Err()
|
|
}
|
|
return validators.Result{}, origin
|
|
}
|
|
var out map[string]any
|
|
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
|
|
PromptID: prompts.PromptIDValidatorEditorialReview,
|
|
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
|
|
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
|
|
}, &out)
|
|
return validators.Result{}, err
|
|
}}
|
|
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
|
|
}}
|
|
sections := []chunking.Section{}
|
|
for i := 0; i < 3; i++ {
|
|
sections = append(sections, chunking.Section{Index: i, Segments: []schema.Segment{{ID: 1, Text: "hello"}}})
|
|
}
|
|
type pipelineOutcome struct {
|
|
result modulePipelineResult
|
|
err error
|
|
}
|
|
done := make(chan pipelineOutcome, 1)
|
|
go func() {
|
|
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections})
|
|
done <- pipelineOutcome{result: result, err: err}
|
|
}()
|
|
select {
|
|
case outcome := <-done:
|
|
if !errors.Is(outcome.err, origin) {
|
|
t.Fatalf("lost originating failure: %v", outcome.err)
|
|
}
|
|
if len(outcome.result.WorkUnits) != 3 {
|
|
t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits)
|
|
}
|
|
var failure *ExecutionFailure
|
|
if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection ||
|
|
failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() ||
|
|
outcome.result.WorkUnits[0].Failure != failure {
|
|
t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits)
|
|
}
|
|
for _, peer := range outcome.result.WorkUnits[1:] {
|
|
if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure {
|
|
t.Fatalf("lost peer interruption: %+v", peer)
|
|
}
|
|
}
|
|
if len(outcome.result.Approved) != 0 {
|
|
t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("validator failure did not cancel and join peers")
|
|
}
|
|
if transport.calls.Load() != 1 {
|
|
t.Fatalf("pending call dispatched: %d", transport.calls.Load())
|
|
}
|
|
snapshot := collector.Snapshot()
|
|
if len(snapshot.Calls) != 2 {
|
|
t.Fatalf("partial observations missing: %+v", snapshot)
|
|
}
|
|
for _, call := range snapshot.Calls {
|
|
if call.Finished.IsZero() || call.Failure != structuredoutput.FailureRuntime {
|
|
t.Fatalf("unfinished/cause lost: %+v", call)
|
|
}
|
|
}
|
|
if got := llm.Summarize(snapshot); got.ProviderAttempts != 1 {
|
|
t.Fatalf("attempt count: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
out, err := New(fakeFactory{}).Run(ctx, RunInput{
|
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}},
|
|
})
|
|
if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" {
|
|
t.Fatalf("canceled entry: result=%+v error=%v", out, err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
|
|
origin := errors.New("earlier proposal failed")
|
|
laterValidationStarted := make(chan struct{})
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
module := outcomeModule{
|
|
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
|
|
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
|
|
close(laterValidationStarted)
|
|
<-ctx.Done()
|
|
return validators.Result{}, ctx.Err()
|
|
}},
|
|
}},
|
|
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
|
if req.Section.Index == 0 {
|
|
select {
|
|
case <-laterValidationStarted:
|
|
return contracts.ProposalResult{}, origin
|
|
case <-ctx.Done():
|
|
return contracts.ProposalResult{}, ctx.Err()
|
|
}
|
|
}
|
|
return contracts.ProposalResult{
|
|
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
|
|
SegmentReferences: []int{1},
|
|
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
|
|
}, nil
|
|
},
|
|
}
|
|
cfg := runnerTestConfig()
|
|
cfg.Chunking.MaxSectionTokens = 3
|
|
cfg.Chunking.MinSectionTokens = 0
|
|
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
|
Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
|
})
|
|
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
|
|
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
|
|
}
|
|
var failure *ExecutionFailure
|
|
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
|
|
failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure {
|
|
t.Fatalf("lost originating proposal failure: %+v", failure)
|
|
}
|
|
candidate := out.Candidates[0]
|
|
later := out.WorkUnits[1]
|
|
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
|
|
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
|
|
out.FinalTranscript.Segments[1].Text != "other text" {
|
|
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
|
|
}
|
|
}
|
|
|
|
func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil
|
|
}
|
|
}
|
|
last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
|
|
cancel()
|
|
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")},
|
|
"second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}},
|
|
}})
|
|
out, err := r.Run(ctx, RunInput{
|
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}},
|
|
})
|
|
var failure *ExecutionFailure
|
|
if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 ||
|
|
len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed ||
|
|
len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 ||
|
|
!out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure)
|
|
}
|
|
}
|