Files
audita/internal/framework/runner/window_test.go

306 lines
12 KiB
Go
Raw Normal View History

package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"reflect"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
ck "go.maximumdirect.net/configkit"
)
type sharedBackendModule struct {
*grammar.Module
validator validators.Validator
headEntered <-chan struct{}
}
func (m sharedBackendModule) Validators() []validators.Validator {
return []validators.Validator{m.validator}
}
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index > 0 {
select {
case <-m.headEntered:
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
return m.Module.Propose(ctx, req)
}
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
dir := t.TempDir()
path := filepath.Join(dir, "config.yml")
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
if err != nil {
t.Fatal(err)
}
release := make(chan struct{})
entered := make(chan struct{})
validationStarted := make(chan struct{}, 1)
var other, validating, active, peak atomic.Int32
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
body, err := io.ReadAll(r.Body)
if err != nil {
return nil, err
}
n := active.Add(1)
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
}
defer active.Add(-1)
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
switch {
case bytes.Contains(body, []byte(`"validations"`)):
validating.Add(1)
select {
case validationStarted <- struct{}{}:
default:
}
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
case bytes.Contains(body, []byte("marker-000")):
close(entered)
select {
case <-release:
case <-r.Context().Done():
return nil, r.Context().Err()
}
default:
other.Add(1)
}
content, _ := json.Marshal(output)
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
})}
runtime, err := llm.NewRuntime(ctx, loaded, client)
if err != nil {
t.Fatal(err)
}
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{}
for i := 0; i < 36; i++ {
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
}
type result struct {
out RunOutput
err error
}
finished := make(chan result, 1)
go func() {
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
})
finished <- result{out, err}
}()
select {
case <-entered:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
select {
case <-validationStarted:
case <-ctx.Done():
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
}
close(release)
completed := <-finished
if completed.err != nil {
t.Fatal(completed.err)
}
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
}
changes := completed.out.ModuleResults[0].AppliedChanges
if len(changes) != len(transcript.Segments) {
t.Fatalf("got %d applied changes", len(changes))
}
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
t.Fatalf("change %d: %+v", i, change)
}
}
}
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
for _, canceled := range []bool{false, true} {
name := "completed"
if canceled {
name = "canceled"
}
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
laterRevalidated := make(chan struct{})
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
if section == 0 {
select {
case <-laterRevalidated:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
text := []string{"alpha", "bravo"}[section]
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
}}
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
})
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
first := section == 0
revision := req.Context.RepairAttempt != nil
if first && canceled {
cancel()
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
lastRevision := revision && *req.Context.RepairAttempt == 2
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
CorrectionIndex: "1", Approved: first || lastRevision, Reason: "checked",
}}
if lastRevision {
close(laterRevalidated)
}
return contracts.StructuredCompletionResponse{}, nil
})
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
// The module's transport gates are supplied by the scripted calls above.
started := make(chan struct{})
close(started)
module.headEntered = started
var captured []*structuredoutput.PreparedSnapshot
repair := &repairClient{t: t, capturePrepared: func(p *structuredoutput.PreparedSnapshot) {
captured = append(captured, p)
}, outputs: []string{
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"BRAVO"}}}`,
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
}}
dir := t.TempDir()
if canceled {
blocked := filepath.Join(dir, "grammar", "grammar_section-0001_proposal-0001_attempt-0001_repair-request-payload.json")
if err := os.MkdirAll(blocked, 0o755); err != nil {
t.Fatal(err)
}
}
writer := llm.NewDiagnosticsWriter(dir)
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: writer, Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 2}},
}},
})
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
t.Fatal(err)
}
later := out.WorkUnits[1]
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 2 {
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
}
if len(captured) != 2 || len(captured[1].Messages) <= len(captured[0].Messages) {
t.Fatalf("missing growing repair history: %+v", captured)
}
for i, attempt := range later.Repairs[0].Attempts {
if i == 0 && canceled {
if attempt.Artifacts.RequestPayloadPath != "" || len(writer.Warnings()) != 1 {
t.Fatalf("expected one optional request write failure: %+v, %v", attempt.Artifacts, writer.Warnings())
}
continue
}
var request structuredoutput.PreparedSnapshot
readInteraction(t, attempt.Artifacts.RequestPayloadPath, &request)
if !reflect.DeepEqual(&request, captured[i]) {
t.Fatalf("request %d did not preserve prepared evidence", i+1)
}
}
attempt := later.Repairs[0].Attempts[1]
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
t.Fatalf("lost ordered repair evidence: %+v", attempt)
}
var repaired struct {
Decoded proposal_generation.RepairResponse
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
}
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
t.Fatalf("repair artifact uses different identities: %+v", repaired)
}
var checked struct {
Decoded validators.LLMValidationResponse
Reconciliation struct{ Decisions []validators.Decision }
}
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
t.Fatalf("validation evidence uses different identities: %+v", checked)
}
})
}
}
func readInteraction(t *testing.T, path string, result any) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(raw, result); err != nil {
t.Fatal(err)
}
}