306 lines
12 KiB
Go
306 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
|
ck "go.maximumdirect.net/configkit"
|
|
)
|
|
|
|
type sharedBackendModule struct {
|
|
*grammar.Module
|
|
validator validators.Validator
|
|
headEntered <-chan struct{}
|
|
}
|
|
|
|
func (m sharedBackendModule) Validators() []validators.Validator {
|
|
return []validators.Validator{m.validator}
|
|
}
|
|
|
|
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
|
if req.Section.Index > 0 {
|
|
select {
|
|
case <-m.headEntered:
|
|
case <-ctx.Done():
|
|
return contracts.ProposalResult{}, ctx.Err()
|
|
}
|
|
}
|
|
return m.Module.Propose(ctx, req)
|
|
}
|
|
|
|
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
|
defer cancel()
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "config.yml")
|
|
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
|
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release := make(chan struct{})
|
|
entered := make(chan struct{})
|
|
validationStarted := make(chan struct{}, 1)
|
|
var other, validating, active, peak atomic.Int32
|
|
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
|
|
body, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
n := active.Add(1)
|
|
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
|
|
}
|
|
defer active.Add(-1)
|
|
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
|
|
switch {
|
|
case bytes.Contains(body, []byte(`"validations"`)):
|
|
validating.Add(1)
|
|
select {
|
|
case validationStarted <- struct{}{}:
|
|
default:
|
|
}
|
|
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
|
case bytes.Contains(body, []byte("marker-000")):
|
|
close(entered)
|
|
select {
|
|
case <-release:
|
|
case <-r.Context().Done():
|
|
return nil, r.Context().Err()
|
|
}
|
|
default:
|
|
other.Add(1)
|
|
}
|
|
content, _ := json.Marshal(output)
|
|
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
|
|
})}
|
|
runtime, err := llm.NewRuntime(ctx, loaded, client)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
|
cfg := runnerTestConfig()
|
|
cfg.Chunking.MaxSectionTokens = 1
|
|
cfg.Chunking.MinSectionTokens = 0
|
|
transcript := &schema.Transcript{}
|
|
for i := 0; i < 36; i++ {
|
|
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
|
|
}
|
|
type result struct {
|
|
out RunOutput
|
|
err error
|
|
}
|
|
finished := make(chan result, 1)
|
|
go func() {
|
|
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
|
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
|
|
})
|
|
finished <- result{out, err}
|
|
}()
|
|
select {
|
|
case <-entered:
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
select {
|
|
case <-validationStarted:
|
|
case <-ctx.Done():
|
|
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
|
|
}
|
|
close(release)
|
|
completed := <-finished
|
|
if completed.err != nil {
|
|
t.Fatal(completed.err)
|
|
}
|
|
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
|
|
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
|
|
}
|
|
changes := completed.out.ModuleResults[0].AppliedChanges
|
|
if len(changes) != len(transcript.Segments) {
|
|
t.Fatalf("got %d applied changes", len(changes))
|
|
}
|
|
for i, change := range changes {
|
|
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
|
|
t.Fatalf("change %d: %+v", i, change)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
|
for _, canceled := range []bool{false, true} {
|
|
name := "completed"
|
|
if canceled {
|
|
name = "canceled"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
laterRevalidated := make(chan struct{})
|
|
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
|
section := *req.Context.SectionIndex
|
|
if section == 0 {
|
|
select {
|
|
case <-laterRevalidated:
|
|
case <-ctx.Done():
|
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
|
}
|
|
}
|
|
text := []string{"alpha", "bravo"}[section]
|
|
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
|
|
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
|
|
}}
|
|
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
|
|
})
|
|
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
|
section := *req.Context.SectionIndex
|
|
first := section == 0
|
|
revision := req.Context.RepairAttempt != nil
|
|
if first && canceled {
|
|
cancel()
|
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
|
}
|
|
lastRevision := revision && *req.Context.RepairAttempt == 2
|
|
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
|
|
CorrectionIndex: "1", Approved: first || lastRevision, Reason: "checked",
|
|
}}
|
|
if lastRevision {
|
|
close(laterRevalidated)
|
|
}
|
|
return contracts.StructuredCompletionResponse{}, nil
|
|
})
|
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
|
|
// The module's transport gates are supplied by the scripted calls above.
|
|
started := make(chan struct{})
|
|
close(started)
|
|
module.headEntered = started
|
|
var captured []*structuredoutput.PreparedSnapshot
|
|
repair := &repairClient{t: t, capturePrepared: func(p *structuredoutput.PreparedSnapshot) {
|
|
captured = append(captured, p)
|
|
}, outputs: []string{
|
|
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"BRAVO"}}}`,
|
|
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
|
|
}}
|
|
dir := t.TempDir()
|
|
if canceled {
|
|
blocked := filepath.Join(dir, "grammar", "grammar_section-0001_proposal-0001_attempt-0001_repair-request-payload.json")
|
|
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writer := llm.NewDiagnosticsWriter(dir)
|
|
cfg := runnerTestConfig()
|
|
cfg.Chunking.MaxSectionTokens = 1
|
|
cfg.Chunking.MinSectionTokens = 0
|
|
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
|
Config: &cfg, Interactions: writer, Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
|
ModuleSpecs: []contracts.ModuleRunSpec{{
|
|
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
|
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 2}},
|
|
}},
|
|
})
|
|
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
|
|
t.Fatal(err)
|
|
}
|
|
later := out.WorkUnits[1]
|
|
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
|
|
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 2 {
|
|
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
|
|
}
|
|
if len(captured) != 2 || len(captured[1].Messages) <= len(captured[0].Messages) {
|
|
t.Fatalf("missing growing repair history: %+v", captured)
|
|
}
|
|
for i, attempt := range later.Repairs[0].Attempts {
|
|
if i == 0 && canceled {
|
|
if attempt.Artifacts.RequestPayloadPath != "" || len(writer.Warnings()) != 1 {
|
|
t.Fatalf("expected one optional request write failure: %+v, %v", attempt.Artifacts, writer.Warnings())
|
|
}
|
|
continue
|
|
}
|
|
var request structuredoutput.PreparedSnapshot
|
|
readInteraction(t, attempt.Artifacts.RequestPayloadPath, &request)
|
|
if !reflect.DeepEqual(&request, captured[i]) {
|
|
t.Fatalf("request %d did not preserve prepared evidence", i+1)
|
|
}
|
|
}
|
|
attempt := later.Repairs[0].Attempts[1]
|
|
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
|
|
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
|
|
t.Fatalf("lost ordered repair evidence: %+v", attempt)
|
|
}
|
|
var repaired struct {
|
|
Decoded proposal_generation.RepairResponse
|
|
Revision *proposals.EnrichedCorrectionProposal
|
|
Decisions []ValidatorDecisionRecord
|
|
}
|
|
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
|
|
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
|
|
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
|
|
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
|
|
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
|
|
t.Fatalf("repair artifact uses different identities: %+v", repaired)
|
|
}
|
|
var checked struct {
|
|
Decoded validators.LLMValidationResponse
|
|
Reconciliation struct{ Decisions []validators.Decision }
|
|
}
|
|
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
|
|
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
|
|
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
|
|
t.Fatalf("validation evidence uses different identities: %+v", checked)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func readInteraction(t *testing.T, path string, result any) {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(raw, result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|