Unify ready pipeline jobs under one worker pool and remove the global LLM scheduler
This commit is contained in:
@@ -170,9 +170,5 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
|
||||
}
|
||||
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create LLM scheduler: %w", err)
|
||||
}
|
||||
return llm.NewScheduledClient(client, scheduler), nil, nil
|
||||
return client, nil, nil
|
||||
}
|
||||
|
||||
@@ -659,16 +659,6 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
|
||||
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid scheduler concurrency", func(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := config.Default()
|
||||
cfg.Concurrency.TotalLLM = 0
|
||||
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
|
||||
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
|
||||
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
|
||||
|
||||
@@ -425,7 +425,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
Checkpoint: checkpointLoader,
|
||||
CheckpointPolicy: checkpointPolicy,
|
||||
Debug: debugRecorder,
|
||||
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
|
||||
Workers: cfg.Concurrency.Workers,
|
||||
})
|
||||
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if diagnosticErr == nil {
|
||||
|
||||
@@ -16,7 +16,7 @@ type ConcurrencyConfig struct {
|
||||
Workers int `json:"workers"`
|
||||
}
|
||||
|
||||
const defaultWorkers = 64
|
||||
const defaultWorkers = pipeline.DefaultWorkers
|
||||
|
||||
type OutputConfig struct {
|
||||
Directory string `json:"directory"`
|
||||
|
||||
@@ -405,7 +405,7 @@ type effectiveExtractor struct{ key string }
|
||||
|
||||
func (m effectiveExtractor) Key() string { return m.key }
|
||||
func (m effectiveExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (m effectiveExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[effectiveArtifact], error) {
|
||||
func (m effectiveExtractor) Extract(context.Context, contracts.TypedExtractionRequest[effectiveArtifact]) (contracts.TypedExtractionResult[effectiveArtifact], error) {
|
||||
return contracts.TypedExtractionResult[effectiveArtifact]{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -1594,105 +1595,6 @@ func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
|
||||
fake := &fakePromptKitLLM{
|
||||
content: `{"ok":true}`,
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
client := newTestPromptKitClient(t, fake)
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
scheduled := NewScheduledClient(client, scheduler)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if callErr != nil {
|
||||
t.Errorf("CompleteStructured() error = %v, want nil", callErr)
|
||||
}
|
||||
}()
|
||||
}
|
||||
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||
t.Fatalf("max in-flight calls = %d, want <= 1", got)
|
||||
}
|
||||
close(fake.block)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
|
||||
attempts := 1
|
||||
fake := &fakePromptKitLLM{
|
||||
responses: []promptkit.GenerateResponse{
|
||||
{Content: `{"bad":true}`},
|
||||
{Content: `{"ok":true}`},
|
||||
{Content: `{"ok":true}`},
|
||||
},
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v", err)
|
||||
}
|
||||
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
|
||||
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "repair-session",
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
firstDone <- callErr
|
||||
}()
|
||||
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||
|
||||
secondDone := make(chan error, 1)
|
||||
go func() {
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "queued-session",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
secondDone <- callErr
|
||||
}()
|
||||
close(fake.block)
|
||||
if err := <-firstDone; err != nil {
|
||||
t.Fatalf("repaired completion error = %v", err)
|
||||
}
|
||||
if err := <-secondDone; err != nil {
|
||||
t.Fatalf("queued completion error = %v", err)
|
||||
}
|
||||
|
||||
requests := fake.requestsSnapshot()
|
||||
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
|
||||
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
|
||||
}
|
||||
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientValidatesRequest(t *testing.T) {
|
||||
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
|
||||
var out map[string]any
|
||||
@@ -1785,6 +1687,18 @@ func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
|
||||
return registry
|
||||
}
|
||||
|
||||
func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if atomic.LoadInt32(value) >= want {
|
||||
return
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
t.Fatalf("timed out waiting for value >= %d; got %d", want, atomic.LoadInt32(value))
|
||||
}
|
||||
|
||||
type fakePromptKitLLM struct {
|
||||
content string
|
||||
responses []promptkit.GenerateResponse
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type scheduledClient struct {
|
||||
client contracts.StructuredLLMClient
|
||||
scheduler *Scheduler
|
||||
}
|
||||
|
||||
func NewScheduledClient(client contracts.StructuredLLMClient, scheduler *Scheduler) contracts.StructuredLLMClient {
|
||||
return &scheduledClient{
|
||||
client: client,
|
||||
scheduler: scheduler,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if c == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client must not be nil")
|
||||
}
|
||||
if c.client == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client inner client must not be nil")
|
||||
}
|
||||
if c.scheduler == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client scheduler must not be nil")
|
||||
}
|
||||
|
||||
var response contracts.StructuredCompletionResponse
|
||||
err := c.scheduler.Run(ctx, func(ctx context.Context) error {
|
||||
var callErr error
|
||||
response, callErr = c.client.CompleteStructured(ctx, req, out)
|
||||
return callErr
|
||||
})
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
if c == nil || c.client == nil {
|
||||
return nil
|
||||
}
|
||||
provider, ok := c.client.(contracts.LLMProfileManifestProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return provider.LLMProfileManifests()
|
||||
}
|
||||
|
||||
func (c *scheduledClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) {
|
||||
if c == nil || c.client == nil {
|
||||
return nil, nil
|
||||
}
|
||||
provider, ok := c.client.(CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return provider.LLMCheckpointFingerprints()
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestScheduledClientEnforcesSchedulerLimit(t *testing.T) {
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
inner := &blockingStructuredClient{
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
client := NewScheduledClient(inner, scheduler)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err != nil {
|
||||
t.Errorf("CompleteStructured() error = %v, want nil", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
waitForAtomicAtLeast(t, &inner.calls, 1)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if got := atomic.LoadInt32(&inner.maxInFlight); got > 1 {
|
||||
t.Fatalf("max in-flight calls = %d, want <= 1", got)
|
||||
}
|
||||
close(inner.release)
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&inner.calls); got != 3 {
|
||||
t.Fatalf("calls = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledClientPropagatesClientError(t *testing.T) {
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
expected := errors.New("provider unavailable")
|
||||
client := NewScheduledClient(&errorStructuredClient{err: expected}, scheduler)
|
||||
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &struct{}{})
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("CompleteStructured() error = %v, want %v", err, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledClientPropagatesSchedulerError(t *testing.T) {
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := NewScheduledClient(&errorStructuredClient{}, scheduler)
|
||||
|
||||
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{}, &struct{}{})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledClientPreservesCheckpointFingerprints(t *testing.T) {
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inner := &fingerprintedStructuredClient{
|
||||
fingerprints: []CheckpointFingerprint{
|
||||
{Name: "profile_source", Value: "sha256:one"},
|
||||
{Name: "backend_target", Value: "sha256:two"},
|
||||
},
|
||||
}
|
||||
client := NewScheduledClient(inner, scheduler)
|
||||
provider, ok := client.(CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
t.Fatalf("scheduled client %T does not preserve checkpoint fingerprints", client)
|
||||
}
|
||||
got, err := provider.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != len(inner.fingerprints) ||
|
||||
got[0] != inner.fingerprints[0] ||
|
||||
got[1] != inner.fingerprints[1] {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want %#v", got, inner.fingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
type blockingStructuredClient struct {
|
||||
release chan struct{}
|
||||
inFlight int32
|
||||
maxInFlight int32
|
||||
calls int32
|
||||
}
|
||||
|
||||
func (c *blockingStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
atomic.AddInt32(&c.calls, 1)
|
||||
current := atomic.AddInt32(&c.inFlight, 1)
|
||||
for {
|
||||
seen := atomic.LoadInt32(&c.maxInFlight)
|
||||
if current <= seen || atomic.CompareAndSwapInt32(&c.maxInFlight, seen, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer atomic.AddInt32(&c.inFlight, -1)
|
||||
|
||||
select {
|
||||
case <-c.release:
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
if target, ok := out.(*map[string]any); ok {
|
||||
*target = map[string]any{"ok": true}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: json.RawMessage(`{"ok":true}`),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type errorStructuredClient struct {
|
||||
err error
|
||||
}
|
||||
|
||||
type fingerprintedStructuredClient struct {
|
||||
errorStructuredClient
|
||||
fingerprints []CheckpointFingerprint
|
||||
}
|
||||
|
||||
func (c *fingerprintedStructuredClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) {
|
||||
return append([]CheckpointFingerprint(nil), c.fingerprints...), nil
|
||||
}
|
||||
|
||||
func (c *errorStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
return contracts.StructuredCompletionResponse{}, c.err
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Scheduler bounds concurrent LLM backend calls.
|
||||
type Scheduler struct {
|
||||
maxConcurrency int
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
queue []*waiter
|
||||
}
|
||||
|
||||
type waiter struct {
|
||||
ready chan struct{}
|
||||
queued bool
|
||||
granted bool
|
||||
}
|
||||
|
||||
// NewScheduler creates a scheduler with a fixed concurrency limit.
|
||||
func NewScheduler(maxConcurrency int) (*Scheduler, error) {
|
||||
if maxConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("max concurrency must be greater than zero")
|
||||
}
|
||||
return &Scheduler{
|
||||
maxConcurrency: maxConcurrency,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Acquire blocks until a permit is available or the context is canceled.
|
||||
// The returned release function is safe to call multiple times.
|
||||
func (s *Scheduler) Acquire(ctx context.Context) (func(), error) {
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("scheduler must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if s.inFlight < s.maxConcurrency && len(s.queue) == 0 {
|
||||
s.inFlight++
|
||||
s.mu.Unlock()
|
||||
return s.releaseFunc(), nil
|
||||
}
|
||||
|
||||
w := &waiter{
|
||||
ready: make(chan struct{}),
|
||||
queued: true,
|
||||
}
|
||||
s.queue = append(s.queue, w)
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-w.ready:
|
||||
return s.releaseFunc(), nil
|
||||
case <-ctx.Done():
|
||||
s.mu.Lock()
|
||||
if w.queued {
|
||||
s.removeQueuedWaiterLocked(w)
|
||||
s.mu.Unlock()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if w.granted {
|
||||
s.inFlight--
|
||||
s.grantQueuedLocked()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Run acquires a permit, executes fn, and releases the permit.
|
||||
func (s *Scheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
if fn == nil {
|
||||
return fmt.Errorf("scheduler function must not be nil")
|
||||
}
|
||||
release, err := s.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
func (s *Scheduler) releaseFunc() func() {
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
s.mu.Lock()
|
||||
if s.inFlight > 0 {
|
||||
s.inFlight--
|
||||
s.grantQueuedLocked()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) grantQueuedLocked() {
|
||||
for s.inFlight < s.maxConcurrency && len(s.queue) > 0 {
|
||||
w := s.queue[0]
|
||||
s.queue = s.queue[1:]
|
||||
w.queued = false
|
||||
w.granted = true
|
||||
s.inFlight++
|
||||
close(w.ready)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) removeQueuedWaiterLocked(target *waiter) {
|
||||
for i, w := range s.queue {
|
||||
if w == target {
|
||||
w.queued = false
|
||||
s.queue = append(s.queue[:i], s.queue[i+1:]...)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewSchedulerValidation(t *testing.T) {
|
||||
if _, err := NewScheduler(0); err == nil {
|
||||
t.Fatalf("expected validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerMaxConcurrency(t *testing.T) {
|
||||
s, err := NewScheduler(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
|
||||
var inFlight int32
|
||||
var maxInFlight int32
|
||||
release := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runErr := s.Run(context.Background(), func(context.Context) error {
|
||||
current := atomic.AddInt32(&inFlight, 1)
|
||||
for {
|
||||
seen := atomic.LoadInt32(&maxInFlight)
|
||||
if current <= seen || atomic.CompareAndSwapInt32(&maxInFlight, seen, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
<-release
|
||||
atomic.AddInt32(&inFlight, -1)
|
||||
return nil
|
||||
})
|
||||
if runErr != nil {
|
||||
t.Errorf("Run error: %v", runErr)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
waitForAtomicAtLeast(t, &maxInFlight, 2)
|
||||
close(release)
|
||||
wg.Wait()
|
||||
|
||||
if got := atomic.LoadInt32(&maxInFlight); got > 2 {
|
||||
t.Fatalf("expected max in-flight <= 2, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerCancellationWhileQueued(t *testing.T) {
|
||||
s, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
|
||||
release, err := s.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, acquireErr := s.Acquire(ctx)
|
||||
errCh <- acquireErr
|
||||
}()
|
||||
|
||||
waitForQueueDepth(t, s, 1)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case acquireErr := <-errCh:
|
||||
if !errors.Is(acquireErr, context.Canceled) {
|
||||
t.Fatalf("expected context canceled, got %v", acquireErr)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for queued acquire to cancel")
|
||||
}
|
||||
|
||||
release()
|
||||
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
|
||||
t.Fatalf("expected scheduler to accept work after cancellation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerReleaseFunctionIsIdempotent(t *testing.T) {
|
||||
s, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
|
||||
release, err := s.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
|
||||
release()
|
||||
release()
|
||||
|
||||
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
|
||||
t.Fatalf("expected permit to be released once, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerRunReleasesPermitAfterError(t *testing.T) {
|
||||
s, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
|
||||
expected := errors.New("failed")
|
||||
err = s.Run(context.Background(), func(context.Context) error {
|
||||
return expected
|
||||
})
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("expected %v, got %v", expected, err)
|
||||
}
|
||||
|
||||
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
|
||||
t.Fatalf("expected permit to be released after error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type cancelAfterAdmissionContext struct {
|
||||
done chan struct{}
|
||||
checks atomic.Int32
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func newCancelAfterAdmissionContext() *cancelAfterAdmissionContext {
|
||||
return &cancelAfterAdmissionContext{done: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (*cancelAfterAdmissionContext) Deadline() (time.Time, bool) { return time.Time{}, false }
|
||||
|
||||
func (c *cancelAfterAdmissionContext) Done() <-chan struct{} { return c.done }
|
||||
|
||||
func (c *cancelAfterAdmissionContext) Err() error {
|
||||
if c.checks.Add(1) == 1 {
|
||||
return nil
|
||||
}
|
||||
c.once.Do(func() { close(c.done) })
|
||||
return context.Canceled
|
||||
}
|
||||
|
||||
func (*cancelAfterAdmissionContext) Value(any) any { return nil }
|
||||
|
||||
func TestSchedulerDoesNotDispatchCanceledAdmissionAndReleasesNextWaiter(t *testing.T) {
|
||||
s, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler: %v", err)
|
||||
}
|
||||
hold, err := s.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
defer hold()
|
||||
|
||||
ctx := newCancelAfterAdmissionContext()
|
||||
var canceledCalls atomic.Int32
|
||||
canceledErr := make(chan error, 1)
|
||||
go func() {
|
||||
canceledErr <- s.Run(ctx, func(context.Context) error {
|
||||
canceledCalls.Add(1)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
waitForQueueDepth(t, s, 1)
|
||||
|
||||
nextStarted := make(chan struct{})
|
||||
nextErr := make(chan error, 1)
|
||||
go func() {
|
||||
nextErr <- s.Run(context.Background(), func(context.Context) error {
|
||||
close(nextStarted)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
waitForQueueDepth(t, s, 2)
|
||||
|
||||
hold()
|
||||
if err := <-canceledErr; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() error = %v, want context.Canceled", err)
|
||||
}
|
||||
if got := canceledCalls.Load(); got != 0 {
|
||||
t.Fatalf("canceled callback calls = %d, want none", got)
|
||||
}
|
||||
select {
|
||||
case <-nextStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for next FIFO waiter")
|
||||
}
|
||||
if err := <-nextErr; err != nil {
|
||||
t.Fatalf("next Run() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if atomic.LoadInt32(value) >= want {
|
||||
return
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
t.Fatalf("timed out waiting for value >= %d; got %d", want, atomic.LoadInt32(value))
|
||||
}
|
||||
|
||||
func waitForQueueDepth(t *testing.T, s *Scheduler, want int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
s.mu.Lock()
|
||||
depth := len(s.queue)
|
||||
s.mu.Unlock()
|
||||
if depth >= want {
|
||||
return
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
s.mu.Lock()
|
||||
depth := len(s.queue)
|
||||
s.mu.Unlock()
|
||||
t.Fatalf("timed out waiting for queue depth >= %d; got %d", want, depth)
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
type executionDebugClient struct {
|
||||
@@ -20,7 +19,7 @@ func (c executionDebugClient) CompleteStructured(context.Context, contracts.Stru
|
||||
return c.response, c.err
|
||||
}
|
||||
|
||||
func TestScheduledDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
|
||||
func TestDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
|
||||
providerErr := contracts.NewLLMGenerationError(503, "provider unavailable")
|
||||
const content = `{"ok":true}`
|
||||
for _, test := range []struct {
|
||||
@@ -61,16 +60,12 @@ func TestScheduledDebugClientPreservesExecutionThroughTerminalRecords(t *testing
|
||||
} else {
|
||||
response.Content = json.RawMessage(test.content)
|
||||
}
|
||||
scheduler, err := llm.NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder := newCapturedDebugRecorder()
|
||||
client := WithDebugLLMRecording(llm.NewScheduledClient(executionDebugClient{response, test.err}, scheduler), recorder)
|
||||
client := WithDebugLLMRecording(executionDebugClient{response, test.err}, recorder)
|
||||
ctx, scope := withDebugLLMScope(context.Background(), "attempt")
|
||||
got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{})
|
||||
if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want {
|
||||
t.Fatalf("scheduled debug completion = (%+v, %v), want summary and original error", got, err)
|
||||
t.Fatalf("debug completion = (%+v, %v), want summary and original error", got, err)
|
||||
}
|
||||
terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"})
|
||||
if err := terminal.record(nil, test.err); !errors.Is(err, test.err) {
|
||||
|
||||
315
internal/framework/pipeline/ready_jobs_test.go
Normal file
315
internal/framework/pipeline/ready_jobs_test.go
Normal file
@@ -0,0 +1,315 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type activeJobs struct {
|
||||
active atomic.Int32
|
||||
peak atomic.Int32
|
||||
}
|
||||
|
||||
func (jobs *activeJobs) enter() func() {
|
||||
current := jobs.active.Add(1)
|
||||
for {
|
||||
peak := jobs.peak.Load()
|
||||
if current <= peak || jobs.peak.CompareAndSwap(peak, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return func() { jobs.active.Add(-1) }
|
||||
}
|
||||
|
||||
func TestRunnerSharesWorkerLimitBetweenExtractionAndReadyContinuation(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
var jobs activeJobs
|
||||
var firstLaneExtracts atomic.Int32
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
mergeStarted := make(chan struct{}, 1)
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
defer jobs.enter()()
|
||||
if lane == 0 {
|
||||
firstLaneExtracts.Add(1)
|
||||
} else {
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return erasedTypedResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
|
||||
})
|
||||
current := &prepared.Steps[0].lanes[lane]
|
||||
merge := current.typed.merge
|
||||
current.typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
defer jobs.enter()()
|
||||
if lane == 0 {
|
||||
if firstLaneExtracts.Load() != 2 {
|
||||
return erasedTypedResult{}, errors.New("continuation ran before its extraction prerequisites")
|
||||
}
|
||||
mergeStarted <- struct{}{}
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return erasedTypedResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return merge(ctx, implementation, request)
|
||||
}
|
||||
normalize := current.typed.normalize
|
||||
current.typed.normalize = func(ctx context.Context, implementation any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
defer jobs.enter()()
|
||||
return normalize(ctx, implementation, request)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-mergeStarted:
|
||||
synctest.Wait()
|
||||
case <-ctx.Done():
|
||||
t.Fatal("ready continuation could not run while another lane still had extraction work")
|
||||
}
|
||||
unblock()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := jobs.peak.Load(); got != 2 || jobs.active.Load() != 0 {
|
||||
t.Fatalf("mixed active jobs peaked at %d, remaining %d; want the single limit of 2", got, jobs.active.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func limitedProducerEngine(t *testing.T, limit, queue int, client promptkit.LLMClient) *promptkit.Engine {
|
||||
t.Helper()
|
||||
return nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("test-controlled backend", []byte(fmt.Sprintf(`backends:
|
||||
- id: limited
|
||||
endpoint: https://example.invalid/v1
|
||||
concurrency_limit: %d
|
||||
queue_capacity: %d
|
||||
profiles:
|
||||
- id: base
|
||||
backend: limited
|
||||
model: shared-model
|
||||
api_key_required: false
|
||||
`, limit, queue)))))
|
||||
}
|
||||
|
||||
func installNativeExtraction(prepared *PreparedPipeline, engine *promptkit.Engine, jobs *activeJobs, started chan<- struct{}) {
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
prepared.Steps[0].lanes[lane].resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
if jobs != nil {
|
||||
defer jobs.enter()()
|
||||
}
|
||||
if started != nil {
|
||||
started <- struct{}{}
|
||||
}
|
||||
value, outcome, err := RunProducer(ctx, engine, request.Operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, response *promptkit.RunResult) (*any, []string, error) {
|
||||
_, _, err := interpretProducerNumber(ctx, response)
|
||||
value := typedValueForLane(lane, request.Chunk.Index)
|
||||
return &value, nil, err
|
||||
}, producerCorrection, func(context.Context, *any) (any, error) {
|
||||
if lane == 0 {
|
||||
return codecNotes{Items: []string{}}, nil
|
||||
}
|
||||
return codecScore{}, nil
|
||||
})
|
||||
return erasedTypedResult{Value: value, Outcome: outcome}, err
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerWorkersAndNativeBackendAdmissionAreIndependent(t *testing.T) {
|
||||
for _, limits := range []struct{ workers, backend int }{{1, 1}, {3, 2}} {
|
||||
t.Run(fmt.Sprintf("workers-%d/backend-%d", limits.workers, limits.backend), func(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
var providers, jobs activeJobs
|
||||
var calls atomic.Int32
|
||||
providerStarted := make(chan struct{}, 8)
|
||||
jobStarted := make(chan struct{}, 4)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
engine := limitedProducerEngine(t, limits.backend, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls.Add(1)
|
||||
defer providers.enter()()
|
||||
providerStarted <- struct{}{}
|
||||
select {
|
||||
case <-release:
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}))
|
||||
installNativeExtraction(prepared, engine, &jobs, jobStarted)
|
||||
for lane := range prepared.Steps[0].lanes {
|
||||
prepared.Steps[0].lanes[lane].extractValidators.validators[0].typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
result, err := engine.Run(ctx, nativeProducerInitial())
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
|
||||
}
|
||||
return contracts.ValidationResult{Approved: result.Validation.IsValid}, nil
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: limits.workers})
|
||||
if err == nil && len(output.NormalizeOutputs) != 2 {
|
||||
err = fmt.Errorf("got %d lane outputs", len(output.NormalizeOutputs))
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
for range limits.workers {
|
||||
select {
|
||||
case <-jobStarted:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("worker jobs did not overlap as configured")
|
||||
}
|
||||
}
|
||||
for range limits.backend {
|
||||
select {
|
||||
case <-providerStarted:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("native backend did not admit its configured calls")
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobs.peak.Load() != int32(limits.workers) || providers.peak.Load() != int32(limits.backend) || calls.Load() != 8 || jobs.active.Load() != 0 || providers.active.Load() != 0 {
|
||||
t.Fatalf("worker peak %d, backend peak %d, calls %d; active jobs/providers %d/%d", jobs.peak.Load(), providers.peak.Load(), calls.Load(), jobs.active.Load(), providers.active.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerNativeCapacityFailureUsesOperationModeWithoutAnotherQueue(t *testing.T) {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
started, release := make(chan struct{}, 2), make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
engine := limitedProducerEngine(t, 1, 0, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls.Add(1)
|
||||
started <- struct{}{}
|
||||
select {
|
||||
case <-release:
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
occupant := make(chan error, 1)
|
||||
go func() { _, err := engine.Run(ctx, nativeProducerInitial()); occupant <- err }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("backend occupant did not start")
|
||||
}
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
installNativeExtraction(prepared, engine, nil, nil)
|
||||
for lane := range prepared.Steps[0].lanes {
|
||||
prepared.Steps[0].lanes[lane].resolved.ExtractControls = contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}
|
||||
}
|
||||
zero := 0
|
||||
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Execution: contracts.ExecutionOptions{TransientRetries: &zero}})
|
||||
if mode == contracts.ModeStrict {
|
||||
var capacity *promptkit.CapacityError
|
||||
if !errors.As(err, &capacity) || output.FallbackCount != 0 || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("strict capacity failure = %v, fallbacks %d", err, output.FallbackCount)
|
||||
}
|
||||
} else if err != nil || output.FallbackCount != 2 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("permissive capacity failure = %v, fallbacks %d, outputs %d", err, output.FallbackCount, len(output.NormalizeOutputs))
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("capacity handling invoked provider %d times", calls.Load())
|
||||
}
|
||||
unblock()
|
||||
if err := <-occupant; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := engine.Run(ctx, nativeProducerInitial()); err != nil || calls.Load() != 2 {
|
||||
t.Fatalf("backend admission was not released: %v, calls %d", err, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCancellationDrainsNativeWorkAndReleasesAdmission(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
var jobs activeJobs
|
||||
jobStarted := make(chan struct{}, 4)
|
||||
providerStarted := make(chan struct{})
|
||||
engine := limitedProducerEngine(t, 1, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
close(providerStarted)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
installNativeExtraction(prepared, engine, &jobs, jobStarted)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 3})
|
||||
if output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 {
|
||||
err = fmt.Errorf("cancellation delivered output: %v", err)
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
for range 3 {
|
||||
select {
|
||||
case <-jobStarted:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("native jobs did not start")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-providerStarted:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("native backend did not start")
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) || jobs.active.Load() != 0 || len(jobStarted) != 0 {
|
||||
t.Fatalf("cancel = %v, active jobs %d, extra starts %d", err, jobs.active.Load(), len(jobStarted))
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), nativeProducerInitial()); err != nil || calls.Load() != 2 {
|
||||
t.Fatalf("admission after cancellation = %v, provider calls %d", err, calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,9 @@ type Registries struct {
|
||||
|
||||
type Runner struct{}
|
||||
|
||||
// DefaultWorkers is the shared ready-job limit when the caller omits one.
|
||||
const DefaultWorkers = 64
|
||||
|
||||
func New() *Runner {
|
||||
return &Runner{}
|
||||
}
|
||||
@@ -55,9 +58,9 @@ type RunInput struct {
|
||||
Checkpoint CheckpointLoader
|
||||
CheckpointPolicy CheckpointExecutionPolicy
|
||||
Debug DebugRecorder
|
||||
// ExtractWorkers bounds run-wide extract jobs. Values less than one use a
|
||||
// single worker so direct framework callers retain deterministic behavior.
|
||||
ExtractWorkers int
|
||||
// Workers bounds ready extraction and lane-completion jobs together.
|
||||
// Zero selects DefaultWorkers; configuration rejects explicit zero.
|
||||
Workers int
|
||||
|
||||
pipeline ResolvedPipeline
|
||||
// reuseIneligible carries chunk ancestry; reference eligibility stays scoped.
|
||||
@@ -468,6 +471,9 @@ func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string,
|
||||
}
|
||||
|
||||
func validateRunInput(input RunInput) error {
|
||||
if input.Workers < 0 {
|
||||
return fmt.Errorf("workers must be positive when specified")
|
||||
}
|
||||
if input.Prepared == nil {
|
||||
return fmt.Errorf("prepared pipeline must not be nil")
|
||||
}
|
||||
|
||||
@@ -13,33 +13,8 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
type countingProvider struct {
|
||||
active atomic.Int32
|
||||
maximum atomic.Int32
|
||||
calls atomic.Int32
|
||||
}
|
||||
|
||||
func (p *countingProvider) CompleteStructured(ctx context.Context, _ contracts.StructuredCompletionRequest, _ any) (contracts.StructuredCompletionResponse, error) {
|
||||
p.calls.Add(1)
|
||||
current := p.active.Add(1)
|
||||
defer p.active.Add(-1)
|
||||
for {
|
||||
seen := p.maximum.Load()
|
||||
if current <= seen || p.maximum.CompareAndSwap(seen, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func preparedConcurrentPipeline(t *testing.T, chunkCount int) *PreparedPipeline {
|
||||
t.Helper()
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
@@ -226,7 +201,7 @@ func TestRunnerFailsGeneratedHandoffBeforeConsumerExtraction(t *testing.T) {
|
||||
}}
|
||||
prepared.resolved.Steps[0].ArtifactLanes[0].ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
|
||||
var extractCalls atomic.Int32
|
||||
preparedLane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
preparedLane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
extractCalls.Add(1)
|
||||
return erasedTypedResult{Value: codecNotes{}}, nil
|
||||
}
|
||||
@@ -257,7 +232,7 @@ func TestRunnerExecutesOrderedStepsWithHardBarriers(t *testing.T) {
|
||||
}
|
||||
first := &prepared.Steps[0].lanes[0]
|
||||
second := &prepared.Steps[1].lanes[0]
|
||||
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
|
||||
}
|
||||
originalFirstNormalize := first.typed.normalize
|
||||
@@ -268,12 +243,12 @@ func TestRunnerExecutesOrderedStepsWithHardBarriers(t *testing.T) {
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
record("second-extract-started")
|
||||
return erasedTypedResult{Value: codecScore{Value: 2}}, nil
|
||||
}
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -297,21 +272,21 @@ func TestRunnerStopsLaterOrderedStepsAfterFailure(t *testing.T) {
|
||||
first := &prepared.Steps[0].lanes[0]
|
||||
second := &prepared.Steps[1].lanes[0]
|
||||
third := &prepared.Steps[2].lanes[0]
|
||||
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
|
||||
}
|
||||
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{}, errors.New("ordered step extraction failed")
|
||||
}
|
||||
var thirdCalls atomic.Int32
|
||||
third.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
third.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
thirdCalls.Add(1)
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{"later"}}}, nil
|
||||
}
|
||||
encoder := &countingOrderedOutput{}
|
||||
prepared.output = encoder
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
if err == nil || !strings.Contains(err.Error(), `execute pipeline step "step-2"`) {
|
||||
t.Fatalf("Run() error = %v, want step-scoped failure", err)
|
||||
}
|
||||
@@ -326,8 +301,8 @@ func TestRunnerStopsLaterOrderedStepsAfterFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func installExtractOperation(prepared *PreparedPipeline, laneIndex int, operation func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error)) {
|
||||
prepared.Steps[0].lanes[laneIndex].typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
func installExtractOperation(prepared *PreparedPipeline, laneIndex int, operation func(context.Context, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error)) {
|
||||
prepared.Steps[0].lanes[laneIndex].typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return operation(ctx, request)
|
||||
}
|
||||
}
|
||||
@@ -350,7 +325,7 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
|
||||
var maximum atomic.Int32
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
jobIndex := request.Chunk.Index*2 + lane
|
||||
current := active.Add(1)
|
||||
defer active.Add(-1)
|
||||
@@ -377,7 +352,7 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
|
||||
}
|
||||
done := make(chan outcome, 1)
|
||||
go func() {
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
done <- outcome{output: output, err: err}
|
||||
}()
|
||||
for pair := 0; pair < 3; pair++ {
|
||||
@@ -416,45 +391,6 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerStartsLaneContinuationWhileOtherLaneExtractsRemain(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
releaseOtherLane := make(chan struct{})
|
||||
mergeStarted := make(chan struct{}, 1)
|
||||
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||
})
|
||||
installExtractOperation(prepared, 1, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
select {
|
||||
case <-releaseOtherLane:
|
||||
return erasedTypedResult{Value: typedValueForLane(1, request.Chunk.Index)}, nil
|
||||
case <-ctx.Done():
|
||||
return erasedTypedResult{}, ctx.Err()
|
||||
}
|
||||
})
|
||||
originalMerge := prepared.Steps[0].lanes[0].typed.merge
|
||||
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
select {
|
||||
case mergeStarted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return originalMerge(ctx, implementation, request)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-mergeStarted:
|
||||
close(releaseOtherLane)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first lane merge did not start while the second lane remained in extract")
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
base := prepared.Steps[0].lanes[0]
|
||||
@@ -470,7 +406,7 @@ func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
|
||||
extractor: base.typed.extractor,
|
||||
merger: base.typed.merger,
|
||||
normalizer: base.typed.normalizer,
|
||||
extract: func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
extract: func(_ context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: codecNotes{Items: []string{request.Chunk.ID}}}, nil
|
||||
},
|
||||
merge: func(_ context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
@@ -496,7 +432,7 @@ func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
|
||||
prepared.resolved.Steps[0].ArtifactLanes = resolvedLanes
|
||||
prepared.Steps[0].ArtifactLanes = publicLanes
|
||||
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -514,7 +450,7 @@ func TestRunnerSelectsFrameworkErrorByStableLaneOrder(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, lane, func(context.Context, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
ready <- struct{}{}
|
||||
<-release
|
||||
return erasedTypedResult{}, fmt.Errorf("lane-%d failure", lane)
|
||||
@@ -522,7 +458,7 @@ func TestRunnerSelectsFrameworkErrorByStableLaneOrder(t *testing.T) {
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
done <- err
|
||||
}()
|
||||
<-ready
|
||||
@@ -537,7 +473,7 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
|
||||
})
|
||||
}
|
||||
@@ -555,7 +491,7 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
done <- err
|
||||
}()
|
||||
<-ready
|
||||
@@ -566,103 +502,12 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerKeepsSuccessfulChunksWhenAnotherExtractIsRejected(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 2)
|
||||
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
|
||||
})
|
||||
}
|
||||
validator := &prepared.Steps[0].lanes[0].extractValidators.validators[0]
|
||||
validator.typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
if target.chunk != nil && target.chunk.Index == 0 {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test", CorrectionGuidance: "return an acceptable candidate"}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.Rejected) != 1 || output.Rejected[0].LaneID != prepared.Steps[0].lanes[0].resolved.ID || output.Rejected[0].ChunkIndex != 0 {
|
||||
t.Fatalf("rejections = %#v, want the first lane's first chunk", output.Rejected)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("normalize outputs = %d, want both lanes to continue", len(output.NormalizeOutputs))
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "rejected" {
|
||||
t.Fatalf("validation status = %q, want rejected", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerSeparatelyBoundsExtractJobsAndSharedProviderCalls(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 3)
|
||||
provider := &countingProvider{}
|
||||
scheduler, err := frameworkllm.NewScheduler(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v", err)
|
||||
}
|
||||
client := frameworkllm.NewScheduledClient(provider, scheduler)
|
||||
var jobActive atomic.Int32
|
||||
var jobMaximum atomic.Int32
|
||||
var attempts sync.Map
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
prepared.Steps[0].lanes[lane].resolved.Extract.Retries = 1
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
current := jobActive.Add(1)
|
||||
defer jobActive.Add(-1)
|
||||
for {
|
||||
seen := jobMaximum.Load()
|
||||
if current <= seen || jobMaximum.CompareAndSwap(seen, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, callErr := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, nil); callErr != nil {
|
||||
return erasedTypedResult{}, callErr
|
||||
}
|
||||
key := fmt.Sprintf("%d/%d", lane, request.Chunk.Index)
|
||||
countValue, _ := attempts.LoadOrStore(key, &atomic.Int32{})
|
||||
count := countValue.(*atomic.Int32).Add(1)
|
||||
if lane == 0 && request.Chunk.Index == 0 && count == 1 {
|
||||
return erasedTypedResult{}, errors.New("retry extract")
|
||||
}
|
||||
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
|
||||
})
|
||||
validator := &prepared.Steps[0].lanes[lane].extractValidators.validators[0]
|
||||
validator.typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
|
||||
if _, callErr := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "validate"}, nil); callErr != nil {
|
||||
return contracts.ValidationResult{}, callErr
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("normalize outputs = %d, want 2", len(output.NormalizeOutputs))
|
||||
}
|
||||
if got := jobMaximum.Load(); got != 3 {
|
||||
t.Fatalf("maximum extract jobs = %d, want 3", got)
|
||||
}
|
||||
if got := provider.maximum.Load(); got != 2 {
|
||||
t.Fatalf("maximum provider calls = %d, want 2", got)
|
||||
}
|
||||
if got := provider.calls.Load(); got != 13 {
|
||||
t.Fatalf("provider calls = %d, want 13 including retry and validators", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerReturnsParentCancellationAndStopsQueuedExtracts(t *testing.T) {
|
||||
prepared := preparedConcurrentPipeline(t, 4)
|
||||
started := make(chan struct{}, 8)
|
||||
for laneIndex := range prepared.Steps[0].lanes {
|
||||
lane := laneIndex
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
started <- struct{}{}
|
||||
<-ctx.Done()
|
||||
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, ctx.Err()
|
||||
@@ -671,7 +516,7 @@ func TestRunnerReturnsParentCancellationAndStopsQueuedExtracts(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
|
||||
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
||||
done <- err
|
||||
}()
|
||||
<-started
|
||||
|
||||
@@ -151,22 +151,33 @@ type laneEngineConfig struct {
|
||||
states []*laneExtractState
|
||||
}
|
||||
|
||||
// laneJob is ready to execute: a nil chunk selects the lane's merge/normalize
|
||||
// continuation, otherwise it selects one extraction. Prerequisites are resolved
|
||||
// by the coordinator before dispatch, never by an occupied worker.
|
||||
type laneJob struct {
|
||||
lane *laneExtractState
|
||||
chunk *source.Chunk
|
||||
}
|
||||
|
||||
type laneJobResult struct {
|
||||
extract *extractJobResult
|
||||
completion *laneCompletion
|
||||
}
|
||||
|
||||
type laneEngine struct {
|
||||
runner *Runner
|
||||
laneEngineConfig
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
workerCount int
|
||||
jobs chan extractJob
|
||||
results chan extractJobResult
|
||||
completions chan laneCompletion
|
||||
continuations chan *laneExtractState
|
||||
continuationWorkers sync.WaitGroup
|
||||
pending []*laneExtractState
|
||||
completedOutputs []RunOutput
|
||||
runErrors []orderedRunError
|
||||
launched int
|
||||
completed int
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
workerCount int
|
||||
jobs chan laneJob
|
||||
results chan laneJobResult
|
||||
workers sync.WaitGroup
|
||||
pending []*laneExtractState
|
||||
completedOutputs []RunOutput
|
||||
runErrors []orderedRunError
|
||||
nextExtract int
|
||||
active int
|
||||
}
|
||||
|
||||
func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig) ([]RunOutput, []orderedRunError) {
|
||||
@@ -175,31 +186,20 @@ func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig)
|
||||
}
|
||||
|
||||
func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *laneEngine {
|
||||
workerCount := config.input.ExtractWorkers
|
||||
if workerCount < 1 {
|
||||
workerCount = 1
|
||||
workerCount := config.input.Workers
|
||||
if workerCount == 0 {
|
||||
workerCount = DefaultWorkers
|
||||
}
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
return &laneEngine{
|
||||
runner: r, laneEngineConfig: config, ctx: ctx, cancel: cancel, workerCount: workerCount,
|
||||
jobs: make(chan extractJob, workerCount), results: make(chan extractJobResult, workerCount),
|
||||
completions: make(chan laneCompletion, len(config.states)), continuations: make(chan *laneExtractState, workerCount),
|
||||
jobs: make(chan laneJob), results: make(chan laneJobResult, workerCount),
|
||||
completedOutputs: make([]RunOutput, len(config.states)),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
|
||||
defer e.cancel()
|
||||
e.initializeCollection()
|
||||
e.startExtractWorkers()
|
||||
e.startContinuationWorkers()
|
||||
e.collect()
|
||||
close(e.continuations)
|
||||
e.continuationWorkers.Wait()
|
||||
return e.completedOutputs, e.runErrors
|
||||
}
|
||||
|
||||
func (e *laneEngine) initializeCollection() {
|
||||
for _, state := range e.states {
|
||||
if state.terminal {
|
||||
e.completedOutputs[state.index] = state.output
|
||||
@@ -207,90 +207,96 @@ func (e *laneEngine) initializeCollection() {
|
||||
e.pending = append(e.pending, state)
|
||||
}
|
||||
}
|
||||
e.startWorkers()
|
||||
e.collect()
|
||||
close(e.jobs)
|
||||
e.workers.Wait()
|
||||
return e.completedOutputs, e.runErrors
|
||||
}
|
||||
|
||||
func (e *laneEngine) startExtractWorkers() {
|
||||
var workers sync.WaitGroup
|
||||
func (e *laneEngine) startWorkers() {
|
||||
for i := 0; i < e.workerCount; i++ {
|
||||
workers.Add(1)
|
||||
e.workers.Add(1)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
defer e.workers.Done()
|
||||
for job := range e.jobs {
|
||||
if e.ctx.Err() != nil {
|
||||
if job.chunk != nil {
|
||||
result := e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, extractJob{lane: job.lane, chunk: *job.chunk})
|
||||
e.results <- laneJobResult{extract: &result}
|
||||
continue
|
||||
}
|
||||
e.results <- e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, job)
|
||||
completion := laneCompletion{index: job.lane.index}
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
completion.err = err
|
||||
} else {
|
||||
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
|
||||
}
|
||||
e.results <- laneJobResult{completion: &completion}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go e.dispatchExtractJobs()
|
||||
go func() { workers.Wait(); close(e.results) }()
|
||||
}
|
||||
|
||||
func (e *laneEngine) dispatchExtractJobs() {
|
||||
defer close(e.jobs)
|
||||
for chunkIndex := range e.chunks {
|
||||
for laneIndex := range e.states {
|
||||
state := e.states[laneIndex]
|
||||
if state.terminal || state.decision.Reused {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case e.jobs <- extractJob{lane: state, chunk: e.chunks[chunkIndex]}:
|
||||
case <-e.ctx.Done():
|
||||
// nextJob gives ready continuations priority, then preserves chunk-first,
|
||||
// lane-second extraction order. Dispatch remains lazy rather than creating a
|
||||
// queue or goroutine for every extraction.
|
||||
func (e *laneEngine) nextJob() (laneJob, bool) {
|
||||
if len(e.pending) > 0 {
|
||||
return laneJob{lane: e.pending[0]}, true
|
||||
}
|
||||
for e.nextExtract < len(e.chunks)*len(e.states) {
|
||||
state := e.states[e.nextExtract%len(e.states)]
|
||||
if !state.terminal && !state.decision.Reused {
|
||||
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
|
||||
}
|
||||
e.nextExtract++
|
||||
}
|
||||
return laneJob{}, false
|
||||
}
|
||||
|
||||
// collect alone owns dispatch and result accounting. Cancellation disables new
|
||||
// dispatch, but every dispatched job still returns one result before workers exit.
|
||||
func (e *laneEngine) collect() {
|
||||
for {
|
||||
if e.ctx.Err() != nil {
|
||||
if e.active == 0 {
|
||||
return
|
||||
}
|
||||
e.receiveResult(<-e.results)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *laneEngine) startContinuationWorkers() {
|
||||
for i := 0; i < e.workerCount; i++ {
|
||||
e.continuationWorkers.Add(1)
|
||||
go func() {
|
||||
defer e.continuationWorkers.Done()
|
||||
for state := range e.continuations {
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
e.completions <- laneCompletion{index: state.index, err: err}
|
||||
continue
|
||||
}
|
||||
laneOutput, err := e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, state)
|
||||
e.completions <- laneCompletion{index: state.index, output: laneOutput, err: err}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// collect is the sole owner of pending, launch, and completion accounting. It
|
||||
// must drain closed extract results and one completion for every launched
|
||||
// continuation, even after cancellation.
|
||||
func (e *laneEngine) collect() {
|
||||
resultChannel := (<-chan extractJobResult)(e.results)
|
||||
for resultChannel != nil || len(e.pending) > 0 || e.completed < e.launched {
|
||||
var continuationChannel chan<- *laneExtractState
|
||||
var nextContinuation *laneExtractState
|
||||
if len(e.pending) > 0 && e.ctx.Err() == nil {
|
||||
continuationChannel = e.continuations
|
||||
nextContinuation = e.pending[0]
|
||||
} else if e.ctx.Err() != nil {
|
||||
e.pending = nil
|
||||
job, ready := e.nextJob()
|
||||
if !ready && e.active == 0 {
|
||||
return
|
||||
}
|
||||
var jobs chan<- laneJob
|
||||
if ready && e.active < e.workerCount {
|
||||
jobs = e.jobs
|
||||
}
|
||||
select {
|
||||
case continuationChannel <- nextContinuation:
|
||||
e.pending = e.pending[1:]
|
||||
e.launched++
|
||||
case result, ok := <-resultChannel:
|
||||
if !ok {
|
||||
resultChannel = nil
|
||||
continue
|
||||
case <-e.ctx.Done():
|
||||
case result := <-e.results:
|
||||
e.receiveResult(result)
|
||||
case jobs <- job:
|
||||
e.active++
|
||||
if job.chunk == nil {
|
||||
e.pending = e.pending[1:]
|
||||
} else {
|
||||
e.nextExtract++
|
||||
}
|
||||
e.handleExtractResult(result)
|
||||
case completion := <-e.completions:
|
||||
e.handleCompletion(completion)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *laneEngine) receiveResult(result laneJobResult) {
|
||||
e.active--
|
||||
if result.extract != nil {
|
||||
e.handleExtractResult(*result.extract)
|
||||
} else {
|
||||
e.handleCompletion(*result.completion)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *laneEngine) handleExtractResult(result extractJobResult) {
|
||||
state := e.states[result.laneIndex]
|
||||
state.remaining--
|
||||
@@ -317,7 +323,6 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
|
||||
}
|
||||
|
||||
func (e *laneEngine) handleCompletion(completion laneCompletion) {
|
||||
e.completed++
|
||||
e.completedOutputs[completion.index] = completion.output
|
||||
if completion.err != nil {
|
||||
e.runErrors = append(e.runErrors, classifyLaneError(completion.index, len(e.chunks), completion.err))
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
|
||||
}
|
||||
|
||||
debug := newCapturedDebugRecorder()
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2, Debug: debug})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2, Debug: debug})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ func TestRunnerPropagatesLaneFallbackWithoutCountingDependents(t *testing.T) {
|
||||
}
|
||||
profile := typedResolutionProfile()
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"affected": affected, "unrelated": unrelated}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), ExtractWorkers: 1})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -272,7 +272,7 @@ func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers(t *testing.T)
|
||||
{ID: "build", Artifacts: map[string]ArtifactLaneProfile{"producer": producer, "earlier": earlier}},
|
||||
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer, "later": unrelated}},
|
||||
}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), ExtractWorkers: 1})
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestNativeCombatPipelineHonorsSceneEligibilityWithoutSyntheticRequests(t *t
|
||||
check(err)
|
||||
preparations := 0
|
||||
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
|
||||
Prepared: prepared, SourceID: "combat-session", Workers: 1,
|
||||
RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Aria strikes the ogre."}]}`),
|
||||
Execution: contracts.ExecutionOptions{OnPrepared: func(context.Context, promptkit.PreparedRun) { preparations++ }},
|
||||
})
|
||||
|
||||
@@ -94,7 +94,7 @@ func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
check(err)
|
||||
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
|
||||
Prepared: prepared, SourceID: "combat-session", Workers: 1,
|
||||
RawInput: []byte(`{"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
|
||||
})
|
||||
if len(client.requests) != 1 {
|
||||
|
||||
@@ -137,7 +137,7 @@ func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
|
||||
model := "run-wide-spell-model"
|
||||
var preparations []promptkit.PreparedRun
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", ExtractWorkers: 1,
|
||||
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
|
||||
RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
|
||||
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
|
||||
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
const (
|
||||
concurrentChunkerKey = "test/concurrent-chunks"
|
||||
concurrentExtractorKey = "test/concurrent-extractor"
|
||||
concurrentValidatorKey = "test/concurrent-validator"
|
||||
)
|
||||
|
||||
type integrationConcurrencyTracker struct {
|
||||
extractActive atomic.Int32
|
||||
extractMaximum atomic.Int32
|
||||
providerActive atomic.Int32
|
||||
providerMaximum atomic.Int32
|
||||
providerCalls atomic.Int32
|
||||
validatorCalls atomic.Int32
|
||||
}
|
||||
|
||||
func updateMaximum(maximum *atomic.Int32, current int32) {
|
||||
for {
|
||||
seen := maximum.Load()
|
||||
if current <= seen || maximum.CompareAndSwap(seen, current) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type instrumentedProvider struct {
|
||||
tracker *integrationConcurrencyTracker
|
||||
}
|
||||
|
||||
func (p instrumentedProvider) CompleteStructured(ctx context.Context, _ contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
p.tracker.providerCalls.Add(1)
|
||||
current := p.tracker.providerActive.Add(1)
|
||||
defer p.tracker.providerActive.Add(-1)
|
||||
updateMaximum(&p.tracker.providerMaximum, current)
|
||||
select {
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
content := []byte(`{"approved":true}`)
|
||||
if out != nil {
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
type concurrentChunker struct{}
|
||||
|
||||
func (concurrentChunker) Key() string { return concurrentChunkerKey }
|
||||
func (concurrentChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (concurrentChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
ranges := make([]source.ChunkRange, len(request.Source.Units))
|
||||
for i, unit := range request.Source.Units {
|
||||
ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
|
||||
}
|
||||
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: ranges}}, nil
|
||||
}
|
||||
|
||||
type concurrentExtractor struct {
|
||||
client contracts.StructuredLLMClient
|
||||
tracker *integrationConcurrencyTracker
|
||||
failed atomic.Bool
|
||||
}
|
||||
|
||||
func (*concurrentExtractor) Key() string { return concurrentExtractorKey }
|
||||
func (*concurrentExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (e *concurrentExtractor) Extract(ctx context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
current := e.tracker.extractActive.Add(1)
|
||||
defer e.tracker.extractActive.Add(-1)
|
||||
updateMaximum(&e.tracker.extractMaximum, current)
|
||||
var response map[string]any
|
||||
if _, err := e.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: concurrentExtractorKey}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
if e.failed.CompareAndSwap(false, true) {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, errors.New("retry requested")
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{}}}, nil
|
||||
}
|
||||
|
||||
type concurrentValidator struct {
|
||||
client contracts.StructuredLLMClient
|
||||
tracker *integrationConcurrencyTracker
|
||||
}
|
||||
|
||||
func (*concurrentValidator) Name() string { return concurrentValidatorKey }
|
||||
func (*concurrentValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
func (v *concurrentValidator) Validate(ctx context.Context, _ contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
||||
v.tracker.validatorCalls.Add(1)
|
||||
var response map[string]any
|
||||
if _, err := v.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: concurrentValidatorKey}, &response); err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func TestRunnerIndependentlyBoundsWorkersAndProviderCallsAcrossRegisteredModules(t *testing.T) {
|
||||
tracker := &integrationConcurrencyTracker{}
|
||||
scheduler, err := frameworkllm.NewScheduler(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v", err)
|
||||
}
|
||||
client := frameworkllm.NewScheduledClient(instrumentedProvider{tracker: tracker}, scheduler)
|
||||
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
|
||||
if err := catalog.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: concurrentChunkerKey, Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source.transcript"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return concurrentChunker{}, nil }); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
validateOptions := func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }
|
||||
if err := pipeline.RegisterExtractorBuilder(catalog.Extractors, pipeline.ModuleSpec{Key: concurrentExtractorKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, ArtifactKind: dnd.SpellListKind, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.spell_casts"}}, validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SpellList], error) {
|
||||
return &concurrentExtractor{client: request.Dependencies.LLM, tracker: tracker}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
validators := pipeline.NewValidatorRegistry()
|
||||
catalog.Validators = validators
|
||||
if err := pipeline.RegisterTypedValidatorBuilder(validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: concurrentValidatorKey, ExecutionClass: contracts.ExecutionClassLLMBacked}, validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SpellList], error) {
|
||||
return &concurrentValidator{client: request.Dependencies.LLM, tracker: tracker}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator: %v", err)
|
||||
}
|
||||
|
||||
cfg := loadDNDSpellsPipelineConfig(t)
|
||||
profile := cfg.Pipelines["dnd-spells-fixture"]
|
||||
profile.Chunk = pipeline.Binding(concurrentChunkerKey)
|
||||
validatorOverride := pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(concurrentValidatorKey)}}
|
||||
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{
|
||||
"alpha": {Extract: pipeline.ModuleBinding{Module: concurrentExtractorKey, Retries: 1, Validators: validatorOverride}, Merge: pipeline.Binding(pipeline.DefaultMergeModule), Normalize: pipeline.Binding(pipeline.DefaultNormalizeModule)},
|
||||
"beta": {Extract: pipeline.ModuleBinding{Module: concurrentExtractorKey, Retries: 1, Validators: validatorOverride}, Merge: pipeline.Binding(pipeline.DefaultMergeModule), Normalize: pipeline.Binding(pipeline.DefaultNormalizeModule)},
|
||||
}
|
||||
cfg.Pipelines["dnd-spells-fixture"] = profile
|
||||
resolved, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-spells-fixture", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
registries := pipeline.Registries{Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, ArtifactCodecs: catalog.ArtifactCodecs, ArtifactEvidence: catalog.ArtifactEvidence, Extractors: catalog.Extractors, Mergers: catalog.Mergers, Normalizers: catalog.Normalizers, Validators: catalog.Validators, ValidatorChains: catalog.ValidatorChains, Outputs: catalog.Outputs}
|
||||
output, err := runPreparedPipeline(t, registries, resolved.ResolvedPipeline, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t), ExtractWorkers: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("normalize outputs = %d, want 2", len(output.NormalizeOutputs))
|
||||
}
|
||||
if got := tracker.extractMaximum.Load(); got != 3 {
|
||||
t.Fatalf("maximum extract jobs = %d, want 3", got)
|
||||
}
|
||||
if got := tracker.providerMaximum.Load(); got != 2 {
|
||||
t.Fatalf("maximum provider calls = %d, want 2", got)
|
||||
}
|
||||
if got := tracker.validatorCalls.Load(); got == 0 {
|
||||
t.Fatal("LLM-backed validator was not called")
|
||||
}
|
||||
if got := tracker.providerCalls.Load(); got <= tracker.validatorCalls.Load() {
|
||||
t.Fatalf("provider calls = %d, validator calls = %d; want extractor, retry, and validator calls", got, tracker.validatorCalls.Load())
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
ExtractWorkers: 1,
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
@@ -165,7 +165,7 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
ExtractWorkers: 1,
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
||||
@@ -212,7 +212,7 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
ExtractWorkers: 1,
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
ExtractWorkers: 1,
|
||||
Workers: 1,
|
||||
Checkpoint: checkpoint,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -473,7 +473,7 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: append([]byte(nil), raw...),
|
||||
ExtractWorkers: 1,
|
||||
Workers: 1,
|
||||
Checkpoint: checkpoint,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user