Unify ready pipeline jobs under one worker pool and remove the global LLM scheduler

This commit is contained in:
2026-11-22 01:45:31 +00:00
parent b47208bd57
commit 9a527b77c4
24 changed files with 496 additions and 1160 deletions

View File

@@ -170,9 +170,5 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
if err != nil {
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
}
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
if err != nil {
return nil, nil, fmt.Errorf("create LLM scheduler: %w", err)
}
return llm.NewScheduledClient(client, scheduler), nil, nil
return client, nil, nil
}

View File

@@ -659,16 +659,6 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
})
t.Run("invalid scheduler concurrency", func(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Concurrency.TotalLLM = 0
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
}
})
}
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {

View File

@@ -425,7 +425,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Checkpoint: checkpointLoader,
CheckpointPolicy: checkpointPolicy,
Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
Workers: cfg.Concurrency.Workers,
})
diagnosticProjection, diagnosticErr := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if diagnosticErr == nil {

View File

@@ -16,7 +16,7 @@ type ConcurrencyConfig struct {
Workers int `json:"workers"`
}
const defaultWorkers = 64
const defaultWorkers = pipeline.DefaultWorkers
type OutputConfig struct {
Directory string `json:"directory"`

View File

@@ -405,7 +405,7 @@ type effectiveExtractor struct{ key string }
func (m effectiveExtractor) Key() string { return m.key }
func (m effectiveExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (m effectiveExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[effectiveArtifact], error) {
func (m effectiveExtractor) Extract(context.Context, contracts.TypedExtractionRequest[effectiveArtifact]) (contracts.TypedExtractionResult[effectiveArtifact], error) {
return contracts.TypedExtractionResult[effectiveArtifact]{}, nil
}

View File

@@ -13,6 +13,7 @@ import (
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"sync"
"sync/atomic"
@@ -1594,105 +1595,6 @@ func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
}
}
func TestScheduledPromptKitClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakePromptKitLLM{
content: `{"ok":true}`,
block: make(chan struct{}),
}
client := newTestPromptKitClient(t, fake)
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
scheduled := NewScheduledClient(client, scheduler)
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if callErr != nil {
t.Errorf("CompleteStructured() error = %v, want nil", callErr)
}
}()
}
waitForAtomicAtLeast(t, &fake.calls, 1)
time.Sleep(20 * time.Millisecond)
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
t.Fatalf("max in-flight calls = %d, want <= 1", got)
}
close(fake.block)
wg.Wait()
}
func TestScheduledPromptKitClientHoldsPermitAcrossStructuredOutputRepair(t *testing.T) {
attempts := 1
fake := &fakePromptKitLLM{
responses: []promptkit.GenerateResponse{
{Content: `{"bad":true}`},
{Content: `{"ok":true}`},
{Content: `{"ok":true}`},
},
block: make(chan struct{}),
}
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v", err)
}
scheduled := NewScheduledClient(newTestPromptKitClient(t, fake), scheduler)
firstDone := make(chan error, 1)
go func() {
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "repair-session",
StructuredOutputRepairAttempts: &attempts,
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
firstDone <- callErr
}()
waitForAtomicAtLeast(t, &fake.calls, 1)
secondDone := make(chan error, 1)
go func() {
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "queued-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
secondDone <- callErr
}()
close(fake.block)
if err := <-firstDone; err != nil {
t.Fatalf("repaired completion error = %v", err)
}
if err := <-secondDone; err != nil {
t.Fatalf("queued completion error = %v", err)
}
requests := fake.requestsSnapshot()
if len(requests) != 3 || requests[1].Prompt.SessionID != "repair-session" || requests[2].Prompt.SessionID != "queued-session" {
t.Fatalf("generation order = %#v, want repair before queued completion", requests)
}
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
t.Fatalf("max in-flight calls = %d, want one scheduled logical completion", got)
}
}
func TestPromptKitClientValidatesRequest(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{content: `{"ok":true}`})
var out map[string]any
@@ -1785,6 +1687,18 @@ func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
return registry
}
func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if atomic.LoadInt32(value) >= want {
return
}
runtime.Gosched()
}
t.Fatalf("timed out waiting for value >= %d; got %d", want, atomic.LoadInt32(value))
}
type fakePromptKitLLM struct {
content string
responses []promptkit.GenerateResponse

View File

@@ -1,66 +0,0 @@
package llm
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type scheduledClient struct {
client contracts.StructuredLLMClient
scheduler *Scheduler
}
func NewScheduledClient(client contracts.StructuredLLMClient, scheduler *Scheduler) contracts.StructuredLLMClient {
return &scheduledClient{
client: client,
scheduler: scheduler,
}
}
func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client must not be nil")
}
if c.client == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client inner client must not be nil")
}
if c.scheduler == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scheduled LLM client scheduler must not be nil")
}
var response contracts.StructuredCompletionResponse
err := c.scheduler.Run(ctx, func(ctx context.Context) error {
var callErr error
response, callErr = c.client.CompleteStructured(ctx, req, out)
return callErr
})
if err != nil {
return response, err
}
return response, nil
}
func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.client == nil {
return nil
}
provider, ok := c.client.(contracts.LLMProfileManifestProvider)
if !ok {
return nil
}
return provider.LLMProfileManifests()
}
func (c *scheduledClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) {
if c == nil || c.client == nil {
return nil, nil
}
provider, ok := c.client.(CheckpointFingerprintProvider)
if !ok {
return nil, nil
}
return provider.LLMCheckpointFingerprints()
}

View File

@@ -1,151 +0,0 @@
package llm
import (
"context"
"encoding/json"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestScheduledClientEnforcesSchedulerLimit(t *testing.T) {
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
inner := &blockingStructuredClient{
release: make(chan struct{}),
}
client := NewScheduledClient(inner, scheduler)
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err != nil {
t.Errorf("CompleteStructured() error = %v, want nil", err)
}
}()
}
waitForAtomicAtLeast(t, &inner.calls, 1)
time.Sleep(20 * time.Millisecond)
if got := atomic.LoadInt32(&inner.maxInFlight); got > 1 {
t.Fatalf("max in-flight calls = %d, want <= 1", got)
}
close(inner.release)
wg.Wait()
if got := atomic.LoadInt32(&inner.calls); got != 3 {
t.Fatalf("calls = %d, want 3", got)
}
}
func TestScheduledClientPropagatesClientError(t *testing.T) {
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
expected := errors.New("provider unavailable")
client := NewScheduledClient(&errorStructuredClient{err: expected}, scheduler)
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &struct{}{})
if !errors.Is(err, expected) {
t.Fatalf("CompleteStructured() error = %v, want %v", err, expected)
}
}
func TestScheduledClientPropagatesSchedulerError(t *testing.T) {
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := NewScheduledClient(&errorStructuredClient{}, scheduler)
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{}, &struct{}{})
if !errors.Is(err, context.Canceled) {
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
}
}
func TestScheduledClientPreservesCheckpointFingerprints(t *testing.T) {
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatal(err)
}
inner := &fingerprintedStructuredClient{
fingerprints: []CheckpointFingerprint{
{Name: "profile_source", Value: "sha256:one"},
{Name: "backend_target", Value: "sha256:two"},
},
}
client := NewScheduledClient(inner, scheduler)
provider, ok := client.(CheckpointFingerprintProvider)
if !ok {
t.Fatalf("scheduled client %T does not preserve checkpoint fingerprints", client)
}
got, err := provider.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if len(got) != len(inner.fingerprints) ||
got[0] != inner.fingerprints[0] ||
got[1] != inner.fingerprints[1] {
t.Fatalf("checkpoint fingerprints = %#v, want %#v", got, inner.fingerprints)
}
}
type blockingStructuredClient struct {
release chan struct{}
inFlight int32
maxInFlight int32
calls int32
}
func (c *blockingStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
atomic.AddInt32(&c.calls, 1)
current := atomic.AddInt32(&c.inFlight, 1)
for {
seen := atomic.LoadInt32(&c.maxInFlight)
if current <= seen || atomic.CompareAndSwapInt32(&c.maxInFlight, seen, current) {
break
}
}
defer atomic.AddInt32(&c.inFlight, -1)
select {
case <-c.release:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
if target, ok := out.(*map[string]any); ok {
*target = map[string]any{"ok": true}
}
return contracts.StructuredCompletionResponse{
Content: json.RawMessage(`{"ok":true}`),
}, nil
}
type errorStructuredClient struct {
err error
}
type fingerprintedStructuredClient struct {
errorStructuredClient
fingerprints []CheckpointFingerprint
}
func (c *fingerprintedStructuredClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) {
return append([]CheckpointFingerprint(nil), c.fingerprints...), nil
}
func (c *errorStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return contracts.StructuredCompletionResponse{}, c.err
}

View File

@@ -1,125 +0,0 @@
package llm
import (
"context"
"fmt"
"sync"
)
// Scheduler bounds concurrent LLM backend calls.
type Scheduler struct {
maxConcurrency int
mu sync.Mutex
inFlight int
queue []*waiter
}
type waiter struct {
ready chan struct{}
queued bool
granted bool
}
// NewScheduler creates a scheduler with a fixed concurrency limit.
func NewScheduler(maxConcurrency int) (*Scheduler, error) {
if maxConcurrency <= 0 {
return nil, fmt.Errorf("max concurrency must be greater than zero")
}
return &Scheduler{
maxConcurrency: maxConcurrency,
}, nil
}
// Acquire blocks until a permit is available or the context is canceled.
// The returned release function is safe to call multiple times.
func (s *Scheduler) Acquire(ctx context.Context) (func(), error) {
if s == nil {
return nil, fmt.Errorf("scheduler must not be nil")
}
if err := ctx.Err(); err != nil {
return nil, err
}
s.mu.Lock()
if s.inFlight < s.maxConcurrency && len(s.queue) == 0 {
s.inFlight++
s.mu.Unlock()
return s.releaseFunc(), nil
}
w := &waiter{
ready: make(chan struct{}),
queued: true,
}
s.queue = append(s.queue, w)
s.mu.Unlock()
select {
case <-w.ready:
return s.releaseFunc(), nil
case <-ctx.Done():
s.mu.Lock()
if w.queued {
s.removeQueuedWaiterLocked(w)
s.mu.Unlock()
return nil, ctx.Err()
}
if w.granted {
s.inFlight--
s.grantQueuedLocked()
}
s.mu.Unlock()
return nil, ctx.Err()
}
}
// Run acquires a permit, executes fn, and releases the permit.
func (s *Scheduler) Run(ctx context.Context, fn func(context.Context) error) error {
if fn == nil {
return fmt.Errorf("scheduler function must not be nil")
}
release, err := s.Acquire(ctx)
if err != nil {
return err
}
defer release()
if err := ctx.Err(); err != nil {
return err
}
return fn(ctx)
}
func (s *Scheduler) releaseFunc() func() {
var once sync.Once
return func() {
once.Do(func() {
s.mu.Lock()
if s.inFlight > 0 {
s.inFlight--
s.grantQueuedLocked()
}
s.mu.Unlock()
})
}
}
func (s *Scheduler) grantQueuedLocked() {
for s.inFlight < s.maxConcurrency && len(s.queue) > 0 {
w := s.queue[0]
s.queue = s.queue[1:]
w.queued = false
w.granted = true
s.inFlight++
close(w.ready)
}
}
func (s *Scheduler) removeQueuedWaiterLocked(target *waiter) {
for i, w := range s.queue {
if w == target {
w.queued = false
s.queue = append(s.queue[:i], s.queue[i+1:]...)
return
}
}
}

View File

@@ -1,237 +0,0 @@
package llm
import (
"context"
"errors"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestNewSchedulerValidation(t *testing.T) {
if _, err := NewScheduler(0); err == nil {
t.Fatalf("expected validation error")
}
}
func TestSchedulerMaxConcurrency(t *testing.T) {
s, err := NewScheduler(2)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
var inFlight int32
var maxInFlight int32
release := make(chan struct{})
var wg sync.WaitGroup
for i := 0; i < 12; i++ {
wg.Add(1)
go func() {
defer wg.Done()
runErr := s.Run(context.Background(), func(context.Context) error {
current := atomic.AddInt32(&inFlight, 1)
for {
seen := atomic.LoadInt32(&maxInFlight)
if current <= seen || atomic.CompareAndSwapInt32(&maxInFlight, seen, current) {
break
}
}
<-release
atomic.AddInt32(&inFlight, -1)
return nil
})
if runErr != nil {
t.Errorf("Run error: %v", runErr)
}
}()
}
waitForAtomicAtLeast(t, &maxInFlight, 2)
close(release)
wg.Wait()
if got := atomic.LoadInt32(&maxInFlight); got > 2 {
t.Fatalf("expected max in-flight <= 2, got %d", got)
}
}
func TestSchedulerCancellationWhileQueued(t *testing.T) {
s, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
release, err := s.Acquire(context.Background())
if err != nil {
t.Fatalf("Acquire: %v", err)
}
defer release()
ctx, cancel := context.WithCancel(context.Background())
errCh := make(chan error, 1)
go func() {
_, acquireErr := s.Acquire(ctx)
errCh <- acquireErr
}()
waitForQueueDepth(t, s, 1)
cancel()
select {
case acquireErr := <-errCh:
if !errors.Is(acquireErr, context.Canceled) {
t.Fatalf("expected context canceled, got %v", acquireErr)
}
case <-time.After(time.Second):
t.Fatalf("timed out waiting for queued acquire to cancel")
}
release()
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
t.Fatalf("expected scheduler to accept work after cancellation, got %v", err)
}
}
func TestSchedulerReleaseFunctionIsIdempotent(t *testing.T) {
s, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
release, err := s.Acquire(context.Background())
if err != nil {
t.Fatalf("Acquire: %v", err)
}
release()
release()
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
t.Fatalf("expected permit to be released once, got %v", err)
}
}
func TestSchedulerRunReleasesPermitAfterError(t *testing.T) {
s, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
expected := errors.New("failed")
err = s.Run(context.Background(), func(context.Context) error {
return expected
})
if !errors.Is(err, expected) {
t.Fatalf("expected %v, got %v", expected, err)
}
if err := s.Run(context.Background(), func(context.Context) error { return nil }); err != nil {
t.Fatalf("expected permit to be released after error, got %v", err)
}
}
type cancelAfterAdmissionContext struct {
done chan struct{}
checks atomic.Int32
once sync.Once
}
func newCancelAfterAdmissionContext() *cancelAfterAdmissionContext {
return &cancelAfterAdmissionContext{done: make(chan struct{})}
}
func (*cancelAfterAdmissionContext) Deadline() (time.Time, bool) { return time.Time{}, false }
func (c *cancelAfterAdmissionContext) Done() <-chan struct{} { return c.done }
func (c *cancelAfterAdmissionContext) Err() error {
if c.checks.Add(1) == 1 {
return nil
}
c.once.Do(func() { close(c.done) })
return context.Canceled
}
func (*cancelAfterAdmissionContext) Value(any) any { return nil }
func TestSchedulerDoesNotDispatchCanceledAdmissionAndReleasesNextWaiter(t *testing.T) {
s, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler: %v", err)
}
hold, err := s.Acquire(context.Background())
if err != nil {
t.Fatalf("Acquire: %v", err)
}
defer hold()
ctx := newCancelAfterAdmissionContext()
var canceledCalls atomic.Int32
canceledErr := make(chan error, 1)
go func() {
canceledErr <- s.Run(ctx, func(context.Context) error {
canceledCalls.Add(1)
return nil
})
}()
waitForQueueDepth(t, s, 1)
nextStarted := make(chan struct{})
nextErr := make(chan error, 1)
go func() {
nextErr <- s.Run(context.Background(), func(context.Context) error {
close(nextStarted)
return nil
})
}()
waitForQueueDepth(t, s, 2)
hold()
if err := <-canceledErr; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context.Canceled", err)
}
if got := canceledCalls.Load(); got != 0 {
t.Fatalf("canceled callback calls = %d, want none", got)
}
select {
case <-nextStarted:
case <-time.After(time.Second):
t.Fatalf("timed out waiting for next FIFO waiter")
}
if err := <-nextErr; err != nil {
t.Fatalf("next Run() error = %v", err)
}
}
func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if atomic.LoadInt32(value) >= want {
return
}
runtime.Gosched()
}
t.Fatalf("timed out waiting for value >= %d; got %d", want, atomic.LoadInt32(value))
}
func waitForQueueDepth(t *testing.T, s *Scheduler, want int) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
s.mu.Lock()
depth := len(s.queue)
s.mu.Unlock()
if depth >= want {
return
}
runtime.Gosched()
}
s.mu.Lock()
depth := len(s.queue)
s.mu.Unlock()
t.Fatalf("timed out waiting for queue depth >= %d; got %d", want, depth)
}

View File

@@ -8,7 +8,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
type executionDebugClient struct {
@@ -20,7 +19,7 @@ func (c executionDebugClient) CompleteStructured(context.Context, contracts.Stru
return c.response, c.err
}
func TestScheduledDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
func TestDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
providerErr := contracts.NewLLMGenerationError(503, "provider unavailable")
const content = `{"ok":true}`
for _, test := range []struct {
@@ -61,16 +60,12 @@ func TestScheduledDebugClientPreservesExecutionThroughTerminalRecords(t *testing
} else {
response.Content = json.RawMessage(test.content)
}
scheduler, err := llm.NewScheduler(1)
if err != nil {
t.Fatal(err)
}
recorder := newCapturedDebugRecorder()
client := WithDebugLLMRecording(llm.NewScheduledClient(executionDebugClient{response, test.err}, scheduler), recorder)
client := WithDebugLLMRecording(executionDebugClient{response, test.err}, recorder)
ctx, scope := withDebugLLMScope(context.Background(), "attempt")
got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{})
if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want {
t.Fatalf("scheduled debug completion = (%+v, %v), want summary and original error", got, err)
t.Fatalf("debug completion = (%+v, %v), want summary and original error", got, err)
}
terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"})
if err := terminal.record(nil, test.err); !errors.Is(err, test.err) {

View File

@@ -0,0 +1,315 @@
package pipeline
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type activeJobs struct {
active atomic.Int32
peak atomic.Int32
}
func (jobs *activeJobs) enter() func() {
current := jobs.active.Add(1)
for {
peak := jobs.peak.Load()
if current <= peak || jobs.peak.CompareAndSwap(peak, current) {
break
}
}
return func() { jobs.active.Add(-1) }
}
func TestRunnerSharesWorkerLimitBetweenExtractionAndReadyContinuation(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
var jobs activeJobs
var firstLaneExtracts atomic.Int32
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
mergeStarted := make(chan struct{}, 1)
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
if lane == 0 {
firstLaneExtracts.Add(1)
} else {
select {
case <-release:
case <-ctx.Done():
return erasedTypedResult{}, ctx.Err()
}
}
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
})
current := &prepared.Steps[0].lanes[lane]
merge := current.typed.merge
current.typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
if lane == 0 {
if firstLaneExtracts.Load() != 2 {
return erasedTypedResult{}, errors.New("continuation ran before its extraction prerequisites")
}
mergeStarted <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return erasedTypedResult{}, ctx.Err()
}
}
return merge(ctx, implementation, request)
}
normalize := current.typed.normalize
current.typed.normalize = func(ctx context.Context, implementation any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
return normalize(ctx, implementation, request)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- err
}()
select {
case <-mergeStarted:
synctest.Wait()
case <-ctx.Done():
t.Fatal("ready continuation could not run while another lane still had extraction work")
}
unblock()
if err := <-done; err != nil {
t.Fatal(err)
}
if got := jobs.peak.Load(); got != 2 || jobs.active.Load() != 0 {
t.Fatalf("mixed active jobs peaked at %d, remaining %d; want the single limit of 2", got, jobs.active.Load())
}
})
}
func limitedProducerEngine(t *testing.T, limit, queue int, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
return nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("test-controlled backend", []byte(fmt.Sprintf(`backends:
- id: limited
endpoint: https://example.invalid/v1
concurrency_limit: %d
queue_capacity: %d
profiles:
- id: base
backend: limited
model: shared-model
api_key_required: false
`, limit, queue)))))
}
func installNativeExtraction(prepared *PreparedPipeline, engine *promptkit.Engine, jobs *activeJobs, started chan<- struct{}) {
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
prepared.Steps[0].lanes[lane].resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
if jobs != nil {
defer jobs.enter()()
}
if started != nil {
started <- struct{}{}
}
value, outcome, err := RunProducer(ctx, engine, request.Operation, nativeProducerInitial(),
func(ctx context.Context, response *promptkit.RunResult) (*any, []string, error) {
_, _, err := interpretProducerNumber(ctx, response)
value := typedValueForLane(lane, request.Chunk.Index)
return &value, nil, err
}, producerCorrection, func(context.Context, *any) (any, error) {
if lane == 0 {
return codecNotes{Items: []string{}}, nil
}
return codecScore{}, nil
})
return erasedTypedResult{Value: value, Outcome: outcome}, err
})
}
}
func TestRunnerWorkersAndNativeBackendAdmissionAreIndependent(t *testing.T) {
for _, limits := range []struct{ workers, backend int }{{1, 1}, {3, 2}} {
t.Run(fmt.Sprintf("workers-%d/backend-%d", limits.workers, limits.backend), func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
var providers, jobs activeJobs
var calls atomic.Int32
providerStarted := make(chan struct{}, 8)
jobStarted := make(chan struct{}, 4)
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
engine := limitedProducerEngine(t, limits.backend, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls.Add(1)
defer providers.enter()()
providerStarted <- struct{}{}
select {
case <-release:
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}))
installNativeExtraction(prepared, engine, &jobs, jobStarted)
for lane := range prepared.Steps[0].lanes {
prepared.Steps[0].lanes[lane].extractValidators.validators[0].typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
result, err := engine.Run(ctx, nativeProducerInitial())
if err != nil {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
}
return contracts.ValidationResult{Approved: result.Validation.IsValid}, nil
}
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: limits.workers})
if err == nil && len(output.NormalizeOutputs) != 2 {
err = fmt.Errorf("got %d lane outputs", len(output.NormalizeOutputs))
}
done <- err
}()
for range limits.workers {
select {
case <-jobStarted:
case <-ctx.Done():
t.Fatal("worker jobs did not overlap as configured")
}
}
for range limits.backend {
select {
case <-providerStarted:
case <-ctx.Done():
t.Fatal("native backend did not admit its configured calls")
}
}
unblock()
if err := <-done; err != nil {
t.Fatal(err)
}
if jobs.peak.Load() != int32(limits.workers) || providers.peak.Load() != int32(limits.backend) || calls.Load() != 8 || jobs.active.Load() != 0 || providers.active.Load() != 0 {
t.Fatalf("worker peak %d, backend peak %d, calls %d; active jobs/providers %d/%d", jobs.peak.Load(), providers.peak.Load(), calls.Load(), jobs.active.Load(), providers.active.Load())
}
})
}
}
func TestRunnerNativeCapacityFailureUsesOperationModeWithoutAnotherQueue(t *testing.T) {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
t.Run(string(mode), func(t *testing.T) {
var calls atomic.Int32
started, release := make(chan struct{}, 2), make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
engine := limitedProducerEngine(t, 1, 0, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls.Add(1)
started <- struct{}{}
select {
case <-release:
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}))
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
occupant := make(chan error, 1)
go func() { _, err := engine.Run(ctx, nativeProducerInitial()); occupant <- err }()
select {
case <-started:
case <-ctx.Done():
t.Fatal("backend occupant did not start")
}
prepared := preparedConcurrentPipeline(t, 1)
installNativeExtraction(prepared, engine, nil, nil)
for lane := range prepared.Steps[0].lanes {
prepared.Steps[0].lanes[lane].resolved.ExtractControls = contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}
}
zero := 0
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Execution: contracts.ExecutionOptions{TransientRetries: &zero}})
if mode == contracts.ModeStrict {
var capacity *promptkit.CapacityError
if !errors.As(err, &capacity) || output.FallbackCount != 0 || len(output.OutputFiles) != 0 {
t.Fatalf("strict capacity failure = %v, fallbacks %d", err, output.FallbackCount)
}
} else if err != nil || output.FallbackCount != 2 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("permissive capacity failure = %v, fallbacks %d, outputs %d", err, output.FallbackCount, len(output.NormalizeOutputs))
}
if calls.Load() != 1 {
t.Fatalf("capacity handling invoked provider %d times", calls.Load())
}
unblock()
if err := <-occupant; err != nil {
t.Fatal(err)
}
if _, err := engine.Run(ctx, nativeProducerInitial()); err != nil || calls.Load() != 2 {
t.Fatalf("backend admission was not released: %v, calls %d", err, calls.Load())
}
})
}
}
func TestRunnerCancellationDrainsNativeWorkAndReleasesAdmission(t *testing.T) {
var calls atomic.Int32
var jobs activeJobs
jobStarted := make(chan struct{}, 4)
providerStarted := make(chan struct{})
engine := limitedProducerEngine(t, 1, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if calls.Add(1) == 1 {
close(providerStarted)
<-ctx.Done()
return nil, ctx.Err()
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
prepared := preparedConcurrentPipeline(t, 2)
installNativeExtraction(prepared, engine, &jobs, jobStarted)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 3})
if output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 {
err = fmt.Errorf("cancellation delivered output: %v", err)
}
done <- err
}()
for range 3 {
select {
case <-jobStarted:
case <-ctx.Done():
t.Fatal("native jobs did not start")
}
}
select {
case <-providerStarted:
case <-ctx.Done():
t.Fatal("native backend did not start")
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) || jobs.active.Load() != 0 || len(jobStarted) != 0 {
t.Fatalf("cancel = %v, active jobs %d, extra starts %d", err, jobs.active.Load(), len(jobStarted))
}
if _, err := engine.Run(context.Background(), nativeProducerInitial()); err != nil || calls.Load() != 2 {
t.Fatalf("admission after cancellation = %v, provider calls %d", err, calls.Load())
}
}

View File

@@ -33,6 +33,9 @@ type Registries struct {
type Runner struct{}
// DefaultWorkers is the shared ready-job limit when the caller omits one.
const DefaultWorkers = 64
func New() *Runner {
return &Runner{}
}
@@ -55,9 +58,9 @@ type RunInput struct {
Checkpoint CheckpointLoader
CheckpointPolicy CheckpointExecutionPolicy
Debug DebugRecorder
// ExtractWorkers bounds run-wide extract jobs. Values less than one use a
// single worker so direct framework callers retain deterministic behavior.
ExtractWorkers int
// Workers bounds ready extraction and lane-completion jobs together.
// Zero selects DefaultWorkers; configuration rejects explicit zero.
Workers int
pipeline ResolvedPipeline
// reuseIneligible carries chunk ancestry; reference eligibility stays scoped.
@@ -468,6 +471,9 @@ func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string,
}
func validateRunInput(input RunInput) error {
if input.Workers < 0 {
return fmt.Errorf("workers must be positive when specified")
}
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}

View File

@@ -13,33 +13,8 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
type countingProvider struct {
active atomic.Int32
maximum atomic.Int32
calls atomic.Int32
}
func (p *countingProvider) CompleteStructured(ctx context.Context, _ contracts.StructuredCompletionRequest, _ any) (contracts.StructuredCompletionResponse, error) {
p.calls.Add(1)
current := p.active.Add(1)
defer p.active.Add(-1)
for {
seen := p.maximum.Load()
if current <= seen || p.maximum.CompareAndSwap(seen, current) {
break
}
}
select {
case <-time.After(5 * time.Millisecond):
return contracts.StructuredCompletionResponse{}, nil
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
func preparedConcurrentPipeline(t *testing.T, chunkCount int) *PreparedPipeline {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
@@ -226,7 +201,7 @@ func TestRunnerFailsGeneratedHandoffBeforeConsumerExtraction(t *testing.T) {
}}
prepared.resolved.Steps[0].ArtifactLanes[0].ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
var extractCalls atomic.Int32
preparedLane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
preparedLane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
extractCalls.Add(1)
return erasedTypedResult{Value: codecNotes{}}, nil
}
@@ -257,7 +232,7 @@ func TestRunnerExecutesOrderedStepsWithHardBarriers(t *testing.T) {
}
first := &prepared.Steps[0].lanes[0]
second := &prepared.Steps[1].lanes[0]
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
}
originalFirstNormalize := first.typed.normalize
@@ -268,12 +243,12 @@ func TestRunnerExecutesOrderedStepsWithHardBarriers(t *testing.T) {
}
return result, err
}
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
record("second-extract-started")
return erasedTypedResult{Value: codecScore{Value: 2}}, nil
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
@@ -297,21 +272,21 @@ func TestRunnerStopsLaterOrderedStepsAfterFailure(t *testing.T) {
first := &prepared.Steps[0].lanes[0]
second := &prepared.Steps[1].lanes[0]
third := &prepared.Steps[2].lanes[0]
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
first.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil
}
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
second.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{}, errors.New("ordered step extraction failed")
}
var thirdCalls atomic.Int32
third.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
third.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
thirdCalls.Add(1)
return erasedTypedResult{Value: codecNotes{Items: []string{"later"}}}, nil
}
encoder := &countingOrderedOutput{}
prepared.output = encoder
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
if err == nil || !strings.Contains(err.Error(), `execute pipeline step "step-2"`) {
t.Fatalf("Run() error = %v, want step-scoped failure", err)
}
@@ -326,8 +301,8 @@ func TestRunnerStopsLaterOrderedStepsAfterFailure(t *testing.T) {
}
}
func installExtractOperation(prepared *PreparedPipeline, laneIndex int, operation func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error)) {
prepared.Steps[0].lanes[laneIndex].typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
func installExtractOperation(prepared *PreparedPipeline, laneIndex int, operation func(context.Context, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error)) {
prepared.Steps[0].lanes[laneIndex].typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return operation(ctx, request)
}
}
@@ -350,7 +325,7 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
var maximum atomic.Int32
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
jobIndex := request.Chunk.Index*2 + lane
current := active.Add(1)
defer active.Add(-1)
@@ -377,7 +352,7 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
}
done := make(chan outcome, 1)
go func() {
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- outcome{output: output, err: err}
}()
for pair := 0; pair < 3; pair++ {
@@ -416,45 +391,6 @@ func TestRunnerBoundsExtractJobsAndStabilizesReverseCompletion(t *testing.T) {
}
}
func TestRunnerStartsLaneContinuationWhileOtherLaneExtractsRemain(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
releaseOtherLane := make(chan struct{})
mergeStarted := make(chan struct{}, 1)
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
})
installExtractOperation(prepared, 1, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
select {
case <-releaseOtherLane:
return erasedTypedResult{Value: typedValueForLane(1, request.Chunk.Index)}, nil
case <-ctx.Done():
return erasedTypedResult{}, ctx.Err()
}
})
originalMerge := prepared.Steps[0].lanes[0].typed.merge
prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
select {
case mergeStarted <- struct{}{}:
default:
}
return originalMerge(ctx, implementation, request)
}
done := make(chan error, 1)
go func() {
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
done <- err
}()
select {
case <-mergeStarted:
close(releaseOtherLane)
case <-time.After(2 * time.Second):
t.Fatal("first lane merge did not start while the second lane remained in extract")
}
if err := <-done; err != nil {
t.Fatalf("Run() error = %v", err)
}
}
func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
base := prepared.Steps[0].lanes[0]
@@ -470,7 +406,7 @@ func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
extractor: base.typed.extractor,
merger: base.typed.merger,
normalizer: base.typed.normalizer,
extract: func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extract: func(_ context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{request.Chunk.ID}}}, nil
},
merge: func(_ context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
@@ -496,7 +432,7 @@ func TestRunnerBoundsConcurrentLaneContinuations(t *testing.T) {
prepared.resolved.Steps[0].ArtifactLanes = resolvedLanes
prepared.Steps[0].ArtifactLanes = publicLanes
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
@@ -514,7 +450,7 @@ func TestRunnerSelectsFrameworkErrorByStableLaneOrder(t *testing.T) {
release := make(chan struct{})
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, lane, func(context.Context, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
ready <- struct{}{}
<-release
return erasedTypedResult{}, fmt.Errorf("lane-%d failure", lane)
@@ -522,7 +458,7 @@ func TestRunnerSelectsFrameworkErrorByStableLaneOrder(t *testing.T) {
}
done := make(chan error, 1)
go func() {
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- err
}()
<-ready
@@ -537,7 +473,7 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
})
}
@@ -555,7 +491,7 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
}
done := make(chan error, 1)
go func() {
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- err
}()
<-ready
@@ -566,103 +502,12 @@ func TestRunnerSelectsMergeErrorBeforeEarlierLaneNormalizeError(t *testing.T) {
}
}
func TestRunnerKeepsSuccessfulChunksWhenAnotherExtractIsRejected(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
})
}
validator := &prepared.Steps[0].lanes[0].extractValidators.validators[0]
validator.typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) {
if target.chunk != nil && target.chunk.Index == 0 {
return contracts.ValidationResult{Approved: false, ReasonCode: "expected_rejection", Message: "rejected by test", CorrectionGuidance: "return an acceptable candidate"}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 1 || output.Rejected[0].LaneID != prepared.Steps[0].lanes[0].resolved.ID || output.Rejected[0].ChunkIndex != 0 {
t.Fatalf("rejections = %#v, want the first lane's first chunk", output.Rejected)
}
if len(output.NormalizeOutputs) != 2 {
t.Fatalf("normalize outputs = %d, want both lanes to continue", len(output.NormalizeOutputs))
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("validation status = %q, want rejected", output.Manifest.ValidationStatus)
}
}
func TestRunnerSeparatelyBoundsExtractJobsAndSharedProviderCalls(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 3)
provider := &countingProvider{}
scheduler, err := frameworkllm.NewScheduler(2)
if err != nil {
t.Fatalf("NewScheduler() error = %v", err)
}
client := frameworkllm.NewScheduledClient(provider, scheduler)
var jobActive atomic.Int32
var jobMaximum atomic.Int32
var attempts sync.Map
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
prepared.Steps[0].lanes[lane].resolved.Extract.Retries = 1
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
current := jobActive.Add(1)
defer jobActive.Add(-1)
for {
seen := jobMaximum.Load()
if current <= seen || jobMaximum.CompareAndSwap(seen, current) {
break
}
}
if _, callErr := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, nil); callErr != nil {
return erasedTypedResult{}, callErr
}
key := fmt.Sprintf("%d/%d", lane, request.Chunk.Index)
countValue, _ := attempts.LoadOrStore(key, &atomic.Int32{})
count := countValue.(*atomic.Int32).Add(1)
if lane == 0 && request.Chunk.Index == 0 && count == 1 {
return erasedTypedResult{}, errors.New("retry extract")
}
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
})
validator := &prepared.Steps[0].lanes[lane].extractValidators.validators[0]
validator.typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
if _, callErr := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "validate"}, nil); callErr != nil {
return contracts.ValidationResult{}, callErr
}
return contracts.ValidationResult{Approved: true}, nil
}
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 3})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 2 {
t.Fatalf("normalize outputs = %d, want 2", len(output.NormalizeOutputs))
}
if got := jobMaximum.Load(); got != 3 {
t.Fatalf("maximum extract jobs = %d, want 3", got)
}
if got := provider.maximum.Load(); got != 2 {
t.Fatalf("maximum provider calls = %d, want 2", got)
}
if got := provider.calls.Load(); got != 13 {
t.Fatalf("provider calls = %d, want 13 including retry and validators", got)
}
}
func TestRunnerReturnsParentCancellationAndStopsQueuedExtracts(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 4)
started := make(chan struct{}, 8)
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
started <- struct{}{}
<-ctx.Done()
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, ctx.Err()
@@ -671,7 +516,7 @@ func TestRunnerReturnsParentCancellationAndStopsQueuedExtracts(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2})
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- err
}()
<-started

View File

@@ -151,22 +151,33 @@ type laneEngineConfig struct {
states []*laneExtractState
}
// laneJob is ready to execute: a nil chunk selects the lane's merge/normalize
// continuation, otherwise it selects one extraction. Prerequisites are resolved
// by the coordinator before dispatch, never by an occupied worker.
type laneJob struct {
lane *laneExtractState
chunk *source.Chunk
}
type laneJobResult struct {
extract *extractJobResult
completion *laneCompletion
}
type laneEngine struct {
runner *Runner
laneEngineConfig
ctx context.Context
cancel context.CancelFunc
workerCount int
jobs chan extractJob
results chan extractJobResult
completions chan laneCompletion
continuations chan *laneExtractState
continuationWorkers sync.WaitGroup
pending []*laneExtractState
completedOutputs []RunOutput
runErrors []orderedRunError
launched int
completed int
ctx context.Context
cancel context.CancelFunc
workerCount int
jobs chan laneJob
results chan laneJobResult
workers sync.WaitGroup
pending []*laneExtractState
completedOutputs []RunOutput
runErrors []orderedRunError
nextExtract int
active int
}
func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig) ([]RunOutput, []orderedRunError) {
@@ -175,31 +186,20 @@ func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig)
}
func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *laneEngine {
workerCount := config.input.ExtractWorkers
if workerCount < 1 {
workerCount = 1
workerCount := config.input.Workers
if workerCount == 0 {
workerCount = DefaultWorkers
}
ctx, cancel := context.WithCancel(parent)
return &laneEngine{
runner: r, laneEngineConfig: config, ctx: ctx, cancel: cancel, workerCount: workerCount,
jobs: make(chan extractJob, workerCount), results: make(chan extractJobResult, workerCount),
completions: make(chan laneCompletion, len(config.states)), continuations: make(chan *laneExtractState, workerCount),
jobs: make(chan laneJob), results: make(chan laneJobResult, workerCount),
completedOutputs: make([]RunOutput, len(config.states)),
}
}
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
defer e.cancel()
e.initializeCollection()
e.startExtractWorkers()
e.startContinuationWorkers()
e.collect()
close(e.continuations)
e.continuationWorkers.Wait()
return e.completedOutputs, e.runErrors
}
func (e *laneEngine) initializeCollection() {
for _, state := range e.states {
if state.terminal {
e.completedOutputs[state.index] = state.output
@@ -207,90 +207,96 @@ func (e *laneEngine) initializeCollection() {
e.pending = append(e.pending, state)
}
}
e.startWorkers()
e.collect()
close(e.jobs)
e.workers.Wait()
return e.completedOutputs, e.runErrors
}
func (e *laneEngine) startExtractWorkers() {
var workers sync.WaitGroup
func (e *laneEngine) startWorkers() {
for i := 0; i < e.workerCount; i++ {
workers.Add(1)
e.workers.Add(1)
go func() {
defer workers.Done()
defer e.workers.Done()
for job := range e.jobs {
if e.ctx.Err() != nil {
if job.chunk != nil {
result := e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, extractJob{lane: job.lane, chunk: *job.chunk})
e.results <- laneJobResult{extract: &result}
continue
}
e.results <- e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, job)
completion := laneCompletion{index: job.lane.index}
if err := e.ctx.Err(); err != nil {
completion.err = err
} else {
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
}
e.results <- laneJobResult{completion: &completion}
}
}()
}
go e.dispatchExtractJobs()
go func() { workers.Wait(); close(e.results) }()
}
func (e *laneEngine) dispatchExtractJobs() {
defer close(e.jobs)
for chunkIndex := range e.chunks {
for laneIndex := range e.states {
state := e.states[laneIndex]
if state.terminal || state.decision.Reused {
continue
}
select {
case e.jobs <- extractJob{lane: state, chunk: e.chunks[chunkIndex]}:
case <-e.ctx.Done():
// nextJob gives ready continuations priority, then preserves chunk-first,
// lane-second extraction order. Dispatch remains lazy rather than creating a
// queue or goroutine for every extraction.
func (e *laneEngine) nextJob() (laneJob, bool) {
if len(e.pending) > 0 {
return laneJob{lane: e.pending[0]}, true
}
for e.nextExtract < len(e.chunks)*len(e.states) {
state := e.states[e.nextExtract%len(e.states)]
if !state.terminal && !state.decision.Reused {
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
}
e.nextExtract++
}
return laneJob{}, false
}
// collect alone owns dispatch and result accounting. Cancellation disables new
// dispatch, but every dispatched job still returns one result before workers exit.
func (e *laneEngine) collect() {
for {
if e.ctx.Err() != nil {
if e.active == 0 {
return
}
e.receiveResult(<-e.results)
continue
}
}
}
func (e *laneEngine) startContinuationWorkers() {
for i := 0; i < e.workerCount; i++ {
e.continuationWorkers.Add(1)
go func() {
defer e.continuationWorkers.Done()
for state := range e.continuations {
if err := e.ctx.Err(); err != nil {
e.completions <- laneCompletion{index: state.index, err: err}
continue
}
laneOutput, err := e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, state)
e.completions <- laneCompletion{index: state.index, output: laneOutput, err: err}
}
}()
}
}
// collect is the sole owner of pending, launch, and completion accounting. It
// must drain closed extract results and one completion for every launched
// continuation, even after cancellation.
func (e *laneEngine) collect() {
resultChannel := (<-chan extractJobResult)(e.results)
for resultChannel != nil || len(e.pending) > 0 || e.completed < e.launched {
var continuationChannel chan<- *laneExtractState
var nextContinuation *laneExtractState
if len(e.pending) > 0 && e.ctx.Err() == nil {
continuationChannel = e.continuations
nextContinuation = e.pending[0]
} else if e.ctx.Err() != nil {
e.pending = nil
job, ready := e.nextJob()
if !ready && e.active == 0 {
return
}
var jobs chan<- laneJob
if ready && e.active < e.workerCount {
jobs = e.jobs
}
select {
case continuationChannel <- nextContinuation:
e.pending = e.pending[1:]
e.launched++
case result, ok := <-resultChannel:
if !ok {
resultChannel = nil
continue
case <-e.ctx.Done():
case result := <-e.results:
e.receiveResult(result)
case jobs <- job:
e.active++
if job.chunk == nil {
e.pending = e.pending[1:]
} else {
e.nextExtract++
}
e.handleExtractResult(result)
case completion := <-e.completions:
e.handleCompletion(completion)
}
}
}
func (e *laneEngine) receiveResult(result laneJobResult) {
e.active--
if result.extract != nil {
e.handleExtractResult(*result.extract)
} else {
e.handleCompletion(*result.completion)
}
}
func (e *laneEngine) handleExtractResult(result extractJobResult) {
state := e.states[result.laneIndex]
state.remaining--
@@ -317,7 +323,6 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
}
func (e *laneEngine) handleCompletion(completion laneCompletion) {
e.completed++
e.completedOutputs[completion.index] = completion.output
if completion.err != nil {
e.runErrors = append(e.runErrors, classifyLaneError(completion.index, len(e.chunks), completion.err))

View File

@@ -50,7 +50,7 @@ func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) {
}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ExtractWorkers: 2, Debug: debug})
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}

View File

@@ -173,7 +173,7 @@ func TestRunnerPropagatesLaneFallbackWithoutCountingDependents(t *testing.T) {
}
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"affected": affected, "unrelated": unrelated}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), ExtractWorkers: 1})
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
if err != nil {
t.Fatal(err)
}
@@ -272,7 +272,7 @@ func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers(t *testing.T)
{ID: "build", Artifacts: map[string]ArtifactLaneProfile{"producer": producer, "earlier": earlier}},
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer, "later": unrelated}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), ExtractWorkers: 1})
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
if err != nil {
t.Fatal(err)
}

View File

@@ -109,7 +109,7 @@ func TestNativeCombatPipelineHonorsSceneEligibilityWithoutSyntheticRequests(t *t
check(err)
preparations := 0
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
Prepared: prepared, SourceID: "combat-session", Workers: 1,
RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Aria strikes the ogre."}]}`),
Execution: contracts.ExecutionOptions{OnPrepared: func(context.Context, promptkit.PreparedRun) { preparations++ }},
})

View File

@@ -94,7 +94,7 @@ func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
Prepared: prepared, SourceID: "combat-session", Workers: 1,
RawInput: []byte(`{"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
})
if len(client.requests) != 1 {

View File

@@ -137,7 +137,7 @@ func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
model := "run-wide-spell-model"
var preparations []promptkit.PreparedRun
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", ExtractWorkers: 1,
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},

View File

@@ -1,176 +0,0 @@
package integration_test
import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
const (
concurrentChunkerKey = "test/concurrent-chunks"
concurrentExtractorKey = "test/concurrent-extractor"
concurrentValidatorKey = "test/concurrent-validator"
)
type integrationConcurrencyTracker struct {
extractActive atomic.Int32
extractMaximum atomic.Int32
providerActive atomic.Int32
providerMaximum atomic.Int32
providerCalls atomic.Int32
validatorCalls atomic.Int32
}
func updateMaximum(maximum *atomic.Int32, current int32) {
for {
seen := maximum.Load()
if current <= seen || maximum.CompareAndSwap(seen, current) {
return
}
}
}
type instrumentedProvider struct {
tracker *integrationConcurrencyTracker
}
func (p instrumentedProvider) CompleteStructured(ctx context.Context, _ contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
p.tracker.providerCalls.Add(1)
current := p.tracker.providerActive.Add(1)
defer p.tracker.providerActive.Add(-1)
updateMaximum(&p.tracker.providerMaximum, current)
select {
case <-time.After(5 * time.Millisecond):
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
content := []byte(`{"approved":true}`)
if out != nil {
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
type concurrentChunker struct{}
func (concurrentChunker) Key() string { return concurrentChunkerKey }
func (concurrentChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (concurrentChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
ranges := make([]source.ChunkRange, len(request.Source.Units))
for i, unit := range request.Source.Units {
ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
}
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: ranges}}, nil
}
type concurrentExtractor struct {
client contracts.StructuredLLMClient
tracker *integrationConcurrencyTracker
failed atomic.Bool
}
func (*concurrentExtractor) Key() string { return concurrentExtractorKey }
func (*concurrentExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *concurrentExtractor) Extract(ctx context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
current := e.tracker.extractActive.Add(1)
defer e.tracker.extractActive.Add(-1)
updateMaximum(&e.tracker.extractMaximum, current)
var response map[string]any
if _, err := e.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: concurrentExtractorKey}, &response); err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
if e.failed.CompareAndSwap(false, true) {
return contracts.TypedExtractionResult[dnd.SpellList]{}, errors.New("retry requested")
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{}}}, nil
}
type concurrentValidator struct {
client contracts.StructuredLLMClient
tracker *integrationConcurrencyTracker
}
func (*concurrentValidator) Name() string { return concurrentValidatorKey }
func (*concurrentValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (v *concurrentValidator) Validate(ctx context.Context, _ contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
v.tracker.validatorCalls.Add(1)
var response map[string]any
if _, err := v.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: concurrentValidatorKey}, &response); err != nil {
return contracts.ValidationResult{}, err
}
return contracts.ValidationResult{Approved: true}, nil
}
func TestRunnerIndependentlyBoundsWorkersAndProviderCallsAcrossRegisteredModules(t *testing.T) {
tracker := &integrationConcurrencyTracker{}
scheduler, err := frameworkllm.NewScheduler(2)
if err != nil {
t.Fatalf("NewScheduler() error = %v", err)
}
client := frameworkllm.NewScheduledClient(instrumentedProvider{tracker: tracker}, scheduler)
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
if err := catalog.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: concurrentChunkerKey, Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source.transcript"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return concurrentChunker{}, nil }); err != nil {
t.Fatalf("register chunker: %v", err)
}
validateOptions := func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }
if err := pipeline.RegisterExtractorBuilder(catalog.Extractors, pipeline.ModuleSpec{Key: concurrentExtractorKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, ArtifactKind: dnd.SpellListKind, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.spell_casts"}}, validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SpellList], error) {
return &concurrentExtractor{client: request.Dependencies.LLM, tracker: tracker}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
validators := pipeline.NewValidatorRegistry()
catalog.Validators = validators
if err := pipeline.RegisterTypedValidatorBuilder(validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: concurrentValidatorKey, ExecutionClass: contracts.ExecutionClassLLMBacked}, validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SpellList], error) {
return &concurrentValidator{client: request.Dependencies.LLM, tracker: tracker}, nil
}); err != nil {
t.Fatalf("register validator: %v", err)
}
cfg := loadDNDSpellsPipelineConfig(t)
profile := cfg.Pipelines["dnd-spells-fixture"]
profile.Chunk = pipeline.Binding(concurrentChunkerKey)
validatorOverride := pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(concurrentValidatorKey)}}
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{
"alpha": {Extract: pipeline.ModuleBinding{Module: concurrentExtractorKey, Retries: 1, Validators: validatorOverride}, Merge: pipeline.Binding(pipeline.DefaultMergeModule), Normalize: pipeline.Binding(pipeline.DefaultNormalizeModule)},
"beta": {Extract: pipeline.ModuleBinding{Module: concurrentExtractorKey, Retries: 1, Validators: validatorOverride}, Merge: pipeline.Binding(pipeline.DefaultMergeModule), Normalize: pipeline.Binding(pipeline.DefaultNormalizeModule)},
}
cfg.Pipelines["dnd-spells-fixture"] = profile
resolved, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-spells-fixture", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
registries := pipeline.Registries{Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, ArtifactCodecs: catalog.ArtifactCodecs, ArtifactEvidence: catalog.ArtifactEvidence, Extractors: catalog.Extractors, Mergers: catalog.Mergers, Normalizers: catalog.Normalizers, Validators: catalog.Validators, ValidatorChains: catalog.ValidatorChains, Outputs: catalog.Outputs}
output, err := runPreparedPipeline(t, registries, resolved.ResolvedPipeline, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t), ExtractWorkers: 3})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.NormalizeOutputs) != 2 {
t.Fatalf("normalize outputs = %d, want 2", len(output.NormalizeOutputs))
}
if got := tracker.extractMaximum.Load(); got != 3 {
t.Fatalf("maximum extract jobs = %d, want 3", got)
}
if got := tracker.providerMaximum.Load(); got != 2 {
t.Fatalf("maximum provider calls = %d, want 2", got)
}
if got := tracker.validatorCalls.Load(); got == 0 {
t.Fatal("LLM-backed validator was not called")
}
if got := tracker.providerCalls.Load(); got <= tracker.validatorCalls.Load() {
t.Fatalf("provider calls = %d, validator calls = %d; want extractor, retry, and validator calls", got, tracker.validatorCalls.Load())
}
}

View File

@@ -69,7 +69,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
@@ -165,7 +165,7 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
@@ -212,7 +212,7 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Workers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v", err)

View File

@@ -81,7 +81,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Workers: 1,
Checkpoint: checkpoint,
})
if err != nil {
@@ -473,7 +473,7 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: append([]byte(nil), raw...),
ExtractWorkers: 1,
Workers: 1,
Checkpoint: checkpoint,
})
if err != nil {