Files
notarius/internal/framework/pipeline/ready_jobs_test.go

316 lines
11 KiB
Go

package pipeline
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type activeJobs struct {
active atomic.Int32
peak atomic.Int32
}
func (jobs *activeJobs) enter() func() {
current := jobs.active.Add(1)
for {
peak := jobs.peak.Load()
if current <= peak || jobs.peak.CompareAndSwap(peak, current) {
break
}
}
return func() { jobs.active.Add(-1) }
}
func TestRunnerSharesWorkerLimitBetweenExtractionAndReadyContinuation(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
var jobs activeJobs
var firstLaneExtracts atomic.Int32
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
mergeStarted := make(chan struct{}, 1)
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
if lane == 0 {
firstLaneExtracts.Add(1)
} else {
select {
case <-release:
case <-ctx.Done():
return erasedTypedResult{}, ctx.Err()
}
}
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
})
current := &prepared.Steps[0].lanes[lane]
merge := current.typed.merge
current.typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
if lane == 0 {
if firstLaneExtracts.Load() != 2 {
return erasedTypedResult{}, errors.New("continuation ran before its extraction prerequisites")
}
mergeStarted <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return erasedTypedResult{}, ctx.Err()
}
}
return merge(ctx, implementation, request)
}
normalize := current.typed.normalize
current.typed.normalize = func(ctx context.Context, implementation any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
defer jobs.enter()()
return normalize(ctx, implementation, request)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
done <- err
}()
select {
case <-mergeStarted:
synctest.Wait()
case <-ctx.Done():
t.Fatal("ready continuation could not run while another lane still had extraction work")
}
unblock()
if err := <-done; err != nil {
t.Fatal(err)
}
if got := jobs.peak.Load(); got != 2 || jobs.active.Load() != 0 {
t.Fatalf("mixed active jobs peaked at %d, remaining %d; want the single limit of 2", got, jobs.active.Load())
}
})
}
func limitedProducerEngine(t *testing.T, limit, queue int, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
return nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("test-controlled backend", []byte(fmt.Sprintf(`backends:
- id: limited
endpoint: https://example.invalid/v1
concurrency_limit: %d
queue_capacity: %d
profiles:
- id: base
backend: limited
model: shared-model
api_key_required: false
`, limit, queue)))))
}
func installNativeExtraction(prepared *PreparedPipeline, engine *promptkit.Engine, jobs *activeJobs, started chan<- struct{}) {
for laneIndex := range prepared.Steps[0].lanes {
lane := laneIndex
prepared.Steps[0].lanes[lane].resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
if jobs != nil {
defer jobs.enter()()
}
if started != nil {
started <- struct{}{}
}
value, outcome, err := RunProducer(ctx, engine, request.Operation, nativeProducerInitial(),
func(ctx context.Context, response *promptkit.RunResult) (*any, []string, error) {
_, _, err := interpretProducerNumber(ctx, response)
value := typedValueForLane(lane, request.Chunk.Index)
return &value, nil, err
}, producerCorrection, func(context.Context, *any) (any, error) {
if lane == 0 {
return codecNotes{Items: []string{}}, nil
}
return codecScore{}, nil
})
return erasedTypedResult{Value: value, Outcome: outcome}, err
})
}
}
func TestRunnerWorkersAndNativeBackendAdmissionAreIndependent(t *testing.T) {
for _, limits := range []struct{ workers, backend int }{{1, 1}, {3, 2}} {
t.Run(fmt.Sprintf("workers-%d/backend-%d", limits.workers, limits.backend), func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 2)
var providers, jobs activeJobs
var calls atomic.Int32
providerStarted := make(chan struct{}, 8)
jobStarted := make(chan struct{}, 4)
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
engine := limitedProducerEngine(t, limits.backend, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls.Add(1)
defer providers.enter()()
providerStarted <- struct{}{}
select {
case <-release:
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}))
installNativeExtraction(prepared, engine, &jobs, jobStarted)
for lane := range prepared.Steps[0].lanes {
prepared.Steps[0].lanes[lane].extractValidators.validators[0].typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
result, err := engine.Run(ctx, nativeProducerInitial())
if err != nil {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
}
return contracts.ValidationResult{Approved: result.Validation.IsValid}, nil
}
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: limits.workers})
if err == nil && len(output.normalizeOutputs) != 2 {
err = fmt.Errorf("got %d lane outputs", len(output.normalizeOutputs))
}
done <- err
}()
for range limits.workers {
select {
case <-jobStarted:
case <-ctx.Done():
t.Fatal("worker jobs did not overlap as configured")
}
}
for range limits.backend {
select {
case <-providerStarted:
case <-ctx.Done():
t.Fatal("native backend did not admit its configured calls")
}
}
unblock()
if err := <-done; err != nil {
t.Fatal(err)
}
if jobs.peak.Load() != int32(limits.workers) || providers.peak.Load() != int32(limits.backend) || calls.Load() != 8 || jobs.active.Load() != 0 || providers.active.Load() != 0 {
t.Fatalf("worker peak %d, backend peak %d, calls %d; active jobs/providers %d/%d", jobs.peak.Load(), providers.peak.Load(), calls.Load(), jobs.active.Load(), providers.active.Load())
}
})
}
}
func TestRunnerNativeCapacityFailureUsesOperationModeWithoutAnotherQueue(t *testing.T) {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
t.Run(string(mode), func(t *testing.T) {
var calls atomic.Int32
started, release := make(chan struct{}, 2), make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
engine := limitedProducerEngine(t, 1, 0, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls.Add(1)
started <- struct{}{}
select {
case <-release:
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}))
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
occupant := make(chan error, 1)
go func() { _, err := engine.Run(ctx, nativeProducerInitial()); occupant <- err }()
select {
case <-started:
case <-ctx.Done():
t.Fatal("backend occupant did not start")
}
prepared := preparedConcurrentPipeline(t, 1)
installNativeExtraction(prepared, engine, nil, nil)
for lane := range prepared.Steps[0].lanes {
prepared.Steps[0].lanes[lane].resolved.ExtractControls = contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}
}
zero := 0
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Execution: contracts.ExecutionOptions{TransientRetries: &zero}})
if mode == contracts.ModeStrict {
var capacity *promptkit.CapacityError
if !errors.As(err, &capacity) || output.FallbackCount != 0 || len(output.OutputFiles) != 0 {
t.Fatalf("strict capacity failure = %v, fallbacks %d", err, output.FallbackCount)
}
} else if err != nil || output.FallbackCount != 2 || len(output.normalizeOutputs) != 2 {
t.Fatalf("permissive capacity failure = %v, fallbacks %d, outputs %d", err, output.FallbackCount, len(output.normalizeOutputs))
}
if calls.Load() != 1 {
t.Fatalf("capacity handling invoked provider %d times", calls.Load())
}
unblock()
if err := <-occupant; err != nil {
t.Fatal(err)
}
if _, err := engine.Run(ctx, nativeProducerInitial()); err != nil || calls.Load() != 2 {
t.Fatalf("backend admission was not released: %v, calls %d", err, calls.Load())
}
})
}
}
func TestRunnerCancellationDrainsNativeWorkAndReleasesAdmission(t *testing.T) {
var calls atomic.Int32
var jobs activeJobs
jobStarted := make(chan struct{}, 4)
providerStarted := make(chan struct{})
engine := limitedProducerEngine(t, 1, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if calls.Add(1) == 1 {
close(providerStarted)
<-ctx.Done()
return nil, ctx.Err()
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
prepared := preparedConcurrentPipeline(t, 2)
installNativeExtraction(prepared, engine, &jobs, jobStarted)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() {
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 3})
if output.FallbackCount != 0 || len(output.normalizeOutputs) != 0 {
err = fmt.Errorf("cancellation delivered output: %v", err)
}
done <- err
}()
for range 3 {
select {
case <-jobStarted:
case <-ctx.Done():
t.Fatal("native jobs did not start")
}
}
select {
case <-providerStarted:
case <-ctx.Done():
t.Fatal("native backend did not start")
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) || jobs.active.Load() != 0 || len(jobStarted) != 0 {
t.Fatalf("cancel = %v, active jobs %d, extra starts %d", err, jobs.active.Load(), len(jobStarted))
}
if _, err := engine.Run(context.Background(), nativeProducerInitial()); err != nil || calls.Load() != 2 {
t.Fatalf("admission after cancellation = %v, provider calls %d", err, calls.Load())
}
}