316 lines
11 KiB
Go
316 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type activeJobs struct {
|
|
active atomic.Int32
|
|
peak atomic.Int32
|
|
}
|
|
|
|
func (jobs *activeJobs) enter() func() {
|
|
current := jobs.active.Add(1)
|
|
for {
|
|
peak := jobs.peak.Load()
|
|
if current <= peak || jobs.peak.CompareAndSwap(peak, current) {
|
|
break
|
|
}
|
|
}
|
|
return func() { jobs.active.Add(-1) }
|
|
}
|
|
|
|
func TestRunnerSharesWorkerLimitBetweenExtractionAndReadyContinuation(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
prepared := preparedConcurrentPipeline(t, 2)
|
|
var jobs activeJobs
|
|
var firstLaneExtracts atomic.Int32
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
|
defer unblock()
|
|
mergeStarted := make(chan struct{}, 1)
|
|
for laneIndex := range prepared.Steps[0].lanes {
|
|
lane := laneIndex
|
|
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
|
defer jobs.enter()()
|
|
if lane == 0 {
|
|
firstLaneExtracts.Add(1)
|
|
} else {
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
return erasedTypedResult{}, ctx.Err()
|
|
}
|
|
}
|
|
return erasedTypedResult{Value: typedValueForLane(lane, request.Chunk.Index)}, nil
|
|
})
|
|
current := &prepared.Steps[0].lanes[lane]
|
|
merge := current.typed.merge
|
|
current.typed.merge = func(ctx context.Context, implementation any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
|
defer jobs.enter()()
|
|
if lane == 0 {
|
|
if firstLaneExtracts.Load() != 2 {
|
|
return erasedTypedResult{}, errors.New("continuation ran before its extraction prerequisites")
|
|
}
|
|
mergeStarted <- struct{}{}
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
return erasedTypedResult{}, ctx.Err()
|
|
}
|
|
}
|
|
return merge(ctx, implementation, request)
|
|
}
|
|
normalize := current.typed.normalize
|
|
current.typed.normalize = func(ctx context.Context, implementation any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
|
defer jobs.enter()()
|
|
return normalize(ctx, implementation, request)
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2})
|
|
done <- err
|
|
}()
|
|
select {
|
|
case <-mergeStarted:
|
|
synctest.Wait()
|
|
case <-ctx.Done():
|
|
t.Fatal("ready continuation could not run while another lane still had extraction work")
|
|
}
|
|
unblock()
|
|
if err := <-done; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := jobs.peak.Load(); got != 2 || jobs.active.Load() != 0 {
|
|
t.Fatalf("mixed active jobs peaked at %d, remaining %d; want the single limit of 2", got, jobs.active.Load())
|
|
}
|
|
})
|
|
}
|
|
|
|
func limitedProducerEngine(t *testing.T, limit, queue int, client promptkit.LLMClient) *promptkit.Engine {
|
|
t.Helper()
|
|
return nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("test-controlled backend", []byte(fmt.Sprintf(`backends:
|
|
- id: limited
|
|
endpoint: https://example.invalid/v1
|
|
concurrency_limit: %d
|
|
queue_capacity: %d
|
|
profiles:
|
|
- id: base
|
|
backend: limited
|
|
model: shared-model
|
|
api_key_required: false
|
|
`, limit, queue)))))
|
|
}
|
|
|
|
func installNativeExtraction(prepared *PreparedPipeline, engine *promptkit.Engine, jobs *activeJobs, started chan<- struct{}) {
|
|
for laneIndex := range prepared.Steps[0].lanes {
|
|
lane := laneIndex
|
|
prepared.Steps[0].lanes[lane].resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked
|
|
installExtractOperation(prepared, lane, func(ctx context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
|
if jobs != nil {
|
|
defer jobs.enter()()
|
|
}
|
|
if started != nil {
|
|
started <- struct{}{}
|
|
}
|
|
value, outcome, err := RunProducer(ctx, engine, request.Operation, nativeProducerInitial(),
|
|
func(ctx context.Context, response *promptkit.RunResult) (*any, []string, error) {
|
|
_, _, err := interpretProducerNumber(ctx, response)
|
|
value := typedValueForLane(lane, request.Chunk.Index)
|
|
return &value, nil, err
|
|
}, producerCorrection, func(context.Context, *any) (any, error) {
|
|
if lane == 0 {
|
|
return codecNotes{Items: []string{}}, nil
|
|
}
|
|
return codecScore{}, nil
|
|
})
|
|
return erasedTypedResult{Value: value, Outcome: outcome}, err
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerWorkersAndNativeBackendAdmissionAreIndependent(t *testing.T) {
|
|
for _, limits := range []struct{ workers, backend int }{{1, 1}, {3, 2}} {
|
|
t.Run(fmt.Sprintf("workers-%d/backend-%d", limits.workers, limits.backend), func(t *testing.T) {
|
|
prepared := preparedConcurrentPipeline(t, 2)
|
|
var providers, jobs activeJobs
|
|
var calls atomic.Int32
|
|
providerStarted := make(chan struct{}, 8)
|
|
jobStarted := make(chan struct{}, 4)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
|
defer unblock()
|
|
engine := limitedProducerEngine(t, limits.backend, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls.Add(1)
|
|
defer providers.enter()()
|
|
providerStarted <- struct{}{}
|
|
select {
|
|
case <-release:
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}))
|
|
installNativeExtraction(prepared, engine, &jobs, jobStarted)
|
|
for lane := range prepared.Steps[0].lanes {
|
|
prepared.Steps[0].lanes[lane].extractValidators.validators[0].typedValidate = func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) {
|
|
result, err := engine.Run(ctx, nativeProducerInitial())
|
|
if err != nil {
|
|
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
|
|
}
|
|
return contracts.ValidationResult{Approved: result.Validation.IsValid}, nil
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: limits.workers})
|
|
if err == nil && len(output.normalizeOutputs) != 2 {
|
|
err = fmt.Errorf("got %d lane outputs", len(output.normalizeOutputs))
|
|
}
|
|
done <- err
|
|
}()
|
|
for range limits.workers {
|
|
select {
|
|
case <-jobStarted:
|
|
case <-ctx.Done():
|
|
t.Fatal("worker jobs did not overlap as configured")
|
|
}
|
|
}
|
|
for range limits.backend {
|
|
select {
|
|
case <-providerStarted:
|
|
case <-ctx.Done():
|
|
t.Fatal("native backend did not admit its configured calls")
|
|
}
|
|
}
|
|
unblock()
|
|
if err := <-done; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if jobs.peak.Load() != int32(limits.workers) || providers.peak.Load() != int32(limits.backend) || calls.Load() != 8 || jobs.active.Load() != 0 || providers.active.Load() != 0 {
|
|
t.Fatalf("worker peak %d, backend peak %d, calls %d; active jobs/providers %d/%d", jobs.peak.Load(), providers.peak.Load(), calls.Load(), jobs.active.Load(), providers.active.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerNativeCapacityFailureUsesOperationModeWithoutAnotherQueue(t *testing.T) {
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
|
t.Run(string(mode), func(t *testing.T) {
|
|
var calls atomic.Int32
|
|
started, release := make(chan struct{}, 2), make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
|
defer unblock()
|
|
engine := limitedProducerEngine(t, 1, 0, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls.Add(1)
|
|
started <- struct{}{}
|
|
select {
|
|
case <-release:
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}))
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
occupant := make(chan error, 1)
|
|
go func() { _, err := engine.Run(ctx, nativeProducerInitial()); occupant <- err }()
|
|
select {
|
|
case <-started:
|
|
case <-ctx.Done():
|
|
t.Fatal("backend occupant did not start")
|
|
}
|
|
prepared := preparedConcurrentPipeline(t, 1)
|
|
installNativeExtraction(prepared, engine, nil, nil)
|
|
for lane := range prepared.Steps[0].lanes {
|
|
prepared.Steps[0].lanes[lane].resolved.ExtractControls = contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}
|
|
}
|
|
zero := 0
|
|
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Execution: contracts.ExecutionOptions{TransientRetries: &zero}})
|
|
if mode == contracts.ModeStrict {
|
|
var capacity *promptkit.CapacityError
|
|
if !errors.As(err, &capacity) || output.FallbackCount != 0 || len(output.OutputFiles) != 0 {
|
|
t.Fatalf("strict capacity failure = %v, fallbacks %d", err, output.FallbackCount)
|
|
}
|
|
} else if err != nil || output.FallbackCount != 2 || len(output.normalizeOutputs) != 2 {
|
|
t.Fatalf("permissive capacity failure = %v, fallbacks %d, outputs %d", err, output.FallbackCount, len(output.normalizeOutputs))
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("capacity handling invoked provider %d times", calls.Load())
|
|
}
|
|
unblock()
|
|
if err := <-occupant; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := engine.Run(ctx, nativeProducerInitial()); err != nil || calls.Load() != 2 {
|
|
t.Fatalf("backend admission was not released: %v, calls %d", err, calls.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerCancellationDrainsNativeWorkAndReleasesAdmission(t *testing.T) {
|
|
var calls atomic.Int32
|
|
var jobs activeJobs
|
|
jobStarted := make(chan struct{}, 4)
|
|
providerStarted := make(chan struct{})
|
|
engine := limitedProducerEngine(t, 1, 8, producerClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if calls.Add(1) == 1 {
|
|
close(providerStarted)
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
prepared := preparedConcurrentPipeline(t, 2)
|
|
installNativeExtraction(prepared, engine, &jobs, jobStarted)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 3})
|
|
if output.FallbackCount != 0 || len(output.normalizeOutputs) != 0 {
|
|
err = fmt.Errorf("cancellation delivered output: %v", err)
|
|
}
|
|
done <- err
|
|
}()
|
|
for range 3 {
|
|
select {
|
|
case <-jobStarted:
|
|
case <-ctx.Done():
|
|
t.Fatal("native jobs did not start")
|
|
}
|
|
}
|
|
select {
|
|
case <-providerStarted:
|
|
case <-ctx.Done():
|
|
t.Fatal("native backend did not start")
|
|
}
|
|
cancel()
|
|
if err := <-done; !errors.Is(err, context.Canceled) || jobs.active.Load() != 0 || len(jobStarted) != 0 {
|
|
t.Fatalf("cancel = %v, active jobs %d, extra starts %d", err, jobs.active.Load(), len(jobStarted))
|
|
}
|
|
if _, err := engine.Run(context.Background(), nativeProducerInitial()); err != nil || calls.Load() != 2 {
|
|
t.Fatalf("admission after cancellation = %v, provider calls %d", err, calls.Load())
|
|
}
|
|
}
|