Run typed producers through native continuation and assessment
This commit is contained in:
@@ -448,6 +448,22 @@ unavailable assessment, guidance contract errors, and cancellation.
|
||||
|
||||
## Stage 10: Implement The Small Native Producer Helper
|
||||
|
||||
**Status:** Complete. Added one typed helper using native Run/Coordinate and
|
||||
module-owned interpretation/correction/fallback functions. It checks native
|
||||
structural status before interpretation, reuses the initial assessment on the
|
||||
first coordination callback, and assesses every delivered revision once,
|
||||
including the final allowed turn. Native coordination owns history and budget.
|
||||
Execution/profile/session selections and existing observers/preparation callbacks
|
||||
are preserved without duplicate enrollment. Failed continuations cannot recover
|
||||
earlier candidates; caller cancellation and undelivered responses never become
|
||||
acceptance or fallback. Real-engine offline fake-provider tests cover acceptance,
|
||||
zero turns, final-turn acceptance, exhaustion, nil-error structural failure,
|
||||
operational/unavailable failures, interpretation guidance, cancellation, and
|
||||
typed native operation timeouts. File-scoped helper tests/vet/race checks and
|
||||
contracts tests pass. Full pipeline tests remain blocked by the recorded legacy
|
||||
ConfigKit YAML import and unmigrated consumers; no whole-package success is
|
||||
claimed. Runner integration, caching, and capture remain in their assigned stages.
|
||||
|
||||
**Goal:** delegate turn iteration while retaining typed Notarius acceptance.
|
||||
|
||||
**Inspect/change:** a focused helper alongside pipeline assessment; upstream
|
||||
|
||||
184
internal/framework/pipeline/producer.go
Normal file
184
internal/framework/pipeline/producer.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// RunProducer connects native execution to typed interpretation and application
|
||||
// assessment. Interpretation returns nil when no typed candidate can be built;
|
||||
// actionable findings remain separate from operational errors. Correction builds
|
||||
// only the next assignment, never conversation history. Fallback receives only
|
||||
// the final usable candidate, or nil after an unusable/failed execution.
|
||||
func RunProducer[T any](
|
||||
ctx context.Context,
|
||||
engine *promptkit.Engine,
|
||||
operation contracts.ProducerOperation[T],
|
||||
initial promptkit.RunRequest,
|
||||
interpret func(context.Context, *promptkit.RunResult) (*T, []string, error),
|
||||
correct func(context.Context, []string) (promptkit.RunRequest, error),
|
||||
fallback func(context.Context, *T) (T, error),
|
||||
) (T, contracts.ProducerOutcome, error) {
|
||||
var zero T
|
||||
if ctx == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
}
|
||||
if interpret == nil || operation.Assess == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer interpretation and assessment must be supplied")
|
||||
}
|
||||
if operation.Controls.MaxAdditionalTurns < 0 {
|
||||
return zero, contracts.ProducerOutcome{}, fmt.Errorf("%w: additional turns must not be negative", promptkit.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
var candidate *T
|
||||
var guidance []string
|
||||
var failure error
|
||||
var assessmentError error
|
||||
accepted := false
|
||||
evaluate := func(result *promptkit.RunResult) {
|
||||
candidate, guidance, failure, accepted = nil, nil, nil, false
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
failure = canceled
|
||||
return
|
||||
}
|
||||
if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
|
||||
failure = fmt.Errorf("%w: producer output did not satisfy its native contract", promptkit.ErrValidation)
|
||||
return
|
||||
}
|
||||
candidate, guidance, failure = interpret(ctx, result)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
failure = canceled
|
||||
return
|
||||
}
|
||||
if failure != nil {
|
||||
candidate, guidance = nil, nil
|
||||
return
|
||||
}
|
||||
if candidate == nil {
|
||||
failure = errors.New("producer response could not be interpreted as a usable candidate")
|
||||
return
|
||||
}
|
||||
assessment, err := operation.Assess(ctx, *candidate)
|
||||
if err != nil {
|
||||
assessmentError = err
|
||||
return
|
||||
}
|
||||
validatorGuidance, err := assessment.Guidance()
|
||||
if err != nil {
|
||||
assessmentError = err
|
||||
return
|
||||
}
|
||||
guidance = append(guidance, validatorGuidance...)
|
||||
accepted = assessment.Accepted() && len(guidance) == 0
|
||||
failure = assessment.Err()
|
||||
if !accepted && failure == nil {
|
||||
failure = errors.New("producer candidate requires correction")
|
||||
}
|
||||
}
|
||||
|
||||
request := producerRequest(initial, operation.Execution, true)
|
||||
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
|
||||
result, err := engine.Run(ctx, request)
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
if err != nil {
|
||||
failure = err
|
||||
} else {
|
||||
evaluate(result)
|
||||
}
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
if assessmentError != nil {
|
||||
return zero, contracts.ProducerOutcome{}, assessmentError
|
||||
}
|
||||
|
||||
if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 {
|
||||
coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer,
|
||||
Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
// The first callback has no new result. Its findings already exist.
|
||||
if turn.Result != nil {
|
||||
evaluate(turn.Result)
|
||||
}
|
||||
if assessmentError != nil {
|
||||
return nil, assessmentError
|
||||
}
|
||||
if accepted || len(guidance) == 0 || turn.RemainingTurns == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
// Once another execution is selected, an earlier candidate is not a
|
||||
// terminal fallback source if construction or native execution fails.
|
||||
candidate = nil
|
||||
next, buildErr := correct(ctx, guidance)
|
||||
if buildErr != nil {
|
||||
return nil, buildErr
|
||||
}
|
||||
if initial.ProfileID != "" {
|
||||
next.ProfileID = initial.ProfileID
|
||||
}
|
||||
if initial.SessionID != "" {
|
||||
next.SessionID = initial.SessionID
|
||||
}
|
||||
next = producerRequest(next, operation.Execution, false)
|
||||
return &next, nil
|
||||
},
|
||||
})
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
if coordination.Undelivered != nil {
|
||||
if coordinateErr == nil {
|
||||
coordinateErr = errors.New("native continuation response was not delivered")
|
||||
}
|
||||
return zero, contracts.ProducerOutcome{}, coordinateErr
|
||||
}
|
||||
if assessmentError != nil {
|
||||
return zero, contracts.ProducerOutcome{}, assessmentError
|
||||
}
|
||||
if coordinateErr != nil {
|
||||
candidate, failure, accepted = nil, coordinateErr, false
|
||||
}
|
||||
}
|
||||
if accepted {
|
||||
return *candidate, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil
|
||||
}
|
||||
var construct func(context.Context) (T, error)
|
||||
if fallback != nil {
|
||||
construct = func(ctx context.Context) (T, error) { return fallback(ctx, candidate) }
|
||||
}
|
||||
return finishProducerFailure(ctx, operation.Controls.Mode, failure, construct)
|
||||
}
|
||||
|
||||
// producerRequest applies native options to a value copy. The invocation
|
||||
// observer is enrolled here for the initial run and on CoordinateRequest for
|
||||
// continuations, never twice on the same continuation.
|
||||
func producerRequest(request promptkit.RunRequest, options contracts.ExecutionOptions, initial bool) promptkit.RunRequest {
|
||||
if options.Execution != nil {
|
||||
request.Execution = options.Execution
|
||||
}
|
||||
if options.TransientRetries != nil {
|
||||
request.TransientRetries = options.TransientRetries
|
||||
}
|
||||
if initial {
|
||||
request.Observer = promptkit.ComposeObservers(request.Observer, options.Observer)
|
||||
}
|
||||
if options.OnPrepared != nil {
|
||||
existing := request.OnPrepared
|
||||
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
|
||||
if existing != nil {
|
||||
existing(ctx, details)
|
||||
}
|
||||
options.OnPrepared(ctx, details)
|
||||
}
|
||||
}
|
||||
return request
|
||||
}
|
||||
518
internal/framework/pipeline/producer_test.go
Normal file
518
internal/framework/pipeline/producer_test.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
|
||||
|
||||
func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return f(ctx, request)
|
||||
}
|
||||
|
||||
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
t.Helper()
|
||||
settings := `profiles:
|
||||
- id: base
|
||||
endpoint: https://example.invalid/v1
|
||||
model: initial-model
|
||||
api_key_required: false
|
||||
- id: correction-default
|
||||
endpoint: https://example.invalid/v1
|
||||
model: correction-model
|
||||
api_key_required: false
|
||||
- id: chosen
|
||||
endpoint: https://example.invalid/v1
|
||||
model: chosen-model
|
||||
api_key_required: false
|
||||
prompts:
|
||||
- id: initial
|
||||
default_profile: base
|
||||
inputs: [{name: source, required: true}]
|
||||
messages:
|
||||
- {role: system, content: 'Keep enduring constraints.'}
|
||||
- {role: user, content: 'Source: {{input "source"}}'}
|
||||
output: {format: json, validation_mode: json, repair_attempts: 0}
|
||||
- id: correction
|
||||
default_profile: correction-default
|
||||
inputs: [{name: guidance, required: true}]
|
||||
messages:
|
||||
- {role: user, content: 'Correct: {{input "guidance"}}'}
|
||||
output: {format: json, validation_mode: json, repair_attempts: 0}
|
||||
`
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
|
||||
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func nativeProducerInitial() promptkit.RunRequest {
|
||||
return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}}
|
||||
}
|
||||
|
||||
func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
var candidate struct {
|
||||
Value int `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &candidate.Value, nil, nil
|
||||
}
|
||||
|
||||
func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil
|
||||
}
|
||||
|
||||
func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
|
||||
func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil
|
||||
}
|
||||
|
||||
func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
values []int
|
||||
turns int
|
||||
mode contracts.ExecutionMode
|
||||
wantAccepted bool
|
||||
}{
|
||||
{"initial acceptance", []int{1}, 3, contracts.ModeStrict, true},
|
||||
{"zero turns", []int{0}, 0, contracts.ModeStrict, false},
|
||||
{"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true},
|
||||
{"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true},
|
||||
{"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false},
|
||||
{"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls, assessments, interpreted, fallbacks := 0, 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if calls >= len(test.values) {
|
||||
return nil, errors.New("unexpected extra provider call")
|
||||
}
|
||||
value := test.values[calls]
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true,
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
if value == 1 {
|
||||
return acceptedProducerAssessment(ctx, value)
|
||||
}
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
interpreted++
|
||||
return interpretProducerNumber(ctx, result)
|
||||
},
|
||||
producerCorrection,
|
||||
func(_ context.Context, final *int) (int, error) {
|
||||
fallbacks++
|
||||
if final == nil || *final != test.values[len(test.values)-1] {
|
||||
t.Fatalf("fallback did not receive the final usable candidate: %v", final)
|
||||
}
|
||||
return -1, nil
|
||||
})
|
||||
if calls != len(test.values) || assessments != calls || interpreted != calls {
|
||||
t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values))
|
||||
}
|
||||
switch {
|
||||
case test.wantAccepted:
|
||||
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 {
|
||||
t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
|
||||
}
|
||||
case test.mode == contracts.ModePermissive:
|
||||
if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 {
|
||||
t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks)
|
||||
}
|
||||
default:
|
||||
if err == nil || outcome.Disposition != "" || fallbacks != 0 {
|
||||
t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) {
|
||||
for _, failCorrection := range []bool{false, true} {
|
||||
calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if failCorrection && calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
interpretations++
|
||||
return interpretProducerNumber(ctx, result)
|
||||
},
|
||||
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
corrections++
|
||||
return producerCorrection(ctx, guidance)
|
||||
},
|
||||
func(_ context.Context, candidate *int) (int, error) {
|
||||
fallbacks++
|
||||
if candidate != nil {
|
||||
t.Fatal("structurally failed response recovered a prior candidate")
|
||||
}
|
||||
return -1, nil
|
||||
})
|
||||
want := 0
|
||||
if failCorrection {
|
||||
want = 1
|
||||
}
|
||||
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 {
|
||||
t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, failCorrection := range []bool{false, true} {
|
||||
calls, interpreted, assessed := 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if failCorrection && calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}
|
||||
return nil, failure
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
interpreted++
|
||||
return interpretProducerNumber(ctx, result)
|
||||
},
|
||||
producerCorrection,
|
||||
func(_ context.Context, candidate *int) (int, error) {
|
||||
if candidate != nil {
|
||||
t.Fatal("failed execution recovered an earlier candidate")
|
||||
}
|
||||
return -1, nil
|
||||
})
|
||||
want := 0
|
||||
if failCorrection {
|
||||
want = 1
|
||||
}
|
||||
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want {
|
||||
t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed)
|
||||
}
|
||||
}
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, failure
|
||||
}))
|
||||
_, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||
if !errors.Is(err, failure) {
|
||||
t.Fatalf("native failure cause was lost: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) {
|
||||
calls, corrections := 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5},
|
||||
Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber,
|
||||
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
corrections++
|
||||
return producerCorrection(ctx, guidance)
|
||||
},
|
||||
func(context.Context, *int) (int, error) { return -1, nil })
|
||||
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 {
|
||||
t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) {
|
||||
for _, explicit := range []bool{false, true} {
|
||||
var requests []promptkit.GenerateRequest
|
||||
var profiles []string
|
||||
initialPrepared, correctionPrepared, observed := 0, 0, 0
|
||||
initialObserved, correctionObserved := 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
requests = append(requests, request)
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
initial := nativeProducerInitial()
|
||||
initial.SessionID = "source-session"
|
||||
if explicit {
|
||||
initial.ProfileID = "chosen"
|
||||
}
|
||||
initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ }
|
||||
initial.Observer = func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
initialObserved++
|
||||
}
|
||||
}
|
||||
overrideModel := "override-model"
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment,
|
||||
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel},
|
||||
Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
observed++
|
||||
}
|
||||
},
|
||||
OnPrepared: func(_ context.Context, details promptkit.PreparedRun) {
|
||||
profiles = append(profiles, details.SelectedProfileID)
|
||||
},
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber,
|
||||
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
if !reflect.DeepEqual(guidance, []string{"Return value one."}) {
|
||||
t.Fatalf("unexpected correction guidance: %#v", guidance)
|
||||
}
|
||||
request, err := producerCorrection(ctx, guidance)
|
||||
request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ }
|
||||
request.Observer = func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
correctionObserved++
|
||||
}
|
||||
}
|
||||
return request, err
|
||||
}, func(context.Context, *int) (int, error) { return -1, nil })
|
||||
if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 {
|
||||
t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared)
|
||||
}
|
||||
if initialObserved != 1 || correctionObserved != 2 {
|
||||
t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved)
|
||||
}
|
||||
wantProfiles := []string{"base", "correction-default", "correction-default"}
|
||||
if explicit {
|
||||
wantProfiles = []string{"chosen", "chosen", "chosen"}
|
||||
}
|
||||
if !reflect.DeepEqual(profiles, wantProfiles) {
|
||||
t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles)
|
||||
}
|
||||
for i, request := range requests {
|
||||
if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID {
|
||||
t.Fatalf("selection lost on turn %d: %#v", i, request)
|
||||
}
|
||||
var content []string
|
||||
for _, message := range request.Prompt.Messages {
|
||||
content = append(content, message.Content)
|
||||
}
|
||||
joined := strings.Join(content, "\n")
|
||||
if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i {
|
||||
t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content)
|
||||
}
|
||||
}
|
||||
if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" {
|
||||
t.Fatal("helper mutated the caller's initial request")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) {
|
||||
for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} {
|
||||
t.Run(moment, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
if moment == "inside assessment" {
|
||||
cancel()
|
||||
return acceptedProducerAssessment(ctx, value)
|
||||
}
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
finished++
|
||||
if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}},
|
||||
}
|
||||
if moment == "before initial" {
|
||||
cancel()
|
||||
}
|
||||
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
interpreted++
|
||||
if moment == "inside interpretation" {
|
||||
cancel()
|
||||
}
|
||||
return interpretProducerNumber(ctx, result)
|
||||
},
|
||||
producerCorrection,
|
||||
func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
||||
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 {
|
||||
t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
||||
}
|
||||
switch moment {
|
||||
case "before initial":
|
||||
if calls != 0 || interpreted != 0 || assessed != 0 {
|
||||
t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed)
|
||||
}
|
||||
case "inside assessment":
|
||||
if calls != 1 || interpreted != 1 || assessed != 1 {
|
||||
t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed)
|
||||
}
|
||||
case "after initial execution":
|
||||
if calls != 1 || interpreted != 0 || assessed != 0 {
|
||||
t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed)
|
||||
}
|
||||
case "inside interpretation":
|
||||
if calls != 1 || interpreted != 1 || assessed != 0 {
|
||||
t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed)
|
||||
}
|
||||
default:
|
||||
if calls != 2 || interpreted != 1 || assessed != 1 {
|
||||
t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
for _, continuation := range []bool{false, true} {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
calls, fallbacks := 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if continuation && calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}))
|
||||
initial := nativeProducerInitial()
|
||||
initial.OperationTimeout = time.Second
|
||||
parent := context.Background()
|
||||
_, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment},
|
||||
initial, interpretProducerNumber,
|
||||
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
request, err := producerCorrection(ctx, guidance)
|
||||
request.OperationTimeout = time.Second
|
||||
return request, err
|
||||
},
|
||||
func(_ context.Context, final *int) (int, error) {
|
||||
fallbacks++
|
||||
if final != nil {
|
||||
t.Fatal("timeout recovered an earlier candidate")
|
||||
}
|
||||
return -1, nil
|
||||
})
|
||||
if parent.Err() != nil {
|
||||
t.Fatalf("native timeout canceled caller: %v", parent.Err())
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 {
|
||||
t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
||||
}
|
||||
} else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 {
|
||||
t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) {
|
||||
for _, usable := range []bool{false, true} {
|
||||
calls, interpreted, assessed := 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true,
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return acceptedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
|
||||
interpreted++
|
||||
candidate, _, err := interpretProducerNumber(ctx, result)
|
||||
if interpreted == 1 {
|
||||
if !usable {
|
||||
candidate = nil
|
||||
}
|
||||
return candidate, []string{"Use the supplied candidate handles."}, err
|
||||
}
|
||||
return candidate, nil, err
|
||||
},
|
||||
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) {
|
||||
t.Fatalf("module findings lost: %#v", guidance)
|
||||
}
|
||||
return producerCorrection(ctx, guidance)
|
||||
}, nil)
|
||||
wantAssessed := 1
|
||||
if usable {
|
||||
wantAssessed = 2
|
||||
}
|
||||
if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed {
|
||||
t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) {
|
||||
failure := errors.New("assessment framework failure")
|
||||
fallbacks, calls := 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, failure
|
||||
}}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
||||
if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 {
|
||||
t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user