Migrate combat semantics validation to native execution and unavailable assessments
This commit is contained in:
58
internal/framework/pipeline/native_assessment_test.go
Normal file
58
internal/framework/pipeline/native_assessment_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeAssessmentChainSharesOneLimitedBackendWithoutHoldingProducerAdmission(t *testing.T) {
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if request.Target.BackendID != "limited" {
|
||||
t.Fatal("fixture did not select the shared backend")
|
||||
}
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}), promptkit.SettingsInput(configkit.Bytes("shared assessment backend", []byte(`backends:
|
||||
- id: limited
|
||||
endpoint: https://example.invalid/v1
|
||||
concurrency_limit: 1
|
||||
queue_capacity: 0
|
||||
profiles:
|
||||
- id: base
|
||||
backend: limited
|
||||
model: assessment-model
|
||||
api_key_required: false
|
||||
`))))
|
||||
chain := preparedValidatorChain{}
|
||||
for _, name := range []string{"first-native", "second-native"} {
|
||||
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name)}})
|
||||
}
|
||||
var order []string
|
||||
operation := contracts.ProducerOperation[int]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
|
||||
Assess: func(ctx context.Context, _ int) (contracts.CandidateAssessment, error) {
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
order = append(order, validator.resolved.Binding.Module)
|
||||
result, err := engine.Run(ctx, nativeProducerInitial())
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: fmt.Errorf("native contract failed")}
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})
|
||||
},
|
||||
}
|
||||
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 3 || !reflect.DeepEqual(order, []string{"first-native", "second-native"}) {
|
||||
t.Fatalf("nested assessment=%d %#v %v, native calls=%d order=%v", value, outcome, err, calls, order)
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ func (f producerClientFunc) Generate(ctx context.Context, request promptkit.Gene
|
||||
return f(ctx, request)
|
||||
}
|
||||
|
||||
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine {
|
||||
t.Helper()
|
||||
settings := `profiles:
|
||||
- id: base
|
||||
@@ -52,8 +52,12 @@ prompts:
|
||||
- {role: user, content: 'Correct: {{input "guidance"}}'}
|
||||
output: {format: json, validation_mode: json, repair_attempts: 0}
|
||||
`
|
||||
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}
|
||||
if len(extraInputs) > 0 {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: extraInputs})
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
|
||||
Underlays: underlays,
|
||||
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user