Migrate combat semantics validation to native execution and unavailable assessments

This commit is contained in:
2026-11-22 00:42:20 +00:00
parent 52a1c96327
commit 5d3696d4a0
13 changed files with 571 additions and 493 deletions

View File

@@ -0,0 +1,58 @@
package pipeline
import (
"context"
"fmt"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestNativeAssessmentChainSharesOneLimitedBackendWithoutHoldingProducerAdmission(t *testing.T) {
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.BackendID != "limited" {
t.Fatal("fixture did not select the shared backend")
}
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}), promptkit.SettingsInput(configkit.Bytes("shared assessment backend", []byte(`backends:
- id: limited
endpoint: https://example.invalid/v1
concurrency_limit: 1
queue_capacity: 0
profiles:
- id: base
backend: limited
model: assessment-model
api_key_required: false
`))))
chain := preparedValidatorChain{}
for _, name := range []string{"first-native", "second-native"} {
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name)}})
}
var order []string
operation := contracts.ProducerOperation[int]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
Assess: func(ctx context.Context, _ int) (contracts.CandidateAssessment, error) {
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
order = append(order, validator.resolved.Binding.Module)
result, err := engine.Run(ctx, nativeProducerInitial())
if err != nil {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
}
if !result.Validation.IsValid {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: fmt.Errorf("native contract failed")}
}
return contracts.ValidationResult{Approved: true}, nil
})
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 3 || !reflect.DeepEqual(order, []string{"first-native", "second-native"}) {
t.Fatalf("nested assessment=%d %#v %v, native calls=%d order=%v", value, outcome, err, calls, order)
}
}

View File

@@ -22,7 +22,7 @@ func (f producerClientFunc) Generate(ctx context.Context, request promptkit.Gene
return f(ctx, request)
}
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine {
t.Helper()
settings := `profiles:
- id: base
@@ -52,8 +52,12 @@ prompts:
- {role: user, content: 'Correct: {{input "guidance"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}
if len(extraInputs) > 0 {
underlays = append(underlays, configkit.Underlay{Inputs: extraInputs})
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
Underlays: underlays,
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)