Migrate scene planning to native continuation and whole-source fallback
This commit is contained in:
116
internal/framework/pipeline/chunk_plan_operation_test.go
Normal file
116
internal/framework/pipeline/chunk_plan_operation_test.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type nativePlanChunker struct{ engine *promptkit.Engine }
|
||||
|
||||
type namedPlanValidator struct {
|
||||
name string
|
||||
chunkAssessmentValidator
|
||||
}
|
||||
|
||||
func (v namedPlanValidator) Name() string { return v.name }
|
||||
|
||||
func (nativePlanChunker) Key() string { return "native-plan" }
|
||||
func (nativePlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (c nativePlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
|
||||
value, _, err := interpretProducerNumber(ctx, result)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
plan := typedTestPlan(request.Source)
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/scene-policy": json.RawMessage(fmt.Sprintf("%d", *value))}
|
||||
return &plan, nil, nil
|
||||
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
|
||||
return typedTestPlan(request.Source), nil
|
||||
})
|
||||
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
|
||||
}
|
||||
|
||||
func TestRunnerMaterializesAndFullyAssessesEachNativePlanRevision(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
turns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"final-turn acceptance", contracts.ModeStrict, 2, false, false},
|
||||
{"strict exhaustion", contracts.ModeStrict, 1, true, false},
|
||||
{"permissive exhaustion", contracts.ModePermissive, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
value := calls
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
||||
}))
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "native-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked},
|
||||
rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
|
||||
return nativePlanChunker{request.Dependencies.Engine}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var assessed []string
|
||||
for _, name := range []string{"first-plan-policy", "second-plan-policy"} {
|
||||
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
func() (contracts.ChunkValidator, error) {
|
||||
return namedPlanValidator{name, chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
if len(request.Chunks) != 1 || len(request.Chunks[0].Units) != len(request.Source.Units) || request.Chunks[0].Ref.SourceID != request.Source.ID {
|
||||
t.Fatal("validator did not receive normally materialized chunks")
|
||||
}
|
||||
value := string(request.Chunks[0].Annotations["test/scene-policy"])
|
||||
assessed = append(assessed, name+"/"+value)
|
||||
if value != "2" {
|
||||
return contracts.ValidationResult{ReasonCode: "scene_policy", CorrectionGuidance: "Reconsider the scene transition."}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
probe := newRunnerProbe()
|
||||
profile := typedResolutionProfile()
|
||||
profile.Chunk = Binding("native-plan")
|
||||
profile.Chunk.Mode, profile.Chunk.MaxAdditionalTurns = &test.mode, &test.turns
|
||||
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first-plan-policy"), Binding("second-plan-policy")}}
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input")})
|
||||
if (err != nil) != test.wantError || calls != test.turns+1 {
|
||||
t.Fatalf("native plan output=%#v error=%v calls=%d", output, err, calls)
|
||||
}
|
||||
var want []string
|
||||
for i := 0; i <= test.turns; i++ {
|
||||
want = append(want, fmt.Sprintf("first-plan-policy/%d", i), fmt.Sprintf("second-plan-policy/%d", i))
|
||||
}
|
||||
if !reflect.DeepEqual(assessed, want) {
|
||||
t.Fatalf("ordered plan assessments=%v, want %v", assessed, want)
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || (output.FallbackCount == 1) != test.wantFallback {
|
||||
t.Fatalf("plan disposition=%#v", output)
|
||||
}
|
||||
for _, role := range []string{"extract", "merge", "normalize"} {
|
||||
if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{!test.wantFallback}) {
|
||||
t.Fatalf("derived %s eligibility=%v", role, probe.eligible)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user