206 lines
9.6 KiB
Go
206 lines
9.6 KiB
Go
|
|
package scenes
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"math"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
const wholeScene = `{"scenes":[{"start_unit_id":1,"end_unit_id":4}]}`
|
||
|
|
const splitScenes = `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":4}]}`
|
||
|
|
|
||
|
|
func TestNativePlanAssessesEveryRevisionThroughTheFinalTurn(t *testing.T) {
|
||
|
|
client := &fakeScenesLLMClient{responses: []string{wholeScene, wholeScene, splitScenes}}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 2
|
||
|
|
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "scene-override"}
|
||
|
|
var prepared []promptkit.PreparedRun
|
||
|
|
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||
|
|
chunks, err := source.MaterializeChunkPlan(request.Source, plan)
|
||
|
|
if err != nil {
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
assessments++
|
||
|
|
findings := []contracts.ValidatorAssessment{
|
||
|
|
{ValidatorName: "coverage", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
|
||
|
|
{ValidatorName: "scene-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
|
||
|
|
}
|
||
|
|
if len(chunks) != 2 {
|
||
|
|
findings[1].Decision = contracts.AssessmentRejected
|
||
|
|
findings[1].Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Separate the negotiation from the combat."}
|
||
|
|
}
|
||
|
|
return contracts.CandidateAssessment{Validators: findings}, nil
|
||
|
|
}
|
||
|
|
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||
|
|
len(result.Plan.Ranges) != 2 || assessments != 3 || len(client.requests) != 3 || len(prepared) != 3 {
|
||
|
|
t.Fatalf("final-turn result=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||
|
|
}
|
||
|
|
for i, call := range client.requests {
|
||
|
|
wantID := PromptID
|
||
|
|
if i > 0 {
|
||
|
|
wantID = CorrectionPromptID
|
||
|
|
}
|
||
|
|
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile ||
|
||
|
|
call.Target.Model != "scene-override" || call.Prompt.SessionID != request.SessionID {
|
||
|
|
t.Fatal("native run selections lost")
|
||
|
|
}
|
||
|
|
if i == 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
old, revised := client.requests[i-1].Prompt.Messages, call.Prompt.Messages
|
||
|
|
if len(revised) != len(old)+2 || !reflect.DeepEqual(old, revised[:len(old)]) ||
|
||
|
|
revised[len(old)].Role != "assistant" || revised[len(old)].Content != client.responses[i-1] {
|
||
|
|
t.Fatal("native continuation changed source context/history")
|
||
|
|
}
|
||
|
|
guidance := revised[len(old)+1].Content
|
||
|
|
if !strings.Contains(guidance, "Separate the negotiation") || strings.Contains(guidance, "operator-only-rejection") {
|
||
|
|
t.Fatal("correction did not separate actionable findings from operator diagnostics")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativePlanCanCorrectAnUnresolvableBoundary(t *testing.T) {
|
||
|
|
client := &fakeScenesLLMClient{responses: []string{
|
||
|
|
`{"scenes":[{"start_unit_id":1,"end_unit_id":999}]}`, splitScenes,
|
||
|
|
}}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
_, err := source.MaterializeChunkPlan(request.Source, plan)
|
||
|
|
return contracts.CandidateAssessment{}, err
|
||
|
|
}
|
||
|
|
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
|
||
|
|
t.Fatalf("boundary correction=%#v %v, assessments=%d calls=%d", result, err, assessments, len(client.requests))
|
||
|
|
}
|
||
|
|
messages := client.requests[1].Prompt.Messages
|
||
|
|
if !strings.Contains(messages[len(messages)-1].Content, "999") {
|
||
|
|
t.Fatal("correction omitted the unresolved boundary")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestPlanExhaustionUsesOnlyAnExplicitWholeSourceFallback(t *testing.T) {
|
||
|
|
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||
|
|
t.Run(string(mode), func(t *testing.T) {
|
||
|
|
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{scene(1, 2), scene(3, 4)}}}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.Mode = mode
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||
|
|
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
|
||
|
|
Result: contracts.ValidationResult{CorrectionGuidance: "Reconsider the scene transition."},
|
||
|
|
}}}, nil
|
||
|
|
}
|
||
|
|
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||
|
|
if assessments != 2 || len(client.requests) != 2 {
|
||
|
|
t.Fatal("revision was omitted, restarted, or reassessed")
|
||
|
|
}
|
||
|
|
if mode == contracts.ModeStrict {
|
||
|
|
if err == nil || result.Outcome.Disposition != "" {
|
||
|
|
t.Fatalf("strict rejection=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
want := source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 4}}}
|
||
|
|
if err != nil || !reflect.DeepEqual(result.Plan, want) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||
|
|
t.Fatalf("whole-source fallback=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
|
||
|
|
if err != nil || len(chunks) != 1 || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{1, 2, 3, 4}) ||
|
||
|
|
len(chunks[0].Annotations) != 0 || len(chunks[0].PlanAnnotations) != 0 {
|
||
|
|
t.Fatalf("fallback did not materialize the unannotated complete source: %#v %v", chunks, err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativePlanFailureUsesDocumentOrderWithoutEarlierPlanRecovery(t *testing.T) {
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
for _, failedCorrection := range []bool{false, true} {
|
||
|
|
client := &fakeScenesLLMClient{err: failure}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Source.Units[0].ID, request.Source.Units[1].ID = 30, 10
|
||
|
|
for i := range request.Source.Units {
|
||
|
|
unit := &request.Source.Units[i]
|
||
|
|
unit.Ref.StartUnitID, unit.Ref.EndUnitID = unit.ID, unit.ID
|
||
|
|
}
|
||
|
|
if failedCorrection {
|
||
|
|
client.err = nil
|
||
|
|
client.responses = []string{`{"scenes":[{"start_unit_id":30,"end_unit_id":10},{"start_unit_id":3,"end_unit_id":4}]}`}
|
||
|
|
client.errors = []error{nil, failure}
|
||
|
|
}
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 1
|
||
|
|
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||
|
|
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||
|
|
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
|
||
|
|
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the encounter together."},
|
||
|
|
}}}, nil
|
||
|
|
}
|
||
|
|
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible ||
|
||
|
|
!reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 30, EndUnitID: 4}}) {
|
||
|
|
t.Fatalf("failure fallback=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
|
||
|
|
if err != nil || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{30, 10, 3, 4}) {
|
||
|
|
t.Fatalf("nonconsecutive source order lost: %#v %v", chunks, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeStructuralFailureSkipsSemanticAssessment(t *testing.T) {
|
||
|
|
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{}}}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
request.Operation.Controls.MaxAdditionalTurns = 2
|
||
|
|
assessments := 0
|
||
|
|
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
return contracts.CandidateAssessment{}, nil
|
||
|
|
}
|
||
|
|
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||
|
|
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || assessments != 0 {
|
||
|
|
t.Fatalf("structural failure=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCancellationAndMaterializationFailureDoNotPublishFallback(t *testing.T) {
|
||
|
|
t.Run("canceled native delivery", func(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
client := &fakeScenesLLMClient{response: validSceneResponse(), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
result, err := newChunker(t, client).Plan(ctx, request)
|
||
|
|
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||
|
|
t.Fatalf("cancellation=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
t.Run("unmaterializable source", func(t *testing.T) {
|
||
|
|
request := chunkRequest()
|
||
|
|
request.Operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
request.Source.Units[0].Metadata = map[string]any{"invalid": math.NaN()}
|
||
|
|
failure := errors.New("provider unavailable")
|
||
|
|
result, err := newChunker(t, &fakeScenesLLMClient{err: failure}).Plan(context.Background(), request)
|
||
|
|
if err == nil || result.Outcome.Disposition != "" || !errors.Is(err, failure) {
|
||
|
|
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|