Migrate scene planning to native continuation and whole-source fallback
This commit is contained in:
116
internal/framework/pipeline/chunk_plan_operation_test.go
Normal file
116
internal/framework/pipeline/chunk_plan_operation_test.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type nativePlanChunker struct{ engine *promptkit.Engine }
|
||||
|
||||
type namedPlanValidator struct {
|
||||
name string
|
||||
chunkAssessmentValidator
|
||||
}
|
||||
|
||||
func (v namedPlanValidator) Name() string { return v.name }
|
||||
|
||||
func (nativePlanChunker) Key() string { return "native-plan" }
|
||||
func (nativePlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (c nativePlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, nativeProducerInitial(),
|
||||
func(ctx context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
|
||||
value, _, err := interpretProducerNumber(ctx, result)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
plan := typedTestPlan(request.Source)
|
||||
plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/scene-policy": json.RawMessage(fmt.Sprintf("%d", *value))}
|
||||
return &plan, nil, nil
|
||||
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
|
||||
return typedTestPlan(request.Source), nil
|
||||
})
|
||||
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
|
||||
}
|
||||
|
||||
func TestRunnerMaterializesAndFullyAssessesEachNativePlanRevision(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
turns int
|
||||
wantError, wantFallback bool
|
||||
}{
|
||||
{"final-turn acceptance", contracts.ModeStrict, 2, false, false},
|
||||
{"strict exhaustion", contracts.ModeStrict, 1, true, false},
|
||||
{"permissive exhaustion", contracts.ModePermissive, 1, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
value := calls
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
||||
}))
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "native-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked},
|
||||
rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
|
||||
return nativePlanChunker{request.Dependencies.Engine}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var assessed []string
|
||||
for _, name := range []string{"first-plan-policy", "second-plan-policy"} {
|
||||
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
func() (contracts.ChunkValidator, error) {
|
||||
return namedPlanValidator{name, chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||
if len(request.Chunks) != 1 || len(request.Chunks[0].Units) != len(request.Source.Units) || request.Chunks[0].Ref.SourceID != request.Source.ID {
|
||||
t.Fatal("validator did not receive normally materialized chunks")
|
||||
}
|
||||
value := string(request.Chunks[0].Annotations["test/scene-policy"])
|
||||
assessed = append(assessed, name+"/"+value)
|
||||
if value != "2" {
|
||||
return contracts.ValidationResult{ReasonCode: "scene_policy", CorrectionGuidance: "Reconsider the scene transition."}, nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
})}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
probe := newRunnerProbe()
|
||||
profile := typedResolutionProfile()
|
||||
profile.Chunk = Binding("native-plan")
|
||||
profile.Chunk.Mode, profile.Chunk.MaxAdditionalTurns = &test.mode, &test.turns
|
||||
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first-plan-policy"), Binding("second-plan-policy")}}
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input")})
|
||||
if (err != nil) != test.wantError || calls != test.turns+1 {
|
||||
t.Fatalf("native plan output=%#v error=%v calls=%d", output, err, calls)
|
||||
}
|
||||
var want []string
|
||||
for i := 0; i <= test.turns; i++ {
|
||||
want = append(want, fmt.Sprintf("first-plan-policy/%d", i), fmt.Sprintf("second-plan-policy/%d", i))
|
||||
}
|
||||
if !reflect.DeepEqual(assessed, want) {
|
||||
t.Fatalf("ordered plan assessments=%v, want %v", assessed, want)
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || (output.FallbackCount == 1) != test.wantFallback {
|
||||
t.Fatalf("plan disposition=%#v", output)
|
||||
}
|
||||
for _, role := range []string{"extract", "merge", "normalize"} {
|
||||
if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{!test.wantFallback}) {
|
||||
t.Fatalf("derived %s eligibility=%v", role, probe.eligible)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,11 @@ package scenes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -11,6 +15,8 @@ import (
|
||||
)
|
||||
|
||||
const Key = "dnd/scenes"
|
||||
const PromptID = "notarius/dnd.scenes"
|
||||
const CorrectionPromptID = "notarius/dnd.scenes.repair"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"source.transcript",
|
||||
@@ -28,19 +34,18 @@ var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||
}
|
||||
|
||||
var _ contracts.Chunker = (*Chunker)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Chunker struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
engine *promptkit.Engine
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Chunker, error) {
|
||||
if llmClient == nil {
|
||||
return nil, chunkerErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options) (*Chunker, error) {
|
||||
if engine == nil {
|
||||
return nil, chunkerErrorf("native engine must not be nil")
|
||||
}
|
||||
return &Chunker{llm: llmClient}, nil
|
||||
return &Chunker{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (c *Chunker) Key() string {
|
||||
@@ -51,32 +56,12 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return shared.ReferenceSlots(referenceSlotDescriptions)
|
||||
}
|
||||
|
||||
func (c *Chunker) ManifestMetadata() map[string]any {
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
promptSHA = ""
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": ResponseSchemaVersion,
|
||||
"prompt_sha256": promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
}
|
||||
if schema, err := loadResponseSchema(); err == nil {
|
||||
metadata["response_schema_version"] = schema.Version
|
||||
metadata["response_schema_sha256"] = schema.SHA256
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
if c == nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil")
|
||||
}
|
||||
if c.llm == nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("LLM client must not be nil")
|
||||
if c.engine == nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("native engine must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil")
|
||||
@@ -93,41 +78,52 @@ func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contrac
|
||||
if err := source.ValidateDocument(req.Source); err != nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err)
|
||||
}
|
||||
var response chunkResponse
|
||||
completion, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: shared.PromptInputs(req.SourceInput, req.References),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||
materials := shared.PromptInputs(req.SourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
plan, outcome, err := pipeline.RunProducer(ctx, c.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
|
||||
var response chunkResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, chunkerErrorf("decode native output: %w", err)
|
||||
}
|
||||
plan, err := planFromResponse(req.Source, response)
|
||||
if err != nil {
|
||||
return nil, []string{fmt.Sprintf("Revise the scene boundaries: %v. Use source-unit IDs from the supplied transcript and cover every unit in document order, without gaps or overlaps.", err)}, nil
|
||||
}
|
||||
return &plan, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
|
||||
plan := source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{
|
||||
StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
}}}
|
||||
if _, err := source.MaterializeChunkPlan(req.Source, plan); err != nil {
|
||||
return source.ChunkPlan{}, chunkerErrorf("materialize whole-source fallback: %w", err)
|
||||
}
|
||||
return plan, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("capture model candidate: %w", err)
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("produce scene plan: %w", err)
|
||||
}
|
||||
|
||||
plan, err := planFromResponse(req.Source, response)
|
||||
if err != nil {
|
||||
return contracts.ChunkPlanResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||
}
|
||||
return contracts.ChunkPlanResult{Plan: plan, ModelCandidate: candidate}, nil
|
||||
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +133,7 @@ func Register(registry *pipeline.ChunkerRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
return New(request.Dependencies.Engine, options)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -8,10 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNewModuleSpecAndRegister(t *testing.T) {
|
||||
@@ -22,18 +25,19 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
|
||||
}
|
||||
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks"},
|
||||
ReferenceSlots: wantReferenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks"},
|
||||
ReferenceSlots: wantReferenceSlots(),
|
||||
}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
got := ModuleSpec()
|
||||
got.Prompts[0] = "changed"
|
||||
got.Requires[0] = "changed"
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
@@ -52,7 +56,7 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
|
||||
if !reflect.DeepEqual(registered, want) {
|
||||
t.Fatalf("registered spec = %#v, want %#v", registered, want)
|
||||
}
|
||||
built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{LLM: client}})
|
||||
built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{Engine: chunker.engine}})
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
@@ -68,8 +72,8 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestConstructionRejectsMissingDependencyAndUnknownOptions(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
t.Fatalf("New(nil) error = %v, want LLM client error", err)
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v, want native engine error", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
|
||||
@@ -122,7 +126,10 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
|
||||
},
|
||||
}
|
||||
|
||||
result, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||
request := chunkRequest()
|
||||
var prepared promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Plan() error = %v, want nil", err)
|
||||
}
|
||||
@@ -130,34 +137,21 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
req := client.requests[0]
|
||||
if req.StageName != Key {
|
||||
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
|
||||
call := client.requests[0]
|
||||
if prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native prompt/profile/session not preserved")
|
||||
}
|
||||
if req.PromptID != PromptID || req.PromptVersion != ResponseSchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", req.PromptID, req.PromptVersion, PromptID, ResponseSchemaVersion)
|
||||
if result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible {
|
||||
t.Fatalf("plan not ordinarily accepted: %#v", result)
|
||||
}
|
||||
if req.SessionID != "session-123" || req.ProfileID != "profile-scenes" {
|
||||
t.Fatalf("session/profile = %q/%q, want session-123/profile-scenes", req.SessionID, req.ProfileID)
|
||||
var transcriptFound bool
|
||||
for _, message := range call.Prompt.Messages {
|
||||
if strings.Contains(message.Content, sceneTranscriptJSON) {
|
||||
transcriptFound = true
|
||||
}
|
||||
}
|
||||
transcript, ok := req.Inputs["transcript"]
|
||||
if !ok {
|
||||
t.Fatalf("transcript input missing from %#v", req.Inputs)
|
||||
}
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" {
|
||||
t.Fatalf("transcript metadata = %#v", transcript)
|
||||
}
|
||||
if got := string(transcript.Content); got != sceneTranscriptJSON {
|
||||
t.Fatalf("transcript content = %q, want original source input", got)
|
||||
}
|
||||
if got := string(req.Inputs["players"].Content); got != " " {
|
||||
t.Fatalf("players input = %q, want empty reference placeholder", got)
|
||||
}
|
||||
if got := string(req.Inputs["party"].Content); got != " " {
|
||||
t.Fatalf("party input = %q, want empty reference placeholder", got)
|
||||
}
|
||||
if got := string(req.Inputs["glossary"].Content); got != " " {
|
||||
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
|
||||
if !transcriptFound {
|
||||
t.Fatal("full source input missing from native prompt")
|
||||
}
|
||||
|
||||
if result.Plan.SourceDigest != "sha256:source" {
|
||||
@@ -179,13 +173,7 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
|
||||
if len(result.Plan.Annotations) != 0 {
|
||||
t.Fatalf("plan annotations = %#v, want absent", result.Plan.Annotations)
|
||||
}
|
||||
wantCandidate, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal expected candidate: %v", err)
|
||||
}
|
||||
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || !reflect.DeepEqual(result.ModelCandidate.Response, wantCandidate) {
|
||||
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) {
|
||||
@@ -223,7 +211,7 @@ func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) {
|
||||
scene(20, 20),
|
||||
}}}
|
||||
_, err := newChunker(t, client).Plan(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "appears after") {
|
||||
if err == nil {
|
||||
t.Fatalf("Plan() error = %v, want document-position reversal", err)
|
||||
}
|
||||
})
|
||||
@@ -262,19 +250,16 @@ func TestPlanPassesReferencesAsPromptInputs(t *testing.T) {
|
||||
if _, err := newChunker(t, client).Plan(context.Background(), req); err != nil {
|
||||
t.Fatalf("Plan() error = %v, want nil", err)
|
||||
}
|
||||
request := client.requests[0]
|
||||
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
|
||||
t.Fatalf("players input = %q, want reference content", got)
|
||||
var messages string
|
||||
for _, message := range client.requests[0].Prompt.Messages {
|
||||
messages += message.Content
|
||||
}
|
||||
if got := string(request.Inputs["party"].Content); got != "Aria: cleric" {
|
||||
t.Fatalf("party input = %q, want reference content", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" {
|
||||
t.Fatalf("glossary input = %q, want reference content", got)
|
||||
}
|
||||
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria: cleric") {
|
||||
t.Fatalf("transcript input contains reference content")
|
||||
for _, reference := range []string{"Alice: Aria", "Aria: cleric", "Brightmantle: local temple"} {
|
||||
if strings.Count(messages, reference) != 1 {
|
||||
t.Fatalf("reference not rendered once: %q", reference)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
|
||||
@@ -297,35 +282,6 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
|
||||
metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata()
|
||||
|
||||
tests := map[string]string{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": ResponseSchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": ResponseSchemaVersion,
|
||||
}
|
||||
for key, want := range tests {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||
value, ok := metadata[key].(string)
|
||||
if !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
|
||||
if _, ok := metadata[forbidden]; ok {
|
||||
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRejectsInvalidRequests(t *testing.T) {
|
||||
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||
validReq := chunkRequest()
|
||||
@@ -357,14 +313,14 @@ func TestPlanRejectsInvalidRequests(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Plan() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("Plan() error = %q, want module context and %q", err.Error(), tt.want)
|
||||
if !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("Plan() error = %v, want %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRejectsMalformedStructuredOutput(t *testing.T) {
|
||||
func TestPlanFromResponseRejectsMalformedSceneRanges(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response chunkResponse
|
||||
@@ -413,8 +369,7 @@ func TestPlanRejectsMalformedStructuredOutput(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := &fakeScenesLLMClient{response: tt.response}
|
||||
_, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||
_, err := planFromResponse(sceneSourceDocument(), tt.response)
|
||||
if err == nil {
|
||||
t.Fatal("Plan() error = nil, want error")
|
||||
}
|
||||
@@ -443,6 +398,13 @@ func chunkRequest() contracts.ChunkRequest {
|
||||
SourceInput: sceneSourceInput(),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-scenes",
|
||||
Operation: contracts.ProducerOperation[source.ChunkPlan]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass},
|
||||
ReuseEligible: true,
|
||||
Assess: func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,11 +414,33 @@ func sceneSourceInput() contracts.LLMInputMaterial {
|
||||
return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
|
||||
}
|
||||
|
||||
func newChunker(t *testing.T, client contracts.StructuredLLMClient) *Chunker {
|
||||
func newChunker(t *testing.T, client promptkit.LLMClient) *Chunker {
|
||||
t.Helper()
|
||||
chunker, err := New(client, Options{})
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("scene fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: profile-scenes
|
||||
endpoint: https://example.invalid/v1
|
||||
model: scene-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chunker, err := New(engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return chunker
|
||||
}
|
||||
@@ -513,47 +497,32 @@ func unitIDs(units []source.SourceUnit) []int {
|
||||
}
|
||||
|
||||
type fakeScenesLLMClient struct {
|
||||
response chunkResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response chunkResponse
|
||||
responses []string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeScenesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
|
||||
target, ok := out.(*chunkResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
req.Vars = cloneVars(req.Vars)
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneVars(in map[string]any) map[string]any {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
if index < len(client.responses) {
|
||||
content = []byte(client.responses[index])
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
21
internal/modules/dnd/chunk/scenes/model_test.go
Normal file
21
internal/modules/dnd/chunk/scenes/model_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) {
|
||||
raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`)
|
||||
|
||||
var response chunkResponse
|
||||
if err := json.Unmarshal(raw, &response); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if got := response.Scenes[0].StartUnitID.String(); got != "1" {
|
||||
t.Fatalf("StartUnitID = %q, want 1", got)
|
||||
}
|
||||
if got := response.Scenes[0].EndUnitID.String(); got != "3" {
|
||||
t.Fatalf("EndUnitID = %q, want 3", got)
|
||||
}
|
||||
}
|
||||
205
internal/modules/dnd/chunk/scenes/native_execution_test.go
Normal file
205
internal/modules/dnd/chunk/scenes/native_execution_test.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const wholeScene = `{"scenes":[{"start_unit_id":1,"end_unit_id":4}]}`
|
||||
const splitScenes = `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":4}]}`
|
||||
|
||||
func TestNativePlanAssessesEveryRevisionThroughTheFinalTurn(t *testing.T) {
|
||||
client := &fakeScenesLLMClient{responses: []string{wholeScene, wholeScene, splitScenes}}
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.MaxAdditionalTurns = 2
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "scene-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
chunks, err := source.MaterializeChunkPlan(request.Source, plan)
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
assessments++
|
||||
findings := []contracts.ValidatorAssessment{
|
||||
{ValidatorName: "coverage", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
|
||||
{ValidatorName: "scene-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
|
||||
}
|
||||
if len(chunks) != 2 {
|
||||
findings[1].Decision = contracts.AssessmentRejected
|
||||
findings[1].Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Separate the negotiation from the combat."}
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: findings}, nil
|
||||
}
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
|
||||
len(result.Plan.Ranges) != 2 || assessments != 3 || len(client.requests) != 3 || len(prepared) != 3 {
|
||||
t.Fatalf("final-turn result=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
wantID := PromptID
|
||||
if i > 0 {
|
||||
wantID = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile ||
|
||||
call.Target.Model != "scene-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native run selections lost")
|
||||
}
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
old, revised := client.requests[i-1].Prompt.Messages, call.Prompt.Messages
|
||||
if len(revised) != len(old)+2 || !reflect.DeepEqual(old, revised[:len(old)]) ||
|
||||
revised[len(old)].Role != "assistant" || revised[len(old)].Content != client.responses[i-1] {
|
||||
t.Fatal("native continuation changed source context/history")
|
||||
}
|
||||
guidance := revised[len(old)+1].Content
|
||||
if !strings.Contains(guidance, "Separate the negotiation") || strings.Contains(guidance, "operator-only-rejection") {
|
||||
t.Fatal("correction did not separate actionable findings from operator diagnostics")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativePlanCanCorrectAnUnresolvableBoundary(t *testing.T) {
|
||||
client := &fakeScenesLLMClient{responses: []string{
|
||||
`{"scenes":[{"start_unit_id":1,"end_unit_id":999}]}`, splitScenes,
|
||||
}}
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
_, err := source.MaterializeChunkPlan(request.Source, plan)
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
|
||||
t.Fatalf("boundary correction=%#v %v, assessments=%d calls=%d", result, err, assessments, len(client.requests))
|
||||
}
|
||||
messages := client.requests[1].Prompt.Messages
|
||||
if !strings.Contains(messages[len(messages)-1].Content, "999") {
|
||||
t.Fatal("correction omitted the unresolved boundary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanExhaustionUsesOnlyAnExplicitWholeSourceFallback(t *testing.T) {
|
||||
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{scene(1, 2), scene(3, 4)}}}
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.Mode = mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{CorrectionGuidance: "Reconsider the scene transition."},
|
||||
}}}, nil
|
||||
}
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if assessments != 2 || len(client.requests) != 2 {
|
||||
t.Fatal("revision was omitted, restarted, or reassessed")
|
||||
}
|
||||
if mode == contracts.ModeStrict {
|
||||
if err == nil || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("strict rejection=%#v %v", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 4}}}
|
||||
if err != nil || !reflect.DeepEqual(result.Plan, want) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
|
||||
t.Fatalf("whole-source fallback=%#v %v", result, err)
|
||||
}
|
||||
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
|
||||
if err != nil || len(chunks) != 1 || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{1, 2, 3, 4}) ||
|
||||
len(chunks[0].Annotations) != 0 || len(chunks[0].PlanAnnotations) != 0 {
|
||||
t.Fatalf("fallback did not materialize the unannotated complete source: %#v %v", chunks, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativePlanFailureUsesDocumentOrderWithoutEarlierPlanRecovery(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, failedCorrection := range []bool{false, true} {
|
||||
client := &fakeScenesLLMClient{err: failure}
|
||||
request := chunkRequest()
|
||||
request.Source.Units[0].ID, request.Source.Units[1].ID = 30, 10
|
||||
for i := range request.Source.Units {
|
||||
unit := &request.Source.Units[i]
|
||||
unit.Ref.StartUnitID, unit.Ref.EndUnitID = unit.ID, unit.ID
|
||||
}
|
||||
if failedCorrection {
|
||||
client.err = nil
|
||||
client.responses = []string{`{"scenes":[{"start_unit_id":30,"end_unit_id":10},{"start_unit_id":3,"end_unit_id":4}]}`}
|
||||
client.errors = []error{nil, failure}
|
||||
}
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
|
||||
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the encounter together."},
|
||||
}}}, nil
|
||||
}
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible ||
|
||||
!reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 30, EndUnitID: 4}}) {
|
||||
t.Fatalf("failure fallback=%#v %v", result, err)
|
||||
}
|
||||
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
|
||||
if err != nil || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{30, 10, 3, 4}) {
|
||||
t.Fatalf("nonconsecutive source order lost: %#v %v", chunks, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeStructuralFailureSkipsSemanticAssessment(t *testing.T) {
|
||||
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{}}}
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 2
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newChunker(t, client).Plan(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || assessments != 0 {
|
||||
t.Fatalf("structural failure=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationAndMaterializationFailureDoNotPublishFallback(t *testing.T) {
|
||||
t.Run("canceled native delivery", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &fakeScenesLLMClient{response: validSceneResponse(), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
result, err := newChunker(t, client).Plan(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
})
|
||||
t.Run("unmaterializable source", func(t *testing.T) {
|
||||
request := chunkRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Source.Units[0].Metadata = map[string]any{"invalid": math.NaN()}
|
||||
failure := errors.New("provider unavailable")
|
||||
result, err := newChunker(t, &fakeScenesLLMClient{err: failure}).Plan(context.Background(), request)
|
||||
if err == nil || result.Outcome.Disposition != "" || !errors.Is(err, failure) {
|
||||
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.scenes",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-transcript-full.md",
|
||||
"common-dnd-references.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/scenes")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope scene assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare scene prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope scene schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,138 +0,0 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPromptAssetsPrepareTranscriptWithInstructionOrdering(t *testing.T) {
|
||||
if !slices.Contains(promptAssetManifest.SharedFiles, "common-dnd-transcript-full.md") {
|
||||
t.Fatalf("shared prompt assets = %#v, want complete-session transcript scope", promptAssetManifest.SharedFiles)
|
||||
}
|
||||
transcript := []byte(`{"sentinel":"scene-transcript"}`)
|
||||
prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary")
|
||||
|
||||
if prepared.PromptID != PromptID {
|
||||
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "dnd_scenes_llm.v1.json" {
|
||||
t.Fatalf("scene output schema path = %q, want private LLM schema", prepared.OutputContract.SchemaPath)
|
||||
}
|
||||
if prepared.Messages[0].Role != "system" || prepared.Messages[1].Role != "user" {
|
||||
t.Fatalf("initial prompt roles = %#v, want system followed by campaign references", prepared.Messages[:2])
|
||||
}
|
||||
if cache := prepared.Messages[1].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("campaign-reference cache control = %#v, want ephemeral", cache)
|
||||
}
|
||||
if cache := prepared.Messages[0].CacheControl; cache != nil {
|
||||
t.Fatalf("system cache control = %#v, want nil", cache)
|
||||
}
|
||||
if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") {
|
||||
t.Fatalf("reference message = %q, want supplied reference inputs", references)
|
||||
}
|
||||
transcriptIndex := -1
|
||||
for index, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, "scene-transcript") {
|
||||
transcriptIndex = index
|
||||
}
|
||||
}
|
||||
if transcriptIndex != len(prepared.Messages)-1 {
|
||||
t.Fatalf("transcript message index = %d, want final message in %#v", transcriptIndex, prepared.Messages)
|
||||
}
|
||||
if cache := prepared.Messages[transcriptIndex].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("transcript cache control = %#v, want ephemeral", cache)
|
||||
}
|
||||
instructionsIndex := transcriptIndex - 1
|
||||
if instructionsIndex <= 1 || prepared.Messages[instructionsIndex].CacheControl != nil {
|
||||
t.Fatalf("instruction message = %#v, want uncached message after references and before transcript", prepared.Messages[instructionsIndex])
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) {
|
||||
t.Errorf("message %d unexpectedly rendered reference input", index)
|
||||
}
|
||||
if index != transcriptIndex && strings.Contains(message.Content, "scene-transcript") {
|
||||
t.Errorf("message %d unexpectedly rendered transcript input", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetDiagnosticsOmitRawMaterials(t *testing.T) {
|
||||
transcript := []byte(`{"secret":"source text"}`)
|
||||
prepared := prepareScenesPrompt(t, transcript, "private player note", "private party note", "private glossary note")
|
||||
metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata()
|
||||
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"prepared": map[string]any{
|
||||
"prompt_id": prepared.PromptID,
|
||||
"prompt_version": prepared.PromptVersion,
|
||||
"prompt_hash": prepared.PromptHash,
|
||||
"rendered_prompt_hash": prepared.RenderedPromptHash,
|
||||
"selected_profile_id": prepared.SelectedProfileID,
|
||||
"output_contract": prepared.OutputContract,
|
||||
"input_hashes": prepared.InputHashes,
|
||||
"effective_model_params": prepared.EffectiveModelParams,
|
||||
},
|
||||
"manifest": metadata,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal diagnostics: %v", err)
|
||||
}
|
||||
diagnostics := string(payload)
|
||||
for _, forbidden := range []string{
|
||||
"source text",
|
||||
"private player note",
|
||||
"private party note",
|
||||
"private glossary note",
|
||||
`"properties"`,
|
||||
"start_unit_id",
|
||||
} {
|
||||
if strings.Contains(diagnostics, forbidden) {
|
||||
t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics)
|
||||
}
|
||||
}
|
||||
if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != ResponseSchemaVersion {
|
||||
t.Fatalf("manifest prompt metadata = %#v", metadata)
|
||||
}
|
||||
if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") {
|
||||
t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"])
|
||||
}
|
||||
}
|
||||
|
||||
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register scene prompt assets: %v", err)
|
||||
}
|
||||
engine := newScenesPromptEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: "scene-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"players": promptkit.Inline(players),
|
||||
"party": promptkit.Inline(party),
|
||||
"glossary": promptkit.Inline(glossary),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"),
|
||||
})
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package scenes
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.scenes"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes_llm")
|
||||
ResponseSchemaID = "notarius.dnd.scenes.llm"
|
||||
ResponseSchemaVersion = "v1"
|
||||
ResponseSchemaName = "notarius_dnd_scenes_llm_v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: ResponseSchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_scenes_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaForScenes(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if got, want := string(schema.Key), "dnd_scenes_llm"; got != want {
|
||||
t.Fatalf("schema.Key = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := schema.ID, "notarius.dnd.scenes.llm"; got != want {
|
||||
t.Fatalf("schema.ID = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := schema.Version, "v1"; got != want {
|
||||
t.Fatalf("schema.Version = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := schema.Name, "notarius_dnd_scenes_llm_v1"; got != want {
|
||||
t.Fatalf("schema.Name = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.HasPrefix(schema.SHA256, "sha256:") {
|
||||
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
|
||||
}
|
||||
if !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
valid := validSceneSchemaResponse()
|
||||
validJSON, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err)
|
||||
}
|
||||
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid scene response rejected: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(map[string]any)
|
||||
}{
|
||||
{
|
||||
name: "missing scenes",
|
||||
mutate: func(response map[string]any) {
|
||||
delete(response, "scenes")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty scenes",
|
||||
mutate: func(response map[string]any) {
|
||||
response["scenes"] = []any{}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-positive start unit",
|
||||
mutate: func(response map[string]any) {
|
||||
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-positive end unit",
|
||||
mutate: func(response map[string]any) {
|
||||
response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-integer endpoint",
|
||||
mutate: func(response map[string]any) {
|
||||
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown top-level field",
|
||||
mutate: func(response map[string]any) {
|
||||
response["boundary_caveats"] = []any{}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown scene field",
|
||||
mutate: func(response map[string]any) {
|
||||
response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract"
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
response := validSceneSchemaResponse()
|
||||
tt.mutate(response)
|
||||
content, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||
t.Fatal("validateJSONSchema() error = nil, want rejected response")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) {
|
||||
raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`)
|
||||
|
||||
var response chunkResponse
|
||||
if err := json.Unmarshal(raw, &response); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if got := response.Scenes[0].StartUnitID.String(); got != "1" {
|
||||
t.Fatalf("StartUnitID = %q, want 1", got)
|
||||
}
|
||||
if got := response.Scenes[0].EndUnitID.String(); got != "3" {
|
||||
t.Fatalf("EndUnitID = %q, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if !json.Valid(second.JSONSchema) {
|
||||
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
|
||||
}
|
||||
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
|
||||
t.Fatal("schema JSON did not use defensive copy")
|
||||
}
|
||||
}
|
||||
|
||||
func validSceneSchemaResponse() map[string]any {
|
||||
return map[string]any{
|
||||
"scenes": []any{
|
||||
map[string]any{
|
||||
"start_unit_id": 1,
|
||||
"end_unit_id": 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse instance: %w", err)
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse schema: %w", err)
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return fmt.Errorf("load schema: %w", err)
|
||||
}
|
||||
schema, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("compile schema: %w", err)
|
||||
}
|
||||
return schema.Validate(instance)
|
||||
}
|
||||
Reference in New Issue
Block a user