Migrate scene planning to native continuation and whole-source fallback

This commit is contained in:
2026-11-22 00:34:33 +00:00
parent 4f82d01534
commit 52a1c96327
13 changed files with 721 additions and 599 deletions

View File

@@ -0,0 +1,8 @@
Revise the previous scene-boundary plan using these findings:
{{ input "guidance" }}
Return a complete replacement scene plan for the original full transcript, not
only the changed ranges. Keep the original source-unit IDs and cover every unit
in document order without gaps or overlaps. Return only JSON matching the original
response contract.

View File

@@ -21,7 +21,7 @@ prompts:
content_file: ../../shared/prompts/common-dnd-references.md
cache_control:
type: ephemeral
- role: user
- role: system
content_file: ./instructions.md
- role: user
content_file: ../../shared/prompts/common-dnd-transcript-full.md
@@ -31,3 +31,17 @@ prompts:
format: json
validation_mode: json_schema
schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json
- id: dnd.scenes.repair
default_profile: notarius/dnd-extraction
inputs:
- name: guidance
required: true
content_type: text/plain
messages:
- role: user
content_file: ./correction.md
output:
format: json
validation_mode: json_schema
schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json

184
assets/scenes_test.go Normal file
View File

@@ -0,0 +1,184 @@
package assets_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"github.com/santhosh-tekuri/jsonschema/v6"
"go.maximumdirect.net/promptkit"
)
func TestScenePromptPairPreservesFullTranscriptAndContext(t *testing.T) {
_, engine := loadNativeAssets(t)
const transcript = `{"sentinel":"scene-transcript"}`
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scenes", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(transcript),
"players": promptkit.Inline("scene-players"),
"party": promptkit.Inline("scene-party"),
"glossary": promptkit.Inline("scene-glossary"),
},
})
if err != nil {
t.Fatal(err)
}
defer initial.Discard()
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scenes.repair", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep the encounter together.")},
})
if err != nil {
t.Fatal(err)
}
defer correction.Discard()
first, next := initial.Details(), correction.Details()
if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID ||
first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
t.Fatal("scene correction pair lost its shared native defaults/output contract")
}
var system, all string
transcriptIndex := -1
for i, message := range first.Messages {
all += message.Content
if message.Role == "system" {
system += message.Content
}
if strings.Contains(message.Content, transcript) {
transcriptIndex = i
}
if strings.Contains(message.Content, transcript) || strings.Contains(message.Content, "scene-players") {
if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral {
t.Fatal("reference/full-transcript native cache hint lost")
}
}
}
if transcriptIndex != len(first.Messages)-1 || !strings.Contains(system, "no gaps or overlaps") || !strings.Contains(system, "coherent unit of play") {
t.Fatal("scene principles or complete final transcript lost")
}
for _, content := range []string{transcript, "scene-players", "scene-party", "scene-glossary"} {
if strings.Count(all, content) != 1 {
t.Fatalf("fixed context not rendered once: %q", content)
}
}
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Keep the encounter together.") ||
strings.Contains(next.Messages[0].Content, "scene-transcript") {
t.Fatal("correction should supply findings and a replacement assignment only")
}
}
func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scenes/schemas/dnd_scenes_llm.v1.json")
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
valid := validSceneSchemaResponse()
validJSON, err := json.Marshal(valid)
if err != nil {
t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err)
}
if err := validateJSONSchema(validJSON, schema); err != nil {
t.Fatalf("valid scene response rejected: %v", err)
}
tests := []struct {
name string
mutate func(map[string]any)
}{
{
name: "missing scenes",
mutate: func(response map[string]any) {
delete(response, "scenes")
},
},
{
name: "empty scenes",
mutate: func(response map[string]any) {
response["scenes"] = []any{}
},
},
{
name: "non-positive start unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0
},
},
{
name: "non-positive end unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0
},
},
{
name: "non-integer endpoint",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5
},
},
{
name: "unknown top-level field",
mutate: func(response map[string]any) {
response["boundary_caveats"] = []any{}
},
},
{
name: "unknown scene field",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
response := validSceneSchemaResponse()
tt.mutate(response)
content, err := json.Marshal(response)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
if err := validateJSONSchema(content, schema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want rejected response")
}
})
}
}
func validSceneSchemaResponse() map[string]any {
return map[string]any{
"scenes": []any{
map[string]any{
"start_unit_id": 1,
"end_unit_id": 3,
},
},
}
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return fmt.Errorf("parse instance: %w", err)
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return fmt.Errorf("parse schema: %w", err)
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return fmt.Errorf("load schema: %w", err)
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return fmt.Errorf("compile schema: %w", err)
}
return schema.Validate(instance)
}

View File

@@ -717,6 +717,29 @@ provider fakes and retain domain identity/reconciliation regression coverage.
## Stage 16: Migrate D&D Scene Planning And Chunk Assessment
**Status:** Complete. Scene planning now prepares the full transcript/reference
inputs once and uses the shared native producer helper with initial/correction
prompts, matching schema/profile defaults, and static dependencies. Enduring
scene principles remain in initial system context. Removed the private completion,
response-schema/asset registration, and manifest metadata paths. Existing
source-order, coverage, gap/overlap, and boundary interpretation rules remain.
Unresolvable boundaries supply actionable correction guidance; unusable native
structural results never become semantic candidates. Permissive terminal handling
constructs one unannotated whole-source plan using normal materialization; invalid
input, cancellation, and fallback-construction failures do not publish a fallback.
The runner already binds materialization and complete chunk assessment to the
native operation without an outer attempt/replay loop. Added a runnable runner
contract proving ordered full-chain assessment through final-turn acceptance,
strict/permissive exhaustion, no fallback revalidation, and derived reuse exclusion.
Migrated scene regressions to a native provider fake and added correction/history,
override, structural-failure, whole-source fallback, and cancellation cases.
Scene prompt-pair/schema checks run in the asset package.
Asset/source tests and vet/race checks, focused pipeline tests and vet/race checks,
and `git diff --check` pass. Scene-chunker package tests remain blocked at setup
by shared legacy helpers importing old ConfigKit's removed YAML `plugin/limit`;
their runtime results are not claimed passing. Scheduled dependency cleanup and
final verification must execute them. Chunk-cache replacement remains Stage 28.
**Goal:** one native chunk producer with deterministic materialization.
**Inspect/change:** `internal/modules/dnd/chunk/scenes`, chunk-plan interpretation/

View File

@@ -0,0 +1,116 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type nativePlanChunker struct{ engine *promptkit.Engine }
type namedPlanValidator struct {
name string
chunkAssessmentValidator
}
func (v namedPlanValidator) Name() string { return v.name }
func (nativePlanChunker) Key() string { return "native-plan" }
func (nativePlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c nativePlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
value, _, err := interpretProducerNumber(ctx, result)
if err != nil {
return nil, nil, err
}
plan := typedTestPlan(request.Source)
plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/scene-policy": json.RawMessage(fmt.Sprintf("%d", *value))}
return &plan, nil, nil
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
return typedTestPlan(request.Source), nil
})
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
}
func TestRunnerMaterializesAndFullyAssessesEachNativePlanRevision(t *testing.T) {
for _, test := range []struct {
name string
mode contracts.ExecutionMode
turns int
wantError, wantFallback bool
}{
{"final-turn acceptance", contracts.ModeStrict, 2, false, false},
{"strict exhaustion", contracts.ModeStrict, 1, true, false},
{"permissive exhaustion", contracts.ModePermissive, 1, false, true},
} {
t.Run(test.name, func(t *testing.T) {
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
value := calls
calls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "native-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked},
rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
return nativePlanChunker{request.Dependencies.Engine}, nil
}); err != nil {
t.Fatal(err)
}
var assessed []string
for _, name := range []string{"first-plan-policy", "second-plan-policy"} {
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.ChunkValidator, error) {
return namedPlanValidator{name, chunkAssessmentValidator(func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
if len(request.Chunks) != 1 || len(request.Chunks[0].Units) != len(request.Source.Units) || request.Chunks[0].Ref.SourceID != request.Source.ID {
t.Fatal("validator did not receive normally materialized chunks")
}
value := string(request.Chunks[0].Annotations["test/scene-policy"])
assessed = append(assessed, name+"/"+value)
if value != "2" {
return contracts.ValidationResult{ReasonCode: "scene_policy", CorrectionGuidance: "Reconsider the scene transition."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
})}, nil
}); err != nil {
t.Fatal(err)
}
}
probe := newRunnerProbe()
profile := typedResolutionProfile()
profile.Chunk = Binding("native-plan")
profile.Chunk.Mode, profile.Chunk.MaxAdditionalTurns = &test.mode, &test.turns
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first-plan-policy"), Binding("second-plan-policy")}}
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input")})
if (err != nil) != test.wantError || calls != test.turns+1 {
t.Fatalf("native plan output=%#v error=%v calls=%d", output, err, calls)
}
var want []string
for i := 0; i <= test.turns; i++ {
want = append(want, fmt.Sprintf("first-plan-policy/%d", i), fmt.Sprintf("second-plan-policy/%d", i))
}
if !reflect.DeepEqual(assessed, want) {
t.Fatalf("ordered plan assessments=%v, want %v", assessed, want)
}
if test.wantError {
return
}
if len(output.NormalizeOutputs) != 1 || (output.FallbackCount == 1) != test.wantFallback {
t.Fatalf("plan disposition=%#v", output)
}
for _, role := range []string{"extract", "merge", "normalize"} {
if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{!test.wantFallback}) {
t.Fatalf("derived %s eligibility=%v", role, probe.eligible)
}
}
})
}
}

View File

@@ -2,7 +2,11 @@ package scenes
import (
"context"
"encoding/json"
"fmt"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -11,6 +15,8 @@ import (
)
const Key = "dnd/scenes"
const PromptID = "notarius/dnd.scenes"
const CorrectionPromptID = "notarius/dnd.scenes.repair"
var requiredCapabilities = []string{
"source.transcript",
@@ -28,19 +34,18 @@ var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
}
var _ contracts.Chunker = (*Chunker)(nil)
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
type Options struct{}
type Chunker struct {
llm contracts.StructuredLLMClient
engine *promptkit.Engine
}
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Chunker, error) {
if llmClient == nil {
return nil, chunkerErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options) (*Chunker, error) {
if engine == nil {
return nil, chunkerErrorf("native engine must not be nil")
}
return &Chunker{llm: llmClient}, nil
return &Chunker{engine: engine}, nil
}
func (c *Chunker) Key() string {
@@ -51,32 +56,12 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return shared.ReferenceSlots(referenceSlotDescriptions)
}
func (c *Chunker) ManifestMetadata() map[string]any {
promptSHA, err := promptAssetMetadata()
if err != nil {
promptSHA = ""
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"prompt_sha256": promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
}
if schema, err := loadResponseSchema(); err == nil {
metadata["response_schema_version"] = schema.Version
metadata["response_schema_sha256"] = schema.SHA256
}
return metadata
}
func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
if c == nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil")
}
if c.llm == nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("LLM client must not be nil")
if c.engine == nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("native engine must not be nil")
}
if ctx == nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil")
@@ -93,41 +78,52 @@ func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contrac
if err := source.ValidateDocument(req.Source); err != nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err)
}
var response chunkResponse
completion, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: ResponseSchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: shared.PromptInputs(req.SourceInput, req.References),
}, &response)
if err != nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("complete structured output: %w", err)
materials := shared.PromptInputs(req.SourceInput, req.References)
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
plan, outcome, err := pipeline.RunProducer(ctx, c.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
var response chunkResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, chunkerErrorf("decode native output: %w", err)
}
plan, err := planFromResponse(req.Source, response)
if err != nil {
return nil, []string{fmt.Sprintf("Revise the scene boundaries: %v. Use source-unit IDs from the supplied transcript and cover every unit in document order, without gaps or overlaps.", err)}, nil
}
return &plan, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
plan := source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{
StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
}}}
if _, err := source.MaterializeChunkPlan(req.Source, plan); err != nil {
return source.ChunkPlan{}, chunkerErrorf("materialize whole-source fallback: %w", err)
}
return plan, nil
})
if err != nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("capture model candidate: %w", err)
return contracts.ChunkPlanResult{}, chunkerErrorf("produce scene plan: %w", err)
}
plan, err := planFromResponse(req.Source, response)
if err != nil {
return contracts.ChunkPlanResult{}, chunkerErrorf("malformed structured output: %w", err)
}
return contracts.ChunkPlanResult{Plan: plan, ModelCandidate: candidate}, nil
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions),
Key: Key,
Stage: pipeline.StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions),
}
}
@@ -137,7 +133,7 @@ func Register(registry *pipeline.ChunkerRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options)
return New(request.Dependencies.Engine, options)
})
}

View File

@@ -8,10 +8,13 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestNewModuleSpecAndRegister(t *testing.T) {
@@ -22,18 +25,19 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: []string{"source.transcript"},
Provides: []string{"chunks"},
ReferenceSlots: wantReferenceSlots(),
Key: Key,
Stage: pipeline.StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: []string{"source.transcript"},
Provides: []string{"chunks"},
ReferenceSlots: wantReferenceSlots(),
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got := ModuleSpec()
got.Prompts[0] = "changed"
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
@@ -52,7 +56,7 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
if !reflect.DeepEqual(registered, want) {
t.Fatalf("registered spec = %#v, want %#v", registered, want)
}
built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{LLM: client}})
built, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Dependencies: pipeline.ModuleDependencies{Engine: chunker.engine}})
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
@@ -68,8 +72,8 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
}
func TestConstructionRejectsMissingDependencyAndUnknownOptions(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
t.Fatalf("New(nil) error = %v, want LLM client error", err)
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v, want native engine error", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
@@ -122,7 +126,10 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
},
}
result, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
request := chunkRequest()
var prepared promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
result, err := newChunker(t, client).Plan(context.Background(), request)
if err != nil {
t.Fatalf("Plan() error = %v, want nil", err)
}
@@ -130,34 +137,21 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
req := client.requests[0]
if req.StageName != Key {
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
call := client.requests[0]
if prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || call.Prompt.SessionID != request.SessionID {
t.Fatal("native prompt/profile/session not preserved")
}
if req.PromptID != PromptID || req.PromptVersion != ResponseSchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", req.PromptID, req.PromptVersion, PromptID, ResponseSchemaVersion)
if result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible {
t.Fatalf("plan not ordinarily accepted: %#v", result)
}
if req.SessionID != "session-123" || req.ProfileID != "profile-scenes" {
t.Fatalf("session/profile = %q/%q, want session-123/profile-scenes", req.SessionID, req.ProfileID)
var transcriptFound bool
for _, message := range call.Prompt.Messages {
if strings.Contains(message.Content, sceneTranscriptJSON) {
transcriptFound = true
}
}
transcript, ok := req.Inputs["transcript"]
if !ok {
t.Fatalf("transcript input missing from %#v", req.Inputs)
}
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" {
t.Fatalf("transcript metadata = %#v", transcript)
}
if got := string(transcript.Content); got != sceneTranscriptJSON {
t.Fatalf("transcript content = %q, want original source input", got)
}
if got := string(req.Inputs["players"].Content); got != " " {
t.Fatalf("players input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["party"].Content); got != " " {
t.Fatalf("party input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["glossary"].Content); got != " " {
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
if !transcriptFound {
t.Fatal("full source input missing from native prompt")
}
if result.Plan.SourceDigest != "sha256:source" {
@@ -179,13 +173,7 @@ func TestPlanReturnsAnnotationFreeSceneRangesFromStructuredOutput(t *testing.T)
if len(result.Plan.Annotations) != 0 {
t.Fatalf("plan annotations = %#v, want absent", result.Plan.Annotations)
}
wantCandidate, err := json.Marshal(client.response)
if err != nil {
t.Fatalf("marshal expected candidate: %v", err)
}
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || !reflect.DeepEqual(result.ModelCandidate.Response, wantCandidate) {
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
}
}
func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) {
@@ -223,7 +211,7 @@ func TestPlanUsesDocumentOrderForNonconsecutiveUnitIDs(t *testing.T) {
scene(20, 20),
}}}
_, err := newChunker(t, client).Plan(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "appears after") {
if err == nil {
t.Fatalf("Plan() error = %v, want document-position reversal", err)
}
})
@@ -262,19 +250,16 @@ func TestPlanPassesReferencesAsPromptInputs(t *testing.T) {
if _, err := newChunker(t, client).Plan(context.Background(), req); err != nil {
t.Fatalf("Plan() error = %v, want nil", err)
}
request := client.requests[0]
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
t.Fatalf("players input = %q, want reference content", got)
var messages string
for _, message := range client.requests[0].Prompt.Messages {
messages += message.Content
}
if got := string(request.Inputs["party"].Content); got != "Aria: cleric" {
t.Fatalf("party input = %q, want reference content", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" {
t.Fatalf("glossary input = %q, want reference content", got)
}
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria: cleric") {
t.Fatalf("transcript input contains reference content")
for _, reference := range []string{"Alice: Aria", "Aria: cleric", "Brightmantle: local temple"} {
if strings.Count(messages, reference) != 1 {
t.Fatalf("reference not rendered once: %q", reference)
}
}
}
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
@@ -297,35 +282,6 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
}
}
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": ResponseSchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
if _, ok := metadata[forbidden]; ok {
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
}
}
}
func TestPlanRejectsInvalidRequests(t *testing.T) {
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
validReq := chunkRequest()
@@ -357,14 +313,14 @@ func TestPlanRejectsInvalidRequests(t *testing.T) {
if err == nil {
t.Fatal("Plan() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Plan() error = %q, want module context and %q", err.Error(), tt.want)
if !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Plan() error = %v, want %q", err, tt.want)
}
})
}
}
func TestPlanRejectsMalformedStructuredOutput(t *testing.T) {
func TestPlanFromResponseRejectsMalformedSceneRanges(t *testing.T) {
tests := []struct {
name string
response chunkResponse
@@ -413,8 +369,7 @@ func TestPlanRejectsMalformedStructuredOutput(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &fakeScenesLLMClient{response: tt.response}
_, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
_, err := planFromResponse(sceneSourceDocument(), tt.response)
if err == nil {
t.Fatal("Plan() error = nil, want error")
}
@@ -443,6 +398,13 @@ func chunkRequest() contracts.ChunkRequest {
SourceInput: sceneSourceInput(),
SessionID: "session-123",
LLMProfile: "profile-scenes",
Operation: contracts.ProducerOperation[source.ChunkPlan]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass},
ReuseEligible: true,
Assess: func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
},
},
}
}
@@ -452,11 +414,33 @@ func sceneSourceInput() contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", "application/json", []byte(sceneTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
}
func newChunker(t *testing.T, client contracts.StructuredLLMClient) *Chunker {
func newChunker(t *testing.T, client promptkit.LLMClient) *Chunker {
t.Helper()
chunker, err := New(client, Options{})
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("scene fixture", []byte(`promptkit:
profiles:
- id: profile-scenes
endpoint: https://example.invalid/v1
model: scene-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
chunker, err := New(engine, Options{})
if err != nil {
t.Fatal(err)
}
return chunker
}
@@ -513,47 +497,32 @@ func unitIDs(units []source.SourceUnit) []int {
}
type fakeScenesLLMClient struct {
response chunkResponse
err error
requests []contracts.StructuredCompletionRequest
response chunkResponse
responses []string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeScenesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*chunkResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
}
req.Correction = correction
req.Vars = cloneVars(req.Vars)
return req
}
func cloneVars(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
}
out := make(map[string]any, len(in))
for key, value := range in {
out[key] = value
}
return out
if index < len(client.responses) {
content = []byte(client.responses[index])
}
return &promptkit.GenerateResponse{Content: string(content)}, nil
}

View File

@@ -0,0 +1,21 @@
package scenes
import (
"encoding/json"
"testing"
)
func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) {
raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`)
var response chunkResponse
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if got := response.Scenes[0].StartUnitID.String(); got != "1" {
t.Fatalf("StartUnitID = %q, want 1", got)
}
if got := response.Scenes[0].EndUnitID.String(); got != "3" {
t.Fatalf("EndUnitID = %q, want 3", got)
}
}

View File

@@ -0,0 +1,205 @@
package scenes
import (
"context"
"errors"
"math"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
const wholeScene = `{"scenes":[{"start_unit_id":1,"end_unit_id":4}]}`
const splitScenes = `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":4}]}`
func TestNativePlanAssessesEveryRevisionThroughTheFinalTurn(t *testing.T) {
client := &fakeScenesLLMClient{responses: []string{wholeScene, wholeScene, splitScenes}}
request := chunkRequest()
request.Operation.Controls.MaxAdditionalTurns = 2
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "scene-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
assessments := 0
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
chunks, err := source.MaterializeChunkPlan(request.Source, plan)
if err != nil {
return contracts.CandidateAssessment{}, err
}
assessments++
findings := []contracts.ValidatorAssessment{
{ValidatorName: "coverage", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
{ValidatorName: "scene-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}},
}
if len(chunks) != 2 {
findings[1].Decision = contracts.AssessmentRejected
findings[1].Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Separate the negotiation from the combat."}
}
return contracts.CandidateAssessment{Validators: findings}, nil
}
result, err := newChunker(t, client).Plan(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
len(result.Plan.Ranges) != 2 || assessments != 3 || len(client.requests) != 3 || len(prepared) != 3 {
t.Fatalf("final-turn result=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
}
for i, call := range client.requests {
wantID := PromptID
if i > 0 {
wantID = CorrectionPromptID
}
if prepared[i].PromptID != wantID || prepared[i].SelectedProfileID != request.LLMProfile ||
call.Target.Model != "scene-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native run selections lost")
}
if i == 0 {
continue
}
old, revised := client.requests[i-1].Prompt.Messages, call.Prompt.Messages
if len(revised) != len(old)+2 || !reflect.DeepEqual(old, revised[:len(old)]) ||
revised[len(old)].Role != "assistant" || revised[len(old)].Content != client.responses[i-1] {
t.Fatal("native continuation changed source context/history")
}
guidance := revised[len(old)+1].Content
if !strings.Contains(guidance, "Separate the negotiation") || strings.Contains(guidance, "operator-only-rejection") {
t.Fatal("correction did not separate actionable findings from operator diagnostics")
}
}
}
func TestNativePlanCanCorrectAnUnresolvableBoundary(t *testing.T) {
client := &fakeScenesLLMClient{responses: []string{
`{"scenes":[{"start_unit_id":1,"end_unit_id":999}]}`, splitScenes,
}}
request := chunkRequest()
request.Operation.Controls.MaxAdditionalTurns = 1
assessments := 0
request.Operation.Assess = func(_ context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
assessments++
_, err := source.MaterializeChunkPlan(request.Source, plan)
return contracts.CandidateAssessment{}, err
}
result, err := newChunker(t, client).Plan(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
t.Fatalf("boundary correction=%#v %v, assessments=%d calls=%d", result, err, assessments, len(client.requests))
}
messages := client.requests[1].Prompt.Messages
if !strings.Contains(messages[len(messages)-1].Content, "999") {
t.Fatal("correction omitted the unresolved boundary")
}
}
func TestPlanExhaustionUsesOnlyAnExplicitWholeSourceFallback(t *testing.T) {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
t.Run(string(mode), func(t *testing.T) {
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{scene(1, 2), scene(3, 4)}}}
request := chunkRequest()
request.Operation.Controls.Mode = mode
request.Operation.Controls.MaxAdditionalTurns = 1
assessments := 0
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
assessments++
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
Result: contracts.ValidationResult{CorrectionGuidance: "Reconsider the scene transition."},
}}}, nil
}
result, err := newChunker(t, client).Plan(context.Background(), request)
if assessments != 2 || len(client.requests) != 2 {
t.Fatal("revision was omitted, restarted, or reassessed")
}
if mode == contracts.ModeStrict {
if err == nil || result.Outcome.Disposition != "" {
t.Fatalf("strict rejection=%#v %v", result, err)
}
return
}
want := source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 4}}}
if err != nil || !reflect.DeepEqual(result.Plan, want) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible {
t.Fatalf("whole-source fallback=%#v %v", result, err)
}
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
if err != nil || len(chunks) != 1 || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{1, 2, 3, 4}) ||
len(chunks[0].Annotations) != 0 || len(chunks[0].PlanAnnotations) != 0 {
t.Fatalf("fallback did not materialize the unannotated complete source: %#v %v", chunks, err)
}
})
}
}
func TestNativePlanFailureUsesDocumentOrderWithoutEarlierPlanRecovery(t *testing.T) {
failure := errors.New("provider unavailable")
for _, failedCorrection := range []bool{false, true} {
client := &fakeScenesLLMClient{err: failure}
request := chunkRequest()
request.Source.Units[0].ID, request.Source.Units[1].ID = 30, 10
for i := range request.Source.Units {
unit := &request.Source.Units[i]
unit.Ref.StartUnitID, unit.Ref.EndUnitID = unit.ID, unit.ID
}
if failedCorrection {
client.err = nil
client.responses = []string{`{"scenes":[{"start_unit_id":30,"end_unit_id":10},{"start_unit_id":3,"end_unit_id":4}]}`}
client.errors = []error{nil, failure}
}
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{
ValidatorName: "scene-policy", Decision: contracts.AssessmentRejected,
Result: contracts.ValidationResult{CorrectionGuidance: "Keep the encounter together."},
}}}, nil
}
result, err := newChunker(t, client).Plan(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 30, EndUnitID: 4}}) {
t.Fatalf("failure fallback=%#v %v", result, err)
}
chunks, err := source.MaterializeChunkPlan(request.Source, result.Plan)
if err != nil || !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{30, 10, 3, 4}) {
t.Fatalf("nonconsecutive source order lost: %#v %v", chunks, err)
}
}
}
func TestNativeStructuralFailureSkipsSemanticAssessment(t *testing.T) {
client := &fakeScenesLLMClient{response: chunkResponse{Scenes: []sceneResponse{}}}
request := chunkRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 2
assessments := 0
request.Operation.Assess = func(context.Context, source.ChunkPlan) (contracts.CandidateAssessment, error) {
assessments++
return contracts.CandidateAssessment{}, nil
}
result, err := newChunker(t, client).Plan(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || assessments != 0 {
t.Fatalf("structural failure=%#v %v", result, err)
}
}
func TestCancellationAndMaterializationFailureDoNotPublishFallback(t *testing.T) {
t.Run("canceled native delivery", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &fakeScenesLLMClient{response: validSceneResponse(), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
request := chunkRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
result, err := newChunker(t, client).Plan(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
})
t.Run("unmaterializable source", func(t *testing.T) {
request := chunkRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Source.Units[0].Metadata = map[string]any{"invalid": math.NaN()}
failure := errors.New("provider unavailable")
result, err := newChunker(t, &fakeScenesLLMClient{err: failure}).Plan(context.Background(), request)
if err == nil || result.Outcome.Disposition != "" || !errors.Is(err, failure) {
t.Fatalf("unusable fallback=%#v %v", result, err)
}
})
}

View File

@@ -1,72 +0,0 @@
package scenes
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.scenes",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-transcript-full.md",
"common-dnd-references.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/scenes")
if err != nil {
return nil, fmt.Errorf("scope scene assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare scene prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope scene schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,138 +0,0 @@
package scenes
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptAssetsPrepareTranscriptWithInstructionOrdering(t *testing.T) {
if !slices.Contains(promptAssetManifest.SharedFiles, "common-dnd-transcript-full.md") {
t.Fatalf("shared prompt assets = %#v, want complete-session transcript scope", promptAssetManifest.SharedFiles)
}
transcript := []byte(`{"sentinel":"scene-transcript"}`)
prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
}
if prepared.OutputContract.SchemaPath != "dnd_scenes_llm.v1.json" {
t.Fatalf("scene output schema path = %q, want private LLM schema", prepared.OutputContract.SchemaPath)
}
if prepared.Messages[0].Role != "system" || prepared.Messages[1].Role != "user" {
t.Fatalf("initial prompt roles = %#v, want system followed by campaign references", prepared.Messages[:2])
}
if cache := prepared.Messages[1].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("campaign-reference cache control = %#v, want ephemeral", cache)
}
if cache := prepared.Messages[0].CacheControl; cache != nil {
t.Fatalf("system cache control = %#v, want nil", cache)
}
if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") {
t.Fatalf("reference message = %q, want supplied reference inputs", references)
}
transcriptIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "scene-transcript") {
transcriptIndex = index
}
}
if transcriptIndex != len(prepared.Messages)-1 {
t.Fatalf("transcript message index = %d, want final message in %#v", transcriptIndex, prepared.Messages)
}
if cache := prepared.Messages[transcriptIndex].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("transcript cache control = %#v, want ephemeral", cache)
}
instructionsIndex := transcriptIndex - 1
if instructionsIndex <= 1 || prepared.Messages[instructionsIndex].CacheControl != nil {
t.Fatalf("instruction message = %#v, want uncached message after references and before transcript", prepared.Messages[instructionsIndex])
}
for index, message := range prepared.Messages {
if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) {
t.Errorf("message %d unexpectedly rendered reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "scene-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
}
}
func TestPromptAssetDiagnosticsOmitRawMaterials(t *testing.T) {
transcript := []byte(`{"secret":"source text"}`)
prepared := prepareScenesPrompt(t, transcript, "private player note", "private party note", "private glossary note")
metadata := newChunker(t, &fakeScenesLLMClient{}).ManifestMetadata()
payload, err := json.Marshal(map[string]any{
"prepared": map[string]any{
"prompt_id": prepared.PromptID,
"prompt_version": prepared.PromptVersion,
"prompt_hash": prepared.PromptHash,
"rendered_prompt_hash": prepared.RenderedPromptHash,
"selected_profile_id": prepared.SelectedProfileID,
"output_contract": prepared.OutputContract,
"input_hashes": prepared.InputHashes,
"effective_model_params": prepared.EffectiveModelParams,
},
"manifest": metadata,
})
if err != nil {
t.Fatalf("marshal diagnostics: %v", err)
}
diagnostics := string(payload)
for _, forbidden := range []string{
"source text",
"private player note",
"private party note",
"private glossary note",
`"properties"`,
"start_unit_id",
} {
if strings.Contains(diagnostics, forbidden) {
t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics)
}
}
if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != ResponseSchemaVersion {
t.Fatalf("manifest prompt metadata = %#v", metadata)
}
if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") {
t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"])
}
}
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register scene prompt assets: %v", err)
}
engine := newScenesPromptEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: ResponseSchemaVersion,
ProfileID: "scene-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
return prepared
}
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"),
})
}

View File

@@ -1,25 +0,0 @@
package scenes
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scenes"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes_llm")
ResponseSchemaID = "notarius.dnd.scenes.llm"
ResponseSchemaVersion = "v1"
ResponseSchemaName = "notarius_dnd_scenes_llm_v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: ResponseSchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_scenes_llm.v1.json",
})
}

View File

@@ -1,179 +0,0 @@
package scenes
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestLoadResponseSchemaForScenes(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if got, want := string(schema.Key), "dnd_scenes_llm"; got != want {
t.Fatalf("schema.Key = %q, want %q", got, want)
}
if got, want := schema.ID, "notarius.dnd.scenes.llm"; got != want {
t.Fatalf("schema.ID = %q, want %q", got, want)
}
if got, want := schema.Version, "v1"; got != want {
t.Fatalf("schema.Version = %q, want %q", got, want)
}
if got, want := schema.Name, "notarius_dnd_scenes_llm_v1"; got != want {
t.Fatalf("schema.Name = %q, want %q", got, want)
}
if !strings.HasPrefix(schema.SHA256, "sha256:") {
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
}
}
func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
valid := validSceneSchemaResponse()
validJSON, err := json.Marshal(valid)
if err != nil {
t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err)
}
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
t.Fatalf("valid scene response rejected: %v", err)
}
tests := []struct {
name string
mutate func(map[string]any)
}{
{
name: "missing scenes",
mutate: func(response map[string]any) {
delete(response, "scenes")
},
},
{
name: "empty scenes",
mutate: func(response map[string]any) {
response["scenes"] = []any{}
},
},
{
name: "non-positive start unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0
},
},
{
name: "non-positive end unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0
},
},
{
name: "non-integer endpoint",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5
},
},
{
name: "unknown top-level field",
mutate: func(response map[string]any) {
response["boundary_caveats"] = []any{}
},
},
{
name: "unknown scene field",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
response := validSceneSchemaResponse()
tt.mutate(response)
content, err := json.Marshal(response)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want rejected response")
}
})
}
}
func TestResponseStructAcceptsIntegerBoundaries(t *testing.T) {
raw := []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":3}]}`)
var response chunkResponse
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if got := response.Scenes[0].StartUnitID.String(); got != "1" {
t.Fatalf("StartUnitID = %q, want 1", got)
}
if got := response.Scenes[0].EndUnitID.String(); got != "3" {
t.Fatalf("EndUnitID = %q, want 3", got)
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if !json.Valid(second.JSONSchema) {
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
}
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
t.Fatal("schema JSON did not use defensive copy")
}
}
func validSceneSchemaResponse() map[string]any {
return map[string]any{
"scenes": []any{
map[string]any{
"start_unit_id": 1,
"end_unit_id": 3,
},
},
}
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return fmt.Errorf("parse instance: %w", err)
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return fmt.Errorf("parse schema: %w", err)
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return fmt.Errorf("load schema: %w", err)
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return fmt.Errorf("compile schema: %w", err)
}
return schema.Validate(instance)
}