From 9074547e33ba96a3832e90814fd99d6d5eabfb9b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 09:37:16 +0000 Subject: [PATCH] Keep cached scene identities aligned with the current plan --- docs/roadmap/remediation.md | 4 + internal/cli/scene_cache_identity_test.go | 124 ++++++++++++++++++ .../extract/scenedescriptions/cache_test.go | 77 +++++++++++ .../extract/scenedescriptions/extractor.go | 18 +++ 4 files changed, 223 insertions(+) create mode 100644 internal/cli/scene_cache_identity_test.go create mode 100644 internal/modules/dnd/extract/scenedescriptions/cache_test.go diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 9a6aa97c..ce7fe7ca 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -159,6 +159,10 @@ retained. ## Finding 3: Rebind Cached Scene Identity To The Current Chunk Plan +**Status: Complete.** Scene hits require one exact current-range scene and +reattach its current ordinal identity without model work or reassessment. +Expansion/collapse integration regressions and scene-module tests pass. + Priority: P1. Relevant stages: 11, 23, 27–29, and 37. ### Evidence And Impact diff --git a/internal/cli/scene_cache_identity_test.go b/internal/cli/scene_cache_identity_test.go new file mode 100644 index 00000000..416ad555 --- /dev/null +++ b/internal/cli/scene_cache_identity_test.go @@ -0,0 +1,124 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "go.maximumdirect.net/promptkit" +) + +type auditChangingPartitionClient struct { + plans atomic.Int32 + descriptions atomic.Int32 + collapse bool +} + +func (client *auditChangingPartitionClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if productionSchemaHasProperty(request, "scenes") { + if (client.plans.Add(1) == 1) != client.collapse { + return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil + } + return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":1},{"start_unit_id":2,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil + } + client.descriptions.Add(1) + return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Travel","summary":"The party continues its journey."}`}, nil +} + +func TestSceneCacheRemainsUsableAfterPartitionRenumbering(t *testing.T) { + t.Run("expand", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, false) }) + t.Run("collapse", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, false) }) + t.Run("expand-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, true) }) + t.Run("collapse-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, true) }) +} + +func auditSceneCacheAfterRepartition(t *testing.T, collapse, bypass bool) { + directory := t.TempDir() + configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json") + configuration := fmt.Sprintf(`version: 6 +concurrency: {workers: 1} +output: {directory: %q} +cache_directory: %q +promptkit: + profiles: + - id: audit + endpoint: https://example.invalid/v1 + model: audit + api_key_required: false +pipelines: + audit: + llm_profile: audit + input: {module: seriatim} + chunk: + module: dnd/scenes + cache: refresh + artifacts: + scene-descriptions: + extract: {module: dnd/scene-descriptions} + merge: {module: appendorder} + normalize: {module: dnd/scene-descriptions} +`, filepath.Join(directory, "output"), filepath.Join(directory, "cache")) + if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil { + t.Fatal(err) + } + transcript := `{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party departs."},{"id":2,"start":1,"end":2,"speaker":"GM","text":"The party travels."},{"id":3,"start":2,"end":3,"speaker":"GM","text":"The party arrives."}]}` + if err := os.WriteFile(inputPath, []byte(transcript), 0o600); err != nil { + t.Fatal(err) + } + client := &auditChangingPartitionClient{collapse: collapse} + for run := 1; run <= 2; run++ { + var stdout, stderr bytes.Buffer + args := []string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--format", "json", "--progress=false"} + if bypass { + args = append(args, "--cache", "bypass") + } + status := RunWithOptions(args, &stdout, &stderr, Options{LLMClient: client}) + if status != 0 { + t.Fatalf("run %d: status=%d plans=%d descriptions=%d stderr=%s", run, status, client.plans.Load(), client.descriptions.Load(), stderr.String()) + } + if run == 2 { + var report struct { + Directory string `json:"output_directory"` + } + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(filepath.Join(report.Directory, "lanes", "scene-descriptions.json")) + if err != nil { + t.Fatal(err) + } + var artifact struct { + Scenes []struct { + ID string `json:"id"` + } `json:"scenes"` + } + if err := json.Unmarshal(content, &artifact); err != nil { + t.Fatal(err) + } + wantCount := 3 + if collapse { + wantCount = 2 + } + if len(artifact.Scenes) != wantCount { + t.Fatalf("published scenes: %s", content) + } + for i, scene := range artifact.Scenes { + if scene.ID != fmt.Sprintf("chunk-%06d", i+1) { + t.Fatalf("published stale scene identity after repartitioning: %s", content) + } + } + } + } + wantCalls := int32(4) // The unchanged final scene is reused across runs. + if bypass { + wantCalls = 5 + } + if client.descriptions.Load() != wantCalls { + t.Fatalf("description calls=%d, want %d", client.descriptions.Load(), wantCalls) + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/cache_test.go b/internal/modules/dnd/extract/scenedescriptions/cache_test.go new file mode 100644 index 00000000..38462666 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/cache_test.go @@ -0,0 +1,77 @@ +package scenedescriptions + +import ( + "context" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" +) + +func TestSceneCacheAdaptsOnlyUsableCurrentRange(t *testing.T) { + for _, name := range []string{"stale identity", "wrong range", "multiple scenes", "untrimmed title"} { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + request := extractionRequest() + store, err := artifactcache.NewFilesystemStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + codec := scenecodec.New() + var key string + request.Operation.Controls.Cache = contracts.CacheAuto + request.Operation.Cache = &contracts.ProducerCache[dnd.SceneDescriptionList]{Store: store, Encode: codec.Encode, Decode: codec.Decode, + Observe: func(_ context.Context, observation contracts.CacheObservation) error { + key = observation.MessageContentHash + return nil + }, + } + client := &fakeSceneDescriptionsLLMClient{raw: narrativeScene} + extractor := newExtractor(t, client) + first, err := extractor.Extract(ctx, request) + if err != nil { + t.Fatal(err) + } + cached := first.Value + cached.Scenes[0].ID = "old-chunk-identity" + switch name { + case "wrong range": + cached.Scenes[0].SourceRef.EndUnitID = 2 + case "multiple scenes": + other := cached.Scenes[0] + other.ID = "another-old-scene" + cached.Scenes = append(cached.Scenes, other) + case "untrimmed title": + cached.Scenes[0].Title = " Untrimmed " + } + content, err := codec.Encode(cached) + if err != nil { + t.Fatal(err) + } + if err := store.Store(ctx, contracts.CacheRefresh, key, content); err != nil { + t.Fatal(err) + } + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{}, nil + } + result, err := extractor.Extract(ctx, request) + if err != nil { + t.Fatal(err) + } + if len(result.Value.Scenes) != 1 || result.Value.Scenes[0].ID != request.Chunk.ID || result.Value.Scenes[0].SourceRef != request.Chunk.Ref { + t.Fatalf("scene does not match current chunk: %#v", result.Value) + } + wantCalls, wantAssessments := 2, 1 + if name == "stale identity" { + wantCalls, wantAssessments = 1, 0 + } + if len(client.requests) != wantCalls || assessments != wantAssessments { + t.Fatalf("model calls=%d assessments=%d, want %d/%d", len(client.requests), assessments, wantCalls, wantAssessments) + } + }) + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor.go b/internal/modules/dnd/extract/scenedescriptions/extractor.go index a8558222..9f54e143 100644 --- a/internal/modules/dnd/extract/scenedescriptions/extractor.go +++ b/internal/modules/dnd/extract/scenedescriptions/extractor.go @@ -79,6 +79,24 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe chunk := source.Chunk{ID: req.Chunk.ID, Ref: req.Chunk.Ref} index := source.NewDocumentIndex(req.Source) + if cache := req.Operation.Cache; cache != nil && cache.Decode != nil { + owned := *cache + decode := cache.Decode + owned.Decode = func(content []byte) (dnd.SceneDescriptionList, error) { + value, err := decode(content) + if err != nil { + return value, err + } + if len(value.Scenes) != 1 || value.Scenes[0].SourceRef != chunk.Ref { + return dnd.SceneDescriptionList{}, fmt.Errorf("cached scene does not describe the current chunk range") + } + // Chunk ordinals can change without changing the semantic prompt. + value.Scenes = append([]dnd.SceneDescription(nil), value.Scenes...) + value.Scenes[0].ID = chunk.ID + return sceneFallback(&value, chunk, index) + } + req.Operation.Cache = &owned + } materials := shared.PromptInputs(sourceInput, req.References) inputs := make(map[string]promptkit.ArtifactRef, len(materials)) for name, material := range materials {