package pipeline import ( "context" "encoding/json" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) // This producer uses real native preparation/continuation and returns domain // plans; only the provider is faked by the tests. type cachedPlanChunker struct{ engine *promptkit.Engine } func (cachedPlanChunker) Key() string { return "cached-plan" } func (cachedPlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (c cachedPlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { references := map[string][]string{} for name, slot := range request.References.Slots { for _, item := range slot.Items { references[name] = append(references[name], string(item.Content)) } } referenceContent, err := json.Marshal(references) if err != nil { return contracts.ChunkPlanResult{}, err } initial := nativeProducerInitial() initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID initial.Inputs["source"] = promptkit.Inline(string(request.SourceInput.Content) + "\n" + string(referenceContent)) plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, initial, func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) { var plan source.ChunkPlan err := json.Unmarshal([]byte(result.RawOutput), &plan) return &plan, nil, err }, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) { return source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: request.Source.Units[0].ID, EndUnitID: request.Source.Units[len(request.Source.Units)-1].ID}}}, nil }) return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err } func planCacheRunInput(t *testing.T, engine *promptkit.Engine, validator chunkAssessmentValidator) RunInput { t.Helper() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "cached-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) { return cachedPlanChunker{request.Dependencies.Engine}, nil }); err != nil { t.Fatal(err) } if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "plan-policy", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.ChunkValidator, error) { return namedPlanValidator{"plan-policy", validator}, nil }); err != nil { t.Fatal(err) } profile := typedResolutionProfile() profile.Chunk = Binding("cached-plan") profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("plan-policy")}} prepared := prepareRunnerProfile(t, catalog, profile, engine) return RunInput{Prepared: prepared, pipeline: prepared.resolved} } func planCacheSource(doc *source.SourceDocument) contracts.LLMInputMaterial { var texts []string for _, unit := range doc.Units { texts = append(texts, unit.Text) } return contracts.NewLLMInputMaterial("source", "text/plain", []byte(strings.Join(texts, "\n")), "", "") } func planResponse(t *testing.T, plan source.ChunkPlan) *promptkit.GenerateResponse { t.Helper() content, err := json.Marshal(plan) if err != nil { t.Fatal(err) } return &promptkit.GenerateResponse{Content: string(content)} } func TestChunkPlanCacheStoresFinalPlanAndReusesWithoutAssessment(t *testing.T) { root, store := filesystemProducerCache(t) doc := typedTestDocumentWithUnits(2) whole := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}} final := typedTestPlan(doc) calls, assessments := 0, 0 client := producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if calls == 1 { return planResponse(t, whole), nil } if calls == 2 { if len(request.Prompt.Messages) != 4 { t.Fatalf("correction history=%#v", request.Prompt.Messages) } } return planResponse(t, final), nil }) engine := nativeProducerEngine(t, client) input := planCacheRunInput(t, engine, func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { assessments++ if len(request.Chunks) == 1 { return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Use separate scenes."}, nil } return contracts.ValidationResult{Approved: true}, nil }) input.Cache = store var keys, decisions []string input.ObserveCache = func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error { if origin.Stage != contracts.DiagnosticOriginStageChunk || origin.ModuleKey != "cached-plan" { t.Fatalf("cache scope=%#v", origin) } if observation.Prepared != nil { keys = append(keys, observation.MessageContentHash) } decisions = append(decisions, observation.Decision) return nil } store.beforeWrite = func(_ string, _ []byte) { if calls != 2 || assessments != 2 { t.Fatal("plan stored before correction acceptance") } } first, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "first-session") if err != nil || !first.accepted || len(first.chunks) != 2 || calls != 2 || assessments != 2 || store.writes != 1 { t.Fatalf("first plan=%#v err=%v calls=%d assessments=%d", first, err, calls, assessments) } bytes, err := os.ReadFile(filepath.Join(root, keys[0]+".json")) var saved source.ChunkPlan if err != nil || json.Unmarshal(bytes, &saved) != nil || !reflect.DeepEqual(saved, *first.plan) { t.Fatalf("cache is not the final raw plan: %s %v", bytes, err) } var fields map[string]json.RawMessage if err := json.Unmarshal(bytes, &fields); err != nil { t.Fatal(err) } for _, field := range []string{"producer", "checkpoint", "document", "provenance", "usage"} { if _, ok := fields[field]; ok { t.Fatalf("cache contains reporting field %q", field) } } store.beforeWrite = nil input.pipeline.Chunk.LLMProfile = "chosen" input.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"} second, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "changed-session") if err != nil || !second.outcome.ReuseEligible || len(second.chunks) != 2 || calls != 2 || assessments != 2 || len(keys) != 2 || keys[0] != keys[1] { t.Fatalf("settings-only reuse=%#v %v calls=%d assessments=%d keys=%v", second, err, calls, assessments, keys) } if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) { t.Fatalf("cache decisions=%v", decisions) } // Materialization is against the current source, not cached transcript bytes. changed := *doc changed.ID = "current-source" changed.Units = append([]source.SourceUnit(nil), doc.Units...) for i := range changed.Units { changed.Units[i].Ref.SourceID = changed.ID } changed.Digest, _ = source.DigestDocument(&changed) // Identity is part of ordinary plan usability if it changes the source digest. // Supplying the matching digest lets this check focus on rematerialized refs. saved.SourceDigest = changed.Digest replacement, _ := json.Marshal(saved) if err := store.inner.Store(context.Background(), contracts.CacheRefresh, keys[0], replacement); err != nil { t.Fatal(err) } third, err := New().runChunkPlan(context.Background(), input, &changed, planCacheSource(doc), "session") if err != nil || calls != 2 || assessments != 2 || third.chunks[0].Ref.SourceID != changed.ID || third.chunks[0].Units[0].Ref.SourceID != changed.ID { t.Fatalf("rematerialization=%#v %v", third, err) } } func TestChunkPlanCacheUsesMessageContentRatherThanSourceIdentity(t *testing.T) { _, store := filesystemProducerCache(t) doc := typedTestDocumentWithUnits(2) calls, assessments := 0, 0 client := producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ plan := typedTestPlan(doc) if calls > 1 { plan.Ranges = []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}} } return planResponse(t, plan), nil }) validator := chunkAssessmentValidator(func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { assessments++ return contracts.ValidationResult{Approved: true}, nil }) input := planCacheRunInput(t, nativeProducerEngine(t, client), validator) input.Cache = store var keys []string input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error { if observation.Prepared != nil { keys = append(keys, observation.MessageContentHash) } return nil } run := func(wantChunks int) { t.Helper() result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session") if err != nil || len(result.chunks) != wantChunks { t.Fatalf("chunks=%d err=%v", len(result.chunks), err) } } run(2) input.pipeline.ChunkReferences.ReferenceSet = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte("new scene context")}}}}} run(1) // Replace only the prompt text, retaining source and reference bytes. input.Prepared.chunker = cachedPlanChunker{nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("changed prompt", []byte(`prompts: - id: initial default_profile: base inputs: [{name: source, required: true}] messages: - {role: system, content: 'A changed scene assignment.'} - {role: user, content: 'Source: {{input "source"}}'} output: {format: json, validation_mode: json, repair_attempts: 0} `))))} run(1) if calls != 3 || assessments != 3 || len(keys) != 3 || keys[0] == keys[1] || keys[1] == keys[2] { t.Fatalf("content reuse: calls=%d assessments=%d keys=%v", calls, assessments, keys) } } func TestChunkPlanCacheUnusableEntriesAreMissesNotAssistantCandidates(t *testing.T) { for _, defect := range []string{"source digest", "source range", "malformed", "legacy envelope"} { t.Run(defect, func(t *testing.T) { root, store := filesystemProducerCache(t) doc := typedTestDocument() calls, assessments := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if len(request.Prompt.Messages) != 2 { t.Fatal("unusable cache replayed as an assistant candidate") } return planResponse(t, typedTestPlan(doc)), nil })) input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { assessments++ return contracts.ValidationResult{Approved: true}, nil }) input.Cache = store var key string input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error { if observation.Prepared != nil { key = observation.MessageContentHash } return nil } if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil { t.Fatal(err) } bad := typedTestPlan(doc) switch defect { case "source digest": bad.SourceDigest = "not-current" case "source range": bad.Ranges[0].EndUnitID = 999 } content, _ := json.Marshal(bad) if defect == "malformed" { content = []byte("{") } if defect == "legacy envelope" { content = []byte(`{"plan":{},"producer":{"key":"legacy"}}`) } // Simulate an existing corrupt entry without going through valid writes. if err := os.WriteFile(filepath.Join(root, key+".json"), content, 0644); err != nil { t.Fatal(err) } result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session") if err != nil || !result.accepted || calls != 2 || assessments != 2 || len(result.diagnostics) != 1 { t.Fatalf("unusable plan=%#v err=%v calls=%d assessments=%d", result, err, calls, assessments) } }) } } func TestChunkPlanCacheModesAndTerminalExclusions(t *testing.T) { for _, test := range []struct { name string cache contracts.CacheMode mode contracts.ExecutionMode reject, ineligible bool reads, writes int fallback bool }{ {"auto", contracts.CacheAuto, contracts.ModeStrict, false, false, 1, 1, false}, {"bypass", contracts.CacheBypass, contracts.ModeStrict, false, false, 0, 0, false}, {"refresh", contracts.CacheRefresh, contracts.ModeStrict, false, false, 0, 1, false}, {"ineligible", contracts.CacheAuto, contracts.ModeStrict, false, true, 0, 0, false}, {"strict rejection", contracts.CacheAuto, contracts.ModeStrict, true, false, 1, 0, false}, {"permissive fallback", contracts.CacheAuto, contracts.ModePermissive, true, false, 1, 0, true}, } { t.Run(test.name, func(t *testing.T) { _, store := filesystemProducerCache(t) doc := typedTestDocumentWithUnits(2) calls, assessments := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return planResponse(t, typedTestPlan(doc)), nil })) input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { assessments++ return contracts.ValidationResult{Approved: !test.reject, ReasonCode: "policy", CorrectionGuidance: "Revise the scene partition."}, nil }) input.Cache = store input.reuseIneligible = test.ineligible input.pipeline.ChunkControls = contracts.ProducerControls{Cache: test.cache, Mode: test.mode, MaxAdditionalTurns: 0} result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session") if (err != nil) != (test.reject && !test.fallback) || calls != 1 || assessments != 1 || store.reads != test.reads || store.writes != test.writes { t.Fatalf("result=%#v err=%v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes) } if test.fallback && (result.outcome.Disposition != contracts.ProducerFallback || result.outcome.ReuseEligible || len(result.chunks) != 1 || len(result.chunks[0].Units) != 2) { t.Fatalf("invalid whole-source fallback: %#v", result) } }) } } func TestChunkPlanFailedRefreshPreservesAcceptanceAndIgnoresLegacyRecords(t *testing.T) { root, store := filesystemProducerCache(t) legacyNames := []string{"checkpoints/run/manifest.json", "chunk-plans/source/plan.json"} for _, name := range legacyNames { filename := filepath.Join(root, name) if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filename, []byte("unread legacy bytes"), 0644); err != nil { t.Fatal(err) } } doc := typedTestDocumentWithUnits(2) reject := false calls, assessments := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return planResponse(t, typedTestPlan(doc)), nil })) input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { assessments++ return contracts.ValidationResult{Approved: !reject, ReasonCode: "partition", CorrectionGuidance: "Reconsider the partition."}, nil }) input.Cache = store input.pipeline.ChunkControls.MaxAdditionalTurns = 0 var key string input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error { if observation.Prepared != nil { key = observation.MessageContentHash } return nil } if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil { t.Fatal(err) } original, err := os.ReadFile(filepath.Join(root, key+".json")) if err != nil { t.Fatal(err) } reject = true input.pipeline.ChunkControls.Cache = contracts.CacheRefresh for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { input.pipeline.ChunkControls.Mode = mode result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session") if (err != nil) != (mode == contracts.ModeStrict) { t.Fatalf("refresh mode=%s result=%#v error=%v", mode, result, err) } if mode == contracts.ModePermissive && (result.outcome.ReuseEligible || result.outcome.Disposition != contracts.ProducerFallback) { t.Fatalf("fallback=%#v", result.outcome) } current, err := os.ReadFile(filepath.Join(root, key+".json")) if err != nil || string(current) != string(original) { t.Fatal("failed refresh changed old acceptance") } } input.pipeline.ChunkControls.Cache = contracts.CacheAuto result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session") if err != nil || result.outcome.Disposition != contracts.ProducerAccepted || calls != 3 || assessments != 3 || store.reads != 2 || store.writes != 1 { t.Fatalf("old acceptance not reused: %#v %v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes) } for _, name := range legacyNames { content, err := os.ReadFile(filepath.Join(root, name)) if err != nil || string(content) != "unread legacy bytes" { t.Fatalf("legacy record modified: %s %q %v", name, content, err) } } } func TestDeterministicChunkingIgnoresCacheAndLegacyRecords(t *testing.T) { root, store := filesystemProducerCache(t) legacy := map[string][]byte{"checkpoints/identity/manifest.json": []byte("old checkpoint"), "chunk-plans/source.json": []byte("old plan")} for name, content := range legacy { filename := filepath.Join(root, name) if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filename, content, 0644); err != nil { t.Fatal(err) } } catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil) input := RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, ObserveCache: func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error { return fmt.Errorf("deterministic chunker prepared a synthetic request") }} for range 2 { if _, err := New().Run(context.Background(), input); err != nil { t.Fatal(err) } } if store.reads != 0 || store.writes != 0 { t.Fatalf("deterministic cache I/O=%d/%d", store.reads, store.writes) } for name, want := range legacy { got, err := os.ReadFile(filepath.Join(root, name)) if err != nil || string(got) != string(want) { t.Fatalf("legacy record modified: %s %q %v", name, got, err) } } }