From 5e68b74469c30ce26393ad0dc39cffc0fdd1f5b5 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 05:01:09 +0000 Subject: [PATCH] Exercise production CLI contracts through native loaders and providers --- docs/roadmap/implementation.md | 21 +- internal/cli/dnd_combat_contract_test.go | 4 +- internal/cli/dnd_npc_contract_test.go | 4 +- internal/cli/dnd_occurrences_contract_test.go | 6 +- .../dnd_scene_descriptions_contract_test.go | 90 +-- internal/cli/npc_registry_contract_test.go | 76 +- internal/cli/production_contract_test.go | 720 +++++++----------- 7 files changed, 343 insertions(+), 578 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 253c634d..318fbb8a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1689,11 +1689,22 @@ race checks pass, as do framework/integration vet checks. The production executable builds. Real executable route and text/JSON/YAML/ fallback checks passed with a local fake HTTP provider after removal of the old runtime wrapper. Focused native CLI tests pass, but the full CLI suite and -whole-tree compilation remain blocked by legacy CLI fixtures/imports; the current -setup failure is old-path ConfigKit importing the removed YAML `plugin/limit`. -Remaining CLI test migration, dependency tidy/old-path removal, notice -reconciliation, and whole-tree validation are required before completion. -This stage is not marked finished. +whole-tree compilation remain blocked by legacy CLI fixtures/imports. + +Migrated the shared production CLI fixture to the real native application loader, +engine, provider requests, version 6 test inputs, and native pipeline validation. +Retained catalog/typed registration, validator-chain replacement, reference +materialization/limits, configured backend selection, scene description profile +selection, source ranges, and published chunk-map provenance. Provider observations +and codec-decoded files replace profile manifests and normalized-output ledgers. +Oversized references prevent model work/publication without requiring the retired +runtime-factory construction order. Removed only tests of deleted asset/fingerprint +factories; native captured-loading tests cover shared configuration ownership. +Focused production CLI tests, vet, and repeated race checks pass. Whole CLI setup now stops at +remaining fixtures importing the removed reporting package, rather than the +legacy ConfigKit YAML import. Remaining CLI test migration, dependency tidy/ +old-path removal, notice reconciliation, and whole-tree validation are required +before completion. This stage is not marked finished. **Goal:** delete the old architecture rather than leave dormant alternatives. diff --git a/internal/cli/dnd_combat_contract_test.go b/internal/cli/dnd_combat_contract_test.go index 6debf8a5..d35b876c 100644 --- a/internal/cli/dnd_combat_contract_test.go +++ b/internal/cli/dnd_combat_contract_test.go @@ -27,7 +27,7 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) { t.Fatalf("artifact lanes = %#v, want one combat lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes) } lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0] - if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key { + if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key { t.Fatalf("resolved combat lane = %#v, want typed production composition", lane) } @@ -172,7 +172,7 @@ func productionCombatContractConfig() config.Config { References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")}, Artifacts: map[string]pipeline.ArtifactLaneProfile{ "combat": { - Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2}, + Extract: pipeline.ModuleBinding{Module: combatextract.Key}, Normalize: pipeline.Binding(combatnormalize.Key), }, }, diff --git a/internal/cli/dnd_npc_contract_test.go b/internal/cli/dnd_npc_contract_test.go index ed4c3480..e34cfc26 100644 --- a/internal/cli/dnd_npc_contract_test.go +++ b/internal/cli/dnd_npc_contract_test.go @@ -28,7 +28,7 @@ func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) { t.Fatalf("artifact lanes = %#v, want one NPC lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes) } lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0] - if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key { + if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key { t.Fatalf("resolved NPC lane = %#v, want typed production composition", lane) } if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 { @@ -129,7 +129,7 @@ func productionNPCContractConfig() config.Config { Chunk: pipeline.Binding(pipeline.DefaultChunkModule), Artifacts: map[string]pipeline.ArtifactLaneProfile{ "npc_registry": { - Extract: pipeline.ModuleBinding{Module: npcextract.Key, Retries: 2}, + Extract: pipeline.ModuleBinding{Module: npcextract.Key}, Normalize: pipeline.Binding(npcnormalize.Key), }, }, diff --git a/internal/cli/dnd_occurrences_contract_test.go b/internal/cli/dnd_occurrences_contract_test.go index 59d0c5eb..e55fe825 100644 --- a/internal/cli/dnd_occurrences_contract_test.go +++ b/internal/cli/dnd_occurrences_contract_test.go @@ -44,7 +44,7 @@ func TestProductionNPCOccurrencePipelineResolvesAndPrepares(t *testing.T) { t.Fatalf("generated bindings = %#v", bindings) } } - if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { + if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil { t.Fatalf("Prepare() error = %v", err) } @@ -120,7 +120,7 @@ func TestProductionNPCOccurrenceReferencesRejectIncompatibleExternalRegistries(t if err != nil { t.Fatalf("MaterializeReferences() error = %v", err) } - if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err == nil || !strings.Contains(err.Error(), test.want) { + if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("Prepare() error = %v, want %q", err, test.want) } }) @@ -175,7 +175,7 @@ func TestProductionOccurrencePipelinesAcceptCompatibleExternalRegistries(t *test if err != nil || len(warnings) != 0 { t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) } - if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { + if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil { t.Fatalf("Prepare() error = %v", err) } }) diff --git a/internal/cli/dnd_scene_descriptions_contract_test.go b/internal/cli/dnd_scene_descriptions_contract_test.go index aa51db75..97b6364c 100644 --- a/internal/cli/dnd_scene_descriptions_contract_test.go +++ b/internal/cli/dnd_scene_descriptions_contract_test.go @@ -2,22 +2,19 @@ package cli import ( "context" - "encoding/json" "fmt" "reflect" "strings" - "sync" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions" + "go.maximumdirect.net/promptkit" ) func TestProductionSceneDescriptionWorkflow(t *testing.T) { @@ -48,37 +45,23 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) { } llmClient := &sceneDescriptionLLM{} - prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient}) + prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient, "scene-description-profile")}) if err != nil { t.Fatalf("Prepare() error = %v", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), - ChunkCacheMode: pipeline.ChunkCacheBypass, + Prepared: prepared, + RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), }) if err != nil { t.Fatalf("Run() error = %v", err) } - if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { - t.Fatalf("run output = %#v, want one approved normalized artifact", output) + if len(output.OutputFiles) != 1 || output.FallbackCount != 0 || output.OutputFiles[0].Name != "lanes/scene-descriptions.json" { + t.Fatalf("output=%#v, want published scene descriptions", output) } - wantProfiles := []artifacts.LLMProfileManifest{{ - ID: "scene-description-profile", - Provider: "promptkit", - Model: "deterministic", - }} - if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) { - t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles) - } - normalizedOutput := output.NormalizeOutputs[0] - if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion { - t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput) - } - - var value dnd.SceneDescriptionList - if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil { - t.Fatalf("decode normalized artifact: %v", err) + value, err := scenecodec.New().Decode(output.OutputFiles[0].Bytes) + if err != nil { + t.Fatal(err) } want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{ {ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."}, @@ -87,61 +70,34 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) { if !reflect.DeepEqual(value, want) { t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want) } - durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json") - if !reflect.DeepEqual(durable, want) { - t.Fatalf("durable output payload = %#v, want %#v", durable, want) - } if len(output.Diagnostics.Groups) != 0 { t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics) } } -type sceneDescriptionLLM struct { - mu sync.Mutex - profile *artifacts.LLMProfileManifest -} +type sceneDescriptionLLM struct{} -func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (*sceneDescriptionLLM) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if req.PromptID != sceneextract.PromptID { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) + if !productionSchemaHasProperty(request, "kind") || request.Target.Model != "scene-description-profile-model" { + return nil, fmt.Errorf("unexpected native scene-description request for %q", request.Target.Model) + } + transcript, err := productionJSONInput(request, "units") + if err != nil { + return nil, err } - transcript := string(req.Inputs["transcript"].Content) var content string switch { - case strings.Contains(transcript, "Cure Wounds"): + case strings.Contains(string(transcript), "Cure Wounds"): content = `{"kind":"narrative","title":" Aria casts Cure Wounds ","summary":" Aria casts Cure Wounds. "}` - case strings.Contains(transcript, "Shield"): + case strings.Contains(string(transcript), "Shield"): content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}` default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript) + return nil, fmt.Errorf("unexpected transcript material %q", transcript) } - if err := json.Unmarshal([]byte(content), out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err) - } - profile := artifacts.LLMProfileManifest{ - ID: req.ProfileID, - Provider: "promptkit", - Model: "deterministic", - } - client.mu.Lock() - client.profile = &profile - client.mu.Unlock() - return contracts.StructuredCompletionResponse{ - Content: []byte(content), - Provider: profile.Provider, - Model: profile.Model, - ProfileID: profile.ID, - }, nil + return &promptkit.GenerateResponse{Content: content}, nil } -func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest { - client.mu.Lock() - defer client.mu.Unlock() - if client.profile == nil { - return nil - } - return []artifacts.LLMProfileManifest{*client.profile} -} +var _ promptkit.LLMClient = (*sceneDescriptionLLM)(nil) diff --git a/internal/cli/npc_registry_contract_test.go b/internal/cli/npc_registry_contract_test.go index f1a82cf1..a83c0a8c 100644 --- a/internal/cli/npc_registry_contract_test.go +++ b/internal/cli/npc_registry_contract_test.go @@ -1,7 +1,6 @@ package cli import ( - "context" "errors" "fmt" "io/fs" @@ -9,75 +8,44 @@ import ( "path/filepath" "strings" "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) -func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *testing.T) { - components := productionTestComponents(t) - npcPath := filepath.Join(t.TempDir(), "npcs.json") - if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil { +func TestOversizedNPCRegistryFailsBeforeModelWorkAndPublication(t *testing.T) { + root := t.TempDir() + npcPath := filepath.Join(root, "npcs.json") + if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0600); err != nil { t.Fatal(err) } - checkpointRoot := filepath.Join(t.TempDir(), "checkpoints") - content := fmt.Sprintf(`version: 5 -cache: - chunk_plans: - mode: bypass - checkpoints: - enabled: true - directory: %q + outputRoot, cacheRoot := filepath.Join(root, "output"), filepath.Join(root, "cache") + content := fmt.Sprintf(`version: 6 +output: {directory: %q} +cache_directory: %q pipelines: dnd-session: - input: - module: seriatim + input: {module: seriatim} artifacts: spells: extract: module: dnd/spells references: - npc_registry: - path: %q - normalize: - module: dnd/spells -`, checkpointRoot, npcPath) - configPath := filepath.Join(t.TempDir(), "config.yml") - if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - - llmConstructed := false - chunkStoreConstructed := false - options := Options{ - Catalog: catalogFromRegistries(components.registries), - Registries: components.registries, - LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - llmConstructed = true - return nil, nil, errors.New("LLM client must not be constructed") - }, - ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) { - chunkStoreConstructed = true - return nil, errors.New("chunk-plan store must not be constructed") - }, - } + npc_registry: {path: %q} + normalize: {module: dnd/spells} +`, outputRoot, cacheRoot, npcPath) + configPath := writeProductionContractConfig(t, content) + client := &productionFakeLLMClient{} var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "run", "dnd-session", "--config", configPath, - "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), - "--chunk_cache", "bypass", "--output-dir", t.TempDir(), - }, &stdout, &stderr, options) + code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json")}, &stdout, &stderr, productionRunOptions(t, client)) for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npc_registry"`, "1048577 bytes", "limit 1048576"} { if code == 0 || !strings.Contains(stderr.String(), fragment) { - t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment) + t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment) } } - if llmConstructed || chunkStoreConstructed { - t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed) + if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 { + t.Fatal("invalid reference reached provider or completion reporting") } - if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) { - t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err) + for _, path := range []string{outputRoot, cacheRoot} { + if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("state allocated before invalid reference: %s: %v", path, err) + } } } diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index d46fcc4f..d6f9db51 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -16,17 +16,12 @@ import ( "sync" "sync/atomic" "testing" - "testing/fstest" "time" - configkit "gitea.maximumdirect.net/eric/ConfigKit" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" @@ -60,8 +55,9 @@ import ( scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" ) func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) { @@ -217,126 +213,98 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) { } func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) { + path := writeProductionSpellCatalogContractConfig(t) var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "config", "validate", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--pipeline", "dnd-session", - }, &stdout, &stderr, Options{}) - if code != 0 || stderr.Len() != 0 { - t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, Options{}) + if code != 0 { + t.Fatalf("validate production composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) { - components := productionTestComponents(t) - newEngine := func(profileFile string) (*promptkit.Engine, error) { - t.Helper() - sources, options, err := components.assets.PromptKitConfigurationSources() - if err != nil { - return nil, err - } - if profileFile != "" { - sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ - Source: configkit.File(profileFile), Imports: true, - }}}) - } - return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...) + for _, test := range []struct { + name, companion string + invalid bool + }{ + {name: "embedded defaults"}, + {name: "operator override", companion: `collections: + notarius: + profiles: + - id: dnd-extraction + endpoint: https://operator.example.invalid/v1 + model: operator-model + timeout_seconds: 75 +`}, + {name: "invalid operator override", companion: `collections: + notarius: + profiles: + - id: dnd-extraction + model: [invalid] +`, invalid: true}, + } { + t.Run(test.name, func(t *testing.T) { + path := writeProductionContractConfig(t, "version: 6\n") + if test.companion != "" { + if err := os.WriteFile(filepath.Join(filepath.Dir(path), "prompts.yml"), []byte(test.companion), 0600); err != nil { + t.Fatal(err) + } + } + loaders := &nativeLoaders{lookupEnv: emptyLookup} + engine, err := loaders.engine(context.Background(), productionSelection(path)) + if test.invalid { + if err == nil { + t.Fatal("invalid operator resource fell back to embedded defaults") + } + return + } + if err != nil { + t.Fatal(err) + } + profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction") + if err != nil { + t.Fatal(err) + } + if test.companion == "" && (profile.Backend == nil || *profile.Backend != "openrouter" || profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil) { + t.Fatalf("lost embedded profile defaults: %#v", profile) + } + model, timeout := "openai/gpt-5.6-luna", 240 + if test.companion != "" { + model, timeout = "operator-model", 75 + if profile.Endpoint == nil || *profile.Endpoint != "https://operator.example.invalid/v1" { + t.Fatalf("operator endpoint=%#v", profile.Endpoint) + } + } + if profile.Model == nil || *profile.Model != model || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != timeout { + t.Fatalf("profile=%#v", profile) + } + }) } - - t.Run("fallback", func(t *testing.T) { - engine, err := newEngine("") - if err != nil { - t.Fatal(err) - } - inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction") - if err != nil { - t.Fatalf("InspectProfile() error = %v, want fallback profile", err) - } - params := inspection.EffectiveModelParams - if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" { - t.Fatalf("fallback profile parameters = %#v", params) - } - if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 { - t.Fatalf("fallback profile selected optional provider controls: %#v", params) - } - }) - - t.Run("valid operator profile wins", func(t *testing.T) { - profilePath := filepath.Join(t.TempDir(), "profiles.yaml") - if err := os.WriteFile(profilePath, []byte(`profiles: - dnd-extraction: - endpoint: http://operator.example.test/v1 - model: operator-model - timeout_seconds: 75 -`), 0o600); err != nil { - t.Fatal(err) - } - engine, err := newEngine(profilePath) - if err != nil { - t.Fatal(err) - } - inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction") - if err != nil { - t.Fatalf("InspectProfile() error = %v, want operator profile", err) - } - params := inspection.EffectiveModelParams - if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" { - t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params) - } - }) - - t.Run("invalid operator profile does not fall through", func(t *testing.T) { - profilePath := filepath.Join(t.TempDir(), "profiles.yaml") - if err := os.WriteFile(profilePath, []byte("profiles:\n dnd-extraction:\n endpoint: http://operator.example.test/v1\n model: operator-model\n unknown: value\n"), 0o600); err != nil { - t.Fatal(err) - } - engine, err := newEngine(profilePath) - if err == nil { - _, err = engine.InspectProfile(context.Background(), "dnd-extraction") - } - if err == nil { - t.Fatal("operator profile error = nil, want failure instead of fallback") - } - }) } -func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) { +func TestProductionModulesPrepareWithoutProviderCalls(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{ - ID: "dnd-scenes", - Input: pipeline.Binding("seriatim"), - Chunk: pipeline.Binding("dnd/scenes"), - Artifacts: map[string]pipeline.ArtifactLaneProfile{ - "spells": {Extract: pipeline.Binding("dnd/spells")}, - }, + ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"), + Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: pipeline.Binding("dnd/spells")}}, } effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)}) if err != nil { - t.Fatalf("resolve production scene pipeline: %v", err) + t.Fatal(err) } - if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { - t.Fatalf("prepare production scene and spell modules: %v", err) + client := &productionFakeLLMClient{} + if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)}); err != nil { + t.Fatal(err) } - - schemaFS, err := components.assets.SchemaFS() - if err != nil { - t.Fatalf("production schema assets: %v", err) - } - if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil { - t.Fatalf("generic reconciliation schema asset: %v", err) - } - engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{ - "assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"), - }) - if engine == nil { - t.Fatal("production PromptKit engine is nil") + if len(client.requestsSnapshot()) != 0 { + t.Fatal("module preparation invoked provider") } } func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) - effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve production spell configuration: %v", err) } @@ -355,7 +323,7 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) { if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) { t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems) } - if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { + if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil { t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err) } } @@ -365,7 +333,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution configPath := writeProductionSpellCatalogContractConfig(t) resolve := func(t *testing.T) pipeline.ResolvedPipeline { t.Helper() - effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve production spell configuration: %v", err) } @@ -390,7 +358,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution if err != nil { t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err) } - _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) + _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}) for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} { if err == nil || !strings.Contains(err.Error(), fragment) { t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment) @@ -406,7 +374,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] slot.Items = append(slot.Items, slot.Items[0]) materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot - _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) + _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}) for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} { if err == nil || !strings.Contains(err.Error(), fragment) { t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment) @@ -419,52 +387,27 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil { t.Fatal(err) } - checkpointRoot := filepath.Join(t.TempDir(), "checkpoints") content := productionSpellCatalogContractConfig(t) catalogSource := repositoryPath("examples", "dnd-spell-catalog.json") - if count := strings.Count(content, catalogSource); count != 2 { - t.Fatalf("spell catalog source occurs %d times, want extract and normalize bindings", count) - } content = strings.Replace(content, catalogSource, "__extract_catalog__", 1) content = replaceRequiredOnce(t, content, catalogSource, catalogPath) content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource) - content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot) - configFile := filepath.Join(t.TempDir(), "config.yml") - if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - - llmConstructed := false - chunkStoreConstructed := false - options := Options{ - Catalog: catalogFromRegistries(components.registries), - Registries: components.registries, - LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - llmConstructed = true - return nil, nil, errors.New("LLM client must not be constructed") - }, - ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) { - chunkStoreConstructed = true - return nil, errors.New("chunk-plan store must not be constructed") - }, - } + configFile := writeProductionContractConfig(t, content) + client := &productionFakeLLMClient{} var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "run", "dnd-session", "--config", configFile, - "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), - }, &stdout, &stderr, options) - errText := stderr.String() + code := RunWithOptions([]string{"run", "dnd-session", "--config", configFile, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass"}, &stdout, &stderr, productionRunOptions(t, client)) for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} { - if code == 0 || !strings.Contains(errText, fragment) { - t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment) + if code == 0 || !strings.Contains(stderr.String(), fragment) { + t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment) } } - if llmConstructed || chunkStoreConstructed { - t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed) + if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 { + t.Fatal("invalid reference reached provider or completion reporting") } - if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) { - t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err) + if _, err := os.Stat(filepath.Join(filepath.Dir(configFile), "notarius-output")); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("output allocation before invalid reference: %v", err) } + }) } @@ -487,186 +430,65 @@ func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPip resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings } -func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) { - components := productionTestComponents(t) - client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) - if err != nil { - t.Fatalf("build production LLM runtime: %v", err) - } - if client == nil { - t.Fatal("production LLM runtime returned a nil client") - } - if len(manifests) != 0 { - t.Fatalf("eager profile manifests = %#v, want none", manifests) - } - fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) - if !ok { - t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client) - } - fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() - if err != nil || len(fingerprints) != 1 { - t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err) - } - if _, ok := client.(contracts.LLMProfileManifestProvider); !ok { - t.Fatalf("production LLM client %T does not provide profile manifests", client) - } -} - -func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *testing.T) { - opts, err := normalizeOptions(Options{ - Catalog: pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry()}, - }) - if err != nil { - t.Fatal(err) - } - if opts.promptKitAssets == nil || opts.LLMClientFactory == nil { - t.Fatalf("normalized options = %#v, want shared profile assets and default runtime factory", opts) - } - if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"dnd-extraction"}, opts.promptKitAssets); err != nil { - t.Fatalf("inspect application fallback profile: %v", err) - } - - client, _, err := opts.LLMClientFactory(context.Background(), config.Default(), "dnd-extraction", LLMRuntimeOverrides{}) - if err != nil { - t.Fatalf("build default runtime: %v", err) - } - fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) - if !ok { - t.Fatalf("default runtime client %T does not provide checkpoint fingerprints", client) - } - runtimeFingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() - if err != nil { - t.Fatal(err) - } - directClient, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{Assets: opts.promptKitAssets}) - if err != nil { - t.Fatal(err) - } - inspectionFingerprints, err := directClient.LLMCheckpointFingerprints() - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) { - t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints) - } -} - -func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) { +func TestProductionNativeEngineUsesConfiguredBackend(t *testing.T) { var providerCalls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - call := providerCalls.Add(1) + providerCalls.Add(1) if r.URL.Path != "/v1/chat/completions" { - t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path) + t.Errorf("provider path=%q", r.URL.Path) } w.Header().Set("Content-Type", "application/json") - if call == 1 { - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"error":{"message":"transient"}}`)) - return - } - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} - }`)) + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":3,"completion_tokens":4,"total_tokens":7}}`)) })) defer server.Close() - - profilePath := filepath.Join(t.TempDir(), "profiles.yml") - if err := os.WriteFile(profilePath, []byte(`profiles: - local-profile: - backend: local - model: local-model -`), 0o600); err != nil { + path := writeProductionContractConfig(t, fmt.Sprintf(`version: 6 +promptkit: + backends: + - id: operator + endpoint: %q + concurrency_limit: 2 + profiles: + - id: local-profile + backend: operator + model: local-model + api_key_required: false + prompts: + - id: local-request + default_profile: local-profile + inputs: [{name: transcript, required: true}] + messages: [{role: user, content: '{{input "transcript"}}'}] + output: {format: json, validation_mode: json} +`, server.URL+"/v1")) + loaders := &nativeLoaders{lookupEnv: emptyLookup} + engine, err := loaders.engine(context.Background(), productionSelection(path)) + if err != nil { t.Fatal(err) } - assets := llm.NewAssetRegistry() - if err := assets.RegisterPromptFS(fstest.MapFS{ - "production.local.yaml": {Data: []byte(`prompts: - production.local: - v1: - inputs: - - name: transcript - required: true - messages: - - role: user - content: '{{ input "transcript" }}' - output: - format: json - validation_mode: json -`)}, - }, "."); err != nil { - t.Fatalf("register prompt assets: %v", err) - } - - cfg := config.Default() - cfg.PromptKit.ProfileFile = profilePath - cfg.PromptKit.TransientRetries = 1 - cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{ - Endpoint: server.URL + "/v1", - ConcurrencyLimit: 2, - } - client, manifests, err := productionLLMClientFactoryWithAssets(assets)( - context.Background(), - cfg, - "local-profile", - LLMRuntimeOverrides{}, - ) + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "local-request", Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("local request")}}) if err != nil { - t.Fatalf("build production LLM runtime: %v", err) + t.Fatal(err) } - if len(manifests) != 0 { - t.Fatalf("eager profile manifests = %#v, want none", manifests) - } - - var out map[string]any - _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - PromptID: "production.local", - ProfileID: "local-profile", - Inputs: contracts.LLMInputSet{ - "transcript": contracts.NewLLMInputMaterial("transcript", "text/plain", []byte("local request"), "", ""), - }, - }, &out) - if err != nil { - t.Fatalf("CompleteStructured() error = %v, want nil", err) - } - if providerCalls.Load() != 2 { - t.Fatalf("provider calls = %d, want 2", providerCalls.Load()) - } - provider, ok := client.(contracts.LLMProfileManifestProvider) - if !ok { - t.Fatalf("production client %T does not provide profile manifests", client) - } - recorded := provider.LLMProfileManifests() - if len(recorded) != 1 || recorded[0].BackendID != promptkit.BackendLocal { - t.Fatalf("production profile manifests = %#v, want local backend", recorded) + if result == nil || !result.Validation.IsValid || strings.TrimSpace(result.RawOutput) != `{"ok":true}` || providerCalls.Load() != 1 { + t.Fatalf("native result=%#v calls=%d", result, providerCalls.Load()) } } -func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) { - t.Run("canceled context", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - components := productionTestComponents(t) - client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{}) - if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 { - t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err) - } - }) - - t.Run("nil assets", func(t *testing.T) { - client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) - if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 { - t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err) - } - }) +func TestProductionNativeLoadHonorsCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + loaders := &nativeLoaders{lookupEnv: emptyLookup} + engine, err := loaders.engine(ctx, productionSelection(filepath.Join(t.TempDir(), "missing.yml"))) + if !errors.Is(err, context.Canceled) || engine != nil { + t.Fatalf("engine=%v error=%v", engine, err) + } } func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) { - base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml")) + base := productionSpellCatalogContractConfig(t) validPath := writeProductionContractConfig(t, base) options := productionCLIOptions(t) var stdout, stderr strings.Builder - if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 { + if code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", validPath}, &stdout, &stderr, options); code != 0 { t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } tests := []struct { @@ -689,9 +511,9 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) }, { name: "invalid artifact variant", - content: base, + content: replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: noop\n"), options: productionCLIOptionsWithoutSpellNormalizer(t), - fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"}, + fragments: []string{"normalizer", "noop", string(dnd.SpellListKind)}, }, { name: "deterministic validator with profile", @@ -704,8 +526,8 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) t.Run(tt.name, func(t *testing.T) { path := writeProductionContractConfig(t, tt.content) var stdout, stderr strings.Builder - code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options) - if code != 1 { + code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, tt.options) + if code != 2 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } for _, fragment := range tt.fragments { @@ -718,11 +540,11 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) } func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) { - base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml")) + base := productionSpellCatalogContractConfig(t) content := replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n") path := writeProductionContractConfig(t, content) components := productionTestComponents(t) - effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + effective, err := loadMaintainedExample(t, path).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve normalize override: %v", err) } @@ -738,67 +560,48 @@ func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key) } -func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testing.T) { +func TestProductionSceneRunPublishesAnnotationFreeChunkPlan(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") - configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes")) - fake := &productionFakeLLMClient{} - options := productionRunOptions(t, fake) + path := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes")) + client := &productionFakeLLMClient{} var stdout, stderr strings.Builder - code := RunWithOptions([]string{ - "run", "dnd-session", "--config", configPath, - "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), - "--chunk_cache", "bypass", "--session-id", "offline-session", - }, &stdout, &stderr, options) + code := RunWithOptions([]string{"run", "dnd-session", "--config", path, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass", "--llm-session-id", "offline-session", "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client)) if code != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } - manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json")) - if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key { - t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan) + var report runResult + if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil { + t.Fatal(err) } - if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID { - t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID) + if report.ArtifactCount != 4 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) { + t.Fatalf("completion=%#v", report) } - if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID { - t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID) - } - index := readProductionJSON[productionChunkMapIndex](t, filepath.Join(outputRoot, productionRunID, "index.json")) - if index.ChunkMap == nil || index.ChunkMap.ArtifactKind != chunkmap.ArtifactKind || index.ChunkMap.File != "chunk-map.json" || index.ChunkMap.MediaType != chunkmap.MediaType || index.ChunkMap.SchemaID != chunkmap.SchemaID || index.ChunkMap.SchemaName != chunkmap.SchemaName || index.ChunkMap.SchemaVersion != chunkmap.SchemaVersion { - t.Fatalf("chunk map index = %#v, want fixed chunk map descriptor", index.ChunkMap) - } - for _, output := range index.OutputFiles { - if output.File == index.ChunkMap.File { - t.Fatalf("lane output files = %#v, want no chunk map", index.OutputFiles) - } - } - content, err := os.ReadFile(filepath.Join(outputRoot, productionRunID, index.ChunkMap.File)) + content, err := os.ReadFile(filepath.Join(report.OutputDirectory, "chunk-map.json")) if err != nil { t.Fatal(err) } - chunkMap, err := chunkmap.New().Decode(content) + value, err := chunkmap.New().Decode(content) if err != nil { - t.Fatalf("Decode(chunk map) error = %v", err) + t.Fatal(err) } - if chunkMap.SourceID != "session-alpha" || chunkMap.SourceDigest != manifest.ChunkPlan.SourceDigest || chunkMap.PlanDigest != manifest.ChunkPlan.PlanDigest || chunkMap.RequestedChunker != scenes.Key || chunkMap.Producer.InputModule != "seriatim" || chunkMap.Producer.ChunkModule != scenes.Key || chunkMap.Producer.LLMProfile != manifest.ChunkPlan.ProducerLLMProfile { - t.Fatalf("chunk map identity and producer = %#v, want accepted scene plan provenance", chunkMap) + if value.SourceID != "session-alpha" || value.SourceDigest == "" || value.PlanDigest == "" || value.RequestedChunker != scenes.Key || value.Producer.InputModule != "seriatim" || value.Producer.ChunkModule != scenes.Key { + t.Fatalf("chunk map=%#v", value) } - if len(chunkMap.Chunks) != 1 || chunkMap.Chunks[0].ID != "chunk-000001" || chunkMap.Chunks[0].Index != 0 || chunkMap.Chunks[0].SourceRef.SourceID != "session-alpha" || chunkMap.Chunks[0].SourceRef.StartUnitID != 1 || chunkMap.Chunks[0].SourceRef.EndUnitID != 2 || chunkMap.Chunks[0].UnitCount != 2 { - t.Fatalf("chunk map chunks = %#v, want one stable accepted scene range", chunkMap.Chunks) + if len(value.Chunks) != 1 || value.Chunks[0].ID != "chunk-000001" || value.Chunks[0].Index != 0 || value.Chunks[0].SourceRef.SourceID != "session-alpha" || value.Chunks[0].SourceRef.StartUnitID != 1 || value.Chunks[0].SourceRef.EndUnitID != 2 || value.Chunks[0].UnitCount != 2 { + t.Fatalf("chunks=%#v", value.Chunks) } - if len(chunkMap.PlanAnnotations) != 0 { - t.Fatalf("chunk map plan annotations = %#v, want none", chunkMap.PlanAnnotations) + if len(value.PlanAnnotations) != 0 || len(value.Chunks[0].Annotations) != 0 { + t.Fatalf("unexpected annotations: %#v", value) } - if len(chunkMap.Chunks[0].Annotations) != 0 { - t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations) + for _, property := range []string{"scenes", "spell_casts", "occurrences"} { + if len(client.requestsWithProperty(property)) != 1 { + t.Fatalf("%s provider calls=%d", property, len(client.requestsWithProperty(property))) + } } - warnings := readProductionJSON[struct { - Groups []contracts.DiagnosticGroup `json:"groups"` - }](t, filepath.Join(outputRoot, productionRunID, "warnings.json")) - if len(warnings.Groups) != 0 { - t.Fatalf("warnings = %#v, want none", warnings.Groups) - } - if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 { - t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts()) + for _, name := range []string{"manifest.json", "index.json", "warnings.json"} { + if _, err := os.Stat(filepath.Join(report.OutputDirectory, name)); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("retired reporting sidecar %s: %v", name, err) + } } } @@ -819,13 +622,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample { func productionSpellCatalogContractConfig(t *testing.T) string { t.Helper() - return fmt.Sprintf(`version: 5 -cache: - chunk_plans: - mode: bypass - checkpoints: - enabled: false - directory: "" + return fmt.Sprintf(`version: 6 pipelines: dnd-session: input: @@ -839,7 +636,6 @@ pipelines: spells: extract: module: dnd/spells - retries: 2 references: spell_catalog: path: %q @@ -886,10 +682,9 @@ func productionCLIOptions(t *testing.T) Options { func productionOptionsFromComponents(components productionComponents) Options { return Options{ - Catalog: catalogFromRegistries(components.registries), - Registries: components.registries, - LookupEnv: emptyLookup, - promptKitAssets: components.assets, + Catalog: catalogFromRegistries(components.registries), + Registries: components.registries, + LookupEnv: emptyLookup, } } @@ -901,35 +696,29 @@ func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options { if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil { t.Fatalf("register mismatched normalizer: %v", err) } - return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets}) + return productionOptionsFromComponents(productionComponents{registries: registries}) } const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef" -func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options { +func productionRunOptions(t *testing.T, client promptkit.LLMClient) Options { t.Helper() options := productionCLIOptions(t) options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() } options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil } - options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") } - options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - return fake, nil, nil - } + options.LLMClient = client return options } func productionRunConfig(outputRoot, chunkModule string) string { - return fmt.Sprintf(`version: 5 + return fmt.Sprintf(`version: 6 output: directory: %q -cache: - chunk_plans: - mode: bypass - checkpoints: {} debug: directory: %q pipelines: dnd-session: + cache: bypass input: module: seriatim chunk: @@ -959,21 +748,7 @@ pipelines: item-occurrences: extract: module: dnd/item-occurrences -`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) -} - -type productionChunkMapIndex struct { - OutputFiles []struct { - File string `json:"file"` - } `json:"output_files"` - ChunkMap *struct { - ArtifactKind contracts.ArtifactKind `json:"artifact_kind"` - File string `json:"file"` - MediaType string `json:"media_type"` - SchemaID string `json:"schema_id"` - SchemaName string `json:"schema_name"` - SchemaVersion string `json:"schema_version"` - } `json:"chunk_map"` +`+productionProviderConfiguration, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) } func writeProductionContractConfig(t *testing.T, content string) string { @@ -1015,74 +790,129 @@ func readProductionJSON[T any](t *testing.T, path string) T { return value } +const productionProviderConfiguration = ` +promptkit: + collections: + notarius: + profiles: + - id: dnd-extraction + endpoint: https://example.invalid/v1 + model: production-test-model + api_key_required: false +` + +func productionSelection(path string) ckconfig.Selection { + return ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: path}} +} + +func productionNativeEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine { + t.Helper() + profiles := make([]map[string]any, len(profileIDs)) + for index, id := range profileIDs { + profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": id + "-model", "api_key_required": false} + } + content, err := json.Marshal(map[string]any{"version": 6, "promptkit": map[string]any{ + "profiles": profiles, + "collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "production-test-model", "api_key_required": false}}}}, + }}) + if err != nil { + t.Fatal(err) + } + path := writeProductionContractConfig(t, string(content)) + loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}} + engine, err := loaders.engine(context.Background(), productionSelection(path)) + if err != nil { + t.Fatal(err) + } + return engine +} + +func productionJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) { + for _, message := range request.Prompt.Messages { + for offset := range len(message.Content) { + if message.Content[offset] != '{' { + continue + } + decoder := json.NewDecoder(strings.NewReader(message.Content[offset:])) + var object map[string]json.RawMessage + if decoder.Decode(&object) == nil && object[field] != nil { + return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil + } + } + } + return nil, fmt.Errorf("native request has no JSON input with field %q", field) +} + type productionFakeLLMClient struct { mu sync.Mutex - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest spellResponse string } -func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (client *productionFakeLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - var content []byte - switch req.PromptID { - case scenes.PromptID: - content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`) - case spells.PromptID: - if client.spellResponse != "" { - content = []byte(client.spellResponse) - } else { - content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) - } - case itemoccurrenceextract.PromptID: - content = []byte(`{"occurrences":[]}`) - case itemregistryextract.PromptID: - content = []byte(`{"items":[]}`) - case itemregistrynormalize.PromptID: - content = []byte(`{"duplicate_groups":[]}`) - default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) - } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) - } - snapshot, err := contracts.CloneStructuredCompletionRequest(req) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err) - } - client.mu.Lock() - client.requests = append(client.requests, snapshot) - client.mu.Unlock() - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil -} - -func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest { client.mu.Lock() defer client.mu.Unlock() - var requests []contracts.StructuredCompletionRequest - for _, req := range client.requests { - if req.PromptID == promptID { - snapshot, err := contracts.CloneStructuredCompletionRequest(req) - if err != nil { - panic(err) - } - requests = append(requests, snapshot) + client.requests = append(client.requests, request) + var content string + switch { + case productionSchemaHasProperty(request, "scenes"): + content = `{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}` + case productionSchemaHasProperty(request, "spell_casts"): + content = client.spellResponse + if content == "" { + content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` + } + case productionSchemaHasProperty(request, "occurrences"): + content = `{"occurrences":[]}` + case productionSchemaHasProperty(request, "items"): + content = `{"items":[]}` + case productionSchemaHasProperty(request, "duplicate_groups"): + content = `{"duplicate_groups":[]}` + default: + return nil, fmt.Errorf("unexpected native response contract") + } + return &promptkit.GenerateResponse{Content: content}, nil +} + +func (client *productionFakeLLMClient) requestsSnapshot() []promptkit.GenerateRequest { + client.mu.Lock() + defer client.mu.Unlock() + return append([]promptkit.GenerateRequest(nil), client.requests...) +} + +func (client *productionFakeLLMClient) requestsWithProperty(property string) []promptkit.GenerateRequest { + var requests []promptkit.GenerateRequest + for _, request := range client.requestsSnapshot() { + if productionSchemaHasProperty(request, property) { + requests = append(requests, request) } } return requests } -func (client *productionFakeLLMClient) requestPrompts() []string { - client.mu.Lock() - defer client.mu.Unlock() - prompts := make([]string, 0, len(client.requests)) - for _, req := range client.requests { - prompts = append(prompts, req.PromptID) +func productionSchemaHasProperty(request promptkit.GenerateRequest, property string) bool { + if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil { + return false } - return prompts + content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema) + if err != nil { + return false + } + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + } + if json.Unmarshal(content, &schema) != nil { + return false + } + _, ok := schema.Properties[property] + return ok } +var _ promptkit.LLMClient = (*productionFakeLLMClient)(nil) + func repositoryPath(parts ...string) string { _, file, _, _ := runtime.Caller(0) return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)