Exercise production CLI contracts through native loaders and providers

This commit is contained in:
2026-11-22 05:01:09 +00:00
parent 572b267c5d
commit 5e68b74469
7 changed files with 343 additions and 578 deletions

View File

@@ -1689,11 +1689,22 @@ race checks pass, as do framework/integration vet checks.
The production executable builds. Real executable route and text/JSON/YAML/
fallback checks passed with a local fake HTTP provider after removal of the old
runtime wrapper. Focused native CLI tests pass, but the full CLI suite and
whole-tree compilation remain blocked by legacy CLI fixtures/imports; the current
setup failure is old-path ConfigKit importing the removed YAML `plugin/limit`.
Remaining CLI test migration, dependency tidy/old-path removal, notice
reconciliation, and whole-tree validation are required before completion.
This stage is not marked finished.
whole-tree compilation remain blocked by legacy CLI fixtures/imports.
Migrated the shared production CLI fixture to the real native application loader,
engine, provider requests, version 6 test inputs, and native pipeline validation.
Retained catalog/typed registration, validator-chain replacement, reference
materialization/limits, configured backend selection, scene description profile
selection, source ranges, and published chunk-map provenance. Provider observations
and codec-decoded files replace profile manifests and normalized-output ledgers.
Oversized references prevent model work/publication without requiring the retired
runtime-factory construction order. Removed only tests of deleted asset/fingerprint
factories; native captured-loading tests cover shared configuration ownership.
Focused production CLI tests, vet, and repeated race checks pass. Whole CLI setup now stops at
remaining fixtures importing the removed reporting package, rather than the
legacy ConfigKit YAML import. Remaining CLI test migration, dependency tidy/
old-path removal, notice reconciliation, and whole-tree validation are required
before completion. This stage is not marked finished.
**Goal:** delete the old architecture rather than leave dormant alternatives.

View File

@@ -27,7 +27,7 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("artifact lanes = %#v, want one combat lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
}
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key {
if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key {
t.Fatalf("resolved combat lane = %#v, want typed production composition", lane)
}
@@ -172,7 +172,7 @@ func productionCombatContractConfig() config.Config {
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
Extract: pipeline.ModuleBinding{Module: combatextract.Key},
Normalize: pipeline.Binding(combatnormalize.Key),
},
},

View File

@@ -28,7 +28,7 @@ func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("artifact lanes = %#v, want one NPC lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
}
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key {
if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key {
t.Fatalf("resolved NPC lane = %#v, want typed production composition", lane)
}
if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
@@ -129,7 +129,7 @@ func productionNPCContractConfig() config.Config {
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"npc_registry": {
Extract: pipeline.ModuleBinding{Module: npcextract.Key, Retries: 2},
Extract: pipeline.ModuleBinding{Module: npcextract.Key},
Normalize: pipeline.Binding(npcnormalize.Key),
},
},

View File

@@ -44,7 +44,7 @@ func TestProductionNPCOccurrencePipelineResolvesAndPrepares(t *testing.T) {
t.Fatalf("generated bindings = %#v", bindings)
}
}
if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("Prepare() error = %v", err)
}
@@ -120,7 +120,7 @@ func TestProductionNPCOccurrenceReferencesRejectIncompatibleExternalRegistries(t
if err != nil {
t.Fatalf("MaterializeReferences() error = %v", err)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err == nil || !strings.Contains(err.Error(), test.want) {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
}
})
@@ -175,7 +175,7 @@ func TestProductionOccurrencePipelinesAcceptCompatibleExternalRegistries(t *test
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("Prepare() error = %v", err)
}
})

View File

@@ -2,22 +2,19 @@ package cli
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
"go.maximumdirect.net/promptkit"
)
func TestProductionSceneDescriptionWorkflow(t *testing.T) {
@@ -48,37 +45,23 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
}
llmClient := &sceneDescriptionLLM{}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient, "scene-description-profile")})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
if len(output.OutputFiles) != 1 || output.FallbackCount != 0 || output.OutputFiles[0].Name != "lanes/scene-descriptions.json" {
t.Fatalf("output=%#v, want published scene descriptions", output)
}
wantProfiles := []artifacts.LLMProfileManifest{{
ID: "scene-description-profile",
Provider: "promptkit",
Model: "deterministic",
}}
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
}
normalizedOutput := output.NormalizeOutputs[0]
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
}
var value dnd.SceneDescriptionList
if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil {
t.Fatalf("decode normalized artifact: %v", err)
value, err := scenecodec.New().Decode(output.OutputFiles[0].Bytes)
if err != nil {
t.Fatal(err)
}
want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{
{ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."},
@@ -87,61 +70,34 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if !reflect.DeepEqual(value, want) {
t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want)
}
durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json")
if !reflect.DeepEqual(durable, want) {
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
}
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
}
}
type sceneDescriptionLLM struct {
mu sync.Mutex
profile *artifacts.LLMProfileManifest
}
type sceneDescriptionLLM struct{}
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (*sceneDescriptionLLM) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if req.PromptID != sceneextract.PromptID {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
if !productionSchemaHasProperty(request, "kind") || request.Target.Model != "scene-description-profile-model" {
return nil, fmt.Errorf("unexpected native scene-description request for %q", request.Target.Model)
}
transcript, err := productionJSONInput(request, "units")
if err != nil {
return nil, err
}
transcript := string(req.Inputs["transcript"].Content)
var content string
switch {
case strings.Contains(transcript, "Cure Wounds"):
case strings.Contains(string(transcript), "Cure Wounds"):
content = `{"kind":"narrative","title":" Aria casts Cure Wounds ","summary":" Aria casts Cure Wounds. "}`
case strings.Contains(transcript, "Shield"):
case strings.Contains(string(transcript), "Shield"):
content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}`
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript)
return nil, fmt.Errorf("unexpected transcript material %q", transcript)
}
if err := json.Unmarshal([]byte(content), out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
}
profile := artifacts.LLMProfileManifest{
ID: req.ProfileID,
Provider: "promptkit",
Model: "deterministic",
}
client.mu.Lock()
client.profile = &profile
client.mu.Unlock()
return contracts.StructuredCompletionResponse{
Content: []byte(content),
Provider: profile.Provider,
Model: profile.Model,
ProfileID: profile.ID,
}, nil
return &promptkit.GenerateResponse{Content: content}, nil
}
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
client.mu.Lock()
defer client.mu.Unlock()
if client.profile == nil {
return nil
}
return []artifacts.LLMProfileManifest{*client.profile}
}
var _ promptkit.LLMClient = (*sceneDescriptionLLM)(nil)

View File

@@ -1,7 +1,6 @@
package cli
import (
"context"
"errors"
"fmt"
"io/fs"
@@ -9,75 +8,44 @@ import (
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *testing.T) {
components := productionTestComponents(t)
npcPath := filepath.Join(t.TempDir(), "npcs.json")
if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
func TestOversizedNPCRegistryFailsBeforeModelWorkAndPublication(t *testing.T) {
root := t.TempDir()
npcPath := filepath.Join(root, "npcs.json")
if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0600); err != nil {
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := fmt.Sprintf(`version: 5
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: true
directory: %q
outputRoot, cacheRoot := filepath.Join(root, "output"), filepath.Join(root, "cache")
content := fmt.Sprintf(`version: 6
output: {directory: %q}
cache_directory: %q
pipelines:
dnd-session:
input:
module: seriatim
input: {module: seriatim}
artifacts:
spells:
extract:
module: dnd/spells
references:
npc_registry:
path: %q
normalize:
module: dnd/spells
`, checkpointRoot, npcPath)
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
llmConstructed := false
chunkStoreConstructed := false
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
chunkStoreConstructed = true
return nil, errors.New("chunk-plan store must not be constructed")
},
}
npc_registry: {path: %q}
normalize: {module: dnd/spells}
`, outputRoot, cacheRoot, npcPath)
configPath := writeProductionContractConfig(t, content)
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configPath,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--chunk_cache", "bypass", "--output-dir", t.TempDir(),
}, &stdout, &stderr, options)
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json")}, &stdout, &stderr, productionRunOptions(t, client))
for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npc_registry"`, "1048577 bytes", "limit 1048576"} {
if code == 0 || !strings.Contains(stderr.String(), fragment) {
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment)
t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment)
}
}
if llmConstructed || chunkStoreConstructed {
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 {
t.Fatal("invalid reference reached provider or completion reporting")
}
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
for _, path := range []string{outputRoot, cacheRoot} {
if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("state allocated before invalid reference: %s: %v", path, err)
}
}
}

View File

@@ -16,17 +16,12 @@ import (
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
@@ -60,8 +55,9 @@ import (
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
@@ -217,126 +213,98 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
}
func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
path := writeProductionSpellCatalogContractConfig(t)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"config", "validate", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--pipeline", "dnd-session",
}, &stdout, &stderr, Options{})
if code != 0 || stderr.Len() != 0 {
t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("validate production composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
components := productionTestComponents(t)
newEngine := func(profileFile string) (*promptkit.Engine, error) {
t.Helper()
sources, options, err := components.assets.PromptKitConfigurationSources()
if err != nil {
return nil, err
}
if profileFile != "" {
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: configkit.File(profileFile), Imports: true,
}}})
}
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
for _, test := range []struct {
name, companion string
invalid bool
}{
{name: "embedded defaults"},
{name: "operator override", companion: `collections:
notarius:
profiles:
- id: dnd-extraction
endpoint: https://operator.example.invalid/v1
model: operator-model
timeout_seconds: 75
`},
{name: "invalid operator override", companion: `collections:
notarius:
profiles:
- id: dnd-extraction
model: [invalid]
`, invalid: true},
} {
t.Run(test.name, func(t *testing.T) {
path := writeProductionContractConfig(t, "version: 6\n")
if test.companion != "" {
if err := os.WriteFile(filepath.Join(filepath.Dir(path), "prompts.yml"), []byte(test.companion), 0600); err != nil {
t.Fatal(err)
}
}
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if test.invalid {
if err == nil {
t.Fatal("invalid operator resource fell back to embedded defaults")
}
return
}
if err != nil {
t.Fatal(err)
}
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
if err != nil {
t.Fatal(err)
}
if test.companion == "" && (profile.Backend == nil || *profile.Backend != "openrouter" || profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil) {
t.Fatalf("lost embedded profile defaults: %#v", profile)
}
model, timeout := "openai/gpt-5.6-luna", 240
if test.companion != "" {
model, timeout = "operator-model", 75
if profile.Endpoint == nil || *profile.Endpoint != "https://operator.example.invalid/v1" {
t.Fatalf("operator endpoint=%#v", profile.Endpoint)
}
}
if profile.Model == nil || *profile.Model != model || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != timeout {
t.Fatalf("profile=%#v", profile)
}
})
}
t.Run("fallback", func(t *testing.T) {
engine, err := newEngine("")
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want fallback profile", err)
}
params := inspection.EffectiveModelParams
if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" {
t.Fatalf("fallback profile parameters = %#v", params)
}
if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 {
t.Fatalf("fallback profile selected optional provider controls: %#v", params)
}
})
t.Run("valid operator profile wins", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(`profiles:
dnd-extraction:
endpoint: http://operator.example.test/v1
model: operator-model
timeout_seconds: 75
`), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
}
params := inspection.EffectiveModelParams
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
}
})
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte("profiles:\n dnd-extraction:\n endpoint: http://operator.example.test/v1\n model: operator-model\n unknown: value\n"), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err == nil {
_, err = engine.InspectProfile(context.Background(), "dnd-extraction")
}
if err == nil {
t.Fatal("operator profile error = nil, want failure instead of fallback")
}
})
}
func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
func TestProductionModulesPrepareWithoutProviderCalls(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{
ID: "dnd-scenes",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding("dnd/scenes"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: pipeline.Binding("dnd/spells")},
},
ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: pipeline.Binding("dnd/spells")}},
}
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production scene pipeline: %v", err)
t.Fatal(err)
}
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
t.Fatalf("prepare production scene and spell modules: %v", err)
client := &productionFakeLLMClient{}
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)}); err != nil {
t.Fatal(err)
}
schemaFS, err := components.assets.SchemaFS()
if err != nil {
t.Fatalf("production schema assets: %v", err)
}
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
t.Fatalf("generic reconciliation schema asset: %v", err)
}
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
if engine == nil {
t.Fatal("production PromptKit engine is nil")
if len(client.requestsSnapshot()) != 0 {
t.Fatal("module preparation invoked provider")
}
}
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production spell configuration: %v", err)
}
@@ -355,7 +323,7 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) {
t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
}
}
@@ -365,7 +333,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
configPath := writeProductionSpellCatalogContractConfig(t)
resolve := func(t *testing.T) pipeline.ResolvedPipeline {
t.Helper()
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production spell configuration: %v", err)
}
@@ -390,7 +358,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err)
}
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
@@ -406,7 +374,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"]
slot.Items = append(slot.Items, slot.Items[0])
materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
@@ -419,52 +387,27 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := productionSpellCatalogContractConfig(t)
catalogSource := repositoryPath("examples", "dnd-spell-catalog.json")
if count := strings.Count(content, catalogSource); count != 2 {
t.Fatalf("spell catalog source occurs %d times, want extract and normalize bindings", count)
}
content = strings.Replace(content, catalogSource, "__extract_catalog__", 1)
content = replaceRequiredOnce(t, content, catalogSource, catalogPath)
content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource)
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
configFile := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
llmConstructed := false
chunkStoreConstructed := false
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
chunkStoreConstructed = true
return nil, errors.New("chunk-plan store must not be constructed")
},
}
configFile := writeProductionContractConfig(t, content)
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configFile,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
}, &stdout, &stderr, options)
errText := stderr.String()
code := RunWithOptions([]string{"run", "dnd-session", "--config", configFile, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass"}, &stdout, &stderr, productionRunOptions(t, client))
for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} {
if code == 0 || !strings.Contains(errText, fragment) {
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment)
if code == 0 || !strings.Contains(stderr.String(), fragment) {
t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment)
}
}
if llmConstructed || chunkStoreConstructed {
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 {
t.Fatal("invalid reference reached provider or completion reporting")
}
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
if _, err := os.Stat(filepath.Join(filepath.Dir(configFile), "notarius-output")); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("output allocation before invalid reference: %v", err)
}
})
}
@@ -487,186 +430,65 @@ func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPip
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings
}
func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
components := productionTestComponents(t)
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
if client == nil {
t.Fatal("production LLM runtime returned a nil client")
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
}
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil || len(fingerprints) != 1 {
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
}
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
t.Fatalf("production LLM client %T does not provide profile manifests", client)
}
}
func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *testing.T) {
opts, err := normalizeOptions(Options{
Catalog: pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry()},
})
if err != nil {
t.Fatal(err)
}
if opts.promptKitAssets == nil || opts.LLMClientFactory == nil {
t.Fatalf("normalized options = %#v, want shared profile assets and default runtime factory", opts)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"dnd-extraction"}, opts.promptKitAssets); err != nil {
t.Fatalf("inspect application fallback profile: %v", err)
}
client, _, err := opts.LLMClientFactory(context.Background(), config.Default(), "dnd-extraction", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build default runtime: %v", err)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("default runtime client %T does not provide checkpoint fingerprints", client)
}
runtimeFingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
directClient, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{Assets: opts.promptKitAssets})
if err != nil {
t.Fatal(err)
}
inspectionFingerprints, err := directClient.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
}
}
func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
func TestProductionNativeEngineUsesConfiguredBackend(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
call := providerCalls.Add(1)
providerCalls.Add(1)
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path)
t.Errorf("provider path=%q", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
if call == 1 {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":{"message":"transient"}}`))
return
}
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":3,"completion_tokens":4,"total_tokens":7}}`))
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`profiles:
local-profile:
backend: local
model: local-model
`), 0o600); err != nil {
path := writeProductionContractConfig(t, fmt.Sprintf(`version: 6
promptkit:
backends:
- id: operator
endpoint: %q
concurrency_limit: 2
profiles:
- id: local-profile
backend: operator
model: local-model
api_key_required: false
prompts:
- id: local-request
default_profile: local-profile
inputs: [{name: transcript, required: true}]
messages: [{role: user, content: '{{input "transcript"}}'}]
output: {format: json, validation_mode: json}
`, server.URL+"/v1"))
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if err != nil {
t.Fatal(err)
}
assets := llm.NewAssetRegistry()
if err := assets.RegisterPromptFS(fstest.MapFS{
"production.local.yaml": {Data: []byte(`prompts:
production.local:
v1:
inputs:
- name: transcript
required: true
messages:
- role: user
content: '{{ input "transcript" }}'
output:
format: json
validation_mode: json
`)},
}, "."); err != nil {
t.Fatalf("register prompt assets: %v", err)
}
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.TransientRetries = 1
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
client, manifests, err := productionLLMClientFactoryWithAssets(assets)(
context.Background(),
cfg,
"local-profile",
LLMRuntimeOverrides{},
)
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "local-request", Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("local request")}})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
t.Fatal(err)
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "production.local",
ProfileID: "local-profile",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "text/plain", []byte("local request"), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if providerCalls.Load() != 2 {
t.Fatalf("provider calls = %d, want 2", providerCalls.Load())
}
provider, ok := client.(contracts.LLMProfileManifestProvider)
if !ok {
t.Fatalf("production client %T does not provide profile manifests", client)
}
recorded := provider.LLMProfileManifests()
if len(recorded) != 1 || recorded[0].BackendID != promptkit.BackendLocal {
t.Fatalf("production profile manifests = %#v, want local backend", recorded)
if result == nil || !result.Validation.IsValid || strings.TrimSpace(result.RawOutput) != `{"ok":true}` || providerCalls.Load() != 1 {
t.Fatalf("native result=%#v calls=%d", result, providerCalls.Load())
}
}
func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
components := productionTestComponents(t)
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
}
})
t.Run("nil assets", func(t *testing.T) {
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
})
func TestProductionNativeLoadHonorsCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(ctx, productionSelection(filepath.Join(t.TempDir(), "missing.yml")))
if !errors.Is(err, context.Canceled) || engine != nil {
t.Fatalf("engine=%v error=%v", engine, err)
}
}
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
base := productionSpellCatalogContractConfig(t)
validPath := writeProductionContractConfig(t, base)
options := productionCLIOptions(t)
var stdout, stderr strings.Builder
if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
if code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", validPath}, &stdout, &stderr, options); code != 0 {
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
tests := []struct {
@@ -689,9 +511,9 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
},
{
name: "invalid artifact variant",
content: base,
content: replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: noop\n"),
options: productionCLIOptionsWithoutSpellNormalizer(t),
fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"},
fragments: []string{"normalizer", "noop", string(dnd.SpellListKind)},
},
{
name: "deterministic validator with profile",
@@ -704,8 +526,8 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
t.Run(tt.name, func(t *testing.T) {
path := writeProductionContractConfig(t, tt.content)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options)
if code != 1 {
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, tt.options)
if code != 2 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
for _, fragment := range tt.fragments {
@@ -718,11 +540,11 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
}
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
base := productionSpellCatalogContractConfig(t)
content := replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
path := writeProductionContractConfig(t, content)
components := productionTestComponents(t)
effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, path).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve normalize override: %v", err)
}
@@ -738,67 +560,48 @@ func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T)
t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key)
}
func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testing.T) {
func TestProductionSceneRunPublishesAnnotationFreeChunkPlan(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
fake := &productionFakeLLMClient{}
options := productionRunOptions(t, fake)
path := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configPath,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--chunk_cache", "bypass", "--session-id", "offline-session",
}, &stdout, &stderr, options)
code := RunWithOptions([]string{"run", "dnd-session", "--config", path, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass", "--llm-session-id", "offline-session", "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json"))
if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key {
t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan)
var report runResult
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
t.Fatal(err)
}
if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID {
t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID)
if report.ArtifactCount != 4 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("completion=%#v", report)
}
if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID {
t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID)
}
index := readProductionJSON[productionChunkMapIndex](t, filepath.Join(outputRoot, productionRunID, "index.json"))
if index.ChunkMap == nil || index.ChunkMap.ArtifactKind != chunkmap.ArtifactKind || index.ChunkMap.File != "chunk-map.json" || index.ChunkMap.MediaType != chunkmap.MediaType || index.ChunkMap.SchemaID != chunkmap.SchemaID || index.ChunkMap.SchemaName != chunkmap.SchemaName || index.ChunkMap.SchemaVersion != chunkmap.SchemaVersion {
t.Fatalf("chunk map index = %#v, want fixed chunk map descriptor", index.ChunkMap)
}
for _, output := range index.OutputFiles {
if output.File == index.ChunkMap.File {
t.Fatalf("lane output files = %#v, want no chunk map", index.OutputFiles)
}
}
content, err := os.ReadFile(filepath.Join(outputRoot, productionRunID, index.ChunkMap.File))
content, err := os.ReadFile(filepath.Join(report.OutputDirectory, "chunk-map.json"))
if err != nil {
t.Fatal(err)
}
chunkMap, err := chunkmap.New().Decode(content)
value, err := chunkmap.New().Decode(content)
if err != nil {
t.Fatalf("Decode(chunk map) error = %v", err)
t.Fatal(err)
}
if chunkMap.SourceID != "session-alpha" || chunkMap.SourceDigest != manifest.ChunkPlan.SourceDigest || chunkMap.PlanDigest != manifest.ChunkPlan.PlanDigest || chunkMap.RequestedChunker != scenes.Key || chunkMap.Producer.InputModule != "seriatim" || chunkMap.Producer.ChunkModule != scenes.Key || chunkMap.Producer.LLMProfile != manifest.ChunkPlan.ProducerLLMProfile {
t.Fatalf("chunk map identity and producer = %#v, want accepted scene plan provenance", chunkMap)
if value.SourceID != "session-alpha" || value.SourceDigest == "" || value.PlanDigest == "" || value.RequestedChunker != scenes.Key || value.Producer.InputModule != "seriatim" || value.Producer.ChunkModule != scenes.Key {
t.Fatalf("chunk map=%#v", value)
}
if len(chunkMap.Chunks) != 1 || chunkMap.Chunks[0].ID != "chunk-000001" || chunkMap.Chunks[0].Index != 0 || chunkMap.Chunks[0].SourceRef.SourceID != "session-alpha" || chunkMap.Chunks[0].SourceRef.StartUnitID != 1 || chunkMap.Chunks[0].SourceRef.EndUnitID != 2 || chunkMap.Chunks[0].UnitCount != 2 {
t.Fatalf("chunk map chunks = %#v, want one stable accepted scene range", chunkMap.Chunks)
if len(value.Chunks) != 1 || value.Chunks[0].ID != "chunk-000001" || value.Chunks[0].Index != 0 || value.Chunks[0].SourceRef.SourceID != "session-alpha" || value.Chunks[0].SourceRef.StartUnitID != 1 || value.Chunks[0].SourceRef.EndUnitID != 2 || value.Chunks[0].UnitCount != 2 {
t.Fatalf("chunks=%#v", value.Chunks)
}
if len(chunkMap.PlanAnnotations) != 0 {
t.Fatalf("chunk map plan annotations = %#v, want none", chunkMap.PlanAnnotations)
if len(value.PlanAnnotations) != 0 || len(value.Chunks[0].Annotations) != 0 {
t.Fatalf("unexpected annotations: %#v", value)
}
if len(chunkMap.Chunks[0].Annotations) != 0 {
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
for _, property := range []string{"scenes", "spell_casts", "occurrences"} {
if len(client.requestsWithProperty(property)) != 1 {
t.Fatalf("%s provider calls=%d", property, len(client.requestsWithProperty(property)))
}
}
warnings := readProductionJSON[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Groups)
}
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
for _, name := range []string{"manifest.json", "index.json", "warnings.json"} {
if _, err := os.Stat(filepath.Join(report.OutputDirectory, name)); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("retired reporting sidecar %s: %v", name, err)
}
}
}
@@ -819,13 +622,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
func productionSpellCatalogContractConfig(t *testing.T) string {
t.Helper()
return fmt.Sprintf(`version: 5
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
return fmt.Sprintf(`version: 6
pipelines:
dnd-session:
input:
@@ -839,7 +636,6 @@ pipelines:
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog:
path: %q
@@ -886,10 +682,9 @@ func productionCLIOptions(t *testing.T) Options {
func productionOptionsFromComponents(components productionComponents) Options {
return Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
promptKitAssets: components.assets,
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
}
}
@@ -901,35 +696,29 @@ func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options {
if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil {
t.Fatalf("register mismatched normalizer: %v", err)
}
return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets})
return productionOptionsFromComponents(productionComponents{registries: registries})
}
const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef"
func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
func productionRunOptions(t *testing.T, client promptkit.LLMClient) Options {
t.Helper()
options := productionCLIOptions(t)
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return fake, nil, nil
}
options.LLMClient = client
return options
}
func productionRunConfig(outputRoot, chunkModule string) string {
return fmt.Sprintf(`version: 5
return fmt.Sprintf(`version: 6
output:
directory: %q
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug:
directory: %q
pipelines:
dnd-session:
cache: bypass
input:
module: seriatim
chunk:
@@ -959,21 +748,7 @@ pipelines:
item-occurrences:
extract:
module: dnd/item-occurrences
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}
type productionChunkMapIndex struct {
OutputFiles []struct {
File string `json:"file"`
} `json:"output_files"`
ChunkMap *struct {
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
File string `json:"file"`
MediaType string `json:"media_type"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
} `json:"chunk_map"`
`+productionProviderConfiguration, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}
func writeProductionContractConfig(t *testing.T, content string) string {
@@ -1015,74 +790,129 @@ func readProductionJSON[T any](t *testing.T, path string) T {
return value
}
const productionProviderConfiguration = `
promptkit:
collections:
notarius:
profiles:
- id: dnd-extraction
endpoint: https://example.invalid/v1
model: production-test-model
api_key_required: false
`
func productionSelection(path string) ckconfig.Selection {
return ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: path}}
}
func productionNativeEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
t.Helper()
profiles := make([]map[string]any, len(profileIDs))
for index, id := range profileIDs {
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": id + "-model", "api_key_required": false}
}
content, err := json.Marshal(map[string]any{"version": 6, "promptkit": map[string]any{
"profiles": profiles,
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "production-test-model", "api_key_required": false}}}},
}})
if err != nil {
t.Fatal(err)
}
path := writeProductionContractConfig(t, string(content))
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if err != nil {
t.Fatal(err)
}
return engine
}
func productionJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
for _, message := range request.Prompt.Messages {
for offset := range len(message.Content) {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var object map[string]json.RawMessage
if decoder.Decode(&object) == nil && object[field] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
}
}
}
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
}
type productionFakeLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
spellResponse string
}
func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *productionFakeLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
var content []byte
switch req.PromptID {
case scenes.PromptID:
content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`)
case spells.PromptID:
if client.spellResponse != "" {
content = []byte(client.spellResponse)
} else {
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
}
case itemoccurrenceextract.PromptID:
content = []byte(`{"occurrences":[]}`)
case itemregistryextract.PromptID:
content = []byte(`{"items":[]}`)
case itemregistrynormalize.PromptID:
content = []byte(`{"duplicate_groups":[]}`)
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
}
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock()
client.requests = append(client.requests, snapshot)
client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
}
func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest {
client.mu.Lock()
defer client.mu.Unlock()
var requests []contracts.StructuredCompletionRequest
for _, req := range client.requests {
if req.PromptID == promptID {
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
client.requests = append(client.requests, request)
var content string
switch {
case productionSchemaHasProperty(request, "scenes"):
content = `{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`
case productionSchemaHasProperty(request, "spell_casts"):
content = client.spellResponse
if content == "" {
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
}
case productionSchemaHasProperty(request, "occurrences"):
content = `{"occurrences":[]}`
case productionSchemaHasProperty(request, "items"):
content = `{"items":[]}`
case productionSchemaHasProperty(request, "duplicate_groups"):
content = `{"duplicate_groups":[]}`
default:
return nil, fmt.Errorf("unexpected native response contract")
}
return &promptkit.GenerateResponse{Content: content}, nil
}
func (client *productionFakeLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
client.mu.Lock()
defer client.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), client.requests...)
}
func (client *productionFakeLLMClient) requestsWithProperty(property string) []promptkit.GenerateRequest {
var requests []promptkit.GenerateRequest
for _, request := range client.requestsSnapshot() {
if productionSchemaHasProperty(request, property) {
requests = append(requests, request)
}
}
return requests
}
func (client *productionFakeLLMClient) requestPrompts() []string {
client.mu.Lock()
defer client.mu.Unlock()
prompts := make([]string, 0, len(client.requests))
for _, req := range client.requests {
prompts = append(prompts, req.PromptID)
func productionSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
return false
}
return prompts
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
if err != nil {
return false
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
}
if json.Unmarshal(content, &schema) != nil {
return false
}
_, ok := schema.Properties[property]
return ok
}
var _ promptkit.LLMClient = (*productionFakeLLMClient)(nil)
func repositoryPath(parts ...string) string {
_, file, _, _ := runtime.Caller(0)
return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)