Exercise production CLI contracts through native loaders and providers

This commit is contained in:
2026-11-22 05:01:09 +00:00
parent 572b267c5d
commit 5e68b74469
7 changed files with 343 additions and 578 deletions

View File

@@ -27,7 +27,7 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("artifact lanes = %#v, want one combat lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
}
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key {
if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key {
t.Fatalf("resolved combat lane = %#v, want typed production composition", lane)
}
@@ -172,7 +172,7 @@ func productionCombatContractConfig() config.Config {
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
Extract: pipeline.ModuleBinding{Module: combatextract.Key},
Normalize: pipeline.Binding(combatnormalize.Key),
},
},

View File

@@ -28,7 +28,7 @@ func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("artifact lanes = %#v, want one NPC lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
}
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key {
if lane.ID != "npc_registry" || lane.ArtifactKind != dnd.NPCRegistryKind || lane.Extract.Module != npcextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key {
t.Fatalf("resolved NPC lane = %#v, want typed production composition", lane)
}
if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
@@ -129,7 +129,7 @@ func productionNPCContractConfig() config.Config {
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"npc_registry": {
Extract: pipeline.ModuleBinding{Module: npcextract.Key, Retries: 2},
Extract: pipeline.ModuleBinding{Module: npcextract.Key},
Normalize: pipeline.Binding(npcnormalize.Key),
},
},

View File

@@ -44,7 +44,7 @@ func TestProductionNPCOccurrencePipelineResolvesAndPrepares(t *testing.T) {
t.Fatalf("generated bindings = %#v", bindings)
}
}
if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("Prepare() error = %v", err)
}
@@ -120,7 +120,7 @@ func TestProductionNPCOccurrenceReferencesRejectIncompatibleExternalRegistries(t
if err != nil {
t.Fatalf("MaterializeReferences() error = %v", err)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err == nil || !strings.Contains(err.Error(), test.want) {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
}
})
@@ -175,7 +175,7 @@ func TestProductionOccurrencePipelinesAcceptCompatibleExternalRegistries(t *test
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("Prepare() error = %v", err)
}
})

View File

@@ -2,22 +2,19 @@ package cli
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
"go.maximumdirect.net/promptkit"
)
func TestProductionSceneDescriptionWorkflow(t *testing.T) {
@@ -48,37 +45,23 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
}
llmClient := &sceneDescriptionLLM{}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient, "scene-description-profile")})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
ChunkCacheMode: pipeline.ChunkCacheBypass,
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
if len(output.OutputFiles) != 1 || output.FallbackCount != 0 || output.OutputFiles[0].Name != "lanes/scene-descriptions.json" {
t.Fatalf("output=%#v, want published scene descriptions", output)
}
wantProfiles := []artifacts.LLMProfileManifest{{
ID: "scene-description-profile",
Provider: "promptkit",
Model: "deterministic",
}}
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
}
normalizedOutput := output.NormalizeOutputs[0]
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
}
var value dnd.SceneDescriptionList
if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil {
t.Fatalf("decode normalized artifact: %v", err)
value, err := scenecodec.New().Decode(output.OutputFiles[0].Bytes)
if err != nil {
t.Fatal(err)
}
want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{
{ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."},
@@ -87,61 +70,34 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if !reflect.DeepEqual(value, want) {
t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want)
}
durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json")
if !reflect.DeepEqual(durable, want) {
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
}
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want grounded descriptions without diagnostics", output.Diagnostics)
}
}
type sceneDescriptionLLM struct {
mu sync.Mutex
profile *artifacts.LLMProfileManifest
}
type sceneDescriptionLLM struct{}
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (*sceneDescriptionLLM) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if req.PromptID != sceneextract.PromptID {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
if !productionSchemaHasProperty(request, "kind") || request.Target.Model != "scene-description-profile-model" {
return nil, fmt.Errorf("unexpected native scene-description request for %q", request.Target.Model)
}
transcript, err := productionJSONInput(request, "units")
if err != nil {
return nil, err
}
transcript := string(req.Inputs["transcript"].Content)
var content string
switch {
case strings.Contains(transcript, "Cure Wounds"):
case strings.Contains(string(transcript), "Cure Wounds"):
content = `{"kind":"narrative","title":" Aria casts Cure Wounds ","summary":" Aria casts Cure Wounds. "}`
case strings.Contains(transcript, "Shield"):
case strings.Contains(string(transcript), "Shield"):
content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}`
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript)
return nil, fmt.Errorf("unexpected transcript material %q", transcript)
}
if err := json.Unmarshal([]byte(content), out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
}
profile := artifacts.LLMProfileManifest{
ID: req.ProfileID,
Provider: "promptkit",
Model: "deterministic",
}
client.mu.Lock()
client.profile = &profile
client.mu.Unlock()
return contracts.StructuredCompletionResponse{
Content: []byte(content),
Provider: profile.Provider,
Model: profile.Model,
ProfileID: profile.ID,
}, nil
return &promptkit.GenerateResponse{Content: content}, nil
}
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
client.mu.Lock()
defer client.mu.Unlock()
if client.profile == nil {
return nil
}
return []artifacts.LLMProfileManifest{*client.profile}
}
var _ promptkit.LLMClient = (*sceneDescriptionLLM)(nil)

View File

@@ -1,7 +1,6 @@
package cli
import (
"context"
"errors"
"fmt"
"io/fs"
@@ -9,75 +8,44 @@ import (
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *testing.T) {
components := productionTestComponents(t)
npcPath := filepath.Join(t.TempDir(), "npcs.json")
if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
func TestOversizedNPCRegistryFailsBeforeModelWorkAndPublication(t *testing.T) {
root := t.TempDir()
npcPath := filepath.Join(root, "npcs.json")
if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0600); err != nil {
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := fmt.Sprintf(`version: 5
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: true
directory: %q
outputRoot, cacheRoot := filepath.Join(root, "output"), filepath.Join(root, "cache")
content := fmt.Sprintf(`version: 6
output: {directory: %q}
cache_directory: %q
pipelines:
dnd-session:
input:
module: seriatim
input: {module: seriatim}
artifacts:
spells:
extract:
module: dnd/spells
references:
npc_registry:
path: %q
normalize:
module: dnd/spells
`, checkpointRoot, npcPath)
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
llmConstructed := false
chunkStoreConstructed := false
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
chunkStoreConstructed = true
return nil, errors.New("chunk-plan store must not be constructed")
},
}
npc_registry: {path: %q}
normalize: {module: dnd/spells}
`, outputRoot, cacheRoot, npcPath)
configPath := writeProductionContractConfig(t, content)
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configPath,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--chunk_cache", "bypass", "--output-dir", t.TempDir(),
}, &stdout, &stderr, options)
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json")}, &stdout, &stderr, productionRunOptions(t, client))
for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npc_registry"`, "1048577 bytes", "limit 1048576"} {
if code == 0 || !strings.Contains(stderr.String(), fragment) {
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment)
t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment)
}
}
if llmConstructed || chunkStoreConstructed {
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 {
t.Fatal("invalid reference reached provider or completion reporting")
}
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
for _, path := range []string{outputRoot, cacheRoot} {
if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("state allocated before invalid reference: %s: %v", path, err)
}
}
}

View File

@@ -16,17 +16,12 @@ import (
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
@@ -60,8 +55,9 @@ import (
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
@@ -217,126 +213,98 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
}
func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
path := writeProductionSpellCatalogContractConfig(t)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"config", "validate", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--pipeline", "dnd-session",
}, &stdout, &stderr, Options{})
if code != 0 || stderr.Len() != 0 {
t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("validate production composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
components := productionTestComponents(t)
newEngine := func(profileFile string) (*promptkit.Engine, error) {
t.Helper()
sources, options, err := components.assets.PromptKitConfigurationSources()
if err != nil {
return nil, err
}
if profileFile != "" {
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: configkit.File(profileFile), Imports: true,
}}})
}
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
for _, test := range []struct {
name, companion string
invalid bool
}{
{name: "embedded defaults"},
{name: "operator override", companion: `collections:
notarius:
profiles:
- id: dnd-extraction
endpoint: https://operator.example.invalid/v1
model: operator-model
timeout_seconds: 75
`},
{name: "invalid operator override", companion: `collections:
notarius:
profiles:
- id: dnd-extraction
model: [invalid]
`, invalid: true},
} {
t.Run(test.name, func(t *testing.T) {
path := writeProductionContractConfig(t, "version: 6\n")
if test.companion != "" {
if err := os.WriteFile(filepath.Join(filepath.Dir(path), "prompts.yml"), []byte(test.companion), 0600); err != nil {
t.Fatal(err)
}
}
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if test.invalid {
if err == nil {
t.Fatal("invalid operator resource fell back to embedded defaults")
}
return
}
if err != nil {
t.Fatal(err)
}
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
if err != nil {
t.Fatal(err)
}
if test.companion == "" && (profile.Backend == nil || *profile.Backend != "openrouter" || profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil) {
t.Fatalf("lost embedded profile defaults: %#v", profile)
}
model, timeout := "openai/gpt-5.6-luna", 240
if test.companion != "" {
model, timeout = "operator-model", 75
if profile.Endpoint == nil || *profile.Endpoint != "https://operator.example.invalid/v1" {
t.Fatalf("operator endpoint=%#v", profile.Endpoint)
}
}
if profile.Model == nil || *profile.Model != model || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != timeout {
t.Fatalf("profile=%#v", profile)
}
})
}
t.Run("fallback", func(t *testing.T) {
engine, err := newEngine("")
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want fallback profile", err)
}
params := inspection.EffectiveModelParams
if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" {
t.Fatalf("fallback profile parameters = %#v", params)
}
if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 {
t.Fatalf("fallback profile selected optional provider controls: %#v", params)
}
})
t.Run("valid operator profile wins", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(`profiles:
dnd-extraction:
endpoint: http://operator.example.test/v1
model: operator-model
timeout_seconds: 75
`), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
}
params := inspection.EffectiveModelParams
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
}
})
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte("profiles:\n dnd-extraction:\n endpoint: http://operator.example.test/v1\n model: operator-model\n unknown: value\n"), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err == nil {
_, err = engine.InspectProfile(context.Background(), "dnd-extraction")
}
if err == nil {
t.Fatal("operator profile error = nil, want failure instead of fallback")
}
})
}
func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
func TestProductionModulesPrepareWithoutProviderCalls(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{
ID: "dnd-scenes",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding("dnd/scenes"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: pipeline.Binding("dnd/spells")},
},
ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: pipeline.Binding("dnd/spells")}},
}
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production scene pipeline: %v", err)
t.Fatal(err)
}
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
t.Fatalf("prepare production scene and spell modules: %v", err)
client := &productionFakeLLMClient{}
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)}); err != nil {
t.Fatal(err)
}
schemaFS, err := components.assets.SchemaFS()
if err != nil {
t.Fatalf("production schema assets: %v", err)
}
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
t.Fatalf("generic reconciliation schema asset: %v", err)
}
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
if engine == nil {
t.Fatal("production PromptKit engine is nil")
if len(client.requestsSnapshot()) != 0 {
t.Fatal("module preparation invoked provider")
}
}
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
components := productionTestComponents(t)
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production spell configuration: %v", err)
}
@@ -355,7 +323,7 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) {
t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems)
}
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})}); err != nil {
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
}
}
@@ -365,7 +333,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
configPath := writeProductionSpellCatalogContractConfig(t)
resolve := func(t *testing.T) pipeline.ResolvedPipeline {
t.Helper()
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve production spell configuration: %v", err)
}
@@ -390,7 +358,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err)
}
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
@@ -406,7 +374,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"]
slot.Items = append(slot.Items, slot.Items[0])
materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, &productionFakeLLMClient{})})
for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
@@ -419,52 +387,27 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := productionSpellCatalogContractConfig(t)
catalogSource := repositoryPath("examples", "dnd-spell-catalog.json")
if count := strings.Count(content, catalogSource); count != 2 {
t.Fatalf("spell catalog source occurs %d times, want extract and normalize bindings", count)
}
content = strings.Replace(content, catalogSource, "__extract_catalog__", 1)
content = replaceRequiredOnce(t, content, catalogSource, catalogPath)
content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource)
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
configFile := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
llmConstructed := false
chunkStoreConstructed := false
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
chunkStoreConstructed = true
return nil, errors.New("chunk-plan store must not be constructed")
},
}
configFile := writeProductionContractConfig(t, content)
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configFile,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
}, &stdout, &stderr, options)
errText := stderr.String()
code := RunWithOptions([]string{"run", "dnd-session", "--config", configFile, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass"}, &stdout, &stderr, productionRunOptions(t, client))
for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} {
if code == 0 || !strings.Contains(errText, fragment) {
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment)
if code == 0 || !strings.Contains(stderr.String(), fragment) {
t.Fatalf("status=%d stderr=%q, want %q", code, stderr.String(), fragment)
}
}
if llmConstructed || chunkStoreConstructed {
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
if len(client.requestsSnapshot()) != 0 || stdout.Len() != 0 {
t.Fatal("invalid reference reached provider or completion reporting")
}
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
if _, err := os.Stat(filepath.Join(filepath.Dir(configFile), "notarius-output")); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("output allocation before invalid reference: %v", err)
}
})
}
@@ -487,186 +430,65 @@ func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPip
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings
}
func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
components := productionTestComponents(t)
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
if client == nil {
t.Fatal("production LLM runtime returned a nil client")
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
}
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil || len(fingerprints) != 1 {
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
}
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
t.Fatalf("production LLM client %T does not provide profile manifests", client)
}
}
func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *testing.T) {
opts, err := normalizeOptions(Options{
Catalog: pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry()},
})
if err != nil {
t.Fatal(err)
}
if opts.promptKitAssets == nil || opts.LLMClientFactory == nil {
t.Fatalf("normalized options = %#v, want shared profile assets and default runtime factory", opts)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"dnd-extraction"}, opts.promptKitAssets); err != nil {
t.Fatalf("inspect application fallback profile: %v", err)
}
client, _, err := opts.LLMClientFactory(context.Background(), config.Default(), "dnd-extraction", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build default runtime: %v", err)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("default runtime client %T does not provide checkpoint fingerprints", client)
}
runtimeFingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
directClient, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{Assets: opts.promptKitAssets})
if err != nil {
t.Fatal(err)
}
inspectionFingerprints, err := directClient.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
}
}
func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
func TestProductionNativeEngineUsesConfiguredBackend(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
call := providerCalls.Add(1)
providerCalls.Add(1)
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path)
t.Errorf("provider path=%q", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
if call == 1 {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":{"message":"transient"}}`))
return
}
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":3,"completion_tokens":4,"total_tokens":7}}`))
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`profiles:
local-profile:
backend: local
model: local-model
`), 0o600); err != nil {
path := writeProductionContractConfig(t, fmt.Sprintf(`version: 6
promptkit:
backends:
- id: operator
endpoint: %q
concurrency_limit: 2
profiles:
- id: local-profile
backend: operator
model: local-model
api_key_required: false
prompts:
- id: local-request
default_profile: local-profile
inputs: [{name: transcript, required: true}]
messages: [{role: user, content: '{{input "transcript"}}'}]
output: {format: json, validation_mode: json}
`, server.URL+"/v1"))
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if err != nil {
t.Fatal(err)
}
assets := llm.NewAssetRegistry()
if err := assets.RegisterPromptFS(fstest.MapFS{
"production.local.yaml": {Data: []byte(`prompts:
production.local:
v1:
inputs:
- name: transcript
required: true
messages:
- role: user
content: '{{ input "transcript" }}'
output:
format: json
validation_mode: json
`)},
}, "."); err != nil {
t.Fatalf("register prompt assets: %v", err)
}
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.TransientRetries = 1
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
client, manifests, err := productionLLMClientFactoryWithAssets(assets)(
context.Background(),
cfg,
"local-profile",
LLMRuntimeOverrides{},
)
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "local-request", Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("local request")}})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
t.Fatal(err)
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "production.local",
ProfileID: "local-profile",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "text/plain", []byte("local request"), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if providerCalls.Load() != 2 {
t.Fatalf("provider calls = %d, want 2", providerCalls.Load())
}
provider, ok := client.(contracts.LLMProfileManifestProvider)
if !ok {
t.Fatalf("production client %T does not provide profile manifests", client)
}
recorded := provider.LLMProfileManifests()
if len(recorded) != 1 || recorded[0].BackendID != promptkit.BackendLocal {
t.Fatalf("production profile manifests = %#v, want local backend", recorded)
if result == nil || !result.Validation.IsValid || strings.TrimSpace(result.RawOutput) != `{"ok":true}` || providerCalls.Load() != 1 {
t.Fatalf("native result=%#v calls=%d", result, providerCalls.Load())
}
}
func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
components := productionTestComponents(t)
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
}
})
t.Run("nil assets", func(t *testing.T) {
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
})
func TestProductionNativeLoadHonorsCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
loaders := &nativeLoaders{lookupEnv: emptyLookup}
engine, err := loaders.engine(ctx, productionSelection(filepath.Join(t.TempDir(), "missing.yml")))
if !errors.Is(err, context.Canceled) || engine != nil {
t.Fatalf("engine=%v error=%v", engine, err)
}
}
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
base := productionSpellCatalogContractConfig(t)
validPath := writeProductionContractConfig(t, base)
options := productionCLIOptions(t)
var stdout, stderr strings.Builder
if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
if code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", validPath}, &stdout, &stderr, options); code != 0 {
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
tests := []struct {
@@ -689,9 +511,9 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
},
{
name: "invalid artifact variant",
content: base,
content: replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: noop\n"),
options: productionCLIOptionsWithoutSpellNormalizer(t),
fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"},
fragments: []string{"normalizer", "noop", string(dnd.SpellListKind)},
},
{
name: "deterministic validator with profile",
@@ -704,8 +526,8 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
t.Run(tt.name, func(t *testing.T) {
path := writeProductionContractConfig(t, tt.content)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options)
if code != 1 {
code := RunWithOptions([]string{"pipeline", "validate", "dnd-session", "--config", path}, &stdout, &stderr, tt.options)
if code != 2 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
for _, fragment := range tt.fragments {
@@ -718,11 +540,11 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
}
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
base := productionSpellCatalogContractConfig(t)
content := replaceRequiredOnce(t, base, " normalize:\n module: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
path := writeProductionContractConfig(t, content)
components := productionTestComponents(t)
effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
effective, err := loadMaintainedExample(t, path).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("resolve normalize override: %v", err)
}
@@ -738,67 +560,48 @@ func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T)
t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key)
}
func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testing.T) {
func TestProductionSceneRunPublishesAnnotationFreeChunkPlan(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
fake := &productionFakeLLMClient{}
options := productionRunOptions(t, fake)
path := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
client := &productionFakeLLMClient{}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session", "--config", configPath,
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--chunk_cache", "bypass", "--session-id", "offline-session",
}, &stdout, &stderr, options)
code := RunWithOptions([]string{"run", "dnd-session", "--config", path, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--cache", "bypass", "--llm-session-id", "offline-session", "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json"))
if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key {
t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan)
var report runResult
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
t.Fatal(err)
}
if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID {
t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID)
if report.ArtifactCount != 4 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("completion=%#v", report)
}
if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID {
t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID)
}
index := readProductionJSON[productionChunkMapIndex](t, filepath.Join(outputRoot, productionRunID, "index.json"))
if index.ChunkMap == nil || index.ChunkMap.ArtifactKind != chunkmap.ArtifactKind || index.ChunkMap.File != "chunk-map.json" || index.ChunkMap.MediaType != chunkmap.MediaType || index.ChunkMap.SchemaID != chunkmap.SchemaID || index.ChunkMap.SchemaName != chunkmap.SchemaName || index.ChunkMap.SchemaVersion != chunkmap.SchemaVersion {
t.Fatalf("chunk map index = %#v, want fixed chunk map descriptor", index.ChunkMap)
}
for _, output := range index.OutputFiles {
if output.File == index.ChunkMap.File {
t.Fatalf("lane output files = %#v, want no chunk map", index.OutputFiles)
}
}
content, err := os.ReadFile(filepath.Join(outputRoot, productionRunID, index.ChunkMap.File))
content, err := os.ReadFile(filepath.Join(report.OutputDirectory, "chunk-map.json"))
if err != nil {
t.Fatal(err)
}
chunkMap, err := chunkmap.New().Decode(content)
value, err := chunkmap.New().Decode(content)
if err != nil {
t.Fatalf("Decode(chunk map) error = %v", err)
t.Fatal(err)
}
if chunkMap.SourceID != "session-alpha" || chunkMap.SourceDigest != manifest.ChunkPlan.SourceDigest || chunkMap.PlanDigest != manifest.ChunkPlan.PlanDigest || chunkMap.RequestedChunker != scenes.Key || chunkMap.Producer.InputModule != "seriatim" || chunkMap.Producer.ChunkModule != scenes.Key || chunkMap.Producer.LLMProfile != manifest.ChunkPlan.ProducerLLMProfile {
t.Fatalf("chunk map identity and producer = %#v, want accepted scene plan provenance", chunkMap)
if value.SourceID != "session-alpha" || value.SourceDigest == "" || value.PlanDigest == "" || value.RequestedChunker != scenes.Key || value.Producer.InputModule != "seriatim" || value.Producer.ChunkModule != scenes.Key {
t.Fatalf("chunk map=%#v", value)
}
if len(chunkMap.Chunks) != 1 || chunkMap.Chunks[0].ID != "chunk-000001" || chunkMap.Chunks[0].Index != 0 || chunkMap.Chunks[0].SourceRef.SourceID != "session-alpha" || chunkMap.Chunks[0].SourceRef.StartUnitID != 1 || chunkMap.Chunks[0].SourceRef.EndUnitID != 2 || chunkMap.Chunks[0].UnitCount != 2 {
t.Fatalf("chunk map chunks = %#v, want one stable accepted scene range", chunkMap.Chunks)
if len(value.Chunks) != 1 || value.Chunks[0].ID != "chunk-000001" || value.Chunks[0].Index != 0 || value.Chunks[0].SourceRef.SourceID != "session-alpha" || value.Chunks[0].SourceRef.StartUnitID != 1 || value.Chunks[0].SourceRef.EndUnitID != 2 || value.Chunks[0].UnitCount != 2 {
t.Fatalf("chunks=%#v", value.Chunks)
}
if len(chunkMap.PlanAnnotations) != 0 {
t.Fatalf("chunk map plan annotations = %#v, want none", chunkMap.PlanAnnotations)
if len(value.PlanAnnotations) != 0 || len(value.Chunks[0].Annotations) != 0 {
t.Fatalf("unexpected annotations: %#v", value)
}
if len(chunkMap.Chunks[0].Annotations) != 0 {
t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations)
for _, property := range []string{"scenes", "spell_casts", "occurrences"} {
if len(client.requestsWithProperty(property)) != 1 {
t.Fatalf("%s provider calls=%d", property, len(client.requestsWithProperty(property)))
}
}
warnings := readProductionJSON[struct {
Groups []contracts.DiagnosticGroup `json:"groups"`
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
if len(warnings.Groups) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Groups)
}
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemoccurrenceextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-occurrence request", fake.requestPrompts())
for _, name := range []string{"manifest.json", "index.json", "warnings.json"} {
if _, err := os.Stat(filepath.Join(report.OutputDirectory, name)); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("retired reporting sidecar %s: %v", name, err)
}
}
}
@@ -819,13 +622,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
func productionSpellCatalogContractConfig(t *testing.T) string {
t.Helper()
return fmt.Sprintf(`version: 5
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
return fmt.Sprintf(`version: 6
pipelines:
dnd-session:
input:
@@ -839,7 +636,6 @@ pipelines:
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog:
path: %q
@@ -886,10 +682,9 @@ func productionCLIOptions(t *testing.T) Options {
func productionOptionsFromComponents(components productionComponents) Options {
return Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
promptKitAssets: components.assets,
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
}
}
@@ -901,35 +696,29 @@ func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options {
if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil {
t.Fatalf("register mismatched normalizer: %v", err)
}
return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets})
return productionOptionsFromComponents(productionComponents{registries: registries})
}
const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef"
func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
func productionRunOptions(t *testing.T, client promptkit.LLMClient) Options {
t.Helper()
options := productionCLIOptions(t)
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return fake, nil, nil
}
options.LLMClient = client
return options
}
func productionRunConfig(outputRoot, chunkModule string) string {
return fmt.Sprintf(`version: 5
return fmt.Sprintf(`version: 6
output:
directory: %q
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug:
directory: %q
pipelines:
dnd-session:
cache: bypass
input:
module: seriatim
chunk:
@@ -959,21 +748,7 @@ pipelines:
item-occurrences:
extract:
module: dnd/item-occurrences
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}
type productionChunkMapIndex struct {
OutputFiles []struct {
File string `json:"file"`
} `json:"output_files"`
ChunkMap *struct {
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
File string `json:"file"`
MediaType string `json:"media_type"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
} `json:"chunk_map"`
`+productionProviderConfiguration, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}
func writeProductionContractConfig(t *testing.T, content string) string {
@@ -1015,74 +790,129 @@ func readProductionJSON[T any](t *testing.T, path string) T {
return value
}
const productionProviderConfiguration = `
promptkit:
collections:
notarius:
profiles:
- id: dnd-extraction
endpoint: https://example.invalid/v1
model: production-test-model
api_key_required: false
`
func productionSelection(path string) ckconfig.Selection {
return ckconfig.Selection{Config: commandkit.Supplied[string]{Set: true, Value: path}}
}
func productionNativeEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
t.Helper()
profiles := make([]map[string]any, len(profileIDs))
for index, id := range profileIDs {
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": id + "-model", "api_key_required": false}
}
content, err := json.Marshal(map[string]any{"version": 6, "promptkit": map[string]any{
"profiles": profiles,
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "production-test-model", "api_key_required": false}}}},
}})
if err != nil {
t.Fatal(err)
}
path := writeProductionContractConfig(t, string(content))
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
engine, err := loaders.engine(context.Background(), productionSelection(path))
if err != nil {
t.Fatal(err)
}
return engine
}
func productionJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
for _, message := range request.Prompt.Messages {
for offset := range len(message.Content) {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var object map[string]json.RawMessage
if decoder.Decode(&object) == nil && object[field] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
}
}
}
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
}
type productionFakeLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
spellResponse string
}
func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *productionFakeLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
var content []byte
switch req.PromptID {
case scenes.PromptID:
content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`)
case spells.PromptID:
if client.spellResponse != "" {
content = []byte(client.spellResponse)
} else {
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
}
case itemoccurrenceextract.PromptID:
content = []byte(`{"occurrences":[]}`)
case itemregistryextract.PromptID:
content = []byte(`{"items":[]}`)
case itemregistrynormalize.PromptID:
content = []byte(`{"duplicate_groups":[]}`)
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
}
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
}
client.mu.Lock()
client.requests = append(client.requests, snapshot)
client.mu.Unlock()
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
}
func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest {
client.mu.Lock()
defer client.mu.Unlock()
var requests []contracts.StructuredCompletionRequest
for _, req := range client.requests {
if req.PromptID == promptID {
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
if err != nil {
panic(err)
}
requests = append(requests, snapshot)
client.requests = append(client.requests, request)
var content string
switch {
case productionSchemaHasProperty(request, "scenes"):
content = `{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`
case productionSchemaHasProperty(request, "spell_casts"):
content = client.spellResponse
if content == "" {
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
}
case productionSchemaHasProperty(request, "occurrences"):
content = `{"occurrences":[]}`
case productionSchemaHasProperty(request, "items"):
content = `{"items":[]}`
case productionSchemaHasProperty(request, "duplicate_groups"):
content = `{"duplicate_groups":[]}`
default:
return nil, fmt.Errorf("unexpected native response contract")
}
return &promptkit.GenerateResponse{Content: content}, nil
}
func (client *productionFakeLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
client.mu.Lock()
defer client.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), client.requests...)
}
func (client *productionFakeLLMClient) requestsWithProperty(property string) []promptkit.GenerateRequest {
var requests []promptkit.GenerateRequest
for _, request := range client.requestsSnapshot() {
if productionSchemaHasProperty(request, property) {
requests = append(requests, request)
}
}
return requests
}
func (client *productionFakeLLMClient) requestPrompts() []string {
client.mu.Lock()
defer client.mu.Unlock()
prompts := make([]string, 0, len(client.requests))
for _, req := range client.requests {
prompts = append(prompts, req.PromptID)
func productionSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
return false
}
return prompts
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
if err != nil {
return false
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
}
if json.Unmarshal(content, &schema) != nil {
return false
}
_, ok := schema.Properties[property]
return ok
}
var _ promptkit.LLMClient = (*productionFakeLLMClient)(nil)
func repositoryPath(parts ...string) string {
_, file, _, _ := runtime.Caller(0)
return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)