Migrate maintained D&D examples to native configuration

This commit is contained in:
2026-11-22 05:45:31 +00:00
parent 3ef55f4668
commit 854d23b91e
25 changed files with 189 additions and 195 deletions

View File

@@ -3,9 +3,11 @@ package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
@@ -20,6 +22,8 @@ import (
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
@@ -28,8 +32,8 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
cfg := loadMaintainedExample(t, example.path)
if example.name == "complete" && cfg.Concurrency.Workers != 2 {
t.Fatalf("complete example concurrency = %#v, want an explicit shared worker limit of 2", cfg.Concurrency)
if example.name == "complete" && cfg.Concurrency.Workers != 4 {
t.Fatalf("complete example concurrency = %#v, want four shared workers independent of backend admission", cfg.Concurrency)
}
raw, err := os.ReadFile(example.transcriptPath)
if err != nil {
@@ -59,6 +63,9 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
}
if example.name == "complete" {
if pipelineID == "dnd-session-development" {
if materialized.ChunkControls.MaxAdditionalTurns != 0 || materialized.Chunk.LLMProfile != "notarius/dnd-extraction-development" {
t.Fatalf("development native controls=%#v profile=%q", materialized.ChunkControls, materialized.Chunk.LLMProfile)
}
parent, found := completePipelines["dnd-session"]
if !found {
t.Fatal("complete parent pipeline was not resolved before its development variant")
@@ -137,56 +144,104 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
}
}
func TestMaintainedConfigurationExampleSet(t *testing.T) {
entries, err := os.ReadDir(repositoryPath("examples"))
func TestMaintainedExamplesIsolateCompanionsAndInheritNativeResources(t *testing.T) {
client := &productionFakeLLMClient{}
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
ctx := context.Background()
completePath := repositoryPath("examples", "dnd-complete", "config.yml")
complete, err := loaders.engine(ctx, nativeSelection(completePath))
if err != nil {
t.Fatal(err)
}
var names []string
for _, entry := range entries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") {
names = append(names, entry.Name())
base, err := complete.InspectProfile(ctx, "notarius/dnd-extraction")
if err != nil {
t.Fatal(err)
}
example, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-example")
if err != nil {
t.Fatal(err)
}
development, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-development")
if err != nil {
t.Fatal(err)
}
for _, profile := range []*promptkit.ProfileInspection{example, development} {
if !reflect.DeepEqual(profile.Model, base.Model) || !reflect.DeepEqual(profile.TimeoutSeconds, base.TimeoutSeconds) || !reflect.DeepEqual(profile.ServiceTier, base.ServiceTier) || profile.Backend == nil || *profile.Backend != "limited-openrouter" {
t.Fatalf("inherited profile=%#v, embedded parent=%#v", profile, base)
}
}
sort.Strings(names)
if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" {
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
if development.Temperature == nil || *development.Temperature != 0.2 || !reflect.DeepEqual(example.Temperature, base.Temperature) {
t.Fatalf("development customization leaked into parent: %#v %#v", development, example)
}
bundled, err := complete.InspectBackend(ctx, "openrouter")
if err != nil {
t.Fatal(err)
}
limited, err := complete.InspectBackend(ctx, "limited-openrouter")
if err != nil {
t.Fatal(err)
}
if limited.ConcurrencyLimit == nil || *limited.ConcurrencyLimit != 2 || limited.Endpoint != bundled.Endpoint || !reflect.DeepEqual(limited.APIKeyEnv, bundled.APIKeyEnv) || !reflect.DeepEqual(limited.SmokeTests, bundled.SmokeTests) {
t.Fatalf("limited backend=%#v, bundled parent=%#v", limited, bundled)
}
minimal, err := loaders.engine(ctx, nativeSelection(repositoryPath("examples", "dnd-minimal", "config.yml")))
if err != nil {
t.Fatal(err)
}
if _, err := minimal.InspectProfile(ctx, "notarius/dnd-extraction-example"); !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("complete example profile leaked into minimal selection: %v", err)
}
if _, err := minimal.InspectBackend(ctx, "limited-openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) {
t.Fatalf("complete example backend leaked into minimal selection: %v", err)
}
if len(client.requestsSnapshot()) != 0 {
t.Fatal("example inspection called provider")
}
}
overlayEntries, err := os.ReadDir(repositoryPath("examples", "overlays"))
func TestMaintainedDevelopmentOverlayResolvesNativeControlsAndRelativeOutput(t *testing.T) {
path := repositoryPath("examples", "dnd-complete", "config.yml")
selection := nativeSelection(path)
selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"}
loaders := &nativeLoaders{lookupEnv: emptyLookup}
result, err := loaders.application(context.Background(), selection)
if err != nil {
t.Fatal(err)
}
names = names[:0]
for _, entry := range overlayEntries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yml") {
names = append(names, entry.Name())
}
}
sort.Strings(names)
if got := strings.Join(names, ","); got != "development.yml" {
t.Fatalf("maintained configuration overlays = %q, want development.yml", got)
}
llmProfiles, err := os.ReadDir(repositoryPath("examples", "llm-profiles"))
loaded, err := config.FromResult(result)
if err != nil {
t.Fatal(err)
}
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
t.Fatalf("maintained LLM profile files = %#v, want the single combined profile file", llmProfiles)
cfg := loaded.RuntimeConfig()
if cfg.Concurrency.Workers != 1 || cfg.Output.Directory != filepath.Join(filepath.Dir(path), "notarius-output-development") {
t.Fatalf("development application controls=%#v %#v", cfg.Concurrency, cfg.Output)
}
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
if err != nil {
t.Fatal(err)
}
if resolved.ResolvedPipeline.Chunk.LLMProfile != "notarius/dnd-extraction-development" || resolved.ResolvedPipeline.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
t.Fatalf("development chunk selection=%#v controls=%#v", resolved.ResolvedPipeline.Chunk, resolved.ResolvedPipeline.ChunkControls)
}
}
func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
t.Chdir(repositoryPath())
t.Setenv("OPENROUTER_API_KEY", "")
client := &productionFakeLLMClient{}
options := Options{LLMClient: client}
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
var configOut, configErr strings.Builder
if code := RunWithOptions([]string{"config", "validate", "--config", example.path}, &configOut, &configErr, options); code != 0 {
t.Fatalf("configuration validation: status=%d stdout=%q stderr=%q", code, configOut.String(), configErr.String())
}
for _, pipelineID := range example.pipelineIDs {
t.Run(pipelineID, func(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"pipeline", "validate", pipelineID, "--config", example.path,
}, &stdout, &stderr, Options{})
}, &stdout, &stderr, options)
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), pipelineID) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
@@ -198,11 +253,14 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"promptkit", "show", "profile", "notarius/dnd-extraction-development",
"--config", repositoryPath("examples", "dnd-complete.config.yml"), "--format", "json",
}, &stdout, &stderr, Options{})
"--config", repositoryPath("examples", "dnd-complete", "config.yml"), "--format", "json",
}, &stdout, &stderr, options)
if code != 0 || stderr.Len() != 0 || !json.Valid([]byte(stdout.String())) || !strings.Contains(stdout.String(), "dnd-extraction-development") {
t.Fatalf("development profile inspection: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(client.requestsSnapshot()) != 0 {
t.Fatal("offline example validation or inspection called provider")
}
}
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
@@ -244,7 +302,8 @@ func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) [
}
func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal.config.yml"))
t.Setenv("OPENROUTER_API_KEY", "notarius-test-credential")
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal", "config.yml"))
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
if err != nil {
t.Fatal(err)
@@ -259,8 +318,8 @@ func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
"--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"),
"--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json",
}, &stdout, &stderr, options)
if code != 0 {
@@ -301,7 +360,7 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing
options := productionRunOptions(t, client)
args := []string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
"--input-file", malformed, "--cache", "bypass", "--output-dir", outputRoot,
}
if debug {
@@ -340,7 +399,7 @@ func resolveInputForMaintainedExample(components productionComponents, pipelineI
}
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete", "config.yml"))
catalog := catalogFromRegistries(productionTestComponents(t).registries)
sources := map[string]string{
"party": "/references/party.txt",