Files
notarius/internal/cli/example_contract_test.go

456 lines
21 KiB
Go

package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
locationoccurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences"
locationextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
locationoccurrencenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationoccurrences"
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
components := productionTestComponents(t)
completePipelines := make(map[string]pipeline.ResolvedPipeline)
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
cfg := loadMaintainedExample(t, example.path)
if example.name == "complete" && cfg.Concurrency.Workers != 4 {
t.Fatalf("complete example concurrency = %#v, want four shared workers independent of backend admission", cfg.Concurrency)
}
raw, err := os.ReadFile(example.transcriptPath)
if err != nil {
t.Fatalf("read maintained transcript %q: %v", example.transcriptPath, err)
}
document, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{
Path: example.transcriptPath,
Raw: raw,
})
if err != nil {
t.Fatalf("parse maintained transcript %q: %v", example.transcriptPath, err)
}
if len(document.Units) == 0 {
t.Fatalf("maintained transcript %q has no parsed units", example.transcriptPath)
}
for _, pipelineID := range example.pipelineIDs {
effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID))
if err != nil {
t.Fatalf("resolve maintained example %q: %v", pipelineID, err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
ConfigPath: example.path,
WorkingDir: filepath.Dir(example.path),
})
if err != nil {
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
}
if example.name == "complete" {
if pipelineID == "dnd-session-development" {
if materialized.ChunkControls.MaxAdditionalTurns != 0 || materialized.Chunk.LLMProfile != "notarius/dnd-extraction-development" {
t.Fatalf("development native controls=%#v profile=%q", materialized.ChunkControls, materialized.Chunk.LLMProfile)
}
parent, found := completePipelines["dnd-session"]
if !found {
t.Fatal("complete parent pipeline was not resolved before its development variant")
}
if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") {
t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent))
}
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
}
}
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
}
completePipelines[pipelineID] = materialized
locationLane := referenceContractLane(t, materialized, "location-registry")
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key {
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
}
occurrenceLane := referenceContractLane(t, materialized, "location-occurrences")
if occurrenceLane.ArtifactKind != dnd.LocationOccurrenceListKind || occurrenceLane.Extract.Module != locationoccurrenceextract.Key || occurrenceLane.Merge.Module != pipeline.DefaultMergeModule || occurrenceLane.Normalize.Module != locationoccurrencenormalize.Key {
t.Fatalf("location occurrence lane = %#v, want typed occurrence composition", occurrenceLane)
}
for _, target := range []pipeline.ResolvedReferenceTarget{occurrenceLane.ExtractReferences, occurrenceLane.NormalizeReferences} {
binding, found := generatedReferenceBinding(target.Bindings, "location_registry")
if !found || binding.Artifact.Step != "describe-session" || binding.Artifact.Lane != "location-registry" {
t.Fatalf("location occurrence %s reference = %#v, want generated location registry", target.Stage, binding)
}
}
for _, slot := range []string{"party", "glossary"} {
if len(occurrenceLane.ExtractReferences.ReferenceSet.Slots[slot].Items) != 1 {
t.Fatalf("location occurrence extractor %s reference was not materialized: %#v", slot, occurrenceLane.ExtractReferences)
}
if _, found := occurrenceLane.NormalizeReferences.ReferenceSet.Slots[slot]; found {
t.Fatalf("location occurrence normalizer unexpectedly consumes %s: %#v", slot, occurrenceLane.NormalizeReferences)
}
}
spellLane := referenceContractLane(t, materialized, "spells")
if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane)
}
itemOccurrenceLane := referenceContractLane(t, materialized, "item-occurrences")
for _, references := range []pipeline.ResolvedReferenceTarget{itemOccurrenceLane.ExtractReferences, itemOccurrenceLane.NormalizeReferences} {
if _, found := references.ReferenceSet.Slots["npc_registry"]; found {
t.Fatalf("item occurrence lane unexpectedly depends on generated NPCs: %#v", itemOccurrenceLane)
}
if _, found := references.ReferenceSet.Slots["scene_descriptions"]; found {
t.Fatalf("item occurrence lane unexpectedly depends on generated scene descriptions: %#v", itemOccurrenceLane)
}
}
enemyEventLane := referenceContractLane(t, materialized, "enemy-events")
for slot, want := range map[string]struct{ step, lane string }{
"npc_registry": {step: "describe-session", lane: "npc-registry"},
"scene_descriptions": {step: "describe-session", lane: "scene-descriptions"},
"combat_turns": {step: "extract-events", lane: "combat-turns"},
"npc_occurrences": {step: "extract-events", lane: "npc-occurrences"},
} {
binding, found := generatedReferenceBinding(enemyEventLane.ExtractReferences.Bindings, slot)
if !found || binding.Artifact.Step != want.step || binding.Artifact.Lane != want.lane {
t.Fatalf("enemy event %s reference = %#v, want generated %s/%s artifact", slot, binding, want.step, want.lane)
}
}
}
}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{"pipeline", "list", "--config", example.path, "--format", "json"}, &stdout, &stderr, productionOptionsFromComponents(components))
var ids []string
if err := json.Unmarshal([]byte(stdout.String()), &ids); err != nil {
t.Fatal(err)
}
if code != 0 || strings.Join(ids, ",") != strings.Join(example.pipelineIDs, ",") || stderr.Len() != 0 {
t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestMaintainedExamplesIsolateCompanionsAndInheritNativeResources(t *testing.T) {
client := &productionFakeLLMClient{}
loaders := &nativeLoaders{lookupEnv: emptyLookup, engineOptions: []promptkit.Option{promptkit.WithLLMClient(client)}}
ctx := context.Background()
completePath := repositoryPath("examples", "dnd-complete", "config.yml")
complete, err := loaders.engine(ctx, nativeSelection(completePath))
if err != nil {
t.Fatal(err)
}
base, err := complete.InspectProfile(ctx, "notarius/dnd-extraction")
if err != nil {
t.Fatal(err)
}
example, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-example")
if err != nil {
t.Fatal(err)
}
development, err := complete.InspectProfile(ctx, "notarius/dnd-extraction-development")
if err != nil {
t.Fatal(err)
}
for _, profile := range []*promptkit.ProfileInspection{example, development} {
if !reflect.DeepEqual(profile.Model, base.Model) || !reflect.DeepEqual(profile.TimeoutSeconds, base.TimeoutSeconds) || !reflect.DeepEqual(profile.ServiceTier, base.ServiceTier) || profile.Backend == nil || *profile.Backend != "limited-openrouter" {
t.Fatalf("inherited profile=%#v, embedded parent=%#v", profile, base)
}
}
if development.Temperature == nil || *development.Temperature != 0.2 || !reflect.DeepEqual(example.Temperature, base.Temperature) {
t.Fatalf("development customization leaked into parent: %#v %#v", development, example)
}
bundled, err := complete.InspectBackend(ctx, "openrouter")
if err != nil {
t.Fatal(err)
}
limited, err := complete.InspectBackend(ctx, "limited-openrouter")
if err != nil {
t.Fatal(err)
}
if limited.ConcurrencyLimit == nil || *limited.ConcurrencyLimit != 2 || limited.Endpoint != bundled.Endpoint || !reflect.DeepEqual(limited.APIKeyEnv, bundled.APIKeyEnv) || !reflect.DeepEqual(limited.SmokeTests, bundled.SmokeTests) {
t.Fatalf("limited backend=%#v, bundled parent=%#v", limited, bundled)
}
minimal, err := loaders.engine(ctx, nativeSelection(repositoryPath("examples", "dnd-minimal", "config.yml")))
if err != nil {
t.Fatal(err)
}
if _, err := minimal.InspectProfile(ctx, "notarius/dnd-extraction-example"); !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("complete example profile leaked into minimal selection: %v", err)
}
if _, err := minimal.InspectBackend(ctx, "limited-openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) {
t.Fatalf("complete example backend leaked into minimal selection: %v", err)
}
if len(client.requestsSnapshot()) != 0 {
t.Fatal("example inspection called provider")
}
}
func TestMaintainedDevelopmentOverlayResolvesNativeControlsAndRelativeOutput(t *testing.T) {
path := repositoryPath("examples", "dnd-complete", "config.yml")
selection := nativeSelection(path)
selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"}
loaders := &nativeLoaders{lookupEnv: emptyLookup}
result, err := loaders.application(context.Background(), selection)
if err != nil {
t.Fatal(err)
}
loaded, err := config.FromResult(result)
if err != nil {
t.Fatal(err)
}
cfg := loaded.RuntimeConfig()
if cfg.Concurrency.Workers != 1 || cfg.Output.Directory != filepath.Join(filepath.Dir(path), "notarius-output-development") {
t.Fatalf("development application controls=%#v %#v", cfg.Concurrency, cfg.Output)
}
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
if err != nil {
t.Fatal(err)
}
if resolved.ResolvedPipeline.Chunk.LLMProfile != "notarius/dnd-extraction-development" || resolved.ResolvedPipeline.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
t.Fatalf("development chunk selection=%#v controls=%#v", resolved.ResolvedPipeline.Chunk, resolved.ResolvedPipeline.ChunkControls)
}
}
func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
t.Chdir(repositoryPath())
t.Setenv("OPENROUTER_API_KEY", "")
client := &productionFakeLLMClient{}
options := Options{LLMClient: client}
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
var configOut, configErr strings.Builder
if code := RunWithOptions([]string{"config", "validate", "--config", example.path}, &configOut, &configErr, options); code != 0 {
t.Fatalf("configuration validation: status=%d stdout=%q stderr=%q", code, configOut.String(), configErr.String())
}
for _, pipelineID := range example.pipelineIDs {
t.Run(pipelineID, func(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"pipeline", "validate", pipelineID, "--config", example.path,
}, &stdout, &stderr, options)
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), pipelineID) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
})
}
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"promptkit", "show", "profile", "notarius/dnd-extraction-development",
"--config", repositoryPath("examples", "dnd-complete", "config.yml"), "--format", "json",
}, &stdout, &stderr, options)
if code != 0 || stderr.Len() != 0 || !json.Valid([]byte(stdout.String())) || !strings.Contains(stdout.String(), "dnd-extraction-development") {
t.Fatalf("development profile inspection: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(client.requestsSnapshot()) != 0 {
t.Fatal("offline example validation or inspection called provider")
}
}
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
result := make([]string, 0, len(resolved.Steps))
for _, step := range resolved.Steps {
laneIDs := make([]string, 0, len(step.ArtifactLanes))
for _, lane := range step.ArtifactLanes {
laneIDs = append(laneIDs, lane.ID)
}
sort.Strings(laneIDs)
result = append(result, step.ID+":"+strings.Join(laneIDs, ","))
}
return result
}
func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string {
var descriptions []string
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
for _, target := range []struct {
name string
target pipeline.ResolvedReferenceTarget
}{
{name: "extract", target: lane.ExtractReferences},
{name: "merge", target: lane.MergeReferences},
{name: "normalize", target: lane.NormalizeReferences},
} {
for _, binding := range target.target.Bindings {
if binding.Artifact == nil {
continue
}
descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane))
}
}
}
}
sort.Strings(descriptions)
return descriptions
}
func TestMaintainedMinimalInvocationPublishesDomainArtifacts(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "notarius-test-credential")
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-minimal", "config.yml"))
resolved, err := cfg.Resolve(resolveInputForMaintainedExample(productionTestComponents(t), "dnd-session"))
if err != nil {
t.Fatal(err)
}
lane := referenceContractLane(t, resolved.ResolvedPipeline, "spells")
if lane.Extract.Module != "dnd/spells" || lane.Normalize.Module != spellnormalize.Key || len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
t.Fatalf("minimal spell composition=%#v, want embedded catalog without overlay references", lane)
}
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{}
options := productionRunOptions(t, fake)
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
"--input-file", repositoryPath("examples", "dnd-minimal", "transcript.json"),
"--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json",
}, &stdout, &stderr, options)
if code != 0 {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
var report runResult
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
t.Fatal(err)
}
runRoot := filepath.Join(outputRoot, productionRunID)
if report.Pipeline != "dnd-session" || report.ArtifactCount != 1 || report.OutputDirectory != runRoot || report.FallbackCount != 0 {
t.Fatalf("completion=%#v", report)
}
files := readTree(t, runRoot)
if len(files) != 1 || files["lanes/spells.json"] == nil {
t.Fatalf("published files=%v, want only the spell artifact", files)
}
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes/spells.json"))
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || len(artifact.SpellCasts[0].SourceRefs) != 1 || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
t.Fatalf("artifact=%#v, want one source-linked Cure Wounds cast", artifact)
}
}
func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) {
malformed := filepath.Join(t.TempDir(), "malformed.json")
if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil {
t.Fatal(err)
}
for _, debug := range []bool{false, true} {
name := "without debug"
if debug {
name = "with debug"
}
t.Run(name, func(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
client := &productionFakeLLMClient{}
options := productionRunOptions(t, client)
args := []string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal", "config.yml"),
"--input-file", malformed, "--cache", "bypass", "--output-dir", outputRoot,
}
if debug {
args = append(args, "--debug", "--debug-dir", debugRoot)
}
var stdout, stderr strings.Builder
code := RunWithOptions(args, &stdout, &stderr, options)
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertAbsent(t, outputRoot)
if !debug {
assertAbsent(t, debugRoot)
return
}
bundle := onlyChildDir(t, debugRoot)
outcome := readProductionJSON[struct {
Error string `json:"error"`
Outcome struct {
Pipeline string `json:"pipeline"`
Published bool `json:"published"`
} `json:"outcome"`
}](t, filepath.Join(bundle, "outcome.json"))
if !strings.Contains(outcome.Error, "parse input") || outcome.Outcome.Pipeline != "dnd-session" || outcome.Outcome.Published {
t.Fatalf("failure evidence=%#v", outcome)
}
if len(client.requestsSnapshot()) != 0 {
t.Fatal("malformed input reached provider")
}
})
}
}
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
}
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete", "config.yml"))
catalog := catalogFromRegistries(productionTestComponents(t).registries)
sources := map[string]string{
"party": "/references/party.txt",
"players": "/references/players.txt",
"glossary": "/references/glossary.txt",
"spell_catalog": "/references/spells.json",
}
requests := make([]cliReferenceRequest, 0, len(sources))
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
requests = append(requests, cliReferenceRequest{
Selector: mustParseReferenceSelector(t, slot, "--reference"),
Source: sources[slot],
})
}
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
if err != nil {
t.Fatalf("expand complete D&D references: %v", err)
}
if len(unbinds) != 0 {
t.Fatalf("unbinds = %#v, want none", unbinds)
}
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
for _, binding := range overrides {
actual[referenceContractBindingLabel(binding)] = binding
}
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
if err != nil {
t.Fatal(err)
}
matched := make(map[string]int, len(sources))
for _, target := range targets {
for slot, sourcePath := range sources {
if _, ok := target.slots[slot]; !ok {
continue
}
matched[slot]++
label := targetLabel(target) + "." + slot
binding, ok := actual[label]
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
}
}
}
for slot := range sources {
if matched[slot] < 2 {
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
}
}
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
t.Fatalf("resolve complete D&D CLI references: %v", err)
}
}