package assets_test import ( "context" "encoding/json" "errors" "fmt" "io/fs" "path" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/assets" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" "go.yaml.in/yaml/v4" ) type assetSettings struct { PromptKit promptkit.Settings `yaml:"promptkit"` } type noGeneration struct{ t *testing.T } func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.t.Error("resource preparation unexpectedly invoked generation") return nil, fmt.Errorf("unexpected generation") } func loadNativeAssets(t *testing.T) (*configkit.Result[assetSettings], *promptkit.Engine) { t.Helper() filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) definition := promptkit.Configure(configkit.Definition[assetSettings]{}, func(value *assetSettings) *promptkit.Settings { return &value.PromptKit }) result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[assetSettings]{ Definition: definition, Underlays: []configkit.Underlay{ promptkit.BundledOpenRouterUnderlay(), {Inputs: []configkit.Input{{ Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}, }}}, }, }) if err != nil { var loadError *configkit.LoadError if errors.As(err, &loadError) { for _, diagnostic := range loadError.Diagnostics() { t.Logf("%+v", diagnostic) } } t.Fatal(err) } engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(noGeneration{t})) if err != nil { t.Fatal(err) } return result, engine } func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) { t.Parallel() _, engine := loadNativeAssets(t) // Discover the actual declarations as well as the composed inventory, so an // accidentally omitted import cannot silently remove a module's coverage. err := fs.WalkDir(assets.FS(), ".", func(file string, entry fs.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() || path.Base(file) != "prompt.yaml" { return nil } data, err := fs.ReadFile(assets.FS(), file) if err != nil { return err } var declared struct { Prompts []promptkit.PromptSettings `yaml:"prompts"` } if err := yaml.Unmarshal(data, &declared); err != nil { return err } if len(declared.Prompts) == 0 { return fmt.Errorf("%s has no native prompts", file) } for _, prompt := range declared.Prompts { t.Run(prompt.ID, func(t *testing.T) { inputs := make(map[string]promptkit.ArtifactRef) for _, input := range prompt.Inputs { inputs[input.Name] = promptkit.Inline(input.Name + "-fixture") } request := promptkit.RunRequest{PromptID: "notarius/" + prompt.ID, Inputs: inputs, APIKey: "fixture-key"} if prompt.DefaultProfile == nil { request.ProfileID = "notarius/dnd-extraction" } prepared, err := engine.PrepareExecution(context.Background(), request) if err != nil { t.Fatal(err) } defer prepared.Discard() details := prepared.Details() if details.OutputContract.Format != promptkit.FormatJSON || details.OutputContract.ValidationMode != promptkit.ValidationJSONSchema || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil { t.Fatalf("missing native schema contract: %#v", details.OutputContract) } // Preparation must load the real self-contained schema, not merely retain // a filename or use a separate application schema registry. schemaBytes, err := fs.ReadFile(assets.FS(), details.OutputContract.SchemaPath) if err != nil { t.Fatal(err) } var expected any if err := json.Unmarshal(schemaBytes, &expected); err != nil { t.Fatal(err) } actualBytes, err := json.Marshal(details.StructuredOutput.JSONSchema.Schema) if err != nil { t.Fatal(err) } var actual any if err := json.Unmarshal(actualBytes, &actual); err != nil { t.Fatal(err) } if !reflect.DeepEqual(actual, expected) { t.Fatal("prepared schema differs from the owned asset") } first := prompt.Messages[0] if first.ContentFile == nil { t.Fatal("expected a source-relative initial message") } content, err := fs.ReadFile(assets.FS(), path.Join(path.Dir(file), *first.ContentFile)) if err != nil { t.Fatal(err) } if first.Role == nil || len(details.Messages) == 0 || details.Messages[0].Role != *first.Role { t.Fatal("declared message role was not preserved") } if !strings.Contains(string(content), "{{") && strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) { t.Fatal("source-relative message content was not preserved") } }) } return nil }) if err != nil { t.Fatal(err) } } func TestRegistryNormalizationPromptPairsRetainTheirNativeContracts(t *testing.T) { _, engine := loadNativeAssets(t) for _, test := range []struct { id string guidance string rules []string }{ {"notarius/dnd.npc_registry.normalize", "Preserve distinct individuals.", []string{"same individual", "unadorned proper name"}}, {"notarius/dnd.location_registry.normalize", "Keep parent and child places separate.", []string{"same physical place", "parent and child places"}}, } { t.Run(test.id, func(t *testing.T) { initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ PromptID: test.id, APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{ "candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`), "transcript": promptkit.Inline(`{"windows":[]}`), }, }) if err != nil { t.Fatal(err) } defer initial.Discard() correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ PromptID: test.id + ".repair", APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.guidance)}, }) if err != nil { t.Fatal(err) } defer correction.Discard() first, next := initial.Details(), correction.Details() if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID || first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { t.Fatal("normalization pair lost its shared native defaults/output contract") } var system string for _, message := range first.Messages { if message.Role == "system" { system += message.Content } } for _, rule := range append(test.rules, "candidate_number_range", "start_unit_id", "canonical_candidate_number") { if !strings.Contains(system, rule) { t.Fatalf("missing enduring system principle %q", rule) } } if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, test.guidance) { t.Fatal("correction did not supply only the findings and replacement assignment") } }) } } func TestEmbeddedProfileRetainsSelectedExecutionDefaults(t *testing.T) { t.Parallel() _, engine := loadNativeAssets(t) profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction") if err != nil { t.Fatal(err) } if profile.Backend == nil || *profile.Backend != "openrouter" || profile.Model == nil || *profile.Model != "openai/gpt-5.6-luna" || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 240 || profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil { t.Fatalf("changed extraction defaults: %#v", profile) } }