Migrate facade tests to the new configuration contracts
This commit is contained in:
@@ -17,7 +17,7 @@ func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactR
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("ownership", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
})}
|
})}
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
||||||
underlay := configkit.Input{Source: configkit.Mapping("caller underlay", map[string]any{
|
underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller underlay", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
||||||
})}
|
})}
|
||||||
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
||||||
@@ -22,7 +22,7 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||||
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
}
|
}
|
||||||
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller main", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
"profiles": []any{
|
"profiles": []any{
|
||||||
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
||||||
@@ -76,8 +76,8 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
{"replace", "replace", "custom", false, false},
|
{"replace", "replace", "custom", false, false},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("sequence", map[string]any{
|
||||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": test.mode}}},
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": test.mode}}},
|
||||||
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
@@ -99,13 +99,13 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
||||||
profile := configkit.Input{Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
||||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("strict group accepted duplicate records")
|
t.Fatal("strict group accepted duplicate records")
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("replace backends", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replace backends", map[string]any{
|
||||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"backends"}, "mode": "replace"}}},
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "backends"}, "mode": "replace"}}},
|
||||||
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
})}
|
})}
|
||||||
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
||||||
input := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}})}
|
}}})}
|
||||||
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
|||||||
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
t.Fatal("bundle source missing")
|
t.Fatal("bundle source missing")
|
||||||
}
|
}
|
||||||
if explanation, found := view.Explain(configkit.Keys("backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
if explanation, found := view.Explain(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
||||||
t.Fatalf("bundle explanation = %+v", explanation)
|
t.Fatalf("bundle explanation = %+v", explanation)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,7 +94,7 @@ func fmtSources(sources []configkit.SourceInfo) string {
|
|||||||
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
||||||
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
||||||
groups := make([]configkit.Underlay, 1, 2)
|
groups := make([]configkit.Underlay, 1, 2)
|
||||||
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Source: source}}}
|
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}
|
||||||
before := append([]configkit.Underlay(nil), groups...)
|
before := append([]configkit.Underlay(nil), groups...)
|
||||||
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -105,7 +105,7 @@ func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
||||||
input := configkit.Input{Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
||||||
replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
replacement := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
||||||
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -137,7 +137,7 @@ func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
|||||||
if err != nil || mismatch == nil || !mismatch.Completed {
|
if err != nil || mismatch == nil || !mismatch.Completed {
|
||||||
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("application", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
||||||
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
|||||||
@@ -265,7 +265,7 @@ func newBackendCapacityEngine(
|
|||||||
options = append(options, promptkit.WithArtifactReader(reader))
|
options = append(options, promptkit.WithArtifactReader(reader))
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||||
"backends": []any{backend},
|
"backends": []any{backend},
|
||||||
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
|
||||||
"prompts": []any{definition},
|
"prompts": []any{definition},
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
|||||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||||
}
|
}
|
||||||
difference := differences[0]
|
difference := differences[0]
|
||||||
if got, want := difference.Path.String(), "/profiles"; got != want {
|
if got, want := difference.Path.String(), "/promptkit/profiles"; got != want {
|
||||||
t.Fatalf("difference path = %q, want %q", got, want)
|
t.Fatalf("difference path = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||||
@@ -84,7 +84,7 @@ func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
|||||||
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
||||||
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("backend", map[string]any{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("backend", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
||||||
})}}}}}})
|
})}}}}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -104,7 +104,7 @@ func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
changed := inspect("https://other.invalid/v1")
|
changed := inspect("https://other.invalid/v1")
|
||||||
differences, err = left.Diff(changed)
|
differences, err = left.Diff(changed)
|
||||||
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("changed diff = %+v, %v", differences, err)
|
t.Fatalf("changed diff = %+v, %v", differences, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -122,7 +122,7 @@ func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare presence and sequence values: %v", err)
|
t.Fatalf("compare presence and sequence values: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("atomic backend list difference = %+v", differences)
|
t.Fatalf("atomic backend list difference = %+v", differences)
|
||||||
}
|
}
|
||||||
difference := differences[0]
|
difference := differences[0]
|
||||||
@@ -147,7 +147,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare sensitive values: %v", err)
|
t.Fatalf("compare sensitive values: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("sensitive backend list difference = %+v", differences)
|
t.Fatalf("sensitive backend list difference = %+v", differences)
|
||||||
}
|
}
|
||||||
secret := differences[0]
|
secret := differences[0]
|
||||||
@@ -176,7 +176,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
|||||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||||
t.Fatal("mutating returned path reached retained inspection")
|
t.Fatal("mutating returned path reached retained inspection")
|
||||||
}
|
}
|
||||||
if _, exists := againByPath["/backends"]; !exists {
|
if _, exists := againByPath["/promptkit/backends"]; !exists {
|
||||||
t.Fatalf("original backend path was lost after mutation: %+v", again)
|
t.Fatalf("original backend path was lost after mutation: %+v", again)
|
||||||
}
|
}
|
||||||
for _, difference := range again {
|
for _, difference := range again {
|
||||||
@@ -201,7 +201,7 @@ func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("shape replacement differences = %+v", differences)
|
t.Fatalf("shape replacement differences = %+v", differences)
|
||||||
}
|
}
|
||||||
difference := differences[0]
|
difference := differences[0]
|
||||||
@@ -225,7 +225,7 @@ func comparisonInspectionFrom(t *testing.T, source, model string, extraParams ma
|
|||||||
backend["extra_params"] = extraParams
|
backend["extra_params"] = extraParams
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping(source, map[string]any{
|
||||||
"backends": []any{backend},
|
"backends": []any{backend},
|
||||||
"profiles": []any{map[string]any{
|
"profiles": []any{map[string]any{
|
||||||
"id": "offline", "backend": "local",
|
"id": "offline", "backend": "local",
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("defaults", map[string]any{
|
||||||
"backends": []any{map[string]any{
|
"backends": []any{map[string]any{
|
||||||
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
||||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||||
@@ -35,7 +35,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("operator", map[string]any{
|
||||||
"collections": map[string]any{"literal": map[string]any{
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
||||||
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
||||||
@@ -50,21 +50,21 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model")
|
modelPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(0).Key("model")
|
||||||
model, found := view.Explain(modelPath)
|
model, found := view.Explain(modelPath)
|
||||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
||||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||||
}
|
}
|
||||||
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model")
|
inheritedPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(1).Key("model")
|
||||||
inherited, found := view.Explain(inheritedPath)
|
inherited, found := view.Explain(inheritedPath)
|
||||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||||
}
|
}
|
||||||
if inheritedPath.String() != "/collections/literal/profiles/1/model" {
|
if inheritedPath.String() != "/promptkit/collections/literal/profiles/1/model" {
|
||||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||||
}
|
}
|
||||||
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
secretPath := configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("token")
|
||||||
endpointPath := configkit.Keys("backends").Index(0).Key("endpoint")
|
endpointPath := configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")
|
||||||
endpoint, found := view.Explain(endpointPath)
|
endpoint, found := view.Explain(endpointPath)
|
||||||
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
||||||
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
||||||
@@ -73,12 +73,12 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||||
}
|
}
|
||||||
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
namespacedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||||
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||||
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||||
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||||
}
|
}
|
||||||
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
inheritedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||||
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
||||||
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
||||||
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
||||||
@@ -124,7 +124,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
|
|
||||||
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
||||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("invalid backend", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid backend", map[string]any{
|
||||||
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
||||||
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
||||||
}}},
|
}}},
|
||||||
@@ -139,9 +139,9 @@ func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
func TestConfigurationInspectionRetainsApplicationVersionField(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||||
}}})
|
}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine with accepted declaration: %v", err)
|
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||||
@@ -150,7 +150,7 @@ func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
path := configkit.Keys("version")
|
path := configkit.Keys("promptkit", "version")
|
||||||
field, found := inspection.Explain(path)
|
field, found := inspection.Explain(path)
|
||||||
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
||||||
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||||
@@ -171,7 +171,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("nil engine inspection = %v", err)
|
t.Fatalf("nil engine inspection = %v", err)
|
||||||
}
|
}
|
||||||
var empty promptkit.ConfigurationInspection
|
var empty promptkit.ConfigurationInspection
|
||||||
path := configkit.Keys("profiles", "missing")
|
path := configkit.Keys("promptkit", "profiles", "missing")
|
||||||
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
||||||
t.Fatal("zero inspection reported configuration")
|
t.Fatal("zero inspection reported configuration")
|
||||||
}
|
}
|
||||||
@@ -185,7 +185,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("zero explanation writer = %v", err)
|
t.Fatalf("zero explanation writer = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{
|
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
||||||
})}
|
})}
|
||||||
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
@@ -203,7 +203,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("effective writer error = %v", err)
|
t.Fatalf("effective writer error = %v", err)
|
||||||
}
|
}
|
||||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
if err := view.WriteExplanation(writer, configkit.Keys("promptkit", "profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("explanation writer error = %v", err)
|
t.Fatalf("explanation writer error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -230,7 +230,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
client := countingLLMClient{calls: &modelCalls}
|
client := countingLLMClient{calls: &modelCalls}
|
||||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("offline profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
||||||
})}}}},
|
})}}}},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
@@ -238,7 +238,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(otherInputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("comparison profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
||||||
})}}}},
|
})}}}},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
@@ -261,7 +261,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil || len(differences) != 1 {
|
if err != nil || len(differences) != 1 {
|
||||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||||
}
|
}
|
||||||
if differences[0].Path.String() != "/profiles" ||
|
if differences[0].Path.String() != "/promptkit/profiles" ||
|
||||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||||
t.Fatalf("captured difference = %+v", differences[0])
|
t.Fatalf("captured difference = %+v", differences[0])
|
||||||
@@ -273,7 +273,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
if _, found := view.Explain(configkit.Keys("promptkit", "prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
var report bytes.Buffer
|
var report bytes.Buffer
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
endpoint, model := "http://example.test/v1", "model"
|
endpoint, model := "http://example.test/v1", "model"
|
||||||
lowerName := name
|
lowerName := name
|
||||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
||||||
}),
|
}),
|
||||||
}}})
|
}}})
|
||||||
@@ -49,7 +49,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
if test.clear {
|
if test.clear {
|
||||||
env = ""
|
env = ""
|
||||||
}
|
}
|
||||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
cfg.Sources.Main = &configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
||||||
})}
|
})}
|
||||||
var calls int
|
var calls int
|
||||||
@@ -73,7 +73,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
|
|
||||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||||
backendID, model := "workstation", "offline-model"
|
backendID, model := "workstation", "offline-model"
|
||||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
prompt := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{
|
||||||
"prompts": []any{map[string]any{
|
"prompts": []any{map[string]any{
|
||||||
"id": "hello",
|
"id": "hello",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
@@ -86,10 +86,10 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
|
|||||||
localBackend.ID = backendID
|
localBackend.ID = backendID
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
{Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||||
localBackend,
|
localBackend,
|
||||||
})},
|
})},
|
||||||
{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
{ID: "offline", Backend: &backendID, Model: &model},
|
{ID: "offline", Backend: &backendID, Model: &model},
|
||||||
})},
|
})},
|
||||||
}}},
|
}}},
|
||||||
@@ -116,11 +116,11 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
|||||||
model := "configured-model"
|
model := "configured-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
{Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
||||||
{ID: openRouterID, Endpoint: &openRouterEndpoint},
|
{ID: openRouterID, Endpoint: &openRouterEndpoint},
|
||||||
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
|
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
|
||||||
})},
|
})},
|
||||||
{Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
||||||
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
|
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
|
||||||
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
|
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
|
||||||
})},
|
})},
|
||||||
@@ -170,7 +170,7 @@ prompts:
|
|||||||
t.Fatalf("select operator inputs: %v", err)
|
t.Fatalf("select operator inputs: %v", err)
|
||||||
}
|
}
|
||||||
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
|
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
|
||||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
programmatic := configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
||||||
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
|
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
|
||||||
})}
|
})}
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
@@ -183,9 +183,9 @@ prompts:
|
|||||||
{name: "programmatic", apply: true, want: "programmatic-model"},
|
{name: "programmatic", apply: true, want: "programmatic-model"},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
underlays := []configkit.Underlay{{Inputs: embedded}}
|
underlays := []configkit.Underlay{{Inputs: mountSettingsInputs(embedded)}}
|
||||||
if test.apply {
|
if test.apply {
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: operator})
|
underlays = append(underlays, configkit.Underlay{Inputs: mountSettingsInputs(operator)})
|
||||||
}
|
}
|
||||||
if test.name == "programmatic" {
|
if test.name == "programmatic" {
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
||||||
@@ -207,7 +207,7 @@ prompts:
|
|||||||
|
|
||||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": []any{map[string]any{"id": "broken", "unknown": true}}}),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid", map[string]any{"prompts": []any{map[string]any{"id": "broken", "output": map[string]any{"unknown": true}}}}),
|
||||||
}}})
|
}}})
|
||||||
var loadErr *configkit.LoadError
|
var loadErr *configkit.LoadError
|
||||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||||
@@ -257,10 +257,10 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
|
|||||||
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
{Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
||||||
{ID: backendID, Endpoint: &endpoint},
|
{ID: backendID, Endpoint: &endpoint},
|
||||||
})},
|
})},
|
||||||
{Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
||||||
{ID: "application-profile", Backend: &backendID, Model: &model},
|
{ID: "application-profile", Backend: &backendID, Model: &model},
|
||||||
})},
|
})},
|
||||||
}}},
|
}}},
|
||||||
@@ -304,7 +304,7 @@ func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
|||||||
authenticationEndpoint = "https://mutated.example/key"
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func mountSettingsInputs(inputs []configkit.Input) []configkit.Input {
|
||||||
|
result := append([]configkit.Input(nil), inputs...)
|
||||||
|
for i := range result {
|
||||||
|
result[i].Destination = configkit.Keys("promptkit")
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
type fakeLLMClient struct {
|
type fakeLLMClient struct {
|
||||||
response *promptkit.GenerateResponse
|
response *promptkit.GenerateResponse
|
||||||
responses []*promptkit.GenerateResponse
|
responses []*promptkit.GenerateResponse
|
||||||
@@ -59,7 +67,7 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
|
|||||||
|
|
||||||
func simpleConfig(content string) promptkit.Config {
|
func simpleConfig(content string) promptkit.Config {
|
||||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("consumer test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile",
|
"profiles": []any{map[string]any{"id": "profile",
|
||||||
"endpoint": "http://example.test/v1", "model": "model",
|
"endpoint": "http://example.test/v1", "model": "model",
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestContinuationBranchesAcrossEngines(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
|
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
|
||||||
branch, err := scoped.Run(ctx, request)
|
branch, err := scoped.Run(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func TestCoordinateReleasesCapacityBeforeCallback(t *testing.T) {
|
|||||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
calls++
|
calls++
|
||||||
if turn.Result == nil {
|
if turn.Result == nil {
|
||||||
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}}, nil
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}}, nil
|
||||||
}
|
}
|
||||||
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
|
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
|
||||||
t.Fatalf("delivered turn = %#v", turn)
|
t.Fatalf("delivered turn = %#v", turn)
|
||||||
@@ -98,7 +98,7 @@ func TestCoordinateRetriesAndRepairsDeliverOnce(t *testing.T) {
|
|||||||
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
callbacks++
|
callbacks++
|
||||||
if turn.Result == nil {
|
if turn.Result == nil {
|
||||||
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}}, nil
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}, nil
|
||||||
}
|
}
|
||||||
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
|
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
|
||||||
t.Fatalf("turn = %#v", turn)
|
t.Fatalf("turn = %#v", turn)
|
||||||
|
|||||||
@@ -526,6 +526,9 @@ unrelated runtime contracts or adding test-only source interpretation.
|
|||||||
|
|
||||||
## Stage 12 — Migrate Existing Public Facade Call Sites And Fixtures
|
## Stage 12 — Migrate Existing Public Facade Call Sites And Fixtures
|
||||||
|
|
||||||
|
Status: complete. The root and internal test suites passed with updated request
|
||||||
|
inputs, explicit fragment mounts, native paths, and the shared-model boundary.
|
||||||
|
|
||||||
Focus: existing root-package contract tests and their configuration fixtures.
|
Focus: existing root-package contract tests and their configuration fixtures.
|
||||||
|
|
||||||
1. Change request replacements to `OutputContractInput` and pointer budgets;
|
1. Change request replacements to `OutputContractInput` and pointer budgets;
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ func TestExchangeCapturesInvalidFinalOutput(t *testing.T) {
|
|||||||
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
|
||||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
|
||||||
Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}})
|
Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}})
|
||||||
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
|
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
|
||||||
t.Fatalf("invalid result = %#v, %v", result, err)
|
t.Fatalf("invalid result = %#v, %v", result, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ func TestOperationBudgetInterruptsRetryAndRepair(t *testing.T) {
|
|||||||
req := simpleGenerationRequest()
|
req := simpleGenerationRequest()
|
||||||
req.OperationTimeout = time.Second
|
req.OperationTimeout = time.Second
|
||||||
if repair {
|
if repair {
|
||||||
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
}
|
}
|
||||||
_, err = engine.Run(context.Background(), req)
|
_, err = engine.Run(context.Background(), req)
|
||||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ func TestExecutionGroupExplicitMembershipAndTerminalTotals(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
validation := &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
validation := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
callbackCount := 0
|
callbackCount := 0
|
||||||
var branchID, validatorID string
|
var branchID, validatorID string
|
||||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
|||||||
@@ -38,8 +38,8 @@ func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
|||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: inputs},
|
{Inputs: mountSettingsInputs(inputs)},
|
||||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
|
||||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
@@ -144,10 +144,10 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
|||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
schemas := fstest.MapFS{}
|
schemas := fstest.MapFS{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("structured prompt", map[string]any{
|
||||||
"prompts": []any{map[string]any{"id": "structured",
|
"prompts": []any{map[string]any{"id": "structured",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||||
|
|||||||
@@ -79,10 +79,10 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
req := simpleGenerationRequest()
|
req := simpleGenerationRequest()
|
||||||
req.CaptureExchange = true
|
req.CaptureExchange = true
|
||||||
req.Validation = &promptkit.OutputContract{
|
req.Validation = &promptkit.OutputContractInput{
|
||||||
Format: promptkit.FormatJSON,
|
Format: promptkit.FormatJSON,
|
||||||
ValidationMode: promptkit.ValidationJSON,
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
RepairAttempts: 1,
|
RepairAttempts: new(1),
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := engine.Run(context.Background(), req)
|
result, err := engine.Run(context.Background(), req)
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
func TestListIdentitiesBasicContract(t *testing.T) {
|
func TestListIdentitiesBasicContract(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("listing", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("listing", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "configured-profile", "model": "example-model"}},
|
"profiles": []any{map[string]any{"id": "configured-profile", "model": "example-model"}},
|
||||||
"prompts": []any{listingContractPrompt("example-prompt")},
|
"prompts": []any{listingContractPrompt("example-prompt")},
|
||||||
}),
|
}),
|
||||||
@@ -217,7 +217,7 @@ func newNamespaceListingEngine(t *testing.T) *promptkit.Engine {
|
|||||||
"prompts": []any{listingContractPrompt("prompt")},
|
"prompts": []any{listingContractPrompt("prompt")},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("namespaced listing", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("namespaced listing", map[string]any{
|
||||||
"collections": map[string]any{
|
"collections": map[string]any{
|
||||||
"team": collection("team"),
|
"team": collection("team"),
|
||||||
"team-blue": collection("team-blue"),
|
"team-blue": collection("team-blue"),
|
||||||
@@ -238,7 +238,7 @@ func newNamespaceListingEngine(t *testing.T) *promptkit.Engine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestResourceAndCollectionInventories(t *testing.T) {
|
func TestResourceAndCollectionInventories(t *testing.T) {
|
||||||
underlay := configkit.Input{Source: configkit.Mapping("inventory defaults", map[string]any{
|
underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inventory defaults", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "root-backend", "endpoint": "https://root.example.test/v1"}},
|
"backends": []any{map[string]any{"id": "root-backend", "endpoint": "https://root.example.test/v1"}},
|
||||||
"profiles": []any{map[string]any{"id": "root-profile", "endpoint": "https://root.example.test/v1", "model": "root-model"}},
|
"profiles": []any{map[string]any{"id": "root-profile", "endpoint": "https://root.example.test/v1", "model": "root-model"}},
|
||||||
"prompts": []any{listingContractPrompt("root-prompt")},
|
"prompts": []any{listingContractPrompt("root-prompt")},
|
||||||
@@ -257,7 +257,7 @@ func TestResourceAndCollectionInventories(t *testing.T) {
|
|||||||
"override": map[string]any{"release_version": "old"},
|
"override": map[string]any{"release_version": "old"},
|
||||||
},
|
},
|
||||||
})}
|
})}
|
||||||
main := configkit.Input{Source: configkit.Mapping("inventory overlay", map[string]any{
|
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inventory overlay", map[string]any{
|
||||||
"collections": map[string]any{"override": map[string]any{"release_version": "current"}},
|
"collections": map[string]any{"override": map[string]any{"release_version": "current"}},
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
@@ -360,7 +360,7 @@ func TestListIdentitiesInvalidEngine(t *testing.T) {
|
|||||||
|
|
||||||
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
|
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("listing", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("listing", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile"}},
|
"profiles": []any{map[string]any{"id": "profile"}},
|
||||||
"prompts": []any{listingContractPrompt("example")},
|
"prompts": []any{listingContractPrompt("example")},
|
||||||
}),
|
}),
|
||||||
@@ -395,7 +395,7 @@ func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
||||||
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
|
underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("consumer defaults", map[string]any{
|
||||||
"collections": map[string]any{"shared": map[string]any{
|
"collections": map[string]any{"shared": map[string]any{
|
||||||
"profiles": []any{
|
"profiles": []any{
|
||||||
map[string]any{"id": "base", "endpoint": "https://example.test/v1", "model": "base-model"},
|
map[string]any{"id": "base", "endpoint": "https://example.test/v1", "model": "base-model"},
|
||||||
@@ -406,7 +406,7 @@ func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
"prompts": []any{listingContractPrompt("underlay-only")},
|
"prompts": []any{listingContractPrompt("underlay-only")},
|
||||||
})}
|
})}
|
||||||
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
|
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("application overlay", map[string]any{
|
||||||
"collections": map[string]any{"shared": map[string]any{"profiles": []any{
|
"collections": map[string]any{"shared": map[string]any{"profiles": []any{
|
||||||
map[string]any{"id": "overlay", "model": "effective-model"},
|
map[string]any{"id": "overlay", "model": "effective-model"},
|
||||||
}}},
|
}}},
|
||||||
@@ -575,7 +575,7 @@ func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}},
|
||||||
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
|
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -631,7 +631,7 @@ func countListingPrompt(prompts []promptkit.PromptIdentity, id string) int {
|
|||||||
|
|
||||||
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
|
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("listing contract", values)}
|
||||||
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
|
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ func contentHashEngine(t *testing.T, promptID, profileID, backendID, content str
|
|||||||
if cache {
|
if cache {
|
||||||
message["cache_control"] = map[string]any{"type": "ephemeral"}
|
message["cache_control"] = map[string]any{"type": "ephemeral"}
|
||||||
}
|
}
|
||||||
config := promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("hash test", map[string]any{
|
config := promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("hash test", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": backendID, "endpoint": "https://example.test/v1"}},
|
"backends": []any{map[string]any{"id": backendID, "endpoint": "https://example.test/v1"}},
|
||||||
"profiles": []any{map[string]any{"id": profileID, "backend": backendID, "model": "model"}},
|
"profiles": []any{map[string]any{"id": profileID, "backend": backendID, "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": promptID, "default_profile": profileID, "inputs": []any{map[string]any{"name": "input", "required": false}}, "messages": []any{message}, "output": output}},
|
"prompts": []any{map[string]any{"id": promptID, "default_profile": profileID, "inputs": []any{map[string]any{"name": "input", "required": false}}, "messages": []any{message}, "output": output}},
|
||||||
|
|||||||
@@ -304,7 +304,7 @@ func newNamespacedContractEngine(t *testing.T, client promptkit.LLMClient) *prom
|
|||||||
input.Imports = false
|
input.Imports = false
|
||||||
inputs = append(inputs, input)
|
inputs = append(inputs, input)
|
||||||
}
|
}
|
||||||
inputs = append(inputs, configkit.Input{Source: configkit.Mapping("root resources", map[string]any{
|
inputs = append(inputs, configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("root resources", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "root-only",
|
"profiles": []any{map[string]any{"id": "root-only",
|
||||||
"endpoint": "https://root.example/v1", "model": "root-model",
|
"endpoint": "https://root.example/v1", "model": "root-model",
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ func TestObservedNestedErrorsKeepTheParentOperation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return nil, childErr
|
return nil, childErr
|
||||||
})))
|
})))
|
||||||
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
if mode == "prepared" {
|
if mode == "prepared" {
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) {
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
deadline, _ := ctx.Deadline()
|
deadline, _ := ctx.Deadline()
|
||||||
child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("child execution", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("child execution", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "child-model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "child-model"}},
|
||||||
"prompts": []any{map[string]any{"id": "child",
|
"prompts": []any{map[string]any{"id": "child",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
|
|||||||
@@ -169,7 +169,7 @@ func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T)
|
|||||||
var events []promptkit.ExecutionEvent
|
var events []promptkit.ExecutionEvent
|
||||||
request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}
|
request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}
|
||||||
if tc.validation != "" {
|
if tc.validation != "" {
|
||||||
request.Validation = &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: tc.validation}
|
request.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: tc.validation, RepairAttempts: new(0)}
|
||||||
}
|
}
|
||||||
result, err := engine.Run(context.Background(), request)
|
result, err := engine.Run(context.Background(), request)
|
||||||
if tc.err != nil && (result != nil || !errors.Is(err, promptkit.ErrLLMGenerate)) {
|
if tc.err != nil && (result != nil || !errors.Is(err, promptkit.ErrLLMGenerate)) {
|
||||||
@@ -223,8 +223,8 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
|
|||||||
engine := observationEngine(t, client)
|
engine := observationEngine(t, client)
|
||||||
var events []promptkit.ExecutionEvent
|
var events []promptkit.ExecutionEvent
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON,
|
PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON,
|
||||||
ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1},
|
ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)},
|
||||||
Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) },
|
Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) },
|
||||||
})
|
})
|
||||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
@@ -242,7 +242,7 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
|
|||||||
func TestRunObservationValidationPreparationError(t *testing.T) {
|
func TestRunObservationValidationPreparationError(t *testing.T) {
|
||||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||||
var events []promptkit.ExecutionEvent
|
var events []promptkit.ExecutionEvent
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||||
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
||||||
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,40 +15,40 @@ func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
|
|||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
contract promptkit.OutputContract
|
contract promptkit.OutputContractInput
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "cyclic inline schema",
|
name: "cyclic inline schema",
|
||||||
contract: promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: cycle},
|
contract: promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: cycle},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "inline schema cannot be encoded",
|
name: "inline schema cannot be encoded",
|
||||||
contract: promptkit.OutputContract{
|
contract: promptkit.OutputContractInput{
|
||||||
Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema,
|
Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema,
|
||||||
Schema: make(chan int),
|
Schema: make(chan int),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "unsupported format",
|
name: "unsupported format",
|
||||||
contract: promptkit.OutputContract{
|
contract: promptkit.OutputContractInput{
|
||||||
Format: promptkit.OutputFormat("binary"),
|
Format: promptkit.OutputFormat("binary"),
|
||||||
ValidationMode: promptkit.ValidationNone,
|
ValidationMode: promptkit.ValidationNone,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "repair attempts above maximum",
|
name: "repair attempts above maximum",
|
||||||
contract: promptkit.OutputContract{
|
contract: promptkit.OutputContractInput{
|
||||||
Format: promptkit.FormatText,
|
Format: promptkit.FormatText,
|
||||||
ValidationMode: promptkit.ValidationBasic,
|
ValidationMode: promptkit.ValidationBasic,
|
||||||
RepairAttempts: 4,
|
RepairAttempts: new(4),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "none validation with repair attempts",
|
name: "none validation with repair attempts",
|
||||||
contract: promptkit.OutputContract{
|
contract: promptkit.OutputContractInput{
|
||||||
Format: promptkit.FormatText,
|
Format: promptkit.FormatText,
|
||||||
ValidationMode: promptkit.ValidationNone,
|
ValidationMode: promptkit.ValidationNone,
|
||||||
RepairAttempts: 1,
|
RepairAttempts: new(1),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
@@ -82,7 +82,7 @@ func TestInactiveRuntimeSchemaIsNotRetained(t *testing.T) {
|
|||||||
original := schemaValue{Properties: map[string]any{"title": "original"}}
|
original := schemaValue{Properties: map[string]any{"title": "original"}}
|
||||||
for _, mode := range []promptkit.ValidationMode{promptkit.ValidationNone, promptkit.ValidationBasic, promptkit.ValidationJSON} {
|
for _, mode := range []promptkit.ValidationMode{promptkit.ValidationNone, promptkit.ValidationBasic, promptkit.ValidationJSON} {
|
||||||
for _, value := range []any{original, cycle, make(chan int)} {
|
for _, value := range []any{original, cycle, make(chan int)} {
|
||||||
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: mode, Schema: value, SchemaPath: "ignored.json"}}
|
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: mode, Schema: value, SchemaPath: "ignored.json"}}
|
||||||
prepared, err := engine.Prepare(context.Background(), request)
|
prepared, err := engine.Prepare(context.Background(), request)
|
||||||
if err != nil || prepared.OutputContract.Schema != nil || prepared.OutputContract.SchemaPath != "ignored.json" {
|
if err != nil || prepared.OutputContract.Schema != nil || prepared.OutputContract.SchemaPath != "ignored.json" {
|
||||||
t.Fatalf("prepare inactive %s schema: %+v, %v", mode, prepared, err)
|
t.Fatalf("prepare inactive %s schema: %+v, %v", mode, prepared, err)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||||
{Inputs: profileInputs}, {Inputs: promptInputs}, {Inputs: []configkit.Input{schemaRootInput(schemaSource, ".")}},
|
{Inputs: mountSettingsInputs(profileInputs)}, {Inputs: mountSettingsInputs(promptInputs)}, {Inputs: []configkit.Input{schemaRootInput(schemaSource, ".")}},
|
||||||
}}},
|
}}},
|
||||||
promptkit.WithArtifactReader(reader),
|
promptkit.WithArtifactReader(reader),
|
||||||
promptkit.WithLLMClient(client),
|
promptkit.WithLLMClient(client),
|
||||||
@@ -315,10 +315,10 @@ func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) {
|
|||||||
engine := newPreparedContractEngine(t, client, "Summarize the source.")
|
engine := newPreparedContractEngine(t, client, "Summarize the source.")
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "prepared", CaptureExchange: true,
|
PromptID: "prepared", CaptureExchange: true,
|
||||||
Validation: &promptkit.OutputContract{
|
Validation: &promptkit.OutputContractInput{
|
||||||
Format: promptkit.FormatMarkdown,
|
Format: promptkit.FormatMarkdown,
|
||||||
ValidationMode: promptkit.ValidationBasic,
|
ValidationMode: promptkit.ValidationBasic,
|
||||||
RepairAttempts: 1,
|
RepairAttempts: new(1),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -362,7 +362,7 @@ func TestPreparedExecutionIsolatesMutatingClientAcrossRepair(t *testing.T) {
|
|||||||
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{
|
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{
|
||||||
"nested": map[string]any{"value": "original"},
|
"nested": map[string]any{"value": "original"},
|
||||||
}},
|
}},
|
||||||
Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json", RepairAttempts: 1},
|
Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json", RepairAttempts: new(1)},
|
||||||
AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "appended", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}}},
|
AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "appended", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -914,7 +914,7 @@ func newPreparedContractEngine(
|
|||||||
|
|
||||||
func preparedContractConfig(message string, keyRequired bool) promptkit.Config {
|
func preparedContractConfig(message string, keyRequired bool) promptkit.Config {
|
||||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("prepared execution", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prepared execution", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile",
|
"profiles": []any{map[string]any{"id": "profile",
|
||||||
"endpoint": "http://example.test/v1", "model": "model", "api_key_required": keyRequired,
|
"endpoint": "http://example.test/v1", "model": "model", "api_key_required": keyRequired,
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ func TestPreparedObservationCountsCompletedUsageBeforeRepairFailure(t *testing.T
|
|||||||
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{{Content: "invalid json", Usage: promptkit.TokenUsage{PromptTokens: 4}}, nil}}
|
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{{Content: "invalid json", Usage: promptkit.TokenUsage{PromptTokens: 4}}, nil}}
|
||||||
engine := newPreparedContractEngine(t, client, "hello")
|
engine := newPreparedContractEngine(t, client, "hello")
|
||||||
var events []promptkit.ExecutionEvent
|
var events []promptkit.ExecutionEvent
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -133,7 +133,7 @@ func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testin
|
|||||||
const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY"
|
const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY"
|
||||||
t.Setenv(envName, "available")
|
t.Setenv(envName, "available")
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("credential preparation", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("credential preparation", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile",
|
"profiles": []any{map[string]any{"id": "profile",
|
||||||
"endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true,
|
"endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true,
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -24,11 +24,11 @@ func aliasPrompt() map[string]any {
|
|||||||
func TestProfileInheritanceConfiguredBaseAliasWorkflow(t *testing.T) {
|
func TestProfileInheritanceConfiguredBaseAliasWorkflow(t *testing.T) {
|
||||||
baseID, baseEndpoint, baseModel, reasoning, timeout := "base", "https://profile.example/v1", "synthetic-model", "high", 120
|
baseID, baseEndpoint, baseModel, reasoning, timeout := "base", "https://profile.example/v1", "synthetic-model", "high", 120
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", []promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("application alias", []promptkit.ProfileConfig{
|
||||||
{ID: "base", Endpoint: &baseEndpoint, Model: &baseModel},
|
{ID: "base", Endpoint: &baseEndpoint, Model: &baseModel},
|
||||||
{ID: "child", Extends: &baseID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout},
|
{ID: "child", Extends: &baseID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", aliasPrompt())},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct alias engine: %v", err)
|
t.Fatalf("construct alias engine: %v", err)
|
||||||
@@ -65,7 +65,7 @@ func TestProfileInheritanceYAMLAliasOfConfiguredProfile(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -77,7 +77,7 @@ func TestProfileInheritanceYAMLAliasOfConfiguredProfile(t *testing.T) {
|
|||||||
|
|
||||||
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
|
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("credentials", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("credentials", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "credential-backend",
|
"backends": []any{map[string]any{"id": "credential-backend",
|
||||||
"endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY",
|
"endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY",
|
||||||
}},
|
}},
|
||||||
@@ -109,7 +109,7 @@ func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
|
|||||||
{"profiles": []any{map[string]any{"id": "a", "extends": map[string]any{"id": "b"}}, map[string]any{"id": "b", "extends": map[string]any{"id": "a"}}}},
|
{"profiles": []any{map[string]any{"id": "a", "extends": map[string]any{"id": "b"}}, map[string]any{"id": "b", "extends": map[string]any{"id": "a"}}}},
|
||||||
} {
|
} {
|
||||||
invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("invalid inheritance", document),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid inheritance", document),
|
||||||
}}})
|
}}})
|
||||||
if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err)
|
t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err)
|
||||||
@@ -127,8 +127,8 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
|
|||||||
}
|
}
|
||||||
client := &recordingProfileClient{}
|
client := &recordingProfileClient{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", aliasPrompt())},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ func TestInspectProfileUsesLiteralFileKeyForPreparation(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: profiles}},
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(profiles)}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{
|
||||||
"prompts": []any{map[string]any{
|
"prompts": []any{map[string]any{
|
||||||
"id": "prompt",
|
"id": "prompt",
|
||||||
"default_profile": " literal ",
|
"default_profile": " literal ",
|
||||||
@@ -50,7 +50,7 @@ func TestInspectProfileUsesLiteralFileKeyForPreparation(t *testing.T) {
|
|||||||
func TestInspectProfileIsIndependentAndDoesNotInvokeProvider(t *testing.T) {
|
func TestInspectProfileIsIndependentAndDoesNotInvokeProvider(t *testing.T) {
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("inspection", map[string]any{"profiles": []any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inspection", map[string]any{"profiles": []any{
|
||||||
map[string]any{"id": "profile",
|
map[string]any{"id": "profile",
|
||||||
"endpoint": "https://example.test/v1", "model": "model", "api_key_required": true,
|
"endpoint": "https://example.test/v1", "model": "model", "api_key_required": true,
|
||||||
"extra_params": map[string]any{"nested": map[string]any{"value": "original"}},
|
"extra_params": map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
@@ -77,7 +77,7 @@ func TestInspectProfilePublicErrorsAndPartialTarget(t *testing.T) {
|
|||||||
t.Fatalf("nil engine: %v", err)
|
t.Fatalf("nil engine: %v", err)
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("partial", map[string]any{"profiles": []any{map[string]any{"id": "profile", "model": "model"}}}),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("partial", map[string]any{"profiles": []any{map[string]any{"id": "profile", "model": "model"}}}),
|
||||||
}}})
|
}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -30,9 +30,9 @@ func TestTypedSourcesComposeAndCaptureValues(t *testing.T) {
|
|||||||
prompts[0].Messages[1].Content = &empty
|
prompts[0].Messages[1].Content = &empty
|
||||||
prompts[1].ID = "changed"
|
prompts[1].ID = "changed"
|
||||||
profileEndpoint, model := "https://profile.test/v1", "model"
|
profileEndpoint, model := "https://profile.test/v1", "model"
|
||||||
main := configkit.Input{Source: configkit.Mapping("completion", map[string]any{"prompts": []any{map[string]any{"id": "layered", "messages": []any{map[string]any{"role": "user", "content": "layer"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}})}
|
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("completion", map[string]any{"prompts": []any{map[string]any{"id": "layered", "messages": []any{map[string]any{"role": "user", "content": "layer"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}})}
|
||||||
profileSource := promptkit.ProfileSource("profiles", []promptkit.ProfileConfig{{ID: "local", Endpoint: &profileEndpoint, Model: &model}})
|
profileSource := promptkit.ProfileSource("profiles", []promptkit.ProfileConfig{{ID: "local", Endpoint: &profileEndpoint, Model: &model}})
|
||||||
underlays := []configkit.Underlay{{Inputs: []configkit.Input{{Source: backendSource}, {Source: promptSource}, {Source: profileSource}}}}
|
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(backendSource), promptkit.SettingsInput(promptSource), promptkit.SettingsInput(profileSource)}}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &main}}, promptkit.WithBundledOpenRouter(false))
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &main}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -72,7 +72,7 @@ func TestTypedPromptSourceUsesExplicitFilesystemContext(t *testing.T) {
|
|||||||
source := promptkit.PromptSource("file prompts", []promptkit.PromptConfig{{ID: "base", Messages: []promptkit.PromptMessageConfig{{Role: &role, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}, {ID: "child", Extends: &parent}})
|
source := promptkit.PromptSource("file prompts", []promptkit.PromptConfig{{ID: "base", Messages: []promptkit.PromptMessageConfig{{Role: &role, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}, {ID: "child", Extends: &parent}})
|
||||||
files := fstest.MapFS{"templates/body.md": {Data: []byte("captured {{.value}}")}}
|
files := fstest.MapFS{"templates/body.md": {Data: []byte("captured {{.value}}")}}
|
||||||
filesystem := configkit.NewFilesystem("fixture", files)
|
filesystem := configkit.NewFilesystem("fixture", files)
|
||||||
input := configkit.Input{Source: source, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "templates"}}
|
input := configkit.Input{Source: source, Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "templates"}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -87,7 +87,7 @@ func TestTypedPromptSourceUsesExplicitFilesystemContext(t *testing.T) {
|
|||||||
func TestTypedPromptSourceDefersValidation(t *testing.T) {
|
func TestTypedPromptSourceDefersValidation(t *testing.T) {
|
||||||
role, body, path, format, validation := "user", "body", "also-file", "text", "none"
|
role, body, path, format, validation := "user", "body", "also-file", "text", "none"
|
||||||
source := promptkit.PromptSource("invalid", []promptkit.PromptConfig{{ID: "bad", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &body, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}})
|
source := promptkit.PromptSource("invalid", []promptkit.PromptConfig{{ID: "bad", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &body, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}})
|
||||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}})
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source, Destination: configkit.Keys("promptkit")}}})
|
||||||
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
t.Fatalf("invalid final source: %v", err)
|
t.Fatalf("invalid final source: %v", err)
|
||||||
}
|
}
|
||||||
@@ -95,7 +95,7 @@ func TestTypedPromptSourceDefersValidation(t *testing.T) {
|
|||||||
|
|
||||||
func TestNilTypedPromptSourceSuppliesEmptySequence(t *testing.T) {
|
func TestNilTypedPromptSourceSuppliesEmptySequence(t *testing.T) {
|
||||||
source := promptkit.PromptSource("empty", nil)
|
source := promptkit.PromptSource("empty", nil)
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}}, promptkit.WithBundledOpenRouter(false))
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source, Destination: configkit.Keys("promptkit")}}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -239,6 +239,11 @@ func publicSurfaceViolations(pkg *types.Package, modulePath string) []string {
|
|||||||
case *types.Named:
|
case *types.Named:
|
||||||
inspectTypeArgs(value.TypeArgs(), where)
|
inspectTypeArgs(value.TypeArgs(), where)
|
||||||
if isInternalPackage(value.Obj().Pkg(), modulePath) {
|
if isInternalPackage(value.Obj().Pkg(), modulePath) {
|
||||||
|
if isSharedSettingsType(value.Obj(), modulePath) {
|
||||||
|
inspect(value.Underlying(), where)
|
||||||
|
inspectMethods(value, where)
|
||||||
|
return
|
||||||
|
}
|
||||||
record(where, value)
|
record(where, value)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -343,6 +348,18 @@ func publicSurfaceViolations(pkg *types.Package, modulePath string) []string {
|
|||||||
return compactStrings(violations)
|
return compactStrings(violations)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isSharedSettingsType(obj *types.TypeName, modulePath string) bool {
|
||||||
|
if obj == nil || obj.Pkg() == nil || obj.Pkg().Path() != modulePath+"/internal/settings" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch obj.Name() {
|
||||||
|
case "Settings", "CollectionSettings", "BackendSettings", "ProfileSettings", "PromptSettings", "PromptInputSettings", "PromptMessageSettings", "PromptCacheControlSettings", "PromptOutputSettings", "BackendSmokeTestConfig", "ReachabilityProbeConfig", "AuthenticationProbeConfig":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func isInternalPackage(pkg *types.Package, modulePath string) bool {
|
func isInternalPackage(pkg *types.Package, modulePath string) bool {
|
||||||
return pkg != nil && strings.HasPrefix(pkg.Path(), modulePath+"/internal/")
|
return pkg != nil && strings.HasPrefix(pkg.Path(), modulePath+"/internal/")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
|
|||||||
req := simpleGenerationRequest()
|
req := simpleGenerationRequest()
|
||||||
notifications := 0
|
notifications := 0
|
||||||
req.OnPrepared = func(context.Context, promptkit.PreparedRun) { notifications++ }
|
req.OnPrepared = func(context.Context, promptkit.PreparedRun) { notifications++ }
|
||||||
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
||||||
result, err := engine.Run(context.Background(), req)
|
result, err := engine.Run(context.Background(), req)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || calls != 4 || notifications != 1 {
|
if err != nil || result == nil || !result.Validation.IsValid || calls != 4 || notifications != 1 {
|
||||||
@@ -96,7 +96,7 @@ func reliabilityConfig(backends, profiles map[string]any) promptkit.Config {
|
|||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("reliable execution", map[string]any{
|
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("reliable execution", map[string]any{
|
||||||
"backends": records(backends),
|
"backends": records(backends),
|
||||||
"profiles": records(profiles),
|
"profiles": records(profiles),
|
||||||
"prompts": []any{map[string]any{"id": "prompt",
|
"prompts": []any{map[string]any{"id": "prompt",
|
||||||
@@ -164,7 +164,7 @@ func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) {
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
resultA := make(chan capacityRunResult, 1)
|
resultA := make(chan capacityRunResult, 1)
|
||||||
requestA := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}}
|
requestA := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}
|
||||||
go runCapacityRequest(engine, ctx, requestA, resultA)
|
go runCapacityRequest(engine, ctx, requestA, resultA)
|
||||||
select {
|
select {
|
||||||
case <-firstA:
|
case <-firstA:
|
||||||
@@ -301,8 +301,8 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
|
|||||||
endpoint, model := "https://example.invalid/v1", "model"
|
endpoint, model := "https://example.invalid/v1", "model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||||
{Inputs: inputs},
|
{Inputs: mountSettingsInputs(inputs)},
|
||||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
{ID: "profile", Endpoint: &endpoint, Model: &model},
|
{ID: "profile", Endpoint: &endpoint, Model: &model},
|
||||||
})}}},
|
})}}},
|
||||||
}},
|
}},
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ func TestRunRequestFormattingAndJSONProtectDirectCredential(t *testing.T) {
|
|||||||
Inputs: map[string]promptkit.ArtifactRef{"input": {Type: promptkit.ArtifactRefInline, Body: markers[2]}},
|
Inputs: map[string]promptkit.ArtifactRef{"input": {Type: promptkit.ArtifactRefInline, Body: markers[2]}},
|
||||||
AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: markers[3]}},
|
AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: markers[3]}},
|
||||||
Execution: &promptkit.ExecutionTargetOverride{Model: markers[4]},
|
Execution: &promptkit.ExecutionTargetOverride{Model: markers[4]},
|
||||||
Validation: &promptkit.OutputContract{SchemaPath: markers[5]},
|
Validation: &promptkit.OutputContractInput{SchemaPath: markers[5]},
|
||||||
}
|
}
|
||||||
for _, rendered := range []string{
|
for _, rendered := range []string{
|
||||||
fmt.Sprintf("%v", request), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request),
|
fmt.Sprintf("%v", request), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request),
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ func BenchmarkRequirementInspection(b *testing.B) {
|
|||||||
map[string]any{"name": "transcript", "required": true},
|
map[string]any{"name": "transcript", "required": true},
|
||||||
}, "", noValidationOutput)
|
}, "", noValidationOutput)
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
||||||
"prompts": identifiedRequirementRecords(onePrompt("inspection", definition)),
|
"prompts": identifiedRequirementRecords(onePrompt("inspection", definition)),
|
||||||
"profiles": identifiedRequirementRecords(profiles),
|
"profiles": identifiedRequirementRecords(profiles),
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import (
|
|||||||
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("requirements contract", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("requirements contract", map[string]any{
|
||||||
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
|
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
|
||||||
}),
|
}),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestResourceInspectionSelections(t *testing.T) {
|
func TestResourceInspectionSelections(t *testing.T) {
|
||||||
engine, err := NewEngine(Config{Sources: ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
engine, err := NewEngine(Config{Sources: ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1"}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1"}},
|
||||||
"profiles": []any{map[string]any{"id": "partial", "backend": "local"}},
|
"profiles": []any{map[string]any{"id": "partial", "backend": "local"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic"}}},
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
|
|
||||||
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
||||||
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
@@ -37,7 +37,7 @@ func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, field := range []string{"endpoint", "extra_params", "api_key_env"} {
|
for _, field := range []string{"endpoint", "extra_params", "api_key_env"} {
|
||||||
path := configkit.Keys("profiles").Index(1).Key(field)
|
path := configkit.Keys("promptkit", "profiles").Index(1).Key(field)
|
||||||
explanation, ok := view.Explain(path)
|
explanation, ok := view.Explain(path)
|
||||||
if !ok || !strings.Contains(explanation.Effective.YAML, "null") || len(explanation.History) == 0 {
|
if !ok || !strings.Contains(explanation.Effective.YAML, "null") || len(explanation.History) == 0 {
|
||||||
t.Errorf("%s provenance = %+v, %t", field, explanation, ok)
|
t.Errorf("%s provenance = %+v, %t", field, explanation, ok)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestResourceInspectionGenericEncoding(t *testing.T) {
|
func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
|
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) {
|
|||||||
}, opens: map[string]int{}}
|
}, opens: map[string]int{}}
|
||||||
defaultFS := configkit.NewFilesystem("embedded", defaults)
|
defaultFS := configkit.NewFilesystem("embedded", defaults)
|
||||||
operatorFS := configkit.NewFilesystem("operator", operator)
|
operatorFS := configkit.NewFilesystem("operator", operator)
|
||||||
base := configkit.Input{Source: configkit.Mapping("embedded definitions", map[string]any{
|
base := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("embedded definitions", map[string]any{
|
||||||
"schema_root": "app",
|
"schema_root": "app",
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))},
|
"prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))},
|
||||||
@@ -63,11 +63,11 @@ func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) {
|
|||||||
}), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}}
|
}), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}}
|
||||||
operatorOutput := mixedOutput("schema_file", "schema.json")
|
operatorOutput := mixedOutput("schema_file", "schema.json")
|
||||||
operatorOutput["repair_attempts"] = 1
|
operatorOutput["repair_attempts"] = 1
|
||||||
overlay := configkit.Input{Source: configkit.Mapping("operator definitions", map[string]any{
|
overlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("operator definitions", map[string]any{
|
||||||
"collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{
|
"collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{
|
||||||
mixedPrompt("replaced", operatorOutput),
|
mixedPrompt("replaced", operatorOutput),
|
||||||
map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}},
|
map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}},
|
||||||
map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false}},
|
map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false, "repair_attempts": 0}},
|
||||||
mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")),
|
mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")),
|
||||||
}}},
|
}}},
|
||||||
}), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}}
|
}), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|||||||
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
|
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
|
||||||
}
|
}
|
||||||
filesystem := configkit.NewFilesystem("collection", collection)
|
filesystem := configkit.NewFilesystem("collection", collection)
|
||||||
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompts", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
|
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
|
||||||
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
@@ -36,7 +36,7 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|||||||
if len(client.requests) != 0 {
|
if len(client.requests) != 0 {
|
||||||
t.Fatal("preparation called model")
|
t.Fatal("preparation called model")
|
||||||
}
|
}
|
||||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
|
base.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
|
||||||
result, err := engine.Run(context.Background(), base)
|
result, err := engine.Run(context.Background(), base)
|
||||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||||
t.Fatalf("inline replacement = %+v, %v", result, err)
|
t.Fatalf("inline replacement = %+v, %v", result, err)
|
||||||
@@ -49,7 +49,7 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|||||||
if err != nil || result.Validation.Status != promptkit.ValidationFailed || client.requests[1].StructuredOutput.JSONSchema.Schema != false {
|
if err != nil || result.Validation.Status != promptkit.ValidationFailed || client.requests[1].StructuredOutput.JSONSchema.Schema != false {
|
||||||
t.Fatalf("false schema = %+v, %v", result, err)
|
t.Fatalf("false schema = %+v, %v", result, err)
|
||||||
}
|
}
|
||||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
base.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
||||||
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
||||||
t.Fatalf("runtime path validation = %v", err)
|
t.Fatalf("runtime path validation = %v", err)
|
||||||
}
|
}
|
||||||
@@ -64,7 +64,7 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|||||||
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
|
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
|
||||||
t.Fatalf("selected collection root schema = %#v", got)
|
t.Fatalf("selected collection root schema = %#v", got)
|
||||||
}
|
}
|
||||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
|
base.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
|
||||||
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
t.Fatalf("missing replacement selection = %v", err)
|
t.Fatalf("missing replacement selection = %v", err)
|
||||||
}
|
}
|
||||||
@@ -72,7 +72,7 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|||||||
|
|
||||||
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
|
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
|
||||||
engine := newSimpleEngine(t, "hello")
|
engine := newSimpleEngine(t, "hello")
|
||||||
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
|
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
|
||||||
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
t.Fatalf("missing root = %v", err)
|
t.Fatalf("missing root = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
|||||||
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
|
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
@@ -95,7 +95,7 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
|
|||||||
if err := os.WriteFile(path, []byte(schema), 0o600); err != nil {
|
if err := os.WriteFile(path, []byte(schema), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
@@ -127,7 +127,7 @@ func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) {
|
|||||||
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
|
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func TestInlineSchemaOwnershipAcrossPublicSnapshots(t *testing.T) {
|
|||||||
endpoint, model := "https://example.invalid/v1", "model"
|
endpoint, model := "https://example.invalid/v1", "model"
|
||||||
profiles := promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "profile", Endpoint: &endpoint, Model: &model}})
|
profiles := promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "profile", Endpoint: &endpoint, Model: &model}})
|
||||||
client := &schemaOwnershipClient{}
|
client := &schemaOwnershipClient{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: profiles}, {Source: source}}}}}}, promptkit.WithLLMClient(client))
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: profiles}, {Destination: configkit.Keys("promptkit"), Source: source}}}}}}, promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -126,7 +126,7 @@ func TestInlineSchemaOwnershipAcrossPublicSnapshots(t *testing.T) {
|
|||||||
t.Fatalf("inspection = %+v, %v", again, err)
|
t.Fatalf("inspection = %+v, %v", again, err)
|
||||||
}
|
}
|
||||||
runtimeSchema := map[string]any{"title": "captured", "type": "object"}
|
runtimeSchema := map[string]any{"title": "captured", "type": "object"}
|
||||||
runtimeRequest := promptkit.RunRequest{PromptID: "p", ProfileID: "profile", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: runtimeSchema}}
|
runtimeRequest := promptkit.RunRequest{PromptID: "p", ProfileID: "profile", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: runtimeSchema}}
|
||||||
runtimeHandle, err := engine.PrepareExecution(context.Background(), runtimeRequest)
|
runtimeHandle, err := engine.PrepareExecution(context.Background(), runtimeRequest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
func schemaRootInput(fsys fs.FS, root string) configkit.Input {
|
func schemaRootInput(fsys fs.FS, root string) configkit.Input {
|
||||||
filesystem := configkit.NewFilesystem("schemas", fsys)
|
filesystem := configkit.NewFilesystem("schemas", fsys)
|
||||||
return configkit.Input{
|
return configkit.Input{
|
||||||
Source: promptkit.SchemaRootSource("schemas", root),
|
Destination: configkit.Keys("promptkit"), Source: promptkit.SchemaRootSource("schemas", root),
|
||||||
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
|
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -335,7 +335,7 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
|||||||
reader := &failingArtifactReader{}
|
reader := &failingArtifactReader{}
|
||||||
schemaSource := &failingSchemaFS{}
|
schemaSource := &failingSchemaFS{}
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
|
||||||
input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("private diagnostic test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "application-prompt",
|
"prompts": []any{map[string]any{"id": "application-prompt",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
@@ -484,7 +484,7 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
|
|||||||
Request: request,
|
Request: request,
|
||||||
}, nil
|
}, nil
|
||||||
})}
|
})}
|
||||||
input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cross-backend smoke test", map[string]any{
|
||||||
"backends": []any{
|
"backends": []any{
|
||||||
map[string]any{"id": "backend-a",
|
map[string]any{"id": "backend-a",
|
||||||
"endpoint": "http://backend-a.example/v1",
|
"endpoint": "http://backend-a.example/v1",
|
||||||
@@ -539,7 +539,7 @@ func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t
|
|||||||
Request: request,
|
Request: request,
|
||||||
}, nil
|
}, nil
|
||||||
})}
|
})}
|
||||||
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("redirect diagnostic", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}},
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func TestProgrammaticSourcesPreserveOrderAndRejectDuplicates(t *testing.T) {
|
|||||||
})
|
})
|
||||||
params["visible"] = "changed"
|
params["visible"] = "changed"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: backendSource}, {Source: profileSource}}}},
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: backendSource}, {Destination: configkit.Keys("promptkit"), Source: profileSource}}}},
|
||||||
}}, promptkit.WithBundledOpenRouter(false))
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -42,7 +42,7 @@ func TestProgrammaticSourcesPreserveOrderAndRejectDuplicates(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, kind := range []string{"backends", "profiles"} {
|
for _, kind := range []string{"backends", "profiles"} {
|
||||||
for index, id := range []string{"zeta", "alpha"} {
|
for index, id := range []string{"zeta", "alpha"} {
|
||||||
entry, found := view.Explain(configkit.Keys(kind).Index(index).Key("id"))
|
entry, found := view.Explain(configkit.Keys("promptkit", kind).Index(index).Key("id"))
|
||||||
if !found || !strings.Contains(entry.Effective.YAML, id) {
|
if !found || !strings.Contains(entry.Effective.YAML, id) {
|
||||||
t.Fatalf("%s entry %d = %+v, want %s", kind, index, entry, id)
|
t.Fatalf("%s entry %d = %+v, want %s", kind, index, entry, id)
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,7 @@ func TestProgrammaticSourcesPreserveOrderAndRejectDuplicates(t *testing.T) {
|
|||||||
promptkit.BackendSource("duplicate backends", []promptkit.BackendConfig{{ID: "duplicate", Endpoint: &endpoint}, {ID: "duplicate", Endpoint: &endpoint}}),
|
promptkit.BackendSource("duplicate backends", []promptkit.BackendConfig{{ID: "duplicate", Endpoint: &endpoint}, {ID: "duplicate", Endpoint: &endpoint}}),
|
||||||
promptkit.ProfileSource("duplicate profiles", []promptkit.ProfileConfig{{ID: "duplicate"}, {ID: "duplicate"}}),
|
promptkit.ProfileSource("duplicate profiles", []promptkit.ProfileConfig{{ID: "duplicate"}, {ID: "duplicate"}}),
|
||||||
} {
|
} {
|
||||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}})
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: source}}})
|
||||||
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
t.Fatalf("duplicate source error = %v, want ErrInvalidConfig", err)
|
t.Fatalf("duplicate source error = %v, want ErrInvalidConfig", err)
|
||||||
}
|
}
|
||||||
@@ -65,7 +65,7 @@ func TestProgrammaticProfileVariantInherits(t *testing.T) {
|
|||||||
{ID: parent, Endpoint: &endpoint, Model: &model},
|
{ID: parent, Endpoint: &endpoint, Model: &model},
|
||||||
{ID: "child", Extends: &parent, TimeoutSeconds: &timeout},
|
{ID: "child", Extends: &parent, TimeoutSeconds: &timeout},
|
||||||
})
|
})
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: source}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -88,7 +88,7 @@ func TestProgrammaticSourcesCaptureNestedValuesAndDeferInvalidValues(t *testing.
|
|||||||
return promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "local", ExtraParams: params}})
|
return promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "local", ExtraParams: params}})
|
||||||
}
|
}
|
||||||
load := func(source configkit.Source) (*promptkit.Engine, error) {
|
load := func(source configkit.Source) (*promptkit.Engine, error) {
|
||||||
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}}, promptkit.WithBundledOpenRouter(false))
|
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: source}}}, promptkit.WithBundledOpenRouter(false))
|
||||||
}
|
}
|
||||||
t.Run(kind+"/capture", func(t *testing.T) {
|
t.Run(kind+"/capture", func(t *testing.T) {
|
||||||
values := map[string]any{"nested": map[string]any{"items": []any{"original", int64(42), true, nil}}}
|
values := map[string]any{"nested": map[string]any{"items": []any{"original", int64(42), true, nil}}}
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ func TestBuiltInRetryBudgetAppliesToRepairGeneration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
req := simpleGenerationRequest()
|
req := simpleGenerationRequest()
|
||||||
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
result, err := engine.Run(context.Background(), req)
|
result, err := engine.Run(context.Background(), req)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || *calls != 4 {
|
if err != nil || result == nil || !result.Validation.IsValid || *calls != 4 {
|
||||||
t.Fatalf("repair result=%+v, error=%v, calls=%d", result, err, *calls)
|
t.Fatalf("repair result=%+v, error=%v, calls=%d", result, err, *calls)
|
||||||
|
|||||||
Reference in New Issue
Block a user