Compare commits
67 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65546d794e | |||
| 892cb74e6e | |||
| 64cf84c9b5 | |||
| c23393b2cc | |||
| 5aae5df5fb | |||
| aaaa2816a4 | |||
| 5c135c5b28 | |||
| 31cc0069fc | |||
| 68daa61867 | |||
| 159d369b9b | |||
| 23fb55f9df | |||
| a07e95fd17 | |||
| c4d7cf8cd4 | |||
| 5a6f73e0bb | |||
| c149f7274c | |||
| e5c42de06d | |||
| 04569e44df | |||
| 26530808e0 | |||
| fcd0485340 | |||
| 5cc3fa0168 | |||
| b423de12c3 | |||
| 72955aaaa3 | |||
| 85dae9b443 | |||
| 70db0ad614 | |||
| 37a48f2e20 | |||
| 6c14379026 | |||
| 45bbf7bb2c | |||
| ee810b8fbe | |||
| bc95d1f9fd | |||
| c948fef8fc | |||
| cb180e8728 | |||
| de2a7fcd6f | |||
| f20020d9f4 | |||
| 9396183bdb | |||
| dba6d7d68c | |||
| d515a7e49b | |||
| 171fb11f4e | |||
| c9cbe7d3b3 | |||
| 7df7e420aa | |||
| 8ff9aa3efd | |||
| 78b78ea00b | |||
| 63e9f0b731 | |||
| 20b0f78af2 | |||
| 101a2840d4 | |||
| f083ad5309 | |||
| 19a0604874 | |||
| ea31e71280 | |||
| 27966c1686 | |||
| ea63a7b0ff | |||
| b5b9b1f209 | |||
| 129959b24c | |||
| dfe9d6566f | |||
| bf3d2e19bd | |||
| 4cbd4e2da0 | |||
| b0a16ccce8 | |||
| 8add7c49e6 | |||
| 8563379216 | |||
| 951739eda0 | |||
| a0f696181f | |||
| 3951dcf1f3 | |||
| 363920deb2 | |||
| 06315062b3 | |||
| c669f14610 | |||
| 52287ab2d1 | |||
| a43e0f17ee | |||
| ffaaf552c8 | |||
| d7007eb072 |
15
README.md
15
README.md
@@ -45,6 +45,21 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.22.0 release notes](docs/releases/v0.22.0.md) cover captured conversation
|
||||||
|
exchanges, application-directed turn coordination, and explicit execution groups.
|
||||||
|
|
||||||
|
The [v0.21.0 release notes](docs/releases/v0.21.0.md) cover ConfigKit v1.2.0
|
||||||
|
and main-relative companion imports through the existing configuration-source
|
||||||
|
API.
|
||||||
|
|
||||||
|
The [v0.20.0 release notes](docs/releases/v0.20.0.md) cover resolved resource
|
||||||
|
inspection, generic JSON/YAML reports, typed prompt sources, and backend
|
||||||
|
inheritance. Upgrading consumers should review the changed inspection result
|
||||||
|
types and profile null-fallback behavior.
|
||||||
|
|
||||||
|
The [v0.19.0 release notes](docs/releases/v0.19.0.md) cover audit remediation
|
||||||
|
and compatibility changes to artifact ownership, programmatic sources, and schemas.
|
||||||
|
|
||||||
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||||
OpenRouter backend and profiles, recorded configuration normalization, and
|
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||||
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||||
|
|||||||
20
backends.go
20
backends.go
@@ -19,10 +19,13 @@ const BackendLocal = "local"
|
|||||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||||
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||||
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||||
// inputs for explicit null. BackendConfig has no stable JSON representation.
|
// inputs for explicit null. Extends names a sibling backend in the same list.
|
||||||
|
// BackendConfig has no stable JSON representation.
|
||||||
type BackendConfig struct {
|
type BackendConfig struct {
|
||||||
// ID is the local backend identity within its resource list.
|
// ID is the local backend identity within its resource list.
|
||||||
ID string
|
ID string
|
||||||
|
// Extends names a sibling backend to inherit through ConfigKit.
|
||||||
|
Extends *string
|
||||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||||
Endpoint *string
|
Endpoint *string
|
||||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||||
@@ -44,13 +47,13 @@ type BackendSmokeTestConfig struct {
|
|||||||
// DefaultProfile is the exact profile ID selected as the backend's
|
// DefaultProfile is the exact profile ID selected as the backend's
|
||||||
// diagnostic inference default. A supplied value must not be blank; the ID
|
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||||
// is resolved only when that default is selected.
|
// is resolved only when that default is selected.
|
||||||
DefaultProfile *string
|
DefaultProfile *string `json:"default_profile,omitempty" yaml:"default_profile,omitempty"`
|
||||||
// Reachability optionally selects a full HTTP destination for reachability
|
// Reachability optionally selects a full HTTP destination for reachability
|
||||||
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||||
Reachability *ReachabilityProbeConfig
|
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
|
||||||
// Authentication optionally declares a named authentication diagnostic.
|
// Authentication optionally declares a named authentication diagnostic.
|
||||||
// A supplied object requires Kind.
|
// A supplied object requires Kind.
|
||||||
Authentication *AuthenticationProbeConfig
|
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReachabilityProbeConfig configures an optional reachability destination.
|
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||||
@@ -58,7 +61,7 @@ type ReachabilityProbeConfig struct {
|
|||||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
// Nil selects the effective backend endpoint; an explicit blank value is
|
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||||
// invalid.
|
// invalid.
|
||||||
Endpoint *string
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AuthenticationProbeConfig configures an optional named authentication
|
// AuthenticationProbeConfig configures an optional named authentication
|
||||||
@@ -66,11 +69,11 @@ type ReachabilityProbeConfig struct {
|
|||||||
type AuthenticationProbeConfig struct {
|
type AuthenticationProbeConfig struct {
|
||||||
// Kind selects the authentication adapter. Supported values are "none" and
|
// Kind selects the authentication adapter. Supported values are "none" and
|
||||||
// "openrouter_key"; surrounding whitespace and case are normalized.
|
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||||
Kind *string
|
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
|
||||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
// Nil selects the adapter's backend-relative destination; an explicit blank
|
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||||
// value is invalid.
|
// value is invalid.
|
||||||
Endpoint *string
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
||||||
@@ -84,6 +87,9 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
|||||||
definitions := make([]any, 0, len(backends))
|
definitions := make([]any, 0, len(backends))
|
||||||
for _, config := range backends {
|
for _, config := range backends {
|
||||||
fields := map[string]any{"id": config.ID}
|
fields := map[string]any{"id": config.ID}
|
||||||
|
if config.Extends != nil {
|
||||||
|
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||||
|
}
|
||||||
if config.Endpoint != nil {
|
if config.Endpoint != nil {
|
||||||
fields["endpoint"] = *config.Endpoint
|
fields["endpoint"] = *config.Endpoint
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||||
if err != nil || underlayProfile.EffectiveModelParams.Model != "underlay-model" || underlayProfile.EffectiveModelParams.BackendID != "" {
|
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||||
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
}
|
}
|
||||||
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
||||||
@@ -38,29 +38,29 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
inspect := func(id string) promptkit.ExecutionTarget {
|
inspect := func(id string) *promptkit.ProfileInspection {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
profile, err := engine.InspectProfile(context.Background(), id)
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return profile.EffectiveModelParams
|
return profile
|
||||||
}
|
}
|
||||||
if target := inspect("gpt-sol-latest"); target.Model != "main-model" || target.BackendID != "" {
|
if target := inspect("gpt-sol-latest"); target.Model == nil || *target.Model != "main-model" || target.Backend != nil {
|
||||||
t.Fatalf("replacement = %+v", target)
|
t.Fatalf("replacement = %+v", target)
|
||||||
}
|
}
|
||||||
if target := inspect("custom"); target.Model != "custom-model" || target.BackendID != "openrouter" || target.Endpoint != "https://local.invalid/v1" {
|
if target := inspect("custom"); target.Model == nil || *target.Model != "custom-model" || target.Backend == nil || *target.Backend != "openrouter" || target.Endpoint != nil {
|
||||||
t.Fatalf("inherited = %+v", target)
|
t.Fatalf("inherited = %+v", target)
|
||||||
}
|
}
|
||||||
if target := inspect("gpt-luna-latest"); target.Endpoint != "https://local.invalid/v1" || target.APIKeyEnv != "" {
|
if backend, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || backend.Endpoint != "https://local.invalid/v1" || backend.APIKeyEnv != nil {
|
||||||
t.Fatalf("backend replacement = %+v", target)
|
t.Fatalf("backend replacement = %+v, %v", backend, err)
|
||||||
}
|
}
|
||||||
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
|
if target := inspect("openrouter/profile"); target.Backend == nil || *target.Backend != "openrouter/local" || target.Model == nil || *target.Model != "collection-model" {
|
||||||
t.Fatalf("collection = %+v", target)
|
t.Fatalf("collection = %+v", target)
|
||||||
}
|
}
|
||||||
target := inspect("auto-low")
|
target := inspect("auto-low")
|
||||||
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
if target.BackendID != "other" || target.Model != "other-model" || !reflect.DeepEqual(target.ExtraParams, wantPlugins) {
|
if target.Backend == nil || *target.Backend != "other" || target.Model == nil || *target.Model != "other-model" || target.ExtraParams == nil || !reflect.DeepEqual(*target.ExtraParams, wantPlugins) {
|
||||||
t.Errorf("inherited tier = %+v", target)
|
t.Errorf("inherited tier = %+v", target)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
|||||||
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
}
|
}
|
||||||
profile, err := engine.InspectProfile(context.Background(), "auto")
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
if err != nil || profile.EffectiveModelParams.BackendID != "openrouter" {
|
if err != nil || profile.Backend == nil || *profile.Backend != "openrouter" {
|
||||||
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
}
|
}
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
|
|||||||
@@ -58,11 +58,52 @@ func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
|||||||
t.Fatalf("construct engine with native collection import: %v", err)
|
t.Fatalf("construct engine with native collection import: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputUsesMainRelativeContext(t *testing.T) {
|
||||||
|
filesystem := configkit.NewFilesystem("application", fstest.MapFS{
|
||||||
|
"site/main.yaml": {Data: []byte("version: '2'\n")},
|
||||||
|
"site/prompts/greeting.yaml": {Data: []byte(`id: greeting
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: greeting.txt
|
||||||
|
output: {format: text, validation_mode: none}
|
||||||
|
`)},
|
||||||
|
"site/prompts/greeting.txt": {Data: []byte("Hello from the imported prompt's directory.")},
|
||||||
|
})
|
||||||
|
companion, err := promptkit.CollectionInput("support", configkit.Bytes("companion prompts", []byte(`
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- dir: prompts
|
||||||
|
pattern: ["*.yaml", "*.yml"]
|
||||||
|
destination: [prompts]
|
||||||
|
placement: sequence-item
|
||||||
|
required: true
|
||||||
|
`)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
companion.MainRelative = configkit.MainRelative()
|
||||||
|
main := configkit.Input{Source: filesystem.File("site/main.yaml")}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Main: &main,
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{companion}}},
|
||||||
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with main-relative imports: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(t.Context(), "support/greeting")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(inspection.Messages) != 1 || inspection.Messages[0].Content != "Hello from the imported prompt's directory." {
|
||||||
|
t.Fatalf("unexpected imported messages: %+v", inspection.Messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
|
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
|
||||||
@@ -85,7 +126,7 @@ func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
|||||||
t.Fatalf("construct engine with embedded collection import: %v", err)
|
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
|||||||
"rakestrawhome-profile": rakestrawhomeID,
|
"rakestrawhome-profile": rakestrawhomeID,
|
||||||
} {
|
} {
|
||||||
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
|
if err != nil || inspection.Backend == nil || *inspection.Backend != backendID {
|
||||||
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -242,8 +242,9 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configured profile: %v", err)
|
t.Fatalf("inspect configured profile: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
backendInspection, err := engine.InspectBackend(context.Background(), backendID)
|
||||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
if err != nil || inspection.Endpoint != nil || backendInspection.Endpoint != endpoint {
|
||||||
|
t.Fatalf("profile/backend = %+v / %+v, %v", inspection, backendInspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
141
continuation_contract_test.go
Normal file
141
continuation_contract_test.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestContinuationBranchesAcrossEngines(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
firstClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "first answer"}}
|
||||||
|
firstEngine := newSimpleEngine(t, "first question", promptkit.WithLLMClient(firstClient))
|
||||||
|
initial, err := firstEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, APIKey: "synthetic-first-key"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
original := initial.Exchange.Details()
|
||||||
|
secondClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "follow-up answer"}}
|
||||||
|
secondEngine := newNamespacedContractEngine(t, secondClient)
|
||||||
|
scoped, err := secondEngine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
|
||||||
|
branch, err := scoped.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request.AppendedMessages[0].Content = "branch two"
|
||||||
|
sibling, err := scoped.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
next, err := scoped.Run(ctx, promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: branch.Exchange})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if branch.ParentRunID != initial.RunID || sibling.ParentRunID != initial.RunID || next.ParentRunID != branch.RunID || branch.PromptID != "alpha/shared" || branch.SelectedProfileID != "alpha/shared" || branch.SessionID != "new-session" {
|
||||||
|
t.Fatalf("ancestry or selections incorrect: %#v %#v %#v", branch, sibling, next)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||||
|
t.Fatal("branching changed source")
|
||||||
|
}
|
||||||
|
if branch.Validation.Status != promptkit.ValidationFailed || branch.ModelName != "alpha-model" {
|
||||||
|
t.Fatalf("follow-up settings: %#v", branch)
|
||||||
|
}
|
||||||
|
branchMessages := branch.Exchange.Details().Messages
|
||||||
|
siblingMessages := sibling.Exchange.Details().Messages
|
||||||
|
if len(branchMessages) != 5 || len(siblingMessages) != 5 || branchMessages[3].Content != "branch one" || siblingMessages[3].Content != "branch two" || len(next.Exchange.Details().Messages) != 7 {
|
||||||
|
t.Fatalf("branch histories = %#v / %#v", branchMessages, siblingMessages)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(branch.Exchange.Details())
|
||||||
|
if err != nil || !json.Valid(encoded) {
|
||||||
|
t.Fatalf("details JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
branchMessages[0].Content = "changed details"
|
||||||
|
branch.RawOutput = "changed result"
|
||||||
|
if initial.Exchange.Details().Messages[0].Content != "first question" || branch.Exchange.Details().Messages[4].Content != "follow-up answer" {
|
||||||
|
t.Fatal("caller mutation changed captured history")
|
||||||
|
}
|
||||||
|
for _, request := range secondClient.requests {
|
||||||
|
if request.APIKey == "synthetic-first-key" || request.Prompt.Messages[0].Content != "first question" || request.Prompt.Messages[1].Content != "first answer" {
|
||||||
|
t.Fatalf("unexpected follow-up request: %#v", request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationDoesNotReloadCapturedInputs(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
path := filepath.Join(t.TempDir(), "input.txt")
|
||||||
|
if err := os.WriteFile(path, []byte("original input"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: ""}}
|
||||||
|
engine := newSimpleEngine(t, `{{input "source"}}`, promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte("new input"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, ContinueFrom: initial.Exchange}
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "original input"}, {Role: promptkit.RoleAssistant, Content: ""}, {Role: promptkit.RoleUser, Content: "new input"}, {Role: promptkit.RoleAssistant, Content: ""}}
|
||||||
|
if !reflect.DeepEqual(result.Exchange.Details().Messages, want) {
|
||||||
|
t.Fatalf("history = %#v", result.Exchange.Details().Messages)
|
||||||
|
}
|
||||||
|
if result, err := engine.Run(ctx, request); err == nil || result != nil {
|
||||||
|
t.Fatalf("missing new input = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationSourceSupportsConcurrentRuns(t *testing.T) {
|
||||||
|
client := &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const count = 8
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errors := make(chan error, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||||
|
if err == nil && (result.Exchange == nil || result.ParentRunID != initial.RunID) {
|
||||||
|
t.Error("missing continuation ancestry")
|
||||||
|
}
|
||||||
|
errors <- err
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errors)
|
||||||
|
for err := range errors {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(initial.Exchange.Details().Messages) != 2 {
|
||||||
|
t.Fatal("source changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
90
convert.go
90
convert.go
@@ -4,11 +4,19 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
|
var history *domain.Exchange
|
||||||
|
if req.ContinueFrom != nil && req.ContinueFrom.internal == nil {
|
||||||
|
return domain.RunRequest{}, fmt.Errorf("continuation exchange is empty")
|
||||||
|
}
|
||||||
|
if req.ContinueFrom != nil {
|
||||||
|
history = req.ContinueFrom.internal
|
||||||
|
}
|
||||||
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunRequest{}, err
|
return domain.RunRequest{}, err
|
||||||
@@ -23,6 +31,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
||||||
}
|
}
|
||||||
return domain.RunRequest{
|
return domain.RunRequest{
|
||||||
|
CaptureExchange: req.CaptureExchange || req.ContinueFrom != nil,
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
@@ -34,6 +43,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
AppendedMessages: appendedMessages,
|
AppendedMessages: appendedMessages,
|
||||||
TransientRetries: copyIntPtr(req.TransientRetries),
|
TransientRetries: copyIntPtr(req.TransientRetries),
|
||||||
Observer: observer,
|
Observer: observer,
|
||||||
|
ContinueFrom: history,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,6 +80,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return &PreparedRun{
|
return &PreparedRun{
|
||||||
|
ParentRunID: prepared.ParentRunID,
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
@@ -91,7 +102,13 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
if result == nil {
|
if result == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var exchange *Exchange
|
||||||
|
if result.Exchange != nil {
|
||||||
|
exchange = &Exchange{internal: result.Exchange}
|
||||||
|
}
|
||||||
return &RunResult{
|
return &RunResult{
|
||||||
|
Exchange: exchange,
|
||||||
|
ParentRunID: result.ParentRunID,
|
||||||
RunID: result.RunID,
|
RunID: result.RunID,
|
||||||
Artifact: fromDomainArtifact(result.Artifact),
|
Artifact: fromDomainArtifact(result.Artifact),
|
||||||
RawOutput: result.RawOutput,
|
RawOutput: result.RawOutput,
|
||||||
@@ -228,37 +245,58 @@ func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, ext
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||||
if inspection == nil {
|
out := &ProfileInspection{ProfileID: v.ID, Backend: copyStringPtr(v.Backend), Endpoint: copyStringPtr(v.Endpoint), Model: copyStringPtr(v.Model), Temperature: copyFloat64Ptr(v.Temperature), TopP: copyFloat64Ptr(v.TopP), MaxTokens: copyIntPtr(v.MaxTokens), TimeoutSeconds: copyIntPtr(v.TimeoutSeconds), ServiceTier: copyStringPtr(v.ServiceTier), ReasoningEffort: copyStringPtr(v.ReasoningEffort), APIKeyEnv: copyStringPtr(v.APIKeyEnv), APIKeyRequired: copyBoolPtr(v.APIKeyRequired)}
|
||||||
return nil
|
if v.ExtraParams != nil {
|
||||||
}
|
data := copyAnyMap(v.ExtraParams)
|
||||||
return &ProfileInspection{
|
out.ExtraParams = &data
|
||||||
ProfileID: inspection.ProfileID,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
|
||||||
APIKeyRequired: inspection.APIKeyRequired,
|
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||||
if inspection == nil {
|
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||||
|
if v.ExtraParams != nil {
|
||||||
|
data := copyAnyMap(v.ExtraParams)
|
||||||
|
out.ExtraParams = &data
|
||||||
|
}
|
||||||
|
if v.SmokeTests != nil {
|
||||||
|
out.SmokeTests = &BackendSmokeTestConfig{DefaultProfile: copyStringPtr(v.SmokeTests.DefaultProfile)}
|
||||||
|
if v.SmokeTests.Reachability != nil {
|
||||||
|
out.SmokeTests.Reachability = &ReachabilityProbeConfig{Endpoint: copyStringPtr(v.SmokeTests.Reachability.Endpoint)}
|
||||||
|
}
|
||||||
|
if v.SmokeTests.Authentication != nil {
|
||||||
|
out.SmokeTests.Authentication = &AuthenticationProbeConfig{Kind: copyStringPtr(v.SmokeTests.Authentication.Kind), Endpoint: copyStringPtr(v.SmokeTests.Authentication.Endpoint)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||||
|
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}}
|
||||||
|
if v.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||||
|
for i, input := range *v.Inputs {
|
||||||
|
inputs[i] = PromptInputInspection{Name: input.Name, Required: copyBoolPtr(input.Required), ContentType: copyStringPtr(input.ContentType), Description: copyStringPtr(input.Description)}
|
||||||
|
}
|
||||||
|
out.Inputs = &inputs
|
||||||
|
}
|
||||||
|
out.Messages = make([]PromptMessageInspection, len(v.Messages))
|
||||||
|
for i, message := range v.Messages {
|
||||||
|
out.Messages[i] = PromptMessageInspection{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
out.Messages[i].CacheControl = &PromptCacheControlInspection{Type: CacheControlType(message.CacheControl.Type), TTL: copyStringPtr(message.CacheControl.TTL)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBoolPtr(src *bool) *bool {
|
||||||
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
value := *src
|
||||||
for i, input := range inspection.Inputs {
|
return &value
|
||||||
inputs[i] = PromptInputDefinition{
|
|
||||||
Name: input.Name,
|
|
||||||
Required: input.Required,
|
|
||||||
ContentType: input.ContentType,
|
|
||||||
Description: input.Description,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &PromptInspection{
|
|
||||||
PromptID: inspection.PromptID,
|
|
||||||
PromptHash: inspection.PromptHash,
|
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
|
||||||
Inputs: inputs,
|
|
||||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
|
|||||||
82
coordinate.go
Normal file
82
coordinate.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ConversationTurn is delivered before each requested continuation. Result is
|
||||||
|
// nil on the initial callback; RemainingTurns counts additional runs allowed.
|
||||||
|
type ConversationTurn struct {
|
||||||
|
Exchange *Exchange
|
||||||
|
Result *RunResult
|
||||||
|
RemainingTurns int
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoordinateRequest selects a starting exchange, turn budget and callback.
|
||||||
|
// Observer selects continuation runs only, independently of callback calls.
|
||||||
|
type CoordinateRequest struct {
|
||||||
|
Start *Exchange
|
||||||
|
AdditionalTurns int
|
||||||
|
Next func(context.Context, ConversationTurn) (*RunRequest, error)
|
||||||
|
Observer ExecutionObserver
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoordinationOutcome distinguishes caller-directed stopping from exhaustion.
|
||||||
|
type CoordinationOutcome string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// CoordinationStopped means the callback chose not to continue.
|
||||||
|
CoordinationStopped CoordinationOutcome = "stopped"
|
||||||
|
// CoordinationExhausted means another run was requested without budget.
|
||||||
|
CoordinationExhausted CoordinationOutcome = "exhausted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CoordinationResult contains the normal outcome or a response withheld by
|
||||||
|
// cancellation before delivery to the callback.
|
||||||
|
type CoordinationResult struct {
|
||||||
|
Outcome CoordinationOutcome `json:"outcome,omitempty"`
|
||||||
|
Undelivered *RunResult `json:"undelivered,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coordinate advances one branch from Start using ordinary runs. Next is called
|
||||||
|
// initially even for a zero budget and receives each completed result once,
|
||||||
|
// including the final permitted response. The initial exchange is not itself a
|
||||||
|
// newly delivered result. Each started continuation consumes one turn; retries
|
||||||
|
// and structural repairs do not. Callback work runs after admission is released.
|
||||||
|
// A nil request stops; requesting a run without remaining budget exhausts it.
|
||||||
|
// Callback and execution errors retain their identities. Cancellation observed
|
||||||
|
// before delivery returns the single completed response as Undelivered;
|
||||||
|
// cancellation after callback invocation does not replay that response. Context
|
||||||
|
// cancellation takes precedence over a callback's returned action or error.
|
||||||
|
// Start must be a captured exchange, Next non-nil, and AdditionalTurns
|
||||||
|
// nonnegative; invalid values match ErrInvalidRequest. A nil engine returns
|
||||||
|
// ErrInvalidConfig. The selected request is copied and its ContinueFrom is
|
||||||
|
// replaced by the coordinator's current exchange. The request observer runs
|
||||||
|
// before the coordinator observer for each continuation.
|
||||||
|
func (e *Engine) Coordinate(ctx context.Context, req CoordinateRequest) (CoordinationResult, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return CoordinationResult{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
if req.Start == nil || req.Start.internal == nil || req.Next == nil || req.AdditionalTurns < 0 {
|
||||||
|
return CoordinationResult{}, fmt.Errorf("%w: invalid coordination request", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
outcome, err := usecase.Coordinate(ctx, req.Start.internal, req.AdditionalTurns,
|
||||||
|
func(ctx context.Context, turn usecase.ConversationTurn[RunResult]) (*RunRequest, error) {
|
||||||
|
return req.Next(ctx, ConversationTurn{Exchange: &Exchange{internal: turn.Exchange}, Result: turn.Result, RemainingTurns: turn.RemainingTurns})
|
||||||
|
},
|
||||||
|
func(ctx context.Context, current *domain.Exchange, selected *RunRequest) (*RunResult, *domain.Exchange, error) {
|
||||||
|
request := *selected
|
||||||
|
request.ContinueFrom = &Exchange{internal: current}
|
||||||
|
request.Observer = ComposeObservers(request.Observer, req.Observer)
|
||||||
|
result, runErr := e.Run(ctx, request)
|
||||||
|
if runErr != nil {
|
||||||
|
return nil, nil, runErr
|
||||||
|
}
|
||||||
|
return result, result.Exchange.internal, nil
|
||||||
|
})
|
||||||
|
return CoordinationResult{Outcome: CoordinationOutcome(outcome.Outcome), Undelivered: outcome.Undelivered}, err
|
||||||
|
}
|
||||||
61
coordinate_contract_test.go
Normal file
61
coordinate_contract_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinatePublicBoundary(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, request := range []promptkit.CoordinateRequest{{}, {Start: &promptkit.Exchange{}, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}, {Start: initial.Exchange, AdditionalTurns: -1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}} {
|
||||||
|
if _, err := engine.Coordinate(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("invalid request error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := (*promptkit.Engine)(nil).Coordinate(ctx, promptkit.CoordinateRequest{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine error = %v", err)
|
||||||
|
}
|
||||||
|
callbacks := 0
|
||||||
|
selected := &promptkit.RunRequest{PromptID: "prompt", ContinueFrom: &promptkit.Exchange{}}
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 2, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
callbacks++
|
||||||
|
if callbacks == 1 {
|
||||||
|
if turn.Result != nil || turn.RemainingTurns != 2 {
|
||||||
|
t.Fatalf("initial turn = %#v", turn)
|
||||||
|
}
|
||||||
|
return selected, nil
|
||||||
|
}
|
||||||
|
if turn.Result == nil || turn.Exchange.Details().ParentRunID == "" {
|
||||||
|
t.Fatalf("delivered turn = %#v", turn)
|
||||||
|
}
|
||||||
|
if callbacks == 2 {
|
||||||
|
turn.Result.Exchange = nil
|
||||||
|
*turn.Exchange = promptkit.Exchange{}
|
||||||
|
return selected, nil
|
||||||
|
}
|
||||||
|
if turn.Result.ParentRunID == initial.RunID {
|
||||||
|
t.Fatal("callback mutation redirected branch")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 3 || selected.ContinueFrom == nil {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, callbacks)
|
||||||
|
}
|
||||||
|
stop, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }})
|
||||||
|
if err != nil || stop.Outcome != promptkit.CoordinationStopped {
|
||||||
|
t.Fatalf("zero budget = %#v, %v", stop, err)
|
||||||
|
}
|
||||||
|
sentinel := errors.New("callback")
|
||||||
|
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, sentinel }})
|
||||||
|
if !errors.Is(err, sentinel) {
|
||||||
|
t.Fatalf("callback error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
170
coordinate_lifecycle_contract_test.go
Normal file
170
coordinate_lifecycle_contract_test.go
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinateReleasesCapacityBeforeCallback(t *testing.T) {
|
||||||
|
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{{Content: "initial"}, {Content: ""}, {Content: " "}, {Content: "validator"}}}
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}}, nil
|
||||||
|
}
|
||||||
|
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
|
||||||
|
t.Fatalf("delivered turn = %#v", turn)
|
||||||
|
}
|
||||||
|
_, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
return nil, err
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || calls != 2 || len(client.snapshot()) != 4 {
|
||||||
|
t.Fatalf("outcome=%#v error=%v callbacks=%d calls=%d", result, err, calls, len(client.snapshot()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinatePreservesCapacityAndProviderErrors(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
sourceEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := sourceEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 1)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
done := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, done)
|
||||||
|
select {
|
||||||
|
case <-client.started:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
}
|
||||||
|
request := promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||||
|
}}
|
||||||
|
_, err = engine.Coordinate(ctx, request)
|
||||||
|
var capacity *promptkit.CapacityError
|
||||||
|
if !errors.As(err, &capacity) || capacity.BackendID != "limited" {
|
||||||
|
t.Fatalf("capacity error = %v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
if finished := awaitCapacityRun(t, done); finished.err != nil {
|
||||||
|
t.Fatal(finished.err)
|
||||||
|
}
|
||||||
|
providerEngine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, `{"error":{"message":"synthetic"}}`)
|
||||||
|
_, err = providerEngine.Coordinate(ctx, request)
|
||||||
|
var provider *promptkit.GenerationError
|
||||||
|
if !errors.As(err, &provider) || provider.StatusCode() != http.StatusUnprocessableEntity {
|
||||||
|
t.Fatalf("provider error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateRetriesAndRepairsDeliverOnce(t *testing.T) {
|
||||||
|
initialEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := initialEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
responses := []*http.Response{observedHTTPResponse(503, `{}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}]}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)}
|
||||||
|
calls := 0
|
||||||
|
one := 1
|
||||||
|
config := simpleConfig("follow-up")
|
||||||
|
config.TransientRetries = &one
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
response := responses[calls]
|
||||||
|
calls++
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
callbacks := 0
|
||||||
|
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
callbacks++
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}}, nil
|
||||||
|
}
|
||||||
|
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
|
||||||
|
t.Fatalf("turn = %#v", turn)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 2 || calls != 3 {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d attempts=%d", result, err, callbacks, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateCancellationDelivery(t *testing.T) {
|
||||||
|
for _, cancelInside := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "before delivery", true: "inside callback"}[cancelInside], func(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
if turn.Result != nil {
|
||||||
|
cancel()
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if !cancelInside && event.Kind == promptkit.ExecutionFinished {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}}, nil
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, context.Canceled) || result.Outcome != "" || (result.Undelivered != nil) == cancelInside {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if !cancelInside && (calls != 1 || result.Undelivered.Exchange == nil) {
|
||||||
|
t.Fatalf("undelivered result=%#v calls=%d", result, calls)
|
||||||
|
}
|
||||||
|
if cancelInside && calls != 2 {
|
||||||
|
t.Fatalf("callbacks=%d", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateFailureLeavesSiblingUsable(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sentinel := errors.New("provider")
|
||||||
|
client.err = sentinel
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, sentinel) || !errors.Is(err, promptkit.ErrLLMGenerate) || result.Outcome != "" || result.Undelivered != nil {
|
||||||
|
t.Fatalf("failure=%#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = nil
|
||||||
|
if sibling, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}); err != nil || sibling.Exchange == nil {
|
||||||
|
t.Fatalf("sibling=%#v, %v", sibling, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
31
doc.go
31
doc.go
@@ -4,17 +4,20 @@
|
|||||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||||
// sources for prompts, profiles, and backends, and
|
// sources for prompts, profiles, and backends, and
|
||||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
|
||||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
|
||||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
// [Engine.PrepareExecution], [Engine.Run], [Engine.RunPrepared], or
|
||||||
|
// [Engine.Coordinate]. Captured [Exchange] values support independent
|
||||||
|
// continuations, and [ExecutionGroup] collects explicitly observed runs.
|
||||||
|
// Concrete registries, repositories,
|
||||||
// validators, and the built-in OpenAI-compatible client remain internal
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
// implementation details.
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
// InspectRequirements, InspectProfile, InspectBackend, ListProfiles, ListPrompts,
|
||||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
// Prepare, PrepareExecution, Run, RunPrepared, and Coordinate calls. Engine-local backend policies
|
||||||
// bound admitted Run and RunPrepared calls and model generations where
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// configured, while different backend pools and unlimited backends continue
|
// configured, while different backend pools and unlimited backends continue
|
||||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
@@ -23,7 +26,8 @@
|
|||||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
// InspectRequirements, InspectProfile, and InspectBackend return copied
|
||||||
|
// inspection values.
|
||||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||||
// Returned values and values passed to extension interfaces are likewise
|
// Returned values and values passed to extension interfaces are likewise
|
||||||
// isolated from engine state.
|
// isolated from engine state.
|
||||||
@@ -42,7 +46,7 @@
|
|||||||
// that enforces their filesystem, authorization, and size policies.
|
// that enforces their filesystem, authorization, and size policies.
|
||||||
//
|
//
|
||||||
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||||
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
// [ValidationResult.Errors], and [ExchangeDetails.Messages] may contain sensitive data. Credential
|
||||||
// exclusion and redaction do not sanitize those values. Applications and
|
// exclusion and redaction do not sanitize those values. Applications and
|
||||||
// injected collaborators are responsible for access control, retention,
|
// injected collaborators are responsible for access control, retention,
|
||||||
// logging, and secret handling appropriate to their data.
|
// logging, and secret handling appropriate to their data.
|
||||||
@@ -56,17 +60,22 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction, inspection, handle, and error values, including [Config],
|
// Resource inspection results support direct JSON and YAML encoding.
|
||||||
|
// [PromptInspection], [ProfileInspection], [BackendInspection], and their nested
|
||||||
|
// values define field names, nesting, and omission of absent optional fields.
|
||||||
|
// Formatting belongs to the selected encoder.
|
||||||
|
//
|
||||||
|
// Other construction, inspection, handle, and error values, including [Config],
|
||||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||||
// [ProfileConfig], [RunRequest],
|
// [ProfileConfig], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
// [ArtifactRef], [ExecutionTargetOverride],
|
||||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
// [PromptIdentity], [PromptInputDefinition],
|
||||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||||
// [ExecutionEvent], do not have stable JSON
|
// [ExecutionEvent], do not have stable JSON
|
||||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
// every public value.
|
// every public value.
|
||||||
|
//
|
||||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||||
// contain sensitive request or schema fragments. Applications must apply their
|
// contain sensitive request or schema fragments. Applications must apply their
|
||||||
// own disclosure policy before logging, displaying, or returning them.
|
// own disclosure policy before logging, displaying, or returning them.
|
||||||
|
|||||||
@@ -135,8 +135,10 @@ Replace former maps passed to `BackendSource` in the same way, setting each
|
|||||||
inspection paths now address list indexes, for example
|
inspection paths now address list indexes, for example
|
||||||
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
||||||
and diffs expose resolved values, recorded normalization and provenance.
|
and diffs expose resolved values, recorded normalization and provenance.
|
||||||
`InspectProfile` additionally applies backend completion; request preparation
|
`InspectProfile` reports the selected resolved profile without backend completion.
|
||||||
can apply further overrides, so the views are not promised to be identical.
|
Use `Prepare` or `PreparedExecution.Details` for completed request settings,
|
||||||
|
including backend defaults and request overrides; see the
|
||||||
|
[resource inspection guidance](pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import "go.maximumdirect.net/promptkit"
|
|||||||
For migration from the former module path, see
|
For migration from the former module path, see
|
||||||
[Update Go Imports](configuration-migration.md#update-go-imports).
|
[Update Go Imports](configuration-migration.md#update-go-imports).
|
||||||
|
|
||||||
Configuration APIs use `go.maximumdirect.net/configkit` v1.1.0. Its types are
|
Configuration APIs use `go.maximumdirect.net/configkit` v1.2.0. Its types are
|
||||||
not interchangeable with the old ConfigKit module path.
|
not interchangeable with the old ConfigKit module path.
|
||||||
|
|
||||||
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
@@ -94,7 +94,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{
|
|||||||
|
|
||||||
For a single highest-priority document, set `Sources.Main` to a
|
For a single highest-priority document, set `Sources.Main` to a
|
||||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw
|
||||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||||
collaborator options remain available. Construction validates the final
|
collaborator options remain available. Construction validates the final
|
||||||
effective configuration and captures referenced prompt content; construct a
|
effective configuration and captures referenced prompt content; construct a
|
||||||
@@ -102,6 +102,39 @@ new engine to observe configuration edits. See the [constructor GoDoc](../../eng
|
|||||||
for the exact public contract and the [framework format reference](../formats.md)
|
for the exact public contract and the [framework format reference](../formats.md)
|
||||||
for definition fields.
|
for definition fields.
|
||||||
|
|
||||||
|
### Load Companion Prompts Relative To The Main Configuration
|
||||||
|
|
||||||
|
A non-file underlay can use the selected main configuration's directory for
|
||||||
|
caller-declared imports. For example, this loads standalone prompt definitions
|
||||||
|
from the adjacent `prompts/` directory into the `support` collection:
|
||||||
|
|
||||||
|
```go
|
||||||
|
main := configkit.Input{Source: configkit.File(configPath)}
|
||||||
|
companion, err := promptkit.CollectionInput("support", configkit.Bytes("companion prompts", []byte(`
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- dir: prompts
|
||||||
|
pattern: ["*.yaml", "*.yml"]
|
||||||
|
destination: [prompts]
|
||||||
|
placement: sequence-item
|
||||||
|
required: true
|
||||||
|
`)))
|
||||||
|
if err != nil { return err }
|
||||||
|
companion.MainRelative = configkit.MainRelative()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Main: &main,
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{companion}}},
|
||||||
|
}})
|
||||||
|
```
|
||||||
|
|
||||||
|
`CollectionInput` enables imports; for a raw `configkit.Input`, enable them with
|
||||||
|
`Imports: true`. The binding itself does not enable imports, and an explicit
|
||||||
|
`Input.Filesystem` takes precedence. Imported prompt `content_file` references
|
||||||
|
keep their source-local context. See ConfigKit's
|
||||||
|
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#layered-loading)
|
||||||
|
for eligible inputs, fallback contexts, and behavior when the main has no usable
|
||||||
|
filesystem context.
|
||||||
|
|
||||||
### Use Scoped Engine Views
|
### Use Scoped Engine Views
|
||||||
|
|
||||||
Create a view when an application wants short request IDs to resolve within a
|
Create a view when an application wants short request IDs to resolve within a
|
||||||
@@ -177,8 +210,14 @@ input := configkit.Input{Source: document}
|
|||||||
// Add input to a later underlay, or set Sources.Main = &input.
|
// Add input to a later underlay, or set Sources.Main = &input.
|
||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`PromptSource`, `ProfileSource`, and `BackendSource` are typed conveniences when
|
||||||
is not needed. Their provider maps use ConfigKit Mapping input forms: built-in
|
null is not needed. Prompt and backend helpers, like profiles, support sibling
|
||||||
|
inheritance via `Extends`; partial records can be completed by other layers.
|
||||||
|
For relative prompt `content_file` references, supply the ordinary ConfigKit
|
||||||
|
`Input.Filesystem` context for that source, or use a
|
||||||
|
[main-relative underlay](#load-companion-prompts-relative-to-the-main-configuration).
|
||||||
|
The helpers do not read files or
|
||||||
|
validate final definitions. Profile and backend provider maps use ConfigKit Mapping input forms: built-in
|
||||||
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||||
typed containers explicitly to an appropriate supported value, choosing numeric
|
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||||
precision for your application. Mapping captures supported values immediately;
|
precision for your application. Mapping captures supported values immediately;
|
||||||
@@ -264,9 +303,10 @@ is not necessarily safe to share. Inspection uses the engine's captured result
|
|||||||
and does not reopen sources or resolve credentials. It retains effective values,
|
and does not reopen sources or resolve credentials. It retains effective values,
|
||||||
including recorded field normalization and queue defaulting, alongside source
|
including recorded field normalization and queue defaulting, alongside source
|
||||||
provenance.
|
provenance.
|
||||||
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
|
`InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual
|
||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
resolved resources without backend-to-profile execution completion or provenance.
|
||||||
and validation resources. None of those views is a substitute for the others.
|
`Prepare` or `PreparedExecution.Details` shows completed request-specific settings,
|
||||||
|
rendered messages, inputs, and validation resources. None substitutes for the others.
|
||||||
|
|
||||||
To review changes between two independently captured configurations, compare
|
To review changes between two independently captured configurations, compare
|
||||||
their inspection views:
|
their inspection views:
|
||||||
@@ -295,7 +335,7 @@ for _, difference := range differences {
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
and ConfigKit's [v1.1.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs)
|
and ConfigKit's [effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#effective-configuration-diffs)
|
||||||
for the exact comparison contract. The operation compares only frozen effective
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
configuration values: it does not reread external files or compare their
|
configuration values: it does not reread external files or compare their
|
||||||
contents, resolved credential contents, later execution preparation, or
|
contents, resolved credential contents, later execution preparation, or
|
||||||
@@ -380,8 +420,10 @@ if err != nil {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, input := range inspection.Inputs {
|
if inspection.Inputs != nil {
|
||||||
|
for _, input := range *inspection.Inputs {
|
||||||
// Compare the declared input with application configuration.
|
// Compare the declared input with application configuration.
|
||||||
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -653,6 +695,116 @@ application owns any semantic retry budget. When a
|
|||||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||||
|
|
||||||
|
### Capture And Continue A Conversation
|
||||||
|
|
||||||
|
The [request GoDoc](../../types.go), [exchange GoDoc](../../exchange.go), and
|
||||||
|
[prepared-execution GoDoc](../../prepared_execution.go) own the exact capture,
|
||||||
|
continuation, inspection, and lifetime contracts.
|
||||||
|
|
||||||
|
Capture records the messages actually executed and the final assistant content.
|
||||||
|
An exchange is an immutable in-process capability; `Details()` returns detached,
|
||||||
|
encodable history that may contain sensitive content. Ordinary result encoding
|
||||||
|
does not include the exchange. Continuations imply capture and may branch or use
|
||||||
|
a different engine, profile, output contract, credential, or session:
|
||||||
|
|
||||||
|
```go
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "proposal", CaptureExchange: true})
|
||||||
|
if err != nil { return err }
|
||||||
|
left, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "refine", ContinueFrom: initial.Exchange})
|
||||||
|
if err != nil { return err }
|
||||||
|
right, err := anotherEngine.Run(ctx, promptkit.RunRequest{PromptID: "alternative", ContinueFrom: initial.Exchange})
|
||||||
|
if err != nil { return err }
|
||||||
|
_, _ = left, right // Independent branches; each result has its own exchange.
|
||||||
|
```
|
||||||
|
|
||||||
|
For a frozen execution, set `CaptureExchange` in the request passed to
|
||||||
|
`PrepareExecution`, then obtain the exchange from `RunPrepared`'s result.
|
||||||
|
Preparation and validation alone do not produce an exchange. Captured history
|
||||||
|
precedes newly rendered definition messages and appended messages. A completed
|
||||||
|
structurally invalid result still has an exchange; an operational error returns
|
||||||
|
no result. The caller chooses what constitutes acceptance.
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "proposal", CaptureExchange: true})
|
||||||
|
if err != nil { return err }
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
if err != nil { return err }
|
||||||
|
details := result.Exchange.Details()
|
||||||
|
_ = details // Explicitly encode or inspect when appropriate.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Coordinate Application-Directed Turns
|
||||||
|
|
||||||
|
The following sketches use illustrative application functions. The first records
|
||||||
|
each newly delivered response before selecting another question:
|
||||||
|
|
||||||
|
```go
|
||||||
|
outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||||
|
Start: initial.Exchange, AdditionalTurns: 3,
|
||||||
|
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result != nil {
|
||||||
|
if !turn.Result.Validation.IsValid { return nil, fmt.Errorf("invalid response") }
|
||||||
|
analysis.Record(turn.Result)
|
||||||
|
}
|
||||||
|
question, ok := analysis.NextQuestion()
|
||||||
|
if !ok { return nil, nil }
|
||||||
|
return &promptkit.RunRequest{PromptID: "research/investigate", Vars: map[string]string{"question": question}}, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
An application can instead start with existing findings, assess each new
|
||||||
|
candidate, and keep its decisions in a closure:
|
||||||
|
|
||||||
|
```go
|
||||||
|
outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||||
|
Start: proposal.Exchange, AdditionalTurns: 3, Observer: branchGroup.Observe,
|
||||||
|
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result != nil {
|
||||||
|
if !turn.Result.Validation.IsValid { return nil, fmt.Errorf("invalid revision") }
|
||||||
|
candidate, withdrawn, err := decodeRevision(turn.Result.RawOutput)
|
||||||
|
if err != nil { return nil, err }
|
||||||
|
if withdrawn { branch.MarkWithdrawn(); return nil, nil }
|
||||||
|
findings, err := assess(ctx, engine, candidate, validatorGroup.Observe)
|
||||||
|
if err != nil { return nil, err }
|
||||||
|
branch.Update(candidate, findings)
|
||||||
|
if branch.Done() { return nil, nil }
|
||||||
|
}
|
||||||
|
return &promptkit.RunRequest{PromptID: branch.FollowUpPrompt(), Vars: branch.Variables()}, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
`assess` is application code and explicitly selects its observer on any nested
|
||||||
|
run. A completed turn releases its admission lease before the callback, so that
|
||||||
|
assessment may use the same backend. `stopped` and `exhausted` describe control
|
||||||
|
flow, not content acceptance. Cancellation before callback delivery can return one
|
||||||
|
completed response in `Undelivered`; errors retain their ordinary identities.
|
||||||
|
|
||||||
|
### Collect Selected Execution Facts
|
||||||
|
|
||||||
|
Groups are independent of continuation and enrollment is explicit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var group, validatorGroup promptkit.ExecutionGroup
|
||||||
|
request.Observer = promptkit.ComposeObservers(diagnosticObserver, group.Observe)
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request) // no event yet
|
||||||
|
if err != nil { return err }
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
if err != nil { return err }
|
||||||
|
_ = result
|
||||||
|
// Set CoordinateRequest.Observer to group.Observe for its continuation runs.
|
||||||
|
// A nested validator selects validatorGroup.Observe on its own request.
|
||||||
|
summary, validatorSummary := group.Snapshot(), validatorGroup.Snapshot()
|
||||||
|
_, _ = summary, validatorSummary
|
||||||
|
```
|
||||||
|
|
||||||
|
Only the first terminal event per run contributes counts and usage; coverage of
|
||||||
|
currently observed members is separate from token-usage completeness. Empty
|
||||||
|
groups report complete, known zero totals, while members without terminals do
|
||||||
|
not invent totals. Overlapping groups are not generally additive. See the
|
||||||
|
[offline example](../../examples/go-library/observe/main.go) for complete setup.
|
||||||
|
|
||||||
## Inputs, Profiles, And Overrides
|
## Inputs, Profiles, And Overrides
|
||||||
|
|
||||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||||
@@ -707,10 +859,9 @@ composition rules.
|
|||||||
|
|
||||||
### Inspect A Profile Before Prompt Work
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
Use [`Engine.InspectProfile`](../../engine.go) to inspect a resolved profile
|
||||||
profile without constructing a synthetic prompt or placeholder inputs. It
|
without constructing a synthetic prompt or placeholder inputs. It reports the
|
||||||
resolves the profile's effective target but does not prepare or execute a
|
profile's own fields, not a backend-completed target:
|
||||||
prompt:
|
|
||||||
|
|
||||||
```go
|
```go
|
||||||
inspection, err := engine.InspectProfile(ctx, profileID)
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
@@ -718,24 +869,48 @@ if err != nil {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
if inspection.APIKeyRequired != nil && *inspection.APIKeyRequired {
|
||||||
if inspection.APIKeyRequired {
|
|
||||||
// Arrange a usable direct or configured environment credential.
|
// Arrange a usable direct or configured environment credential.
|
||||||
}
|
}
|
||||||
if target.APIKeyEnv != "" {
|
if inspection.APIKeyEnv != nil {
|
||||||
// This is the configured environment lookup source, not its value.
|
// This is the configured environment lookup source, not its value.
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when only the profile and its target need
|
Use [`InspectBackend`](../../engine.go) to inspect the referenced backend
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
separately. Its diagnostics are configured metadata, not live checks or derived
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
destinations; `ErrBackendNotFound` identifies an absent backend. References in
|
||||||
later execution. A reported `APIKeyEnv` identifies a configured source, while
|
all three resource reports remain as authored. Use `Prepare` for completed
|
||||||
`APIKeyRequired` is the explicit requirement; both may be present. The
|
request settings and rendering, or prepared execution when that work must remain
|
||||||
|
tied to later execution. A reported `APIKeyEnv` identifies a configured source,
|
||||||
|
not its value; `APIKeyRequired` can also be present. The
|
||||||
[credential format reference](../formats.md#credentials) and the method's
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
contracts.
|
contracts.
|
||||||
|
|
||||||
|
Resource inspection pointers represent optional fields: nil is absent, while a
|
||||||
|
non-nil pointer preserves zero, false, or an empty string. Pointer-to-collection
|
||||||
|
fields distinguish omission from an empty map or list. Results and nested mutable
|
||||||
|
values belong to the caller. Ordinary generic encoders need no per-field adapter:
|
||||||
|
|
||||||
|
```go
|
||||||
|
jsonBytes, err := json.Marshal(inspection)
|
||||||
|
if err != nil { return err }
|
||||||
|
yamlBytes, err := yaml.Marshal(inspection) // go.yaml.in/yaml/v4
|
||||||
|
if err != nil { return err }
|
||||||
|
_, _ = jsonBytes, yamlBytes
|
||||||
|
```
|
||||||
|
|
||||||
|
For migration, replace `ProfileInspection.EffectiveModelParams` accesses with
|
||||||
|
the optional resource fields, and use `Prepare` or `PreparedExecution.Details`
|
||||||
|
where backend completion and request overrides are needed. Prompt inspection's
|
||||||
|
`DefaultProfileID`, `Inputs`, nested input metadata, and output fields now use
|
||||||
|
optional pointers where presence matters; check for nil before dereferencing.
|
||||||
|
Consumers such as CommandKit can render the results directly instead of copying
|
||||||
|
each field into a parallel report. See the result types' [GoDoc](../../resource_inspection.go)
|
||||||
|
for keys and ownership; resource inspection does not replace `InspectConfiguration`
|
||||||
|
provenance, `InspectRequirements`, or concrete-request `ValidateRequest`.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
Supply a direct session ID when one prompt should be correlated with a
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
|
|||||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||||
how to select sources and use an engine; the
|
how to select sources and use an engine; the
|
||||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||||
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md)
|
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md)
|
||||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||||
This reference describes what Promptkit does with the final effective values.
|
This reference describes what Promptkit does with the final effective values.
|
||||||
|
|
||||||
@@ -110,8 +110,8 @@ and static template syntax at construction. It does not reread configuration or
|
|||||||
message files on ordinary runs; construct a new engine to observe edits.
|
message files on ordinary runs; construct a new engine to observe edits.
|
||||||
Schema and request artifact reads still happen during preparation. Use
|
Schema and request artifact reads still happen during preparation. Use
|
||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, resource inspection for resolved prompt, profile, and backend fields,
|
||||||
for request-specific rendered details. For identity discovery before inspection,
|
and preparation for completed request-specific details. For identity discovery before inspection,
|
||||||
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
@@ -315,17 +315,33 @@ only a model or backend does not clear unrelated explicitly inherited fields.
|
|||||||
| Profile field | Absent | Explicit value | Explicit null |
|
| Profile field | Absent | Explicit value | Explicit null |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Selected backend endpoint |
|
||||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Backend lookup name, if any |
|
||||||
| `api_key_required` | False | Boolean, including false | False |
|
| `api_key_required` | False | Boolean, including false | False |
|
||||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Backend defaults |
|
||||||
|
|
||||||
|
An empty profile mapping does not clear backend map entries. After ConfigKit
|
||||||
|
composition and inheritance, a whole-map null and omission both permit backend
|
||||||
|
defaults. Nested maps merge recursively, while scalar and array values replace;
|
||||||
|
nulls within provider maps or arrays remain literal data.
|
||||||
|
When `api_key_required` is true and the profile source is absent or null,
|
||||||
|
execution suppresses the backend credential source; an explicitly configured
|
||||||
|
profile source remains in use.
|
||||||
|
|
||||||
|
For example, if the backend provides an endpoint, API-key environment name,
|
||||||
|
and provider map, a profile containing `endpoint: null`, `api_key_env: null`,
|
||||||
|
and `extra_params: null` now inherits those backend values after handoff (unless
|
||||||
|
the required-key rule suppresses the credential source). Earlier versions used
|
||||||
|
those nulls to suppress the corresponding fallback. Use `endpoint: ""` or
|
||||||
|
`api_key_env: ""` to clear those individual strings. There is no profile-level
|
||||||
|
whole-map suppression replacement; change the backend map through ordinary
|
||||||
|
ConfigKit composition when different provider defaults are needed. ConfigKit's
|
||||||
|
retained configuration and provenance continue to show the nulls.
|
||||||
|
|
||||||
An empty mapping does not clear inherited map entries; null clears the whole
|
|
||||||
map. Nested maps merge recursively, while scalar and array values replace.
|
|
||||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||||
@@ -413,7 +429,7 @@ references are exact IDs and use their qualified form when they select a
|
|||||||
collection resource. There is no Promptkit-specific depth limit or second
|
collection resource. There is no Promptkit-specific depth limit or second
|
||||||
base-chain resolver. Missing references and cycles fail at construction. For
|
base-chain resolver. Missing references and cycles fail at construction. For
|
||||||
generic inheritance and layer mechanics, consult
|
generic inheritance and layer mechanics, consult
|
||||||
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md).
|
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md).
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
|
|||||||
@@ -93,6 +93,8 @@ target's model must be available.
|
|||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||||
|
For a continuation, captured history precedes the new definition and appended
|
||||||
|
messages. Ancestry is local metadata and is not an upstream request field.
|
||||||
|
|
||||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||||
does so without provider-specific translation. Tool and deprecated function
|
does so without provider-specific translation. Tool and deprecated function
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
|||||||
parses an error string or receives provider bodies, credentials, headers, or
|
parses an error string or receives provider bodies, credentials, headers, or
|
||||||
other content. Events are values with no mutable containers. Callback panics
|
other content. Events are values with no mutable containers. Callback panics
|
||||||
propagate to the caller.
|
propagate to the caller.
|
||||||
|
The immediate parent run ID is set before start delivery and is retained as
|
||||||
|
selection metadata changes. It describes ancestry, not implicit group membership.
|
||||||
Resolved prompt and profile IDs are the exact runtime identities selected for
|
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||||
the invocation, including namespace qualification when supplied by an engine
|
the invocation, including namespace qualification when supplied by an engine
|
||||||
view.
|
view.
|
||||||
@@ -48,6 +50,13 @@ attempts above the first in a generation, and repair generations with at least
|
|||||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||||
attempt events.
|
attempt events.
|
||||||
|
|
||||||
|
An execution group is a zero-ready synchronized collector of explicitly selected
|
||||||
|
events. It enrolls by run ID and aggregates each run's first terminal counters and
|
||||||
|
presence-aware usage exactly once. Snapshots copy members and totals under one
|
||||||
|
lock; terminal coverage and token completeness are distinct. Observer fan-out
|
||||||
|
copies the callback slice and invokes callbacks synchronously without holding a
|
||||||
|
group lock. No group history, registry, or automatic nested membership exists.
|
||||||
|
|
||||||
Smoke diagnostics mask inherited execution-recorder state while invoking
|
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||||
generation or HTTP collaborators. Their private attempt evidence is assembled
|
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||||
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||||
|
|||||||
@@ -11,15 +11,15 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, resource and requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
| `internal/configuration` | Loads root and namespaced resources through ConfigKit; validates final values, captures indexed resource views and source-relative prompt content, compiled templates, requirement metadata, identity inventories, backend and presence-aware execution profiles, and the read-only provenance inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and identity listing directly to the captured configuration rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
@@ -30,8 +30,8 @@ contributor workflow and validation.
|
|||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -30,6 +30,24 @@ internal callers and tests.
|
|||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
|
|
||||||
|
Captured exchanges are immutable parent-linked domain values. Each holds only
|
||||||
|
its new rendered suffix, final assistant message, and run identities; flattening
|
||||||
|
iteratively fills a single detached message buffer for preparation or projects
|
||||||
|
directly into public inspection values. Preparation
|
||||||
|
prepends this history to newly rendered definition messages and appended caller
|
||||||
|
messages before computing the rendered hash. Follow-up selection and credentials
|
||||||
|
remain those of the new request. Prepared execution retains capture and parent
|
||||||
|
references in its single-claim payload, not its durable details; discard clears
|
||||||
|
the payload. Completed runs capture the exact final output, including exhausted
|
||||||
|
structural repair, while operational errors produce no exchange.
|
||||||
|
|
||||||
|
The sequential coordinator uses ordinary facade runs and retains the current
|
||||||
|
immutable exchange independently of callback-owned values. Each run releases its
|
||||||
|
logical admission lease and finishes observation before application callback
|
||||||
|
delivery. Context checks preserve a completed but undelivered result only when
|
||||||
|
cancellation is observed before invoking the callback. Coordinator budgets count
|
||||||
|
started runs, not transport attempts or structural repairs.
|
||||||
|
|
||||||
## Shared Prompt Selection
|
## Shared Prompt Selection
|
||||||
|
|
||||||
The internal engine applies each namespace view to slashless caller selections
|
The internal engine applies each namespace view to slashless caller selections
|
||||||
@@ -45,12 +63,10 @@ captured when the engine was built.
|
|||||||
Rendering uses those captured parses, cloning them per call to bind request
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
input helpers; it does not parse templates during execution.
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
`InspectPrompt` selects captured metadata directly from the repository. It
|
Resource inspection bypasses the runner: the internal engine selects a captured
|
||||||
returns owned metadata without
|
configuration resource, and the facade detaches its values for the caller.
|
||||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
Prompt resource inspection includes captured message text without rendering or
|
||||||
admission, or model collaborators. The root
|
touching profile, artifact, schema, admission, or model collaborators.
|
||||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
|
||||||
exact contract.
|
|
||||||
|
|
||||||
`InspectRequirements` uses the same metadata selection to resolve the
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
request-profile/default identity, then obtains the captured requirement
|
request-profile/default identity, then obtains the captured requirement
|
||||||
@@ -64,17 +80,12 @@ operation's exact contract.
|
|||||||
|
|
||||||
## Shared Profile Selection
|
## Shared Profile Selection
|
||||||
|
|
||||||
The runner uses one profile-selection and target-resolution boundary for
|
The runner selects a request profile or a prompt default and uses its captured,
|
||||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
presence-aware target constructed from final profile and backend configuration.
|
||||||
request profile or a prompt default; inspection begins with its required
|
Request overrides apply afterward; preparation checks endpoint and model
|
||||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
completeness only after those overrides. Resource profile inspection bypasses
|
||||||
from the final effective profile and backend configuration. Request overrides
|
this path and reports the profile before backend completion. Credential
|
||||||
apply afterward; preparation checks endpoint and model completeness only after
|
availability is checked when execution preparation requires it.
|
||||||
those overrides. Inspection can report a partial profile without requiring a
|
|
||||||
runnable target. Neither path checks credential availability or performs prompt,
|
|
||||||
artifact, schema, rendering, admission, or model-client work. The root
|
|
||||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
|
||||||
exact contract.
|
|
||||||
|
|
||||||
## Smoke-Test Diagnostics
|
## Smoke-Test Diagnostics
|
||||||
|
|
||||||
@@ -131,8 +142,8 @@ profile, or backend:
|
|||||||
schema metadata from it when required;
|
schema metadata from it when required;
|
||||||
2. load and hash input artifacts;
|
2. load and hash input artifacts;
|
||||||
3. render messages and the prompt-defined session;
|
3. render messages and the prompt-defined session;
|
||||||
4. apply any direct session ID, then append already-normalized request messages
|
4. apply any direct session ID, then compose captured history, rendered
|
||||||
after the rendered definition messages;
|
definition messages, and already-normalized request messages in that order;
|
||||||
5. hash the effective rendered prompt; and
|
5. hash the effective rendered prompt; and
|
||||||
6. construct the prepared value and preparation timing.
|
6. construct the prepared value and preparation timing.
|
||||||
|
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
## Typed Configuration Loading
|
## Typed Configuration Loading
|
||||||
|
|
||||||
`internal/configuration` is the single adapter from ConfigKit's final
|
`internal/configuration` is the single adapter from ConfigKit's final
|
||||||
effective configuration to execution-owned definitions. It performs one
|
effective configuration to execution-owned definitions and resource views. It performs one
|
||||||
layered load, captures effective presence (including explicit null), records
|
layered load, captures the needed effective presence, records
|
||||||
field-level normalization and derived defaults during resolution, and validates all effective
|
field-level normalization and derived defaults during resolution, and validates all effective
|
||||||
root and collection backend, profile, and prompt values. Replaced lower-layer
|
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||||
domain values are not separately validated. It indexes root resources under
|
domain values are not separately validated. It indexes root resources under
|
||||||
@@ -36,12 +36,12 @@ owns the exact source-header and collection contracts.
|
|||||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
once to ConfigKit v1.1.0 and returns its public difference records. Comparison
|
once to ConfigKit and returns its public difference records. Comparison
|
||||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
both results; reports may still contain ordinary application data. See the
|
both results; reports may still contain ordinary application data. See the
|
||||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
and ConfigKit's [v1.1.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs).
|
and ConfigKit's [comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
After resource materialization and effective-profile snapshot construction, the
|
After resource materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered identity inventories for backends, profiles,
|
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||||
@@ -94,13 +94,13 @@ presence, validation, sensitivity, and resource context; full IDs select
|
|||||||
captured runtime indexes. Unchanged content-file fields retain their origins.
|
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||||
Backend-to-profile and request-time completion remain separate runtime work.
|
Backend-to-profile and request-time completion remain separate runtime work.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and retains
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
lightweight prompt metadata for `InspectRequirements`; the requirement snapshot
|
||||||
internal requirement snapshot lookup uses the selected prompt ID alone.
|
lookup uses the selected prompt ID alone. Separate configuration-owned resource
|
||||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
inspection projects the resolved record and captured messages without rendering
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
or resolving profile, input, or schema resources. These paths do not rescan
|
||||||
schema resources, and its public shape and prompt hashes do not include the
|
configuration or reopen message files. Presence metadata does not change the
|
||||||
captured presence state. Runtime rendering still checks request variables and
|
opaque definition hash. Runtime rendering still checks request variables and
|
||||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
@@ -149,10 +149,12 @@ ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
|||||||
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
The configuration adapter validates final effective resource references and
|
The configuration adapter validates final effective resource references and
|
||||||
field values, then captures backend definitions and profile target snapshots
|
field values, then captures backend definitions and profile target snapshots
|
||||||
with backend defaults and presence/suppression state. A profile can
|
with backend defaults and outbound-control presence. Null profile endpoint,
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
credential source, and whole provider map permit backend fallback after
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
composition, subject to the required-credential rule. A profile can remain
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
partial until runtime overrides complete it. Resource inspection selects indexed
|
||||||
|
resolved backend and profile records without execution completion or prompt,
|
||||||
|
artifact, schema, credential lookup, or provider work. Preparation applies runtime overrides and then checks target
|
||||||
completeness. Identity-only profile selection can identify a partial captured
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
profile without copying its target or requiring readiness. The
|
profile without copying its target or requiring readiness. The
|
||||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
|
|||||||
@@ -26,17 +26,17 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `go.maximumdirect.net/configkit` at `v1.1.0`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT) |
|
||||||
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/LICENSE).
|
||||||
|
|
||||||
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e).
|
||||||
|
|
||||||
## ReleaseKit
|
## ReleaseKit
|
||||||
|
|
||||||
|
|||||||
55
docs/releases/v0.20.0.md
Normal file
55
docs/releases/v0.20.0.md
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Promptkit v0.20.0
|
||||||
|
|
||||||
|
This minor release adds generically renderable resource inspection and typed
|
||||||
|
prompt source construction. The Go consumer minimum remains 1.27.0 and
|
||||||
|
ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Migration
|
||||||
|
|
||||||
|
`InspectProfile` now returns the resolved profile resource instead of the
|
||||||
|
backend-completed `EffectiveModelParams`. Replace accesses to that field with
|
||||||
|
the optional profile fields; use `Prepare` or `PreparedExecution.Details` for
|
||||||
|
completed request settings. `InspectPrompt` now includes captured messages and
|
||||||
|
uses optional pointers for presence-sensitive fields, including default profile,
|
||||||
|
inputs, input metadata, and output settings. Check pointers before dereferencing.
|
||||||
|
All three resource reports support ordinary JSON and YAML rendering without
|
||||||
|
per-field adapters. See the [consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
After configuration composition and inheritance, profile null endpoint,
|
||||||
|
API-key source, and whole provider map now permit backend defaults during
|
||||||
|
execution as well as appearing absent in resource inspection. Use explicit
|
||||||
|
empty strings to clear endpoint or API-key source. There is no replacement for
|
||||||
|
profile-level suppression of an entire backend provider map; edit backend
|
||||||
|
defaults through composition instead. The required-key rule still applies.
|
||||||
|
See [defaults and overrides](../formats.md#defaults-and-overrides) for the
|
||||||
|
migration example and retained ConfigKit null provenance.
|
||||||
|
|
||||||
|
## Resource Inspection And Sources
|
||||||
|
|
||||||
|
`InspectBackend` reports the resolved backend and configured diagnostics without
|
||||||
|
contacting a provider or deriving diagnostic destinations. Prompt and profile
|
||||||
|
inspection likewise report selected resources independently of request readiness;
|
||||||
|
configuration provenance, requirement analysis, and preparation remain separate.
|
||||||
|
`PromptSource` adds typed programmatic prompt definitions, and `BackendConfig`
|
||||||
|
supports sibling inheritance. ConfigKit continues to own source composition,
|
||||||
|
snapshotting, and relative message-file contexts.
|
||||||
|
|
||||||
|
The source helpers' [GoDoc](../../prompts.go) and
|
||||||
|
[backend configuration API](../../backends.go) describe programmatic construction.
|
||||||
|
|
||||||
|
## Refinements
|
||||||
|
|
||||||
|
Package GoDoc and migration guidance now consistently describe the resource
|
||||||
|
inspection contract, and the full package overview is visible in generated
|
||||||
|
documentation. Prompt inspection avoids repeating the same metadata lookup for
|
||||||
|
each declared input.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
After adapting affected inspection consumers and reviewing profile null usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.20.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/docs/oss/attribution.md)
|
||||||
34
docs/releases/v0.21.0.md
Normal file
34
docs/releases/v0.21.0.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Promptkit v0.21.0
|
||||||
|
|
||||||
|
This minor release updates ConfigKit from v1.1.0 to v1.2.0, making main-relative
|
||||||
|
companion imports available through Promptkit's existing configuration-source
|
||||||
|
API. The Go consumer minimum remains 1.27.0.
|
||||||
|
|
||||||
|
## Main-Relative Companion Imports
|
||||||
|
|
||||||
|
Applications can bind a non-file underlay's declared imports to the selected
|
||||||
|
main configuration's filesystem context. For example, an application can load
|
||||||
|
prompt definitions from an adjacent `prompts/` directory without calculating
|
||||||
|
that directory before loading the main configuration.
|
||||||
|
|
||||||
|
Imports remain explicitly selected by the caller; the upgrade does not enable
|
||||||
|
automatic discovery. Imported prompt message files retain their source-local
|
||||||
|
context. The [consumer example](../consumers/pkg-promptkit.md#load-companion-prompts-relative-to-the-main-configuration)
|
||||||
|
shows collection mounting and YAML filename filtering. ConfigKit's
|
||||||
|
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#layered-loading)
|
||||||
|
defines context selection and fallback behavior.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Promptkit's exported methods and request shapes are unchanged. No configuration
|
||||||
|
format migration is required from v0.20.0. Applications constructing ConfigKit
|
||||||
|
`Input` or `LayeredOptions` with positional struct literals must switch to
|
||||||
|
field-named literals because those upstream types gained fields. See the
|
||||||
|
[ConfigKit release notes](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/releases/v1.2.0.md)
|
||||||
|
for upstream changes.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.21.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/oss/attribution.md)
|
||||||
67
docs/releases/v0.22.0.md
Normal file
67
docs/releases/v0.22.0.md
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Promptkit v0.22.0
|
||||||
|
|
||||||
|
This minor release adds captured conversation exchanges, optional sequential
|
||||||
|
coordination, and explicit execution-group summaries. Applications can continue
|
||||||
|
or branch a conversation without reconstructing rendered history, while keeping
|
||||||
|
response interpretation and workflow decisions in application code.
|
||||||
|
|
||||||
|
## Capture And Continue Conversations
|
||||||
|
|
||||||
|
Set `RunRequest.CaptureExchange` to capture a completed run's effective messages
|
||||||
|
and final assistant content. Supply its `RunResult.Exchange` as `ContinueFrom`
|
||||||
|
on a new request to continue it. Exchanges are reusable, support independent
|
||||||
|
branches and different engines, and expose detached history through `Details()`.
|
||||||
|
Prepared execution also supports capture of the exact snapshot it executes.
|
||||||
|
|
||||||
|
Internal structural-repair instructions and intermediate candidates are not
|
||||||
|
conversation history. A completed result with failed structural validation can
|
||||||
|
still be captured; an operational error returning no result creates no exchange.
|
||||||
|
Follow-ups select their own profiles, output contracts, credentials, and sessions
|
||||||
|
under ordinary request rules. See [capture and continuation guidance](../consumers/pkg-promptkit.md#capture-and-continue-a-conversation)
|
||||||
|
and the [public exchange contract](../../exchange.go).
|
||||||
|
|
||||||
|
## Coordinate Application-Directed Turns
|
||||||
|
|
||||||
|
`Engine.Coordinate` advances one conversation branch using an application callback
|
||||||
|
and an additional-turn budget. The application selects the next request or stops;
|
||||||
|
Promptkit handles turn progression and cancellation. Each completed run releases
|
||||||
|
backend admission before the callback, allowing assessment calls to use the same
|
||||||
|
backend. Stop and budget exhaustion are control-flow outcomes, not judgments of
|
||||||
|
response acceptance. Transport retries and structural repairs remain separate
|
||||||
|
from the turn budget.
|
||||||
|
|
||||||
|
See the [coordination walkthrough](../consumers/pkg-promptkit.md#coordinate-application-directed-turns)
|
||||||
|
and [coordinator GoDoc](../../coordinate.go) for callback and cancellation behavior.
|
||||||
|
|
||||||
|
## Collect Selected Execution Facts
|
||||||
|
|
||||||
|
`ExecutionGroup` collects explicitly selected execution observations, while
|
||||||
|
`ComposeObservers` combines groups with existing diagnostic observers. Detached
|
||||||
|
group snapshots report member identities, terminal coverage, attempt/retry/repair
|
||||||
|
counts, and available token totals without counting a run more than once.
|
||||||
|
Terminal coverage remains distinct from token-usage completeness; membership is
|
||||||
|
not inferred from ancestry or nested calls.
|
||||||
|
|
||||||
|
See [execution-group guidance](../consumers/pkg-promptkit.md#collect-selected-execution-facts),
|
||||||
|
the [group GoDoc](../../execution_group.go), and the updated
|
||||||
|
[offline observation example](../../examples/go-library/observe/main.go).
|
||||||
|
|
||||||
|
## Efficiency And Compatibility
|
||||||
|
|
||||||
|
History assembly and inspection avoid intermediate full-history copies. Observer
|
||||||
|
composition retains storage only for selected callbacks and avoids fan-out
|
||||||
|
closures for zero or one selected callback.
|
||||||
|
|
||||||
|
Capture and coordination are opt-in; existing execution workflows and
|
||||||
|
configuration formats need no migration from v0.21.0. The Go consumer minimum
|
||||||
|
remains 1.27.0, and dependency versions are unchanged. Exported request, prepared,
|
||||||
|
result, and observation structs gained fields; consumers using positional struct
|
||||||
|
literals must switch to keyed literals. Continuation metadata adds `parent_run_id`
|
||||||
|
to prepared/result JSON when applicable; the opaque exchange itself is excluded.
|
||||||
|
Exchanges are in-process capabilities, not serialized restartable sessions.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.22.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/docs/oss/attribution.md)
|
||||||
58
engine.go
58
engine.go
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
@@ -30,6 +31,8 @@ var (
|
|||||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
// every configured profile source. It does not also match ErrProfileLoad.
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
// ErrBackendNotFound identifies a backend absent from resource inspection.
|
||||||
|
ErrBackendNotFound = errors.New("backend not found")
|
||||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||||
// nor the selected prompt's default profile is present. Such an error also
|
// nor the selected prompt's default profile is present. Such an error also
|
||||||
// matches ErrInvalidRequest.
|
// matches ErrInvalidRequest.
|
||||||
@@ -81,12 +84,12 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
// Engine inspects resources, enumerates prompt requirements, and
|
||||||
// prepares and runs Promptkit prompt requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||||
// [Engine.InspectConfiguration],
|
// [Engine.InspectConfiguration],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectPrompt], [Engine.InspectBackend],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||||
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||||
@@ -267,15 +270,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
// qualified IDs pass through.
|
// qualified IDs pass through.
|
||||||
//
|
//
|
||||||
// A successful result proves that the selected definition and any referenced
|
// A successful result proves that the selected definition and any referenced
|
||||||
// message content files were structurally loaded. Inputs are returned in
|
// message content files were structurally loaded. Inputs and captured,
|
||||||
// definition order. DefaultProfileID is declared metadata only and is not
|
// unrendered messages are returned in definition order. DefaultProfileID is
|
||||||
|
// declared metadata only and is not
|
||||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
// Schema path when declared but without loading or compiling that schema.
|
// Schema path when declared but without loading or compiling that schema.
|
||||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
// the selected captured definition; its spelling, length,
|
// the selected captured definition; its spelling, length,
|
||||||
// encoding, algorithm, and security properties are not contracts.
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
//
|
//
|
||||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
// This method does not return parsed templates, source paths, schemas,
|
||||||
// rendered messages, or execution settings. It does not resolve a profile or
|
// rendered messages, or execution settings. It does not resolve a profile or
|
||||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
// contact a provider, or generate model output. The returned PromptInspection
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
@@ -299,21 +303,19 @@ func (e *Engine) InspectPrompt(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPromptInspection(inspection), nil
|
return fromPromptResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
// starting execution work.
|
// starting execution work.
|
||||||
//
|
//
|
||||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||||
// in the captured effective configuration. It applies the framework timeout
|
// in the captured effective configuration. It reports the resolved profile
|
||||||
// baseline, selected
|
// resource without backend completion, execution defaults, or request overrides.
|
||||||
// backend, and then selected profile to EffectiveModelParams without a request
|
|
||||||
// override. BackendID is empty for an endpoint-only profile.
|
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned resource is an environment-variable name, never its
|
||||||
// value. APIKeyRequired reports that a credential is required; an explicit
|
// value. APIKeyRequired, when present and true, reports that a credential is
|
||||||
// profile environment source can also be present. InspectProfile neither derives
|
// required; an explicit profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
@@ -337,7 +339,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainProfileInspection(inspection), nil
|
return fromProfileResource(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectBackend selects a resolved backend by exact ID from the captured
|
||||||
|
// configuration, qualifying a slashless ID with the backend namespace. It
|
||||||
|
// reports configured resource fields and diagnostics, not credential values,
|
||||||
|
// derived diagnostic destinations, or runtime capacity occupancy. References
|
||||||
|
// are not resolved and no network request is made. The result and its mutable
|
||||||
|
// values are caller-owned. Source changes require a new engine.
|
||||||
|
//
|
||||||
|
// An invalid engine matches ErrInvalidConfig, an invalid ID matches
|
||||||
|
// ErrInvalidRequest, and an absent backend matches ErrBackendNotFound (not
|
||||||
|
// ErrInvalidRequest). Cancellation remains detectable via errors.Is. No
|
||||||
|
// partial result is returned on error.
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, backendID string) (*BackendInspection, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
inspection, err := e.internal.InspectBackend(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrBackendNotFound, err)
|
||||||
|
}
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromBackendResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
@@ -487,6 +514,9 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
return e.run(ctx, req)
|
return e.run(ctx, req)
|
||||||
}
|
}
|
||||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||||
|
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
|
||||||
|
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
|
||||||
|
}
|
||||||
recorder.Start()
|
recorder.Start()
|
||||||
if idErr != nil {
|
if idErr != nil {
|
||||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
err := fmt.Errorf("failed to create run id: %w", idErr)
|
||||||
|
|||||||
@@ -75,6 +75,10 @@ type summary struct {
|
|||||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||||
AssessmentOutcome string `json:"assessment_outcome"`
|
AssessmentOutcome string `json:"assessment_outcome"`
|
||||||
|
CoordinationOutcome promptkit.CoordinationOutcome `json:"coordination_outcome"`
|
||||||
|
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||||
|
GroupMembers int `json:"group_members"`
|
||||||
|
GroupAttempts uint64 `json:"group_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() (summary, error) {
|
func run() (summary, error) {
|
||||||
@@ -104,8 +108,9 @@ func run() (summary, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var proposalLog eventLog
|
var proposalLog eventLog
|
||||||
|
var executions promptkit.ExecutionGroup
|
||||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record,
|
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
@@ -118,18 +123,22 @@ func run() (summary, error) {
|
|||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
var assessmentLog eventLog
|
var assessmentLog eventLog
|
||||||
assessment, err := engine.Run(context.Background(), promptkit.RunRequest{
|
decision := "rejected"
|
||||||
PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record,
|
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
||||||
})
|
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
||||||
|
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}}, nil
|
||||||
|
}
|
||||||
|
if !turn.Result.Validation.IsValid {
|
||||||
|
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||||
|
}
|
||||||
|
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !assessment.Validation.IsValid {
|
if !assessment.Validation.IsValid {
|
||||||
return summary{}, fmt.Errorf("assessment failed structural validation")
|
return nil, fmt.Errorf("assessment failed structural validation")
|
||||||
}
|
|
||||||
assessmentTerminal, err := assessmentLog.terminal()
|
|
||||||
if err != nil {
|
|
||||||
return summary{}, err
|
|
||||||
}
|
}
|
||||||
var proposalValue struct {
|
var proposalValue struct {
|
||||||
Risk string `json:"risk"`
|
Risk string `json:"risk"`
|
||||||
@@ -137,16 +146,32 @@ func run() (summary, error) {
|
|||||||
var assessmentValue struct {
|
var assessmentValue struct {
|
||||||
Recommendation string `json:"recommendation"`
|
Recommendation string `json:"recommendation"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil {
|
if err := json.Unmarshal([]byte(turn.Result.RawOutput), &proposalValue); err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
decision := "rejected"
|
|
||||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||||
decision = "accepted"
|
decision = "accepted"
|
||||||
}
|
}
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
exhausted, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: proposal.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "example.proposal"}, nil
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
assessmentTerminal, err := assessmentLog.terminal()
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
group := executions.Snapshot()
|
||||||
return summary{
|
return summary{
|
||||||
ProposalAttempts: proposalTerminal.Attempts,
|
ProposalAttempts: proposalTerminal.Attempts,
|
||||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||||
@@ -154,6 +179,10 @@ func run() (summary, error) {
|
|||||||
AssessmentAttempts: assessmentTerminal.Attempts,
|
AssessmentAttempts: assessmentTerminal.Attempts,
|
||||||
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
||||||
AssessmentOutcome: decision,
|
AssessmentOutcome: decision,
|
||||||
|
CoordinationOutcome: coordination.Outcome,
|
||||||
|
ExhaustedOutcome: exhausted.Outcome,
|
||||||
|
GroupMembers: len(group.Members),
|
||||||
|
GroupAttempts: group.Attempts,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,13 +18,17 @@ func TestObserveExample(t *testing.T) {
|
|||||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||||
Outcome string `json:"assessment_outcome"`
|
Outcome string `json:"assessment_outcome"`
|
||||||
|
CoordinationOutcome string `json:"coordination_outcome"`
|
||||||
|
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||||
|
GroupMembers int `json:"group_members"`
|
||||||
|
GroupAttempts int `json:"group_attempts"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
|
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
|
||||||
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
|
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
|
||||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" {
|
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 {
|
||||||
t.Fatalf("unexpected observation summary: %+v", got)
|
t.Fatalf("unexpected observation summary: %+v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,10 +110,10 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
|
if profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 30 {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"later profile override was not applied: timeout is %d seconds",
|
"later profile override was not applied: timeout is %v",
|
||||||
profile.EffectiveModelParams.TimeoutSeconds,
|
profile.TimeoutSeconds,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
namespaces, err := scoped.ListNamespaces(ctx)
|
namespaces, err := scoped.ListNamespaces(ctx)
|
||||||
@@ -156,7 +156,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
SelectedProfile: prepared.SelectedProfileID,
|
SelectedProfile: prepared.SelectedProfileID,
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
MessageCount: len(prepared.Messages),
|
MessageCount: len(prepared.Messages),
|
||||||
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
|
ProfileTimeout: *profile.TimeoutSeconds,
|
||||||
Namespaces: namespaces,
|
Namespaces: namespaces,
|
||||||
Collections: collectionSummaries,
|
Collections: collectionSummaries,
|
||||||
ScopedPrompts: scopedPromptIDs,
|
ScopedPrompts: scopedPromptIDs,
|
||||||
|
|||||||
47
exchange.go
Normal file
47
exchange.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
|
||||||
|
const exchangeString = "promptkit.Exchange{opaque}"
|
||||||
|
|
||||||
|
// Exchange is an immutable, reusable in-process snapshot of a completed
|
||||||
|
// conversation. It is not a persistence or restart format. Different engines
|
||||||
|
// can continue it, and independent branches do not alter one another.
|
||||||
|
//
|
||||||
|
// History preserves the effective request's message content, roles, and cache
|
||||||
|
// metadata, followed by the exact final [RunResult.RawOutput] as an assistant
|
||||||
|
// message. Intermediate structural-repair candidates and corrective instructions
|
||||||
|
// are excluded. A continuation extends that logical history according to
|
||||||
|
// [RunRequest.ContinueFrom], without changing its source exchange.
|
||||||
|
type Exchange struct{ internal *domain.Exchange }
|
||||||
|
|
||||||
|
// ExchangeDetails contains a detached copy of an exchange's logical history.
|
||||||
|
type ExchangeDetails struct {
|
||||||
|
// RunID identifies the execution that produced this exchange.
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
// ParentRunID identifies the immediate source execution, or is empty.
|
||||||
|
ParentRunID string `json:"parent_run_id,omitempty"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns detached history, or zero details for a nil or empty handle.
|
||||||
|
// Messages include caller and model content and require appropriate handling.
|
||||||
|
// Inspection uses only retained state, without rendering, filesystem reads, or
|
||||||
|
// provider calls. Mutating returned messages or cache metadata cannot affect
|
||||||
|
// this exchange, another branch, or subsequent Details calls.
|
||||||
|
func (e *Exchange) Details() ExchangeDetails {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return ExchangeDetails{}
|
||||||
|
}
|
||||||
|
messages := make([]RenderedMessage, e.internal.MessageCount())
|
||||||
|
e.internal.VisitMessages(func(index int, message domain.RenderedMessage) {
|
||||||
|
messages[index] = RenderedMessage{Role: message.Role, Content: message.Content, CacheControl: fromDomainCacheControl(message.CacheControl)}
|
||||||
|
})
|
||||||
|
return ExchangeDetails{RunID: e.internal.RunID(), ParentRunID: e.internal.ParentRunID(), Messages: messages}
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns an opaque representation without conversation content.
|
||||||
|
func (e Exchange) String() string { return exchangeString }
|
||||||
|
|
||||||
|
// GoString returns an opaque representation without conversation content.
|
||||||
|
func (e Exchange) GoString() string { return exchangeString }
|
||||||
113
exchange_capture_contract_test.go
Normal file
113
exchange_capture_contract_test.go
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExchangeCaptureAndPreparedContinuation(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
plain, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || plain.Exchange != nil {
|
||||||
|
t.Fatalf("plain run = %#v, %v", plain, err)
|
||||||
|
}
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil || initial.Exchange == nil {
|
||||||
|
t.Fatalf("captured run = %#v, %v", initial, err)
|
||||||
|
}
|
||||||
|
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "question"}, {Role: promptkit.RoleAssistant, Content: "answer"}}
|
||||||
|
if details := initial.Exchange.Details(); details.RunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||||
|
t.Fatalf("details = %#v", details)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(initial)
|
||||||
|
if err != nil || strings.Contains(string(encoded), "exchange") || strings.Contains(string(encoded), "question") {
|
||||||
|
t.Fatalf("result JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if handle.Details().ParentRunID != initial.RunID {
|
||||||
|
t.Fatal("prepared ancestry missing")
|
||||||
|
}
|
||||||
|
continued, err := engine.RunPrepared(ctx, handle)
|
||||||
|
if err != nil || continued.Exchange == nil || continued.ParentRunID != initial.RunID {
|
||||||
|
t.Fatalf("continued = %#v, %v", continued, err)
|
||||||
|
}
|
||||||
|
want = append(want, promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "question"}, promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: "answer"})
|
||||||
|
if details := continued.Exchange.Details(); details.ParentRunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||||
|
t.Fatalf("continued details = %#v", details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeCapturesInvalidFinalOutput(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
|
||||||
|
Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}})
|
||||||
|
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
|
||||||
|
t.Fatalf("invalid result = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if messages := result.Exchange.Details().Messages; len(messages) != 2 || messages[1].Content != " " {
|
||||||
|
t.Fatalf("history = %#v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeFailuresLeaveSourceReusable(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
original := initial.Exchange.Details()
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}
|
||||||
|
handle, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
if result, err := engine.RunPrepared(ctx, handle); result != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = errors.New("provider failure")
|
||||||
|
if result, err := engine.Run(ctx, request); result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("failure = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = nil
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
if err != nil || result.Exchange == nil || !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||||
|
t.Fatalf("reused source = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyExchangeRejectedBeforePreparation(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
request := promptkit.RunRequest{PromptID: "missing", ContinueFrom: &promptkit.Exchange{}}
|
||||||
|
ctx := context.Background()
|
||||||
|
if _, err := engine.Prepare(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("prepare error = %v", err)
|
||||||
|
}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("validate error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("prepare execution error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("run error = %v", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatal("invalid exchange reached the model")
|
||||||
|
}
|
||||||
|
}
|
||||||
38
exchange_test.go
Normal file
38
exchange_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExchangeDetailsAndRequestConversion(t *testing.T) {
|
||||||
|
var nilExchange *Exchange
|
||||||
|
if got := nilExchange.Details(); got.RunID != "" || got.Messages != nil {
|
||||||
|
t.Fatalf("nil details = %#v", got)
|
||||||
|
}
|
||||||
|
if _, err := toDomainRunRequest(RunRequest{ContinueFrom: &Exchange{}}); err == nil {
|
||||||
|
t.Fatal("empty exchange accepted")
|
||||||
|
}
|
||||||
|
history := domain.CaptureExchange(nil, "source", []domain.RenderedMessage{{Role: domain.RoleUser, Content: "private", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}}}, "answer")
|
||||||
|
handle := &Exchange{internal: history}
|
||||||
|
request, err := toDomainRunRequest(RunRequest{ContinueFrom: handle})
|
||||||
|
if err != nil || request.ContinueFrom != history || !request.CaptureExchange {
|
||||||
|
t.Fatalf("request = %#v, error = %v", request, err)
|
||||||
|
}
|
||||||
|
details := handle.Details()
|
||||||
|
details.Messages[0].CacheControl.TTL = "changed"
|
||||||
|
if handle.Details().Messages[0].CacheControl.TTL != "1h" {
|
||||||
|
t.Fatal("details changed exchange")
|
||||||
|
}
|
||||||
|
if strings.Contains(fmt.Sprintf("%v %#v", handle, handle), "private") {
|
||||||
|
t.Fatal("format exposed history")
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(PreparedRun{ParentRunID: "source"})
|
||||||
|
if err != nil || !strings.Contains(string(encoded), `"parent_run_id":"source"`) {
|
||||||
|
t.Fatalf("prepared JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
56
execution_group.go
Normal file
56
execution_group.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExecutionGroup collects facts from explicitly selected execution events.
|
||||||
|
// Its zero value is ready for concurrent use. Do not copy it after first use.
|
||||||
|
type ExecutionGroup struct{ group observation.Group }
|
||||||
|
|
||||||
|
// ExecutionGroupMember identifies an observed run and its terminal coverage.
|
||||||
|
type ExecutionGroupMember struct {
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
TerminalObserved bool `json:"terminal_observed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionGroupSummary is a detached snapshot of terminal-only totals.
|
||||||
|
// Usage sums known terminal values, with Known as their logical OR and
|
||||||
|
// Complete as their logical AND. Coverage describes whether every observed
|
||||||
|
// member has a terminal, independently of usage completeness. An empty group
|
||||||
|
// has complete coverage and known complete zero usage; a group with members
|
||||||
|
// but no terminals has unknown, incomplete usage. Overlapping groups are not
|
||||||
|
// generally additive.
|
||||||
|
type ExecutionGroupSummary struct {
|
||||||
|
Members []ExecutionGroupMember `json:"members"`
|
||||||
|
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||||
|
Attempts uint64 `json:"attempts"`
|
||||||
|
TransientRetries uint64 `json:"transient_retries"`
|
||||||
|
RepairCalls uint64 `json:"repair_calls"`
|
||||||
|
Usage ObservedTokenUsage `json:"usage"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Observe enrolls an event's run, counting its first terminal event only.
|
||||||
|
func (g *ExecutionGroup) Observe(event ExecutionEvent) {
|
||||||
|
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
|
||||||
|
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||||
|
}
|
||||||
|
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
||||||
|
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns an internally consistent, caller-owned summary. Coverage
|
||||||
|
// describes observed members only, not future members or application progress.
|
||||||
|
func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
|
||||||
|
value := g.group.Snapshot()
|
||||||
|
count := func(c domain.ObservedTokenCount) ObservedTokenCount {
|
||||||
|
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||||
|
}
|
||||||
|
members := make([]ExecutionGroupMember, len(value.Members))
|
||||||
|
for i, member := range value.Members {
|
||||||
|
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
|
||||||
|
}
|
||||||
|
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||||
|
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||||
|
}
|
||||||
151
execution_group_contract_test.go
Normal file
151
execution_group_contract_test.go
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecutionGroupExplicitMembershipAndTerminalTotals(t *testing.T) {
|
||||||
|
responses := []*http.Response{
|
||||||
|
observedHTTPResponse(200, `{"choices":[{"message":{"content":"initial"}}],"usage":{"prompt_tokens":2}}`),
|
||||||
|
observedHTTPResponse(503, `{}`),
|
||||||
|
observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}],"usage":{"prompt_tokens":3}}`),
|
||||||
|
observedHTTPResponse(200, `{"choices":[{"message":{"content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":4}}`),
|
||||||
|
observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}],"usage":{"prompt_tokens":5}}`),
|
||||||
|
observedHTTPResponse(422, `{"error":{}}`),
|
||||||
|
observedHTTPResponse(200, `{"choices":[{"message":{"content":"validator"}}],"usage":{"prompt_tokens":6}}`),
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
one := 1
|
||||||
|
config := simpleConfig("message")
|
||||||
|
config.TransientRetries = &one
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
response := responses[calls]
|
||||||
|
calls++
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
var all, selected promptkit.ExecutionGroup
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, Observer: all.Observe})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
validation := &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
||||||
|
callbackCount := 0
|
||||||
|
var branchID, validatorID string
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
callbackCount++
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: validation}, nil
|
||||||
|
}
|
||||||
|
branchID = turn.Result.RunID
|
||||||
|
return nil, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbackCount != 2 {
|
||||||
|
t.Fatalf("coordinate=%#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if failed, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Validation: validation, Observer: all.Observe}); err == nil || failed != nil {
|
||||||
|
t.Fatalf("failed branch=%#v, %v", failed, err)
|
||||||
|
}
|
||||||
|
validator, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: selected.Observe})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
validatorID = validator.RunID
|
||||||
|
a, b := all.Snapshot(), selected.Snapshot()
|
||||||
|
if calls != 7 || len(a.Members) != 3 || a.Attempts != 6 || a.TransientRetries != 1 || a.RepairCalls != 2 || a.Usage.PromptTokens.Value != 14 || !a.Usage.PromptTokens.Known || !a.TerminalCoverageComplete {
|
||||||
|
t.Fatalf("all summary=%#v calls=%d", a, calls)
|
||||||
|
}
|
||||||
|
if len(b.Members) != 2 || b.Attempts != 4 || b.Usage.PromptTokens.Value != 13 || !b.TerminalCoverageComplete {
|
||||||
|
t.Fatalf("selected summary=%#v", b)
|
||||||
|
}
|
||||||
|
ids := map[string]bool{}
|
||||||
|
for _, member := range b.Members {
|
||||||
|
ids[member.RunID] = true
|
||||||
|
}
|
||||||
|
if !ids[branchID] || !ids[validatorID] || ids[initial.RunID] {
|
||||||
|
t.Fatalf("selected members=%#v", b.Members)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutionGroupPreparedSilenceAndPanicCoverage(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
|
||||||
|
ctx := context.Background()
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared.Discard()
|
||||||
|
_, _ = engine.RunPrepared(ctx, prepared)
|
||||||
|
if len(group.Snapshot().Members) != 0 {
|
||||||
|
t.Fatal("preparation or failed claim enrolled a run")
|
||||||
|
}
|
||||||
|
func() {
|
||||||
|
defer func() {
|
||||||
|
if recover() == nil {
|
||||||
|
t.Error("expected panic")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_, _ = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
|
||||||
|
_ = group.Snapshot()
|
||||||
|
if event.Kind == promptkit.ExecutionStarted {
|
||||||
|
panic("stop")
|
||||||
|
}
|
||||||
|
})})
|
||||||
|
}()
|
||||||
|
result := group.Snapshot()
|
||||||
|
if len(result.Members) != 1 || result.TerminalCoverageComplete || result.Members[0].TerminalObserved || result.Attempts != 0 {
|
||||||
|
t.Fatalf("panic summary=%#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutionGroupPreparedAndCallbackMembership(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
|
||||||
|
ctx := context.Background()
|
||||||
|
var continuations, nested promptkit.ExecutionGroup
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: nested.Observe})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
preparedResult, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var callbackID string
|
||||||
|
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: continuations.Observe, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||||
|
}
|
||||||
|
callbackResult, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: nested.Observe})
|
||||||
|
if err == nil {
|
||||||
|
callbackID = callbackResult.RunID
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, b := continuations.Snapshot(), nested.Snapshot()
|
||||||
|
if len(a.Members) != 1 || len(b.Members) != 2 || !a.TerminalCoverageComplete || !b.TerminalCoverageComplete {
|
||||||
|
t.Fatalf("summaries = %#v %#v", a, b)
|
||||||
|
}
|
||||||
|
ids := map[string]bool{}
|
||||||
|
for _, member := range b.Members {
|
||||||
|
ids[member.RunID] = true
|
||||||
|
}
|
||||||
|
if !ids[preparedResult.RunID] || !ids[callbackID] || ids[initial.RunID] {
|
||||||
|
t.Fatalf("members=%#v", b.Members)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -78,6 +78,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
|||||||
t.Fatalf("NewEngine: %v", err)
|
t.Fatalf("NewEngine: %v", err)
|
||||||
}
|
}
|
||||||
req := simpleGenerationRequest()
|
req := simpleGenerationRequest()
|
||||||
|
req.CaptureExchange = true
|
||||||
req.Validation = &promptkit.OutputContract{
|
req.Validation = &promptkit.OutputContract{
|
||||||
Format: promptkit.FormatJSON,
|
Format: promptkit.FormatJSON,
|
||||||
ValidationMode: promptkit.ValidationJSON,
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
|
|||||||
4
go.mod
4
go.mod
@@ -4,13 +4,13 @@ go 1.27.0
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
go.maximumdirect.net/configkit v1.1.0
|
go.maximumdirect.net/configkit v1.2.0
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
|
||||||
golang.org/x/mod v0.40.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -4,8 +4,8 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV
|
|||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
go.maximumdirect.net/configkit v1.1.0 h1:UzXN2ZMnEkcL0O4YSLqD9qd9M1gb/NWCHQR/H8AaQzA=
|
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8=
|
||||||
go.maximumdirect.net/configkit v1.1.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||||
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
|
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
|
||||||
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
|
|||||||
@@ -119,10 +119,12 @@ const (
|
|||||||
|
|
||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
profiles map[string]map[string]fieldState
|
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
|
promptRepair map[string]bool
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
|
backendRecords map[string]backendConfig
|
||||||
|
profileRecords map[string]profileConfig
|
||||||
indexedProfiles map[string]domain.ExecutionProfile
|
indexedProfiles map[string]domain.ExecutionProfile
|
||||||
backendIDs []string
|
backendIDs []string
|
||||||
profileIDs []string
|
profileIDs []string
|
||||||
@@ -211,8 +213,6 @@ func indexedProfiles(config *document) map[string]profileConfig {
|
|||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
|
||||||
|
|
||||||
// Load captures one immutable effective configuration for internal consumers.
|
// Load captures one immutable effective configuration for internal consumers.
|
||||||
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
return load(main, underlays)
|
return load(main, underlays)
|
||||||
@@ -235,8 +235,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair)
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
|
||||||
}
|
}
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
@@ -254,8 +253,8 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
profiles: make(map[string]map[string]fieldState),
|
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
|
promptRepair: make(map[string]bool),
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
@@ -320,11 +319,12 @@ func checkConfigurationVersion(version configkit.Version, validation *configkit.
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptInputPresence struct {
|
type promptInputPresence struct {
|
||||||
|
required fieldState
|
||||||
contentType fieldState
|
contentType fieldState
|
||||||
description fieldState
|
description fieldState
|
||||||
}
|
}
|
||||||
|
|
||||||
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) {
|
||||||
for promptIndex, prompt := range prompts {
|
for promptIndex, prompt := range prompts {
|
||||||
inputs := prompt.Inputs
|
inputs := prompt.Inputs
|
||||||
states := make([]promptInputPresence, len(inputs))
|
states := make([]promptInputPresence, len(inputs))
|
||||||
@@ -332,11 +332,13 @@ func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, name
|
|||||||
for index := range inputs {
|
for index := range inputs {
|
||||||
path := base.Index(index)
|
path := base.Index(index)
|
||||||
states[index] = promptInputPresence{
|
states[index] = promptInputPresence{
|
||||||
|
required: captureFieldState(path.Key("required"), resolution),
|
||||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||||
description: captureFieldState(path.Key("description"), resolution),
|
description: captureFieldState(path.Key("description"), resolution),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[indexedResourceID(namespace, prompt.ID)] = states
|
destination[indexedResourceID(namespace, prompt.ID)] = states
|
||||||
|
repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -352,24 +354,6 @@ func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func captureFields[T any](entries []T, identity func(T) string, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
|
||||||
for index, entry := range entries {
|
|
||||||
states := make(map[string]fieldState, len(fields))
|
|
||||||
for _, field := range fields {
|
|
||||||
info, exists := resolution.EntryInfo(basePath.Index(index).Key(field))
|
|
||||||
switch {
|
|
||||||
case !exists || !info.Explicit:
|
|
||||||
states[field] = fieldAbsent
|
|
||||||
case info.Null:
|
|
||||||
states[field] = fieldNull
|
|
||||||
default:
|
|
||||||
states[field] = fieldValue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
destination[indexedResourceID(namespace, identity(entry))] = states
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eligibleResourceList leaves invalid identities to the existing validation
|
// eligibleResourceList leaves invalid identities to the existing validation
|
||||||
// boundaries, without mutating a list that cannot be recorded by identity.
|
// boundaries, without mutating a list that cannot be recorded by identity.
|
||||||
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
||||||
|
|||||||
@@ -488,9 +488,6 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
|||||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||||
t.Fatalf("effective values were not retained: %+v", profile)
|
t.Fatalf("effective values were not retained: %+v", profile)
|
||||||
}
|
}
|
||||||
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
|
|
||||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
|
|
||||||
}
|
|
||||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||||
@@ -776,9 +773,8 @@ func TestLoadCapturesProfilePresence(t *testing.T) {
|
|||||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||||
t.Fatalf("profile values were not preserved: %+v", profile)
|
t.Fatalf("profile values were not preserved: %+v", profile)
|
||||||
}
|
}
|
||||||
fields := loaded.profiles["local"]
|
if profile.Model != nil {
|
||||||
if fields["temperature"] != fieldValue || fields["top_p"] != fieldNull || fields["api_key_required"] != fieldValue || fields["model"] != fieldAbsent {
|
t.Fatalf("omitted model was not preserved: %+v", profile)
|
||||||
t.Fatalf("profile presence was not preserved: %+v", fields)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type promptEntry struct {
|
type promptEntry struct {
|
||||||
|
resource promptConfig
|
||||||
definition domain.PromptDefinition
|
definition domain.PromptDefinition
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
requirementInputs []prompt.InputMetadata
|
requirementInputs []prompt.InputMetadata
|
||||||
@@ -86,7 +87,7 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
}
|
}
|
||||||
definition.Inputs = inputs
|
definition.Inputs = inputs
|
||||||
definition.Templates = messages
|
definition.Templates = messages
|
||||||
entry := &promptEntry{requirementInputs: inputMetadata, messages: compiledMessages}
|
entry := &promptEntry{resource: raw, requirementInputs: inputMetadata, messages: compiledMessages}
|
||||||
definition.Validation = contract
|
definition.Validation = contract
|
||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -316,8 +317,8 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.
|
|||||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptInspection returns an independently owned inspection of one
|
// GetPromptInspection returns lightweight metadata used for requirement
|
||||||
// captured prompt without copying its execution templates.
|
// selection, independently of the full resource inspection view.
|
||||||
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -84,6 +84,10 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
files["catalog/body.md"].Data = []byte("Changed")
|
files["catalog/body.md"].Data = []byte("Changed")
|
||||||
files["app/body.md"].Data = []byte("Changed replacement")
|
files["app/body.md"].Data = []byte("Changed replacement")
|
||||||
files["other/body.md"].Data = []byte("Changed other collection")
|
files["other/body.md"].Data = []byte("Changed other collection")
|
||||||
|
resource, err := state.InspectPrompt(context.Background(), "docs/child")
|
||||||
|
if err != nil || resource.Messages[0].Content != "Captured {{.name}}" || resource.Messages[0].CacheControl == nil || resource.Messages[0].CacheControl.TTL == nil || *resource.Messages[0].CacheControl.TTL != "1h" || resource.Inputs == nil || (*resource.Inputs)[0].Required != nil {
|
||||||
|
t.Fatalf("captured resource = %+v, %v", resource, err)
|
||||||
|
}
|
||||||
again, againInspection, err := state.prompt("docs/child")
|
again, againInspection, err := state.prompt("docs/child")
|
||||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||||
@@ -99,6 +103,25 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptResourcePresence(t *testing.T) {
|
||||||
|
state, err := load(mapping("prompt resources", map[string]any{"prompts": []any{
|
||||||
|
map[string]any{"id": "base", "default_profile": "missing", "inputs": []any{map[string]any{"name": "x", "required": false}, map[string]any{"name": "y"}},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}, map[string]any{"role": "assistant", "content": "{{.missing}}", "cache_control": map[string]any{"type": "ephemeral"}}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := state.InspectPrompt(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||||
|
t.Fatalf("prompt resource = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
||||||
|
|||||||
140
internal/configuration/resources.go
Normal file
140
internal/configuration/resources.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProfileResource is the resolved profile before execution completion.
|
||||||
|
// Its fields borrow immutable configuration captured by the adapter.
|
||||||
|
type ProfileResource struct {
|
||||||
|
ID string
|
||||||
|
Backend, Endpoint, Model *string
|
||||||
|
Temperature, TopP *float64
|
||||||
|
MaxTokens, TimeoutSeconds *int
|
||||||
|
ServiceTier, ReasoningEffort *string
|
||||||
|
APIKeyEnv *string
|
||||||
|
APIKeyRequired *bool
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendResource is the resolved backend and its configured diagnostics.
|
||||||
|
// Its fields borrow immutable configuration captured by the adapter.
|
||||||
|
type BackendResource struct {
|
||||||
|
ID, Endpoint string
|
||||||
|
APIKeyEnv *string
|
||||||
|
ConcurrencyLimit, QueueCapacity *int
|
||||||
|
ExtraParams map[string]any
|
||||||
|
SmokeTests *domain.BackendSmokeTestConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptResource is the resolved prompt with already captured message text.
|
||||||
|
type PromptResource struct {
|
||||||
|
ID, Hash string
|
||||||
|
DefaultProfile, Description, SessionID *string
|
||||||
|
Inputs *[]PromptInputResource
|
||||||
|
Messages []PromptMessageResource
|
||||||
|
Output PromptOutputResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptInputResource struct {
|
||||||
|
Name string
|
||||||
|
Required *bool
|
||||||
|
ContentType, Description *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptMessageResource struct {
|
||||||
|
Role, Content string
|
||||||
|
CacheControl *PromptCacheControlResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptCacheControlResource struct {
|
||||||
|
Type domain.CacheControlType
|
||||||
|
TTL *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptOutputResource struct {
|
||||||
|
Format domain.OutputFormat
|
||||||
|
ValidationMode domain.ValidationMode
|
||||||
|
SchemaPath *string
|
||||||
|
RepairAttempts *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entry, err := state.lookupPromptEntry(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, promptLookupError(err)
|
||||||
|
}
|
||||||
|
raw := entry.resource
|
||||||
|
result := &PromptResource{ID: id, Hash: entry.hash, DefaultProfile: raw.DefaultProfile, Description: raw.Description, SessionID: raw.SessionID}
|
||||||
|
if raw.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputResource, len(raw.Inputs))
|
||||||
|
presence := state.promptMetadata[id]
|
||||||
|
for i, input := range raw.Inputs {
|
||||||
|
inputs[i] = PromptInputResource{Name: input.Name, ContentType: input.ContentType, Description: input.Description}
|
||||||
|
if presence[i].required == fieldValue {
|
||||||
|
required := input.Required
|
||||||
|
inputs[i].Required = &required
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Inputs = &inputs
|
||||||
|
}
|
||||||
|
result.Messages = make([]PromptMessageResource, len(entry.definition.Templates))
|
||||||
|
for i, message := range entry.definition.Templates {
|
||||||
|
result.Messages[i] = PromptMessageResource{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, SchemaPath: raw.Output.SchemaPath}
|
||||||
|
if state.promptRepair[id] {
|
||||||
|
value := raw.Output.RepairAttempts
|
||||||
|
result.Output.RepairAttempts = &value
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectProfile(ctx context.Context, id string) (*ProfileResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state == nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q", profile.ErrProfileNotFound, id)
|
||||||
|
}
|
||||||
|
config, ok := state.profileRecords[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: %q", profile.ErrProfileNotFound, id)
|
||||||
|
}
|
||||||
|
return &ProfileResource{
|
||||||
|
ID: id, Backend: config.Backend, Endpoint: config.Endpoint, Model: config.Model,
|
||||||
|
Temperature: config.Temperature, TopP: config.TopP, MaxTokens: config.MaxTokens,
|
||||||
|
TimeoutSeconds: config.TimeoutSeconds, ServiceTier: config.ServiceTier,
|
||||||
|
ReasoningEffort: config.ReasoningEffort, APIKeyEnv: config.APIKeyEnv,
|
||||||
|
APIKeyRequired: config.APIKeyRequired, ExtraParams: config.ExtraParams,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state == nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
config, ok := state.backendRecords[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
return &BackendResource{
|
||||||
|
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv,
|
||||||
|
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
|
||||||
|
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
67
internal/configuration/resources_test.go
Normal file
67
internal/configuration/resources_test.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolvedBackendAndProfileResources(t *testing.T) {
|
||||||
|
state, err := load(mapping("resources", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "concurrency_limit": 2,
|
||||||
|
"extra_params": map[string]any{"backend": true}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "parent", "backend": "local", "temperature": 0.4},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "temperature": 0, "endpoint": "", "extra_params": map[string]any{}},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resource, err := state.InspectProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resource.ID != "child" || resource.Backend == nil || *resource.Backend != "local" || resource.Endpoint == nil || *resource.Endpoint != "" || resource.Temperature == nil || *resource.Temperature != 0 || resource.Model != nil || resource.ExtraParams == nil || len(resource.ExtraParams) != 0 {
|
||||||
|
t.Fatalf("profile resource = %+v", resource)
|
||||||
|
}
|
||||||
|
target, err := state.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil || target.ResolvedTarget.ExtraParams["backend"] != true {
|
||||||
|
t.Fatalf("execution target = %+v, %v", target, err)
|
||||||
|
}
|
||||||
|
resourceBackend, err := state.InspectBackend(context.Background(), "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resourceBackend.Endpoint != "https://example.test/v1" || resourceBackend.QueueCapacity == nil || *resourceBackend.QueueCapacity != 1024 || resourceBackend.SmokeTests == nil || resourceBackend.SmokeTests.DefaultProfile == nil || *resourceBackend.SmokeTests.DefaultProfile != "missing" || resourceBackend.SmokeTests.Reachability == nil || resourceBackend.SmokeTests.Reachability.Endpoint != nil {
|
||||||
|
t.Fatalf("backend resource = %+v", resourceBackend)
|
||||||
|
}
|
||||||
|
if _, err := state.InspectBackend(context.Background(), "missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
t.Fatalf("backend error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := state.InspectProfile(context.Background(), "missing"); !errors.Is(err, profile.ErrProfileNotFound) {
|
||||||
|
t.Fatalf("profile error = %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := state.InspectBackend(ctx, "local"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("cancellation = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendResourcePreservesInheritedDiagnostics(t *testing.T) {
|
||||||
|
state, err := load(mapping("backend diagnostics", map[string]any{"backends": []any{
|
||||||
|
map[string]any{"id": "parent", "endpoint": "https://provider.test/v1", "smoke_tests": map[string]any{"reachability": map[string]any{"endpoint": "https://probe.test/ready"}, "authentication": map[string]any{"kind": "none"}}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := state.InspectBackend(context.Background(), "child")
|
||||||
|
if err != nil || got.ID != "child" || got.ConcurrencyLimit != nil || got.QueueCapacity != nil || got.SmokeTests == nil || got.SmokeTests.Reachability == nil || got.SmokeTests.Reachability.Endpoint == nil || *got.SmokeTests.Reachability.Endpoint != "https://probe.test/ready" || got.SmokeTests.Authentication == nil || got.SmokeTests.Authentication.Kind == nil || *got.SmokeTests.Authentication.Kind != "none" || got.SmokeTests.Authentication.Endpoint != nil {
|
||||||
|
t.Fatalf("inherited diagnostics = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,8 @@ import (
|
|||||||
func (state *loaded) buildExecutionProfiles() error {
|
func (state *loaded) buildExecutionProfiles() error {
|
||||||
backendConfigs := indexedBackends(&state.result.Config)
|
backendConfigs := indexedBackends(&state.result.Config)
|
||||||
profileConfigs := indexedProfiles(&state.result.Config)
|
profileConfigs := indexedProfiles(&state.result.Config)
|
||||||
|
state.backendRecords = backendConfigs
|
||||||
|
state.profileRecords = profileConfigs
|
||||||
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||||
for _, id := range sortedKeys(backendConfigs) {
|
for _, id := range sortedKeys(backendConfigs) {
|
||||||
config := backendConfigs[id]
|
config := backendConfigs[id]
|
||||||
@@ -23,7 +25,7 @@ func (state *loaded) buildExecutionProfiles() error {
|
|||||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||||
for _, id := range sortedKeys(profileConfigs) {
|
for _, id := range sortedKeys(profileConfigs) {
|
||||||
config := profileConfigs[id]
|
config := profileConfigs[id]
|
||||||
profile, err := captureExecutionProfile(id, config, state.profiles[id], state.registry)
|
profile, err := captureExecutionProfile(id, config, state.registry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -62,7 +64,7 @@ func captureBackend(id string, config backendConfig) domain.Backend {
|
|||||||
return definition
|
return definition
|
||||||
}
|
}
|
||||||
|
|
||||||
func captureExecutionProfile(id string, config profileConfig, fields map[string]fieldState, registry *backend.Registry) (domain.ExecutionProfile, error) {
|
func captureExecutionProfile(id string, config profileConfig, registry *backend.Registry) (domain.ExecutionProfile, error) {
|
||||||
target := defaults.ExecutionTargetDefault()
|
target := defaults.ExecutionTargetDefault()
|
||||||
if config.Backend != nil && *config.Backend != "" {
|
if config.Backend != nil && *config.Backend != "" {
|
||||||
selected, err := registry.GetBackend(*config.Backend)
|
selected, err := registry.GetBackend(*config.Backend)
|
||||||
@@ -75,47 +77,44 @@ func captureExecutionProfile(id string, config profileConfig, fields map[string]
|
|||||||
target.ExtraParams = selected.ExtraParams
|
target.ExtraParams = selected.ExtraParams
|
||||||
}
|
}
|
||||||
presence := domain.ExecutionTargetPresence{}
|
presence := domain.ExecutionTargetPresence{}
|
||||||
if fields["endpoint"] != fieldAbsent {
|
if config.Endpoint != nil {
|
||||||
target.Endpoint = valueOrZero(config.Endpoint)
|
target.Endpoint = valueOrZero(config.Endpoint)
|
||||||
}
|
}
|
||||||
if fields["model"] != fieldAbsent {
|
if config.Model != nil {
|
||||||
target.Model = valueOrZero(config.Model)
|
target.Model = valueOrZero(config.Model)
|
||||||
}
|
}
|
||||||
if fields["temperature"] != fieldAbsent {
|
if config.Temperature != nil {
|
||||||
target.Temperature = valueOrZero(config.Temperature)
|
target.Temperature = valueOrZero(config.Temperature)
|
||||||
presence.Temperature = fields["temperature"] == fieldValue
|
presence.Temperature = true
|
||||||
}
|
}
|
||||||
if fields["top_p"] != fieldAbsent {
|
if config.TopP != nil {
|
||||||
target.TopP = valueOrZero(config.TopP)
|
target.TopP = valueOrZero(config.TopP)
|
||||||
presence.TopP = fields["top_p"] == fieldValue
|
presence.TopP = true
|
||||||
}
|
}
|
||||||
if fields["max_tokens"] != fieldAbsent {
|
if config.MaxTokens != nil {
|
||||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
presence.MaxTokens = true
|
||||||
}
|
}
|
||||||
if fields["timeout_seconds"] == fieldValue {
|
if config.TimeoutSeconds != nil {
|
||||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||||
presence.TimeoutSeconds = true
|
presence.TimeoutSeconds = true
|
||||||
}
|
}
|
||||||
if fields["service_tier"] != fieldAbsent {
|
if config.ServiceTier != nil {
|
||||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||||
}
|
}
|
||||||
if fields["reasoning_effort"] != fieldAbsent {
|
if config.ReasoningEffort != nil {
|
||||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||||
}
|
}
|
||||||
if fields["api_key_env"] != fieldAbsent {
|
if config.APIKeyEnv != nil {
|
||||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||||
}
|
}
|
||||||
if fields["api_key_required"] != fieldAbsent {
|
if config.APIKeyRequired != nil {
|
||||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||||
}
|
}
|
||||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
if target.APIKeyRequired && config.APIKeyEnv == nil {
|
||||||
target.APIKeyEnv = ""
|
target.APIKeyEnv = ""
|
||||||
}
|
}
|
||||||
switch fields["extra_params"] {
|
if config.ExtraParams != nil {
|
||||||
case fieldNull:
|
|
||||||
target.ExtraParams = nil
|
|
||||||
case fieldValue:
|
|
||||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
target := child.ResolvedTarget
|
target := child.ResolvedTarget
|
||||||
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
||||||
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
||||||
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "" || target.APIKeyRequired {
|
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "SECOND_KEY" || target.APIKeyRequired {
|
||||||
t.Fatalf("effective child target is wrong: %+v", target)
|
t.Fatalf("effective child target is wrong: %+v", target)
|
||||||
}
|
}
|
||||||
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
||||||
@@ -154,8 +154,8 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
||||||
if noMap.ResolvedTarget.ExtraParams != nil {
|
if !reflect.DeepEqual(noMap.ResolvedTarget.ExtraParams, selected.ExtraParams) {
|
||||||
t.Fatalf("explicit null provider map did not suppress backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
t.Fatalf("explicit null provider map did not inherit backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
||||||
}
|
}
|
||||||
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
||||||
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
||||||
@@ -183,8 +183,8 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
||||||
}
|
}
|
||||||
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
||||||
if clearedEndpoint.ResolvedTarget.Endpoint != "" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
if clearedEndpoint.ResolvedTarget.Endpoint != "https://first.example/v1" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
||||||
t.Fatalf("explicit null recovered backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
t.Fatalf("explicit null did not recover backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
||||||
}
|
}
|
||||||
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
||||||
again, _ := state.GetProfile(context.Background(), "child")
|
again, _ := state.GetProfile(context.Background(), "child")
|
||||||
@@ -193,6 +193,48 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileNullFallbackAndExplicitValues(t *testing.T) {
|
||||||
|
state, err := load(mapping("profile fallback", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://backend.example/v1", "api_key_env": "BACKEND_KEY", "extra_params": map[string]any{"backend": true}}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "parent", "backend": "backend", "endpoint": "https://parent.example/v1", "extra_params": map[string]any{"parent": true}},
|
||||||
|
map[string]any{"id": "inherited-null", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil},
|
||||||
|
map[string]any{"id": "omitted", "backend": "backend"},
|
||||||
|
map[string]any{"id": "null", "backend": "backend", "endpoint": nil, "api_key_env": nil, "extra_params": nil},
|
||||||
|
map[string]any{"id": "empty", "backend": "backend", "endpoint": "", "api_key_env": "", "extra_params": map[string]any{}},
|
||||||
|
map[string]any{"id": "required-null", "backend": "backend", "api_key_required": true, "api_key_env": nil},
|
||||||
|
map[string]any{"id": "required-omitted", "backend": "backend", "api_key_required": true},
|
||||||
|
map[string]any{"id": "required-explicit", "backend": "backend", "api_key_required": true, "api_key_env": "PROFILE_KEY"},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"inherited-null", "omitted", "null", "empty", "required-null", "required-omitted", "required-explicit"} {
|
||||||
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
target := profile.ResolvedTarget
|
||||||
|
wantEndpoint, wantKey := "https://backend.example/v1", "BACKEND_KEY"
|
||||||
|
switch id {
|
||||||
|
case "empty":
|
||||||
|
wantEndpoint, wantKey = "", ""
|
||||||
|
case "required-null", "required-omitted":
|
||||||
|
wantKey = ""
|
||||||
|
case "required-explicit":
|
||||||
|
wantKey = "PROFILE_KEY"
|
||||||
|
}
|
||||||
|
if target.Endpoint != wantEndpoint || target.APIKeyEnv != wantKey || !reflect.DeepEqual(target.ExtraParams, map[string]any{"backend": true}) {
|
||||||
|
t.Errorf("%s target = %+v", id, target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
child := state.result.Config.Profiles[1]
|
||||||
|
if child.Endpoint != nil || child.ExtraParams != nil || child.APIKeyEnv != nil {
|
||||||
|
t.Fatalf("inherited nulls were not retained in resolved values: %+v", child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
||||||
payloads := make(chan map[string]any, 1)
|
payloads := make(chan map[string]any, 1)
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ type CacheControl struct {
|
|||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
// RunRequest represents a request to generate a single artifact.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
|
CaptureExchange bool
|
||||||
|
ContinueFrom *Exchange
|
||||||
PromptID string
|
PromptID string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
SessionID string
|
SessionID string
|
||||||
@@ -75,6 +77,8 @@ type RunRequest struct {
|
|||||||
|
|
||||||
// RunResult represents the complete result of a prompt execution run.
|
// RunResult represents the complete result of a prompt execution run.
|
||||||
type RunResult struct {
|
type RunResult struct {
|
||||||
|
Exchange *Exchange
|
||||||
|
ParentRunID string
|
||||||
RunID string
|
RunID string
|
||||||
Artifact Artifact
|
Artifact Artifact
|
||||||
RawOutput string
|
RawOutput string
|
||||||
@@ -98,6 +102,7 @@ type RunResult struct {
|
|||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
|
ParentRunID string
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
@@ -148,7 +153,7 @@ type PromptDefinition struct {
|
|||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptInspection is the resolved result of exact prompt inspection.
|
// PromptInspection is lightweight metadata for prompt requirement selection.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
@@ -239,13 +244,6 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProfileInspection is the resolved result of exact profile inspection.
|
|
||||||
type ProfileInspection struct {
|
|
||||||
ProfileID string
|
|
||||||
EffectiveModelParams ExecutionTarget
|
|
||||||
APIKeyRequired bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
type OutputContract struct {
|
type OutputContract struct {
|
||||||
Format OutputFormat `yaml:"format"`
|
Format OutputFormat `yaml:"format"`
|
||||||
|
|||||||
92
internal/domain/exchange.go
Normal file
92
internal/domain/exchange.go
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
// Exchange is an immutable logical conversation ending in an assistant message.
|
||||||
|
type Exchange struct {
|
||||||
|
parent *Exchange
|
||||||
|
runID string
|
||||||
|
parentRunID string
|
||||||
|
messages []RenderedMessage
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
|
// CaptureExchange creates a new exchange without modifying its parent.
|
||||||
|
func CaptureExchange(parent *Exchange, runID string, messages []RenderedMessage, output string) *Exchange {
|
||||||
|
turn := make([]RenderedMessage, len(messages)+1)
|
||||||
|
copyRenderedMessages(turn, messages)
|
||||||
|
turn[len(messages)] = RenderedMessage{Role: RoleAssistant, Content: output}
|
||||||
|
exchange := &Exchange{parent: parent, runID: runID, messages: turn, count: len(turn)}
|
||||||
|
if parent != nil {
|
||||||
|
exchange.parentRunID = parent.runID
|
||||||
|
exchange.count += parent.count
|
||||||
|
}
|
||||||
|
return exchange
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunID returns the identity of the run that produced this exchange.
|
||||||
|
func (e *Exchange) RunID() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.runID
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParentRunID returns the immediate source run identity, if present.
|
||||||
|
func (e *Exchange) ParentRunID() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.parentRunID
|
||||||
|
}
|
||||||
|
|
||||||
|
// MessageCount returns the number of messages in the logical history.
|
||||||
|
func (e *Exchange) MessageCount() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Messages returns a detached copy of the complete logical history.
|
||||||
|
func (e *Exchange) Messages() []RenderedMessage {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return e.MessagesWithSuffix()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MessagesWithSuffix returns independently owned history followed by each part.
|
||||||
|
// A nil exchange supplies no history.
|
||||||
|
func (e *Exchange) MessagesWithSuffix(parts ...[]RenderedMessage) []RenderedMessage {
|
||||||
|
count := e.MessageCount()
|
||||||
|
for _, part := range parts {
|
||||||
|
count += len(part)
|
||||||
|
}
|
||||||
|
messages := make([]RenderedMessage, count)
|
||||||
|
e.visitChunks(func(start int, chunk []RenderedMessage) {
|
||||||
|
copyRenderedMessages(messages[start:], chunk)
|
||||||
|
})
|
||||||
|
offset := e.MessageCount()
|
||||||
|
for _, part := range parts {
|
||||||
|
copyRenderedMessages(messages[offset:], part)
|
||||||
|
offset += len(part)
|
||||||
|
}
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// VisitMessages visits indexed history entries without materializing a slice.
|
||||||
|
// Entries and their cache metadata are borrowed read-only values. Visits need
|
||||||
|
// not be in index order; callers can fill a destination using the supplied index.
|
||||||
|
func (e *Exchange) VisitMessages(visit func(int, RenderedMessage)) {
|
||||||
|
e.visitChunks(func(start int, chunk []RenderedMessage) {
|
||||||
|
for index, message := range chunk {
|
||||||
|
visit(start+index, message)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Exchange) visitChunks(visit func(int, []RenderedMessage)) {
|
||||||
|
for current := e; current != nil; current = current.parent {
|
||||||
|
start := current.count - len(current.messages)
|
||||||
|
visit(start, current.messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
75
internal/domain/exchange_test.go
Normal file
75
internal/domain/exchange_test.go
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExchangeHistoryOwnershipAndBranches(t *testing.T) {
|
||||||
|
control := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||||
|
source := []RenderedMessage{{Role: RoleSystem, Content: "", CacheControl: control}, {Role: RoleUser, Content: "question"}}
|
||||||
|
initial := CaptureExchange(nil, "first", source, " answer ")
|
||||||
|
first := CaptureExchange(initial, "second", []RenderedMessage{{Role: RoleUser, Content: "follow-up"}}, "reply")
|
||||||
|
second := CaptureExchange(initial, "third", []RenderedMessage{{Role: RoleDeveloper, Content: "other"}}, "")
|
||||||
|
source[0].Content = "changed"
|
||||||
|
control.TTL = "changed"
|
||||||
|
got := initial.Messages()
|
||||||
|
want := []RenderedMessage{{Role: RoleSystem, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}, {Role: RoleUser, Content: "question"}, {Role: RoleAssistant, Content: " answer "}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("initial messages = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
got[0].CacheControl.TTL = "mutated"
|
||||||
|
got[1].Content = "mutated"
|
||||||
|
if !reflect.DeepEqual(initial.Messages(), want) {
|
||||||
|
t.Fatal("returned messages changed retained history")
|
||||||
|
}
|
||||||
|
if first.RunID() != "second" || first.ParentRunID() != "first" || first.MessageCount() != 5 {
|
||||||
|
t.Fatal("incorrect branch metadata")
|
||||||
|
}
|
||||||
|
if second.ParentRunID() != "first" || second.Messages()[3].Content != "other" || second.Messages()[4].Content != "" {
|
||||||
|
t.Fatal("incorrect sibling history")
|
||||||
|
}
|
||||||
|
if first.Messages()[3].Content != "follow-up" || !reflect.DeepEqual(initial.Messages(), want) {
|
||||||
|
t.Fatal("branches changed each other or their parent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeManyTurns(t *testing.T) {
|
||||||
|
var exchange *Exchange
|
||||||
|
const turns = 2000
|
||||||
|
for i := 0; i < turns; i++ {
|
||||||
|
exchange = CaptureExchange(exchange, fmt.Sprint(i), []RenderedMessage{{Role: RoleUser, Content: fmt.Sprint(i)}}, fmt.Sprint(i))
|
||||||
|
}
|
||||||
|
messages := exchange.Messages()
|
||||||
|
if len(messages) != turns*2 || exchange.MessageCount() != len(messages) {
|
||||||
|
t.Fatalf("unexpected history length: %d", len(messages))
|
||||||
|
}
|
||||||
|
for i := 0; i < turns; i++ {
|
||||||
|
if messages[2*i].Content != fmt.Sprint(i) || messages[2*i+1].Role != RoleAssistant || messages[2*i+1].Content != fmt.Sprint(i) {
|
||||||
|
t.Fatalf("unexpected turn %d: %#v", i, messages[2*i:2*i+2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeMessagesWithSuffix(t *testing.T) {
|
||||||
|
for _, history := range []*Exchange{nil, CaptureExchange(nil, "source", nil, "answer")} {
|
||||||
|
control := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||||
|
first := []RenderedMessage{{Role: RoleUser, Content: "new", CacheControl: control}}
|
||||||
|
last := []RenderedMessage{{Role: RoleDeveloper, Content: "suffix"}}
|
||||||
|
got := history.MessagesWithSuffix(first, nil, last)
|
||||||
|
want := append(history.Messages(), first...)
|
||||||
|
want = append(want, last...)
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("messages = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
got[history.MessageCount()].CacheControl.TTL = "changed"
|
||||||
|
got[history.MessageCount()].Content = "changed"
|
||||||
|
if first[0].Content != "new" || control.TTL != "1h" {
|
||||||
|
t.Fatal("assembled messages alias the caller suffix")
|
||||||
|
}
|
||||||
|
if history != nil && history.Messages()[0].Content != "answer" {
|
||||||
|
t.Fatal("assembly changed retained history")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -61,11 +61,28 @@ type ObservedTokenUsage struct {
|
|||||||
CacheWriteTokens ObservedTokenCount
|
CacheWriteTokens ObservedTokenCount
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExecutionGroupMember identifies an observed run and terminal coverage.
|
||||||
|
type ExecutionGroupMember struct {
|
||||||
|
RunID string
|
||||||
|
TerminalObserved bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionGroupSummary contains terminal-only aggregate execution facts.
|
||||||
|
type ExecutionGroupSummary struct {
|
||||||
|
Members []ExecutionGroupMember
|
||||||
|
TerminalCoverageComplete bool
|
||||||
|
Attempts uint64
|
||||||
|
TransientRetries uint64
|
||||||
|
RepairCalls uint64
|
||||||
|
Usage ObservedTokenUsage
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||||
// error values. Nonterminal events leave aggregate counters at zero.
|
// error values. Nonterminal events leave aggregate counters at zero.
|
||||||
type ExecutionEvent struct {
|
type ExecutionEvent struct {
|
||||||
Kind ExecutionEventKind
|
Kind ExecutionEventKind
|
||||||
RunID string
|
RunID string
|
||||||
|
ParentRunID string
|
||||||
Time time.Time
|
Time time.Time
|
||||||
PromptID string
|
PromptID string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
|
|||||||
@@ -87,6 +87,7 @@ type Engine struct {
|
|||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
smokeTester *usecase.SmokeTester
|
smokeTester *usecase.SmokeTester
|
||||||
inspection configuration.Inspection
|
inspection configuration.Inspection
|
||||||
|
resources resourceInspection
|
||||||
listing identityInventory
|
listing identityInventory
|
||||||
namespaces resourceNamespaces
|
namespaces resourceNamespaces
|
||||||
}
|
}
|
||||||
@@ -105,6 +106,12 @@ type identityInventory interface {
|
|||||||
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type resourceInspection interface {
|
||||||
|
InspectPrompt(context.Context, string) (*configuration.PromptResource, error)
|
||||||
|
InspectProfile(context.Context, string) (*configuration.ProfileResource, error)
|
||||||
|
InspectBackend(context.Context, string) (*configuration.BackendResource, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ListingFilterMode selects namespace membership for one identity listing.
|
// ListingFilterMode selects namespace membership for one identity listing.
|
||||||
type ListingFilterMode uint8
|
type ListingFilterMode uint8
|
||||||
|
|
||||||
@@ -173,6 +180,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
runner: runner,
|
runner: runner,
|
||||||
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
||||||
inspection: snapshot.Inspection(),
|
inspection: snapshot.Inspection(),
|
||||||
|
resources: snapshot,
|
||||||
listing: snapshot,
|
listing: snapshot,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -224,13 +232,16 @@ func fileSource(name string) (fs.FS, string, error) {
|
|||||||
return os.DirFS(directory), filepath.ToSlash(base), nil
|
return os.DirFS(directory), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*configuration.PromptResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
var err error
|
var err error
|
||||||
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return e.runner.InspectPrompt(ctx, id)
|
return e.resources.InspectPrompt(ctx, id)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectRequirements(
|
func (e *Engine) InspectRequirements(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@@ -247,13 +258,27 @@ func (e *Engine) InspectRequirements(
|
|||||||
}
|
}
|
||||||
return e.runner.InspectRequirements(ctx, request)
|
return e.runner.InspectRequirements(ctx, request)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
func (e *Engine) InspectProfile(ctx context.Context, id string) (*configuration.ProfileResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
var err error
|
var err error
|
||||||
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return e.runner.InspectProfile(ctx, id)
|
return e.resources.InspectProfile(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, id string) (*configuration.BackendResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: backend id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
id, err := e.qualifyResourceID(id, e.namespaces.backend, "backend")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.resources.InspectBackend(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||||
|
|||||||
30
internal/observation/compose.go
Normal file
30
internal/observation/compose.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package observation
|
||||||
|
|
||||||
|
// Compose copies and synchronously invokes non-nil observers in order.
|
||||||
|
func Compose[T any, F ~func(T)](observers ...F) F {
|
||||||
|
count := 0
|
||||||
|
var single F
|
||||||
|
for _, observer := range observers {
|
||||||
|
if observer != nil {
|
||||||
|
count++
|
||||||
|
single = observer
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if count == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if count == 1 {
|
||||||
|
return single
|
||||||
|
}
|
||||||
|
selected := make([]F, 0, count)
|
||||||
|
for _, observer := range observers {
|
||||||
|
if observer != nil {
|
||||||
|
selected = append(selected, observer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return func(event T) {
|
||||||
|
for _, observer := range selected {
|
||||||
|
observer(event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
73
internal/observation/group.go
Normal file
73
internal/observation/group.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
package observation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Group incrementally collects terminal facts for explicitly observed runs.
|
||||||
|
// Its zero value is ready; it must not be copied after first use.
|
||||||
|
type Group struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
members map[string]bool
|
||||||
|
terminals int
|
||||||
|
attempts, retries, repairs uint64
|
||||||
|
usage domain.ObservedTokenUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// Observe enrolls a run and incorporates its first terminal event only.
|
||||||
|
func (g *Group) Observe(event domain.ExecutionEvent) {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.members == nil {
|
||||||
|
g.members = make(map[string]bool)
|
||||||
|
}
|
||||||
|
terminal := g.members[event.RunID]
|
||||||
|
if _, exists := g.members[event.RunID]; !exists {
|
||||||
|
g.members[event.RunID] = false
|
||||||
|
}
|
||||||
|
if event.Kind != domain.ExecutionFinished || terminal {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.members[event.RunID] = true
|
||||||
|
g.attempts += event.Attempts
|
||||||
|
g.retries += event.TransientRetries
|
||||||
|
g.repairs += event.RepairCalls
|
||||||
|
if g.terminals == 0 {
|
||||||
|
complete := domain.ObservedTokenCount{Complete: true}
|
||||||
|
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||||
|
}
|
||||||
|
mergeGroupUsage(&g.usage, event.Usage)
|
||||||
|
g.terminals++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a detached consistent view of currently observed members.
|
||||||
|
func (g *Group) Snapshot() domain.ExecutionGroupSummary {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
|
||||||
|
for id, terminal := range g.members {
|
||||||
|
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
|
||||||
|
}
|
||||||
|
if len(g.members) == 0 {
|
||||||
|
zero := domain.ObservedTokenCount{Known: true, Complete: true}
|
||||||
|
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedTokenUsage) {
|
||||||
|
merge := func(a *domain.ObservedTokenCount, b domain.ObservedTokenCount) {
|
||||||
|
if b.Known {
|
||||||
|
a.Value += b.Value
|
||||||
|
}
|
||||||
|
a.Known = a.Known || b.Known
|
||||||
|
a.Complete = a.Complete && b.Complete
|
||||||
|
}
|
||||||
|
merge(&total.PromptTokens, next.PromptTokens)
|
||||||
|
merge(&total.CompletionTokens, next.CompletionTokens)
|
||||||
|
merge(&total.TotalTokens, next.TotalTokens)
|
||||||
|
merge(&total.CachedTokens, next.CachedTokens)
|
||||||
|
merge(&total.CacheWriteTokens, next.CacheWriteTokens)
|
||||||
|
}
|
||||||
77
internal/observation/group_test.go
Normal file
77
internal/observation/group_test.go
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
package observation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGroupTerminalAccounting(t *testing.T) {
|
||||||
|
var group Group
|
||||||
|
empty := group.Snapshot()
|
||||||
|
if !empty.TerminalCoverageComplete || !empty.Usage.PromptTokens.Known || !empty.Usage.PromptTokens.Complete || empty.Usage.PromptTokens.Value != 0 {
|
||||||
|
t.Fatalf("empty = %#v", empty)
|
||||||
|
}
|
||||||
|
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionAttempt, RunID: "pending", Attempts: 99})
|
||||||
|
pending := group.Snapshot()
|
||||||
|
if pending.TerminalCoverageComplete || pending.Attempts != 0 || pending.Usage.PromptTokens.Known || pending.Usage.PromptTokens.Complete {
|
||||||
|
t.Fatalf("pending = %#v", pending)
|
||||||
|
}
|
||||||
|
zero := domain.ObservedTokenCount{Known: true, Complete: true}
|
||||||
|
known := domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}
|
||||||
|
partial := domain.ObservedTokenCount{Value: 2, Known: true, Complete: false}
|
||||||
|
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "first", Outcome: domain.ExecutionOutcomeError, Attempts: 2, TransientRetries: 1, RepairCalls: 1, Usage: domain.ObservedTokenUsage{PromptTokens: known, CompletionTokens: zero, TotalTokens: partial, CachedTokens: zero}})
|
||||||
|
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "first", Attempts: 100})
|
||||||
|
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "first"})
|
||||||
|
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "pending", Attempts: 3, Usage: domain.ObservedTokenUsage{PromptTokens: partial, CompletionTokens: zero, TotalTokens: domain.ObservedTokenCount{}, CachedTokens: zero}})
|
||||||
|
got := group.Snapshot()
|
||||||
|
if !got.TerminalCoverageComplete || len(got.Members) != 2 || got.Attempts != 5 || got.TransientRetries != 1 || got.RepairCalls != 1 {
|
||||||
|
t.Fatalf("summary = %#v", got)
|
||||||
|
}
|
||||||
|
if got.Usage.PromptTokens != (domain.ObservedTokenCount{Value: 6, Known: true}) || got.Usage.CompletionTokens != zero || got.Usage.TotalTokens != partial || got.Usage.CachedTokens != zero || got.Usage.CacheWriteTokens != (domain.ObservedTokenCount{}) {
|
||||||
|
t.Fatalf("usage = %#v", got.Usage)
|
||||||
|
}
|
||||||
|
got.Members[0].RunID = "changed"
|
||||||
|
for _, member := range group.Snapshot().Members {
|
||||||
|
if member.RunID == "changed" || !member.TerminalObserved {
|
||||||
|
t.Fatalf("member = %#v", member)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupConcurrentSnapshots(t *testing.T) {
|
||||||
|
var group Group
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
const count = 100
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
id := fmt.Sprint(i)
|
||||||
|
group.Observe(domain.ExecutionEvent{RunID: id, Kind: domain.ExecutionStarted})
|
||||||
|
group.Observe(domain.ExecutionEvent{RunID: id, Kind: domain.ExecutionFinished, Attempts: 1})
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
snapshot := group.Snapshot()
|
||||||
|
seen, terminals := map[string]bool{}, 0
|
||||||
|
for _, member := range snapshot.Members {
|
||||||
|
if seen[member.RunID] {
|
||||||
|
t.Fatal("duplicate member")
|
||||||
|
}
|
||||||
|
seen[member.RunID] = true
|
||||||
|
if member.TerminalObserved {
|
||||||
|
terminals++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if snapshot.Attempts != uint64(terminals) || snapshot.TerminalCoverageComplete != (terminals == len(snapshot.Members)) {
|
||||||
|
t.Fatalf("inconsistent snapshot = %#v", snapshot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if final := group.Snapshot(); len(final.Members) != count || final.Attempts != count || !final.TerminalCoverageComplete {
|
||||||
|
t.Fatalf("final = %#v", final)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
// calls methods in execution order; it has no shared history or background work.
|
// calls methods in execution order; it has no shared history or background work.
|
||||||
type Recorder struct {
|
type Recorder struct {
|
||||||
runID string
|
runID string
|
||||||
|
parentRunID string
|
||||||
startedAt time.Time
|
startedAt time.Time
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
observer domain.ExecutionObserver
|
observer domain.ExecutionObserver
|
||||||
@@ -95,6 +96,9 @@ func NewRunID() (string, error) {
|
|||||||
// RunID returns this invocation's identity.
|
// RunID returns this invocation's identity.
|
||||||
func (r *Recorder) RunID() string { return r.runID }
|
func (r *Recorder) RunID() string { return r.runID }
|
||||||
|
|
||||||
|
// SetParentRunID records the immediate source identity before start delivery.
|
||||||
|
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
|
||||||
|
|
||||||
// Start delivers the opening event with only identities already resolved.
|
// Start delivers the opening event with only identities already resolved.
|
||||||
func (r *Recorder) Start() {
|
func (r *Recorder) Start() {
|
||||||
event := r.event(domain.ExecutionStarted)
|
event := r.event(domain.ExecutionStarted)
|
||||||
@@ -177,7 +181,7 @@ func (r *Recorder) Finish(data FinishData) {
|
|||||||
|
|
||||||
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
||||||
return domain.ExecutionEvent{
|
return domain.ExecutionEvent{
|
||||||
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
Kind: kind, RunID: r.runID, ParentRunID: r.parentRunID, Time: r.now(), PromptID: r.selection.PromptID,
|
||||||
ProfileID: r.selection.ProfileID,
|
ProfileID: r.selection.ProfileID,
|
||||||
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
||||||
|
// GetPromptInspection supplies lightweight requirement-selection metadata.
|
||||||
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
||||||
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|||||||
66
internal/usecase/coordinate.go
Normal file
66
internal/usecase/coordinate.go
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ConversationTurn is one delivery to a coordinator callback.
|
||||||
|
type ConversationTurn[R any] struct {
|
||||||
|
Exchange *domain.Exchange
|
||||||
|
Result *R
|
||||||
|
RemainingTurns int
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoordinationOutcome describes a normal coordinator termination.
|
||||||
|
type CoordinationOutcome string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CoordinationStopped CoordinationOutcome = "stopped"
|
||||||
|
CoordinationExhausted CoordinationOutcome = "exhausted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CoordinationResult holds a normal outcome or a completed undelivered result.
|
||||||
|
type CoordinationResult[R any] struct {
|
||||||
|
Outcome CoordinationOutcome
|
||||||
|
Undelivered *R
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coordinate advances one branch using the supplied callback and ordinary run.
|
||||||
|
func Coordinate[Q, R any](ctx context.Context, start *domain.Exchange, turns int,
|
||||||
|
next func(context.Context, ConversationTurn[R]) (*Q, error),
|
||||||
|
run func(context.Context, *domain.Exchange, *Q) (*R, *domain.Exchange, error),
|
||||||
|
) (CoordinationResult[R], error) {
|
||||||
|
current := start
|
||||||
|
var pending *R
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return CoordinationResult[R]{Undelivered: pending}, err
|
||||||
|
}
|
||||||
|
request, err := next(ctx, ConversationTurn[R]{Exchange: current, Result: pending, RemainingTurns: turns})
|
||||||
|
pending = nil
|
||||||
|
if contextErr := ctx.Err(); contextErr != nil {
|
||||||
|
return CoordinationResult[R]{}, contextErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return CoordinationResult[R]{}, err
|
||||||
|
}
|
||||||
|
if request == nil {
|
||||||
|
return CoordinationResult[R]{Outcome: CoordinationStopped}, nil
|
||||||
|
}
|
||||||
|
if turns == 0 {
|
||||||
|
return CoordinationResult[R]{Outcome: CoordinationExhausted}, nil
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return CoordinationResult[R]{}, err
|
||||||
|
}
|
||||||
|
turns--
|
||||||
|
var nextExchange *domain.Exchange
|
||||||
|
pending, nextExchange, err = run(ctx, current, request)
|
||||||
|
if err != nil {
|
||||||
|
return CoordinationResult[R]{}, err
|
||||||
|
}
|
||||||
|
current = nextExchange
|
||||||
|
}
|
||||||
|
}
|
||||||
109
internal/usecase/coordinate_test.go
Normal file
109
internal/usecase/coordinate_test.go
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinateBudgetAndDelivery(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
budget int
|
||||||
|
stop bool
|
||||||
|
outcome CoordinationOutcome
|
||||||
|
runs, callbacks int
|
||||||
|
}{
|
||||||
|
{"zero stop", 0, true, CoordinationStopped, 0, 1},
|
||||||
|
{"zero exhausted", 0, false, CoordinationExhausted, 0, 1},
|
||||||
|
{"stop after last", 2, true, CoordinationStopped, 2, 3},
|
||||||
|
{"request after last", 2, false, CoordinationExhausted, 2, 3},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
start := domain.CaptureExchange(nil, "initial", nil, "start")
|
||||||
|
runs, callbacks := 0, 0
|
||||||
|
result, err := Coordinate(context.Background(), start, tc.budget, func(_ context.Context, turn ConversationTurn[string]) (*string, error) {
|
||||||
|
callbacks++
|
||||||
|
if turn.RemainingTurns != tc.budget-runs || (runs == 0 && turn.Result != nil) || (runs > 0 && (turn.Result == nil || *turn.Result != fmt.Sprint(runs))) {
|
||||||
|
t.Fatalf("unexpected turn: %#v", turn)
|
||||||
|
}
|
||||||
|
if tc.stop && turn.RemainingTurns == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
value := "next"
|
||||||
|
return &value, nil
|
||||||
|
}, func(_ context.Context, current *domain.Exchange, _ *string) (*string, *domain.Exchange, error) {
|
||||||
|
runs++
|
||||||
|
value := fmt.Sprint(runs)
|
||||||
|
return &value, domain.CaptureExchange(current, value, nil, value), nil
|
||||||
|
})
|
||||||
|
if err != nil || result.Outcome != tc.outcome || result.Undelivered != nil || runs != tc.runs || callbacks != tc.callbacks {
|
||||||
|
t.Fatalf("result=%#v error=%v runs=%d callbacks=%d", result, err, runs, callbacks)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateCancellationAndErrors(t *testing.T) {
|
||||||
|
start := domain.CaptureExchange(nil, "initial", nil, "start")
|
||||||
|
request := "continue"
|
||||||
|
for _, mode := range []string{"before", "after run", "inside callback", "callback error", "run error"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
sentinel := errors.New("sentinel")
|
||||||
|
calls := 0
|
||||||
|
if mode == "before" {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
result, err := Coordinate(ctx, start, 1, func(_ context.Context, turn ConversationTurn[string]) (*string, error) {
|
||||||
|
calls++
|
||||||
|
if mode == "inside callback" && turn.Result != nil {
|
||||||
|
cancel()
|
||||||
|
return nil, sentinel
|
||||||
|
}
|
||||||
|
if mode == "callback error" {
|
||||||
|
return nil, fmt.Errorf("callback: %w", sentinel)
|
||||||
|
}
|
||||||
|
return &request, nil
|
||||||
|
}, func(_ context.Context, current *domain.Exchange, _ *string) (*string, *domain.Exchange, error) {
|
||||||
|
if mode == "run error" {
|
||||||
|
return nil, nil, fmt.Errorf("run: %w", sentinel)
|
||||||
|
}
|
||||||
|
value := "response"
|
||||||
|
if mode == "after run" {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return &value, domain.CaptureExchange(current, "next", nil, value), nil
|
||||||
|
})
|
||||||
|
if mode == "before" || mode == "after run" || mode == "inside callback" {
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
} else if !errors.Is(err, sentinel) {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Outcome != "" || (mode == "after run") != (result.Undelivered != nil) {
|
||||||
|
t.Fatalf("result = %#v", result)
|
||||||
|
}
|
||||||
|
if mode == "before" && calls != 0 {
|
||||||
|
t.Fatal("callback invoked after cancellation")
|
||||||
|
}
|
||||||
|
if mode == "inside callback" && calls != 2 {
|
||||||
|
t.Fatalf("callbacks = %d", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinatePropagatesPanic(t *testing.T) {
|
||||||
|
defer func() {
|
||||||
|
if recover() != "callback panic" {
|
||||||
|
t.Error("panic did not propagate")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
Coordinate[string, string](context.Background(), nil, 0, func(context.Context, ConversationTurn[string]) (*string, error) { panic("callback panic") }, nil)
|
||||||
|
}
|
||||||
@@ -30,6 +30,8 @@ type PreparedExecution struct {
|
|||||||
|
|
||||||
type preparedExecutionPayload struct {
|
type preparedExecutionPayload struct {
|
||||||
prepared *domain.PreparedRun
|
prepared *domain.PreparedRun
|
||||||
|
capture bool
|
||||||
|
parent *domain.Exchange
|
||||||
validation validate.PreparedValidation
|
validation validate.PreparedValidation
|
||||||
directKey string
|
directKey string
|
||||||
transientRetries uint64
|
transientRetries uint64
|
||||||
@@ -55,6 +57,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
details: operation.run,
|
details: operation.run,
|
||||||
payload: &preparedExecutionPayload{
|
payload: &preparedExecutionPayload{
|
||||||
prepared: operation.run,
|
prepared: operation.run,
|
||||||
|
capture: operation.capture,
|
||||||
|
parent: operation.parent,
|
||||||
validation: operation.validation,
|
validation: operation.validation,
|
||||||
directKey: state.effectiveModel.APIKey,
|
directKey: state.effectiveModel.APIKey,
|
||||||
transientRetries: state.transientRetries,
|
transientRetries: state.transientRetries,
|
||||||
@@ -108,6 +112,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
if payload.observer != nil {
|
if payload.observer != nil {
|
||||||
var idErr error
|
var idErr error
|
||||||
recorder, idErr = observation.New(payload.observer)
|
recorder, idErr = observation.New(payload.observer)
|
||||||
|
recorder.SetParentRunID(payload.parent.RunID())
|
||||||
observation.RegisterClaim(ctx, recorder)
|
observation.RegisterClaim(ctx, recorder)
|
||||||
ctx = observation.WithRecorder(ctx, recorder)
|
ctx = observation.WithRecorder(ctx, recorder)
|
||||||
selected := payload.prepared
|
selected := payload.prepared
|
||||||
@@ -139,7 +144,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
|
|
||||||
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, payload.transientRetries, runID, start, func(
|
return r.executePreparedRun(ctx, payload.prepared, payload.capture, payload.parent, payload.directKey, payload.transientRetries, runID, start, func(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
artifact *domain.Artifact,
|
artifact *domain.Artifact,
|
||||||
attemptsUsed int,
|
attemptsUsed int,
|
||||||
@@ -174,6 +179,8 @@ func (p *preparedExecutionPayload) clear() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.prepared = nil
|
p.prepared = nil
|
||||||
|
p.capture = false
|
||||||
|
p.parent = nil
|
||||||
p.validation = nil
|
p.validation = nil
|
||||||
p.directKey = ""
|
p.directKey = ""
|
||||||
p.transientRetries = 0
|
p.transientRetries = 0
|
||||||
|
|||||||
38
internal/usecase/profile_fixture_test.go
Normal file
38
internal/usecase/profile_fixture_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
type inspectionProfileRepository struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
r.calls++
|
||||||
|
r.id = id
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
if r.profile == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
profile := *r.profile
|
||||||
|
if profile.ResolvedTarget != nil {
|
||||||
|
target := *profile.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
profile.ResolvedTarget = &target
|
||||||
|
}
|
||||||
|
return &profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := r.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
@@ -85,30 +85,3 @@ func normalizeResolvedExecutionSettings(target domain.ExecutionTarget) (domain.E
|
|||||||
}
|
}
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without prompt or execution work.
|
|
||||||
func (r *Runner) InspectProfile(
|
|
||||||
ctx context.Context,
|
|
||||||
profileID string,
|
|
||||||
) (*domain.ProfileInspection, error) {
|
|
||||||
if strings.TrimSpace(profileID) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, &OperationError{Category: ErrProfileLoad, Cause: ctx.Err()}
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
selection, err := r.resolveProfileSelection(ctx, profileID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
target, _ := resolveExecutionTarget(selection.profile, nil)
|
|
||||||
target.APIKey = ""
|
|
||||||
return &domain.ProfileInspection{
|
|
||||||
ProfileID: selection.id,
|
|
||||||
EffectiveModelParams: target,
|
|
||||||
APIKeyRequired: target.APIKeyRequired,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,105 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/profile"
|
|
||||||
)
|
|
||||||
|
|
||||||
type inspectionProfileRepository struct {
|
|
||||||
profile *domain.ExecutionProfile
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
id string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
r.calls++
|
|
||||||
r.id = id
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
if r.profile == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
profile := *r.profile
|
|
||||||
if profile.ResolvedTarget != nil {
|
|
||||||
target := *profile.ResolvedTarget
|
|
||||||
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
|
||||||
profile.ResolvedTarget = &target
|
|
||||||
}
|
|
||||||
return &profile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
|
||||||
_, err := r.GetProfile(ctx, id)
|
|
||||||
return id, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
|
||||||
configured := domain.ExecutionTarget{
|
|
||||||
BackendID: "backend", Endpoint: "https://backend.example/v1", Model: "model",
|
|
||||||
Temperature: 0, APIKeyEnv: "OPTIONAL_KEY", ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
}
|
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: " literal ", ResolvedTarget: &configured,
|
|
||||||
TargetPresence: domain.ExecutionTargetPresence{Temperature: true},
|
|
||||||
}}
|
|
||||||
runner := &Runner{profiles: profiles}
|
|
||||||
first, err := runner.InspectProfile(context.Background(), " literal ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect exact ID: %v", err)
|
|
||||||
}
|
|
||||||
if profiles.calls != 1 || profiles.id != " literal " || first.ProfileID != profiles.id || !reflect.DeepEqual(first.EffectiveModelParams, configured) {
|
|
||||||
t.Fatalf("inspection=%+v lookup=(%d,%q), want exact snapshot", first, profiles.calls, profiles.id)
|
|
||||||
}
|
|
||||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
second, err := runner.InspectProfile(context.Background(), " literal ")
|
|
||||||
if err != nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" {
|
|
||||||
t.Fatalf("inspection mutated snapshot: (%+v, %v)", second, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
|
|
||||||
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
|
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: "endpoint-only", ResolvedTarget: &domain.ExecutionTarget{
|
|
||||||
Endpoint: "https://profile.example/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
inspection, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "endpoint-only")
|
|
||||||
if err != nil || inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" || inspection.APIKeyRequired {
|
|
||||||
t.Fatalf("unexpected endpoint-only inspection: (%+v, %v)", inspection, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileClassifiesLookupFailures(t *testing.T) {
|
|
||||||
profiles := &inspectionProfileRepository{}
|
|
||||||
runner := &Runner{profiles: profiles}
|
|
||||||
if _, err := runner.InspectProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
|
|
||||||
t.Fatalf("blank inspection=(%v, calls=%d)", err, profiles.calls)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if _, err := runner.InspectProfile(ctx, "profile"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
|
|
||||||
t.Fatalf("canceled inspection=(%v, calls=%d)", err, profiles.calls)
|
|
||||||
}
|
|
||||||
profiles.err = profile.ErrProfileNotFound
|
|
||||||
if _, err := runner.InspectProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
|
|
||||||
t.Fatalf("missing profile error=%v", err)
|
|
||||||
}
|
|
||||||
for _, invalid := range []*Runner{
|
|
||||||
{},
|
|
||||||
{profiles: &inspectionProfileRepository{}},
|
|
||||||
{profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}}},
|
|
||||||
} {
|
|
||||||
if _, err := invalid.InspectProfile(context.Background(), "profile"); !errors.Is(err, ErrProfileLoad) {
|
|
||||||
t.Fatalf("invalid dependency error=%v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -48,17 +48,3 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
|
|||||||
}
|
}
|
||||||
return inspection, nil
|
return inspection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
|
||||||
func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.PromptInspection, error) {
|
|
||||||
if strings.TrimSpace(promptID) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, &OperationError{Category: ErrPromptLoad, Cause: ctx.Err()}
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.resolvePromptInspection(ctx, promptID)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,218 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/prompt"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/promptdef"
|
|
||||||
)
|
|
||||||
|
|
||||||
type inspectionPromptRepository struct {
|
|
||||||
definition *domain.PromptDefinition
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
id string
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptMetadataRepository struct {
|
|
||||||
inspection domain.PromptInspection
|
|
||||||
inspectionErr error
|
|
||||||
lookupID string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptSnapshot(
|
|
||||||
_ context.Context,
|
|
||||||
id string,
|
|
||||||
) (*domain.PromptDefinition, string, error) {
|
|
||||||
r.calls++
|
|
||||||
r.id = id
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, "", r.err
|
|
||||||
}
|
|
||||||
return r.definition, "captured-hash", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
|
||||||
definition, hash, err := r.GetPromptSnapshot(ctx, id)
|
|
||||||
return fixturePromptInspection(definition, hash), err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*inspectionPromptRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
|
||||||
return prompt.RequirementSnapshot{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error) {
|
|
||||||
return nil, "", errors.New("full prompt definition should not be requested")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*promptMetadataRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
|
||||||
return prompt.RequirementSnapshot{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptInspection(
|
|
||||||
_ context.Context,
|
|
||||||
id string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
|
||||||
r.lookupID = id
|
|
||||||
inspection := r.inspection
|
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
|
||||||
return &inspection, r.inspectionErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
|
||||||
definition := &domain.PromptDefinition{
|
|
||||||
ID: "normalized.prompt",
|
|
||||||
DefaultProfile: "not-resolved",
|
|
||||||
Inputs: []domain.PromptInput{
|
|
||||||
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
|
||||||
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
|
|
||||||
},
|
|
||||||
Validation: domain.OutputContract{
|
|
||||||
Format: domain.FormatJSON,
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: "schemas/result.json",
|
|
||||||
RepairAttempts: 2,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
repository := &inspectionPromptRepository{definition: definition}
|
|
||||||
runner := &Runner{promptDefs: repository}
|
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
|
||||||
}
|
|
||||||
if repository.calls != 1 || repository.id != " prompt-id " {
|
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
|
||||||
}
|
|
||||||
if inspection.PromptID != definition.ID ||
|
|
||||||
inspection.PromptHash != "captured-hash" ||
|
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
|
||||||
inspection.OutputContract != definition.Validation {
|
|
||||||
t.Fatalf("inspection=%#v, want definition metadata", inspection)
|
|
||||||
}
|
|
||||||
|
|
||||||
inspection.Inputs[0].Name = "changed"
|
|
||||||
second, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt again: %v", err)
|
|
||||||
}
|
|
||||||
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
|
|
||||||
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedCapturedMetadata(t *testing.T) {
|
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
|
||||||
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
|
||||||
DefaultProfileID: "partial-profile",
|
|
||||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
|
||||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
|
||||||
}}
|
|
||||||
runner := &Runner{promptDefs: repository}
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ")
|
|
||||||
if err != nil || repository.lookupID != " requested-id " ||
|
|
||||||
inspection.PromptID != "captured.prompt" ||
|
|
||||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
|
||||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
|
||||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=%q", inspection, err, repository.lookupID)
|
|
||||||
}
|
|
||||||
inspection.Inputs[0].Name = "caller mutation"
|
|
||||||
inspection.OutputContract.SchemaPath = "caller mutation"
|
|
||||||
second, err := runner.InspectPrompt(context.Background(), "requested-id")
|
|
||||||
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
|
||||||
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesCapturedHashAndMetadata(t *testing.T) {
|
|
||||||
repository := &requirementInspectionPromptRepository{
|
|
||||||
definition: &domain.PromptDefinition{
|
|
||||||
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" ||
|
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
|
||||||
t.Fatalf("captured inspection=(%+v, %v)", inspection, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
|
||||||
t.Run("blank ID", func(t *testing.T) {
|
|
||||||
repository := &inspectionPromptRepository{}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ")
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
|
||||||
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("canceled context", func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
repository := &inspectionPromptRepository{}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
|
||||||
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing prompt", func(t *testing.T) {
|
|
||||||
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
|
||||||
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("defensive prompt dependencies", func(t *testing.T) {
|
|
||||||
var nilRunner *Runner
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
runner *Runner
|
|
||||||
}{
|
|
||||||
{name: "nil runner", runner: nilRunner},
|
|
||||||
{name: "nil repository", runner: &Runner{}},
|
|
||||||
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
_, err := tc.runner.InspectPrompt(context.Background(), "prompt")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
|
||||||
t.Fatalf("inspection error=%v, want prompt load", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
|
|
||||||
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
|
|
||||||
repository := &fakePromptRepo{def: definition}
|
|
||||||
runner := NewRunner(
|
|
||||||
repository,
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
||||||
|
|
||||||
&fakeArtifactReader{},
|
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), definition.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
|
||||||
}
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, ProfileID: "exec"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("prepare prompt: %v", err)
|
|
||||||
}
|
|
||||||
if inspection.PromptHash != prepared.PromptHash {
|
|
||||||
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,20 +9,6 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInspectProfileAllowsIncompleteResolvedTarget(t *testing.T) {
|
|
||||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
|
||||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
|
||||||
}}
|
|
||||||
runner := &Runner{profiles: staticExecutionProfileRepo{profile: profile}}
|
|
||||||
inspection, err := runner.InspectProfile(context.Background(), "partial")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("partial profile inspection failed: %v", err)
|
|
||||||
}
|
|
||||||
if inspection.EffectiveModelParams.Endpoint != "" || inspection.EffectiveModelParams.Model != "" {
|
|
||||||
t.Fatalf("inspection invented a runnable target: %+v", inspection)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
||||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
||||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
||||||
|
|||||||
@@ -79,6 +79,8 @@ type preparationState struct {
|
|||||||
type preparedOperation struct {
|
type preparedOperation struct {
|
||||||
run *domain.PreparedRun
|
run *domain.PreparedRun
|
||||||
validation validate.PreparedValidation
|
validation validate.PreparedValidation
|
||||||
|
capture bool
|
||||||
|
parent *domain.Exchange
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRunner(
|
func NewRunner(
|
||||||
@@ -163,7 +165,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
directAPIKey := state.effectiveModel.APIKey
|
directAPIKey := state.effectiveModel.APIKey
|
||||||
|
|
||||||
return r.executePreparedRun(ctx, operation.run, directAPIKey, state.transientRetries, runID, start, func(
|
return r.executePreparedRun(ctx, operation.run, operation.capture, operation.parent, directAPIKey, state.transientRetries, runID, start, func(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
artifact *domain.Artifact,
|
artifact *domain.Artifact,
|
||||||
attemptsUsed int,
|
attemptsUsed int,
|
||||||
@@ -183,6 +185,8 @@ type preparedValidationFunc func(
|
|||||||
func (r *Runner) executePreparedRun(
|
func (r *Runner) executePreparedRun(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
prepared *domain.PreparedRun,
|
prepared *domain.PreparedRun,
|
||||||
|
capture bool,
|
||||||
|
parent *domain.Exchange,
|
||||||
directAPIKey string,
|
directAPIKey string,
|
||||||
transientRetries uint64,
|
transientRetries uint64,
|
||||||
runID string,
|
runID string,
|
||||||
@@ -252,8 +256,14 @@ func (r *Runner) executePreparedRun(
|
|||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
executionTarget.APIKey = ""
|
executionTarget.APIKey = ""
|
||||||
|
var exchange *domain.Exchange
|
||||||
|
if capture {
|
||||||
|
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
|
||||||
|
}
|
||||||
|
|
||||||
return &domain.RunResult{
|
return &domain.RunResult{
|
||||||
|
Exchange: exchange,
|
||||||
|
ParentRunID: prepared.ParentRunID,
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
Artifact: outputArtifact,
|
Artifact: outputArtifact,
|
||||||
RawOutput: genResp.Content,
|
RawOutput: genResp.Content,
|
||||||
@@ -399,7 +409,7 @@ func (r *Runner) completePreparation(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &preparedOperation{run: prepared, validation: validationPlan}, nil
|
return &preparedOperation{run: prepared, validation: validationPlan, capture: req.CaptureExchange, parent: req.ContinueFrom}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) prepareValidation(
|
func (r *Runner) prepareValidation(
|
||||||
@@ -472,12 +482,13 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
if state.directSessionID != "" {
|
if state.directSessionID != "" {
|
||||||
effectivePrompt.SessionID = state.directSessionID
|
effectivePrompt.SessionID = state.directSessionID
|
||||||
}
|
}
|
||||||
effectivePrompt.Messages = domain.ConcatRenderedMessages(renderedPrompt.Messages, req.AppendedMessages)
|
effectivePrompt.Messages = req.ContinueFrom.MessagesWithSuffix(renderedPrompt.Messages, req.AppendedMessages)
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
effectiveModel := state.effectiveModel
|
effectiveModel := state.effectiveModel
|
||||||
effectiveModel.APIKey = ""
|
effectiveModel.APIKey = ""
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
|
ParentRunID: req.ContinueFrom.RunID(),
|
||||||
PromptID: state.definition.ID,
|
PromptID: state.definition.ID,
|
||||||
PromptHash: state.promptDefinitionHash,
|
PromptHash: state.promptDefinitionHash,
|
||||||
SelectedProfileID: state.selectedProfileID,
|
SelectedProfileID: state.selectedProfileID,
|
||||||
|
|||||||
@@ -1310,6 +1310,39 @@ func TestRunnerComposesAppendedMessagesIntoEveryRenderedPrompt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparesContinuationHistory(t *testing.T) {
|
||||||
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session", Messages: []domain.RenderedMessage{{Role: domain.RoleUser, Content: "new"}}}}
|
||||||
|
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "output"}}
|
||||||
|
runner := NewRunner(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), renderer, client, nil, nil)
|
||||||
|
request := domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), AppendedMessages: []domain.RenderedMessage{{Role: domain.RoleDeveloper, Content: "suffix"}}}
|
||||||
|
plain, err := runner.Prepare(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request.ContinueFrom = domain.CaptureExchange(nil, "parent", []domain.RenderedMessage{{Role: domain.RoleSystem, Content: "old", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}}}, "prior")
|
||||||
|
prepared, err := runner.Prepare(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []domain.RenderedMessage{{Role: domain.RoleSystem, Content: "old", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}}, {Role: domain.RoleAssistant, Content: "prior"}, {Role: domain.RoleUser, Content: "new"}, {Role: domain.RoleDeveloper, Content: "suffix"}}
|
||||||
|
if !reflect.DeepEqual(prepared.Messages, want) || prepared.ParentRunID != "parent" || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
|
||||||
|
t.Fatalf("prepared = %#v, plain = %#v", prepared, plain)
|
||||||
|
}
|
||||||
|
if client.calls != 0 {
|
||||||
|
t.Fatal("preparation called the model")
|
||||||
|
}
|
||||||
|
execution, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if execution.Details().ParentRunID != "parent" || !reflect.DeepEqual(execution.Details().Messages, want) {
|
||||||
|
t.Fatalf("prepared execution = %#v", execution.Details())
|
||||||
|
}
|
||||||
|
if client.calls != 0 {
|
||||||
|
t.Fatal("prepared execution called the model")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRunSuccessful(t *testing.T) {
|
func TestRunnerRunSuccessful(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}
|
||||||
@@ -1696,9 +1729,10 @@ func TestRunnerReleasesAdmissionAcrossRunOutcomes(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
|
CaptureExchange: true,
|
||||||
})
|
})
|
||||||
if tc.wantError == nil {
|
if tc.wantError == nil {
|
||||||
if err != nil || result == nil {
|
if err != nil || result == nil || result.Exchange == nil {
|
||||||
t.Fatalf("successful run=(%+v, %v)", result, err)
|
t.Fatalf("successful run=(%+v, %v)", result, err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -438,11 +438,11 @@ func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
overlay, err := engine.InspectProfile(context.Background(), "shared/overlay")
|
overlay, err := engine.InspectProfile(context.Background(), "shared/overlay")
|
||||||
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
|
if err != nil || overlay == nil || overlay.Model == nil || *overlay.Model != "effective-model" {
|
||||||
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
|
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
|
||||||
}
|
}
|
||||||
child, err := engine.InspectProfile(context.Background(), "shared/child")
|
child, err := engine.InspectProfile(context.Background(), "shared/child")
|
||||||
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
|
if err != nil || child == nil || child.Model == nil || *child.Model != "base-model" {
|
||||||
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
|
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -87,6 +87,26 @@ func TestWithNamespacesRoutesOrdinarySelectionsAndPreservesViews(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectBackendNamespaceSelection(t *testing.T) {
|
||||||
|
engine := newNamespacedContractEngine(t, &fakeLLMClient{})
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{Backend: "alpha"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for id, want := range map[string]string{"backend": "alpha/backend", "beta/backend": "beta/backend"} {
|
||||||
|
got, err := scoped.InspectBackend(context.Background(), id)
|
||||||
|
if err != nil || got.BackendID != want {
|
||||||
|
t.Errorf("backend %q = %+v, %v", id, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := scoped.InspectBackend(context.Background(), "openrouter"); !errors.Is(err, promptkit.ErrBackendNotFound) {
|
||||||
|
t.Fatalf("unexpected fallback: %v", err)
|
||||||
|
}
|
||||||
|
if got, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || got.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("bundled backend = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWithNamespacesHasNoFallbackAndValidatesAtTheCorrectBoundary(t *testing.T) {
|
func TestWithNamespacesHasNoFallbackAndValidatesAtTheCorrectBoundary(t *testing.T) {
|
||||||
engine := newNamespacedContractEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}})
|
engine := newNamespacedContractEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}})
|
||||||
scoped, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -15,6 +16,72 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
|
|||||||
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestContinuationObservationAncestry(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
|
||||||
|
engine := observationEngine(t, client)
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var events []promptkit.ExecutionEvent
|
||||||
|
observer := func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
||||||
|
req := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: observer}
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, req)
|
||||||
|
if err != nil || len(events) != 0 {
|
||||||
|
t.Fatalf("preparation events=%#v error=%v", events, err)
|
||||||
|
}
|
||||||
|
check := func(result *promptkit.RunResult, runErr error) {
|
||||||
|
t.Helper()
|
||||||
|
if runErr != nil || result == nil || len(events) != 3 {
|
||||||
|
t.Fatalf("result=%#v error=%v events=%#v", result, runErr, events)
|
||||||
|
}
|
||||||
|
for _, event := range events {
|
||||||
|
if event.ParentRunID != initial.RunID || event.RunID != result.RunID || strings.Contains(fmt.Sprintf("%+v", event), "private answer") {
|
||||||
|
t.Fatalf("event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
events = nil
|
||||||
|
}
|
||||||
|
result, err := engine.Run(ctx, req)
|
||||||
|
check(result, err)
|
||||||
|
result, err = engine.RunPrepared(ctx, prepared)
|
||||||
|
check(result, err)
|
||||||
|
client.err = errors.New("failure")
|
||||||
|
if result, err := engine.Run(ctx, req); result != nil || err == nil || len(events) != 3 {
|
||||||
|
t.Fatalf("failure result=%#v error=%v events=%#v", result, err, events)
|
||||||
|
}
|
||||||
|
for _, event := range events {
|
||||||
|
if event.ParentRunID != initial.RunID {
|
||||||
|
t.Fatalf("failure event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationNestedRunHasIndependentAncestry(t *testing.T) {
|
||||||
|
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var nested []promptkit.ExecutionEvent
|
||||||
|
var nestedErr error
|
||||||
|
_, err = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if event.Kind == promptkit.ExecutionFinished {
|
||||||
|
_, nestedErr = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: func(value promptkit.ExecutionEvent) { nested = append(nested, value) }})
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
if err != nil || nestedErr != nil || len(nested) == 0 {
|
||||||
|
t.Fatalf("errors=%v, %v; events=%#v", err, nestedErr, nested)
|
||||||
|
}
|
||||||
|
for _, event := range nested {
|
||||||
|
if event.ParentRunID != "" {
|
||||||
|
t.Fatalf("nested event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunObservationCoversFacadeErrors(t *testing.T) {
|
func TestRunObservationCoversFacadeErrors(t *testing.T) {
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ExecutionObserver receives immutable metadata events synchronously. Callers
|
// ExecutionObserver receives immutable metadata events synchronously. Callers
|
||||||
@@ -16,6 +17,12 @@ import (
|
|||||||
// event, and Promptkit keeps no asynchronous queue or durable history.
|
// event, and Promptkit keeps no asynchronous queue or durable history.
|
||||||
type ExecutionObserver func(ExecutionEvent)
|
type ExecutionObserver func(ExecutionEvent)
|
||||||
|
|
||||||
|
// ComposeObservers returns a synchronous ordered fan-out of non-nil observers.
|
||||||
|
// It copies the input slice; a panic propagates and prevents later callbacks.
|
||||||
|
func ComposeObservers(observers ...ExecutionObserver) ExecutionObserver {
|
||||||
|
return observation.Compose[ExecutionEvent](observers...)
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionEventKind identifies one point in an execution's lifecycle.
|
// ExecutionEventKind identifies one point in an execution's lifecycle.
|
||||||
type ExecutionEventKind string
|
type ExecutionEventKind string
|
||||||
|
|
||||||
@@ -102,6 +109,7 @@ type ObservedTokenUsage struct {
|
|||||||
type ExecutionEvent struct {
|
type ExecutionEvent struct {
|
||||||
Kind ExecutionEventKind
|
Kind ExecutionEventKind
|
||||||
RunID string
|
RunID string
|
||||||
|
ParentRunID string
|
||||||
Time time.Time
|
Time time.Time
|
||||||
PromptID string
|
PromptID string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
@@ -129,7 +137,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
|||||||
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||||
}
|
}
|
||||||
return ExecutionEvent{
|
return ExecutionEvent{
|
||||||
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
|
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, ParentRunID: value.ParentRunID, Time: value.Time,
|
||||||
PromptID: value.PromptID, ProfileID: value.ProfileID,
|
PromptID: value.PromptID, ProfileID: value.ProfileID,
|
||||||
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
||||||
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
||||||
|
|||||||
103
observer_composition_contract_test.go
Normal file
103
observer_composition_contract_test.go
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestComposeObserversCopiesAndOrders(t *testing.T) {
|
||||||
|
if promptkit.ComposeObservers() != nil || promptkit.ComposeObservers(nil, nil) != nil {
|
||||||
|
t.Fatal("empty composition is not nil")
|
||||||
|
}
|
||||||
|
var calls []int
|
||||||
|
first := func(promptkit.ExecutionEvent) { calls = append(calls, 1) }
|
||||||
|
second := func(promptkit.ExecutionEvent) { calls = append(calls, 2) }
|
||||||
|
values := []promptkit.ExecutionObserver{first, nil, second, first}
|
||||||
|
observer := promptkit.ComposeObservers(values...)
|
||||||
|
values[0] = second
|
||||||
|
observer(promptkit.ExecutionEvent{})
|
||||||
|
if !reflect.DeepEqual(calls, []int{1, 2, 1}) {
|
||||||
|
t.Fatalf("calls=%v", calls)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if recover() != "panic" || len(calls) != 3 {
|
||||||
|
t.Errorf("panic or calls=%v", calls)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
promptkit.ComposeObservers(func(promptkit.ExecutionEvent) { panic("panic") }, second)(promptkit.ExecutionEvent{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeObserversSparseSelection(t *testing.T) {
|
||||||
|
for _, selected := range []int{0, 1, 2} {
|
||||||
|
calls := 0
|
||||||
|
observers := make([]promptkit.ExecutionObserver, 1024)
|
||||||
|
callback := func(promptkit.ExecutionEvent) { calls++ }
|
||||||
|
for index := 0; index < selected; index++ {
|
||||||
|
observers[len(observers)-1-index] = callback
|
||||||
|
}
|
||||||
|
composed := promptkit.ComposeObservers(observers...)
|
||||||
|
for index := range observers {
|
||||||
|
observers[index] = func(promptkit.ExecutionEvent) { calls += 100 }
|
||||||
|
}
|
||||||
|
if selected == 0 {
|
||||||
|
if composed != nil {
|
||||||
|
t.Fatal("all-nil composition must be nil")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
composed(promptkit.ExecutionEvent{})
|
||||||
|
if calls != selected {
|
||||||
|
t.Fatalf("selected %d callbacks, got %d calls", selected, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateObserverCompositionAndNestedSelection(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var sequence []string
|
||||||
|
var nested int
|
||||||
|
request := &promptkit.RunRequest{PromptID: "prompt", Observer: func(promptkit.ExecutionEvent) { sequence = append(sequence, "request") }}
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: func(promptkit.ExecutionEvent) { sequence = append(sequence, "coordinator") }, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result == nil {
|
||||||
|
return request, nil
|
||||||
|
}
|
||||||
|
_, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: func(promptkit.ExecutionEvent) { nested++ }})
|
||||||
|
return nil, err
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || nested != 3 || !reflect.DeepEqual(sequence, []string{"request", "coordinator", "request", "coordinator", "request", "coordinator"}) {
|
||||||
|
t.Fatalf("result=%#v error=%v events=%v nested=%d", result, err, sequence, nested)
|
||||||
|
}
|
||||||
|
if request.ContinueFrom != nil {
|
||||||
|
t.Fatal("coordinator changed caller request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposedObserversSupportConcurrentRuns(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
count := 0
|
||||||
|
observer := promptkit.ComposeObservers(func(promptkit.ExecutionEvent) { mu.Lock(); count++; mu.Unlock() })
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(&preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: observer}); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if count != 24 {
|
||||||
|
t.Fatalf("event count=%d", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,9 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
|||||||
// A successful PrepareExecution may retain an Observer function in this
|
// A successful PrepareExecution may retain an Observer function in this
|
||||||
// handle's private payload. The callback and other private execution settings
|
// handle's private payload. The callback and other private execution settings
|
||||||
// are cleared on claim completion or Discard. Details never exposes them.
|
// are cleared on claim completion or Discard. Details never exposes them.
|
||||||
|
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
|
||||||
|
// execution of the frozen snapshot. Only RunPrepared can produce its completed
|
||||||
|
// [Exchange]; preparation alone does not.
|
||||||
type PreparedExecution struct {
|
type PreparedExecution struct {
|
||||||
internal *usecase.PreparedExecution
|
internal *usecase.PreparedExecution
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -276,7 +276,7 @@ func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) {
|
|||||||
}}
|
}}
|
||||||
engine := newPreparedContractEngine(t, client, "Summarize the source.")
|
engine := newPreparedContractEngine(t, client, "Summarize the source.")
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "prepared",
|
PromptID: "prepared", CaptureExchange: true,
|
||||||
Validation: &promptkit.OutputContract{
|
Validation: &promptkit.OutputContract{
|
||||||
Format: promptkit.FormatMarkdown,
|
Format: promptkit.FormatMarkdown,
|
||||||
ValidationMode: promptkit.ValidationBasic,
|
ValidationMode: promptkit.ValidationBasic,
|
||||||
@@ -296,6 +296,9 @@ func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) {
|
|||||||
result.Validation.RepairAttempts != 1 || result.Usage != (promptkit.TokenUsage{PromptTokens: 10, CompletionTokens: 16, TotalTokens: 26}) {
|
result.Validation.RepairAttempts != 1 || result.Usage != (promptkit.TokenUsage{PromptTokens: 10, CompletionTokens: 16, TotalTokens: 26}) {
|
||||||
t.Fatalf("repaired result = %+v", result)
|
t.Fatalf("repaired result = %+v", result)
|
||||||
}
|
}
|
||||||
|
if result.Exchange == nil || len(result.Exchange.Details().Messages) != len(details.Messages)+1 || result.Exchange.Details().Messages[len(details.Messages)].Content != "Corrected summary." {
|
||||||
|
t.Fatalf("repaired exchange = %#v", result.Exchange)
|
||||||
|
}
|
||||||
requests := client.snapshot()
|
requests := client.snapshot()
|
||||||
if len(requests) != 2 || len(requests[1].Prompt.Messages) != len(details.Messages)+1 ||
|
if len(requests) != 2 || len(requests[1].Prompt.Messages) != len(details.Messages)+1 ||
|
||||||
!reflect.DeepEqual(requests[1].Prompt.Messages[:len(details.Messages)], details.Messages) ||
|
!reflect.DeepEqual(requests[1].Prompt.Messages[:len(details.Messages)], details.Messages) ||
|
||||||
@@ -316,6 +319,7 @@ func TestPreparedExecutionIsolatesMutatingClientAcrossRepair(t *testing.T) {
|
|||||||
}
|
}
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "prepared",
|
PromptID: "prepared",
|
||||||
|
CaptureExchange: true,
|
||||||
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{
|
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{
|
||||||
"nested": map[string]any{"value": "original"},
|
"nested": map[string]any{"value": "original"},
|
||||||
}},
|
}},
|
||||||
@@ -333,6 +337,9 @@ func TestPreparedExecutionIsolatesMutatingClientAcrossRepair(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(before, prepared.Details()) {
|
if !reflect.DeepEqual(before, prepared.Details()) {
|
||||||
t.Fatal("client mutation changed retained prepared metadata")
|
t.Fatal("client mutation changed retained prepared metadata")
|
||||||
}
|
}
|
||||||
|
if result.Exchange == nil || result.Exchange.Details().Messages[0].Content != "original message" || result.Exchange.Details().Messages[1].CacheControl.TTL != "1h" || result.Exchange.Details().Messages[2].Content != `{"value":3}` {
|
||||||
|
t.Fatalf("mutating client changed exchange history: %#v", result.Exchange.Details())
|
||||||
|
}
|
||||||
result.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "result mutation"
|
result.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "result mutation"
|
||||||
if prepared.Details().EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" {
|
if prepared.Details().EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" {
|
||||||
t.Fatal("result mutation changed retained prepared metadata")
|
t.Fatal("result mutation changed retained prepared metadata")
|
||||||
|
|||||||
@@ -41,9 +41,8 @@ func TestProfileInheritanceConfiguredBaseAliasWorkflow(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect alias: %v", err)
|
t.Fatalf("inspect alias: %v", err)
|
||||||
}
|
}
|
||||||
if child.ProfileID != "child" || child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
|
if child.ProfileID != "child" || child.Backend != nil || base.Backend != nil || child.Model == nil || base.Model == nil || *child.Model != *base.Model ||
|
||||||
child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
|
child.ReasoningEffort == nil || *child.ReasoningEffort != reasoning || child.TimeoutSeconds == nil || *child.TimeoutSeconds != timeout {
|
||||||
child.EffectiveModelParams.ReasoningEffort != reasoning || child.EffectiveModelParams.TimeoutSeconds != timeout {
|
|
||||||
t.Fatalf("alias inspection = %+v, base = %+v", child, base)
|
t.Fatalf("alias inspection = %+v, base = %+v", child, base)
|
||||||
}
|
}
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
@@ -71,7 +70,7 @@ func TestProfileInheritanceYAMLAliasOfConfiguredProfile(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
|
inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
|
||||||
if err != nil || inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
|
if err != nil || inspection.ProfileID != "yaml-alias" || inspection.Model == nil || *inspection.Model == "" {
|
||||||
t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err)
|
t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -92,7 +91,7 @@ func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "credential-child")
|
inspection, err := engine.InspectProfile(context.Background(), "credential-child")
|
||||||
if err != nil || !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
if err != nil || inspection.APIKeyRequired == nil || !*inspection.APIKeyRequired || inspection.APIKeyEnv != nil {
|
||||||
t.Fatalf("credential inspection = (%+v, %v)", inspection, err)
|
t.Fatalf("credential inspection = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -144,7 +143,7 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
|
|||||||
t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model)
|
t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "child")
|
inspection, err := engine.InspectProfile(context.Background(), "child")
|
||||||
if err != nil || inspection.EffectiveModelParams.Model != "first-model" {
|
if err != nil || inspection.Model == nil || *inspection.Model != "first-model" {
|
||||||
t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err)
|
t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ func TestInspectProfileUsesLiteralFileKeyForPreparation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if inspected.ProfileID != " literal " || prepared.SelectedProfileID != inspected.ProfileID || prepared.EffectiveModelParams.Model != inspected.EffectiveModelParams.Model {
|
if inspected.ProfileID != " literal " || inspected.Model == nil || prepared.SelectedProfileID != inspected.ProfileID || prepared.EffectiveModelParams.Model != *inspected.Model {
|
||||||
t.Fatalf("inspection=%+v preparation=%+v", inspected, prepared)
|
t.Fatalf("inspection=%+v preparation=%+v", inspected, prepared)
|
||||||
}
|
}
|
||||||
if _, err := engine.InspectProfile(context.Background(), "literal"); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
if _, err := engine.InspectProfile(context.Background(), "literal"); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||||
@@ -61,12 +61,12 @@ func TestInspectProfileIsIndependentAndDoesNotInvokeProvider(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
first, err := engine.InspectProfile(context.Background(), "profile")
|
first, err := engine.InspectProfile(context.Background(), "profile")
|
||||||
if err != nil || first == nil || !first.APIKeyRequired {
|
if err != nil || first == nil || first.APIKeyRequired == nil || !*first.APIKeyRequired || first.ExtraParams == nil {
|
||||||
t.Fatalf("first inspection=(%+v, %v)", first, err)
|
t.Fatalf("first inspection=(%+v, %v)", first, err)
|
||||||
}
|
}
|
||||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
(*first.ExtraParams)["nested"].(map[string]any)["value"] = "changed"
|
||||||
second, err := engine.InspectProfile(context.Background(), "profile")
|
second, err := engine.InspectProfile(context.Background(), "profile")
|
||||||
if err != nil || second == nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" || len(client.requests) != 0 {
|
if err != nil || second == nil || second.ExtraParams == nil || (*second.ExtraParams)["nested"].(map[string]any)["value"] != "original" || len(client.requests) != 0 {
|
||||||
t.Fatalf("inspection changed snapshot or invoked provider: (%+v, %v), calls=%d", second, err, len(client.requests))
|
t.Fatalf("inspection changed snapshot or invoked provider: (%+v, %v), calls=%d", second, err, len(client.requests))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,7 +94,7 @@ func TestInspectProfilePublicErrorsAndPartialTarget(t *testing.T) {
|
|||||||
t.Fatalf("canceled inspection: %v", err)
|
t.Fatalf("canceled inspection: %v", err)
|
||||||
}
|
}
|
||||||
partial, err := engine.InspectProfile(context.Background(), "profile")
|
partial, err := engine.InspectProfile(context.Background(), "profile")
|
||||||
if err != nil || partial == nil || partial.EffectiveModelParams.Endpoint != "" || partial.EffectiveModelParams.Model != "model" {
|
if err != nil || partial == nil || partial.Endpoint != nil || partial.Model == nil || *partial.Model != "model" {
|
||||||
t.Fatalf("partial inspection=(%+v, %v)", partial, err)
|
t.Fatalf("partial inspection=(%+v, %v)", partial, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
106
prompt_sources_contract_test.go
Normal file
106
prompt_sources_contract_test.go
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTypedSourcesComposeAndCaptureValues(t *testing.T) {
|
||||||
|
endpoint, childEndpoint, base := "https://base.test/v1", "https://child.test/v1", "base"
|
||||||
|
limit, queue := 2, 0
|
||||||
|
backendSource := promptkit.BackendSource("backends", []promptkit.BackendConfig{{ID: "base", Endpoint: &endpoint, ConcurrencyLimit: &limit}, {ID: "child", Extends: &base, Endpoint: &childEndpoint, QueueCapacity: &queue}})
|
||||||
|
role, content, secondContent, empty, cacheType := "user", "hello {{.name}}", "second", "", "ephemeral"
|
||||||
|
format, validation, description, profileID := "text", "none", "description", "local"
|
||||||
|
required, repairs := false, 0
|
||||||
|
prompts := []promptkit.PromptConfig{
|
||||||
|
{ID: "base", DefaultProfile: &profileID, Description: &description, Inputs: []promptkit.PromptInputConfig{{Name: "input", Required: &required, Description: &empty}}, Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content, CacheControl: &promptkit.PromptCacheControlConfig{Type: &cacheType, TTL: &empty}}, {Role: &role, Content: &secondContent}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation, RepairAttempts: &repairs}},
|
||||||
|
{ID: "child", Extends: &base},
|
||||||
|
{ID: "layered"},
|
||||||
|
{ID: "empty-inputs", Inputs: []promptkit.PromptInputConfig{}, Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}},
|
||||||
|
{ID: "omitted-inputs", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}},
|
||||||
|
}
|
||||||
|
promptSource := promptkit.PromptSource("typed prompts", prompts)
|
||||||
|
endpoint, childEndpoint, role, content, description = "changed", "changed", "changed", "changed", "changed"
|
||||||
|
prompts[0].Inputs[0].Name = "changed"
|
||||||
|
prompts[0].Messages[1].Content = &empty
|
||||||
|
prompts[1].ID = "changed"
|
||||||
|
profileEndpoint, model := "https://profile.test/v1", "model"
|
||||||
|
main := configkit.Input{Source: configkit.Mapping("completion", map[string]any{"prompts": []any{map[string]any{"id": "layered", "messages": []any{map[string]any{"role": "user", "content": "layer"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}})}
|
||||||
|
profileSource := promptkit.ProfileSource("profiles", []promptkit.ProfileConfig{{ID: "local", Endpoint: &profileEndpoint, Model: &model}})
|
||||||
|
underlays := []configkit.Underlay{{Inputs: []configkit.Input{{Source: backendSource}, {Source: promptSource}, {Source: profileSource}}}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &main}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
backend, err := engine.InspectBackend(ctx, "child")
|
||||||
|
if err != nil || backend.BackendID != "child" || backend.Endpoint != "https://child.test/v1" || backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit != 2 || backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
|
||||||
|
t.Fatalf("backend = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
prompt, err := engine.InspectPrompt(ctx, "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID != "local" || prompt.Description == nil || *prompt.Description != "description" || prompt.Inputs == nil || len(*prompt.Inputs) != 1 || (*prompt.Inputs)[0].Name != "input" || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required || (*prompt.Inputs)[0].Description == nil || *(*prompt.Inputs)[0].Description != "" || len(prompt.Messages) != 2 || prompt.Messages[0].Content != "hello {{.name}}" || prompt.Messages[1].Content != "second" || prompt.Messages[0].CacheControl.TTL == nil || *prompt.Messages[0].CacheControl.TTL != "" || prompt.OutputContract.RepairAttempts == nil || *prompt.OutputContract.RepairAttempts != 0 {
|
||||||
|
t.Fatalf("prompt = %+v", prompt)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "child", Vars: map[string]string{"name": "reader"}})
|
||||||
|
if err != nil || prepared.Messages[0].Content != "hello reader" {
|
||||||
|
t.Fatalf("preparation = %+v, %v", prepared, err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"layered", "empty-inputs", "omitted-inputs"} {
|
||||||
|
result, err := engine.InspectPrompt(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if id == "empty-inputs" && (result.Inputs == nil || len(*result.Inputs) != 0) {
|
||||||
|
t.Fatalf("empty inputs = %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
if id == "omitted-inputs" && result.Inputs != nil {
|
||||||
|
t.Fatalf("omitted inputs = %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedPromptSourceUsesExplicitFilesystemContext(t *testing.T) {
|
||||||
|
role, path, format, validation, parent := "user", "body.md", "text", "none", "base"
|
||||||
|
source := promptkit.PromptSource("file prompts", []promptkit.PromptConfig{{ID: "base", Messages: []promptkit.PromptMessageConfig{{Role: &role, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}, {ID: "child", Extends: &parent}})
|
||||||
|
files := fstest.MapFS{"templates/body.md": {Data: []byte("captured {{.value}}")}}
|
||||||
|
filesystem := configkit.NewFilesystem("fixture", files)
|
||||||
|
input := configkit.Input{Source: source, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "templates"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
files["templates/body.md"].Data = []byte("changed")
|
||||||
|
prompt, err := engine.InspectPrompt(context.Background(), "child")
|
||||||
|
if err != nil || prompt.Messages[0].Content != "captured {{.value}}" {
|
||||||
|
t.Fatalf("file prompt = %+v, %v", prompt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedPromptSourceDefersValidation(t *testing.T) {
|
||||||
|
role, body, path, format, validation := "user", "body", "also-file", "text", "none"
|
||||||
|
source := promptkit.PromptSource("invalid", []promptkit.PromptConfig{{ID: "bad", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &body, ContentFile: &path}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &validation}}})
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}})
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("invalid final source: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNilTypedPromptSourceSuppliesEmptySequence(t *testing.T) {
|
||||||
|
source := promptkit.PromptSource("empty", nil)
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: source}}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("empty source = %v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
129
prompts.go
Normal file
129
prompts.go
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "go.maximumdirect.net/configkit"
|
||||||
|
|
||||||
|
// PromptConfig is a presence-aware source input for one prompt. Extends names
|
||||||
|
// a sibling prompt. Nil optional pointers and slices are omitted; non-nil
|
||||||
|
// pointers and slices preserve explicit zero, blank, and empty values. These
|
||||||
|
// inputs are mapping constructors, not serialized inspection values. Raw
|
||||||
|
// ConfigKit mappings can express explicit null and other composition controls.
|
||||||
|
type PromptConfig struct {
|
||||||
|
ID string
|
||||||
|
Extends, DefaultProfile, Description, SessionID *string
|
||||||
|
Inputs []PromptInputConfig
|
||||||
|
Messages []PromptMessageConfig
|
||||||
|
Output *PromptOutputConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInputConfig supplies an input declaration. Name is always supplied;
|
||||||
|
// nil optional pointers omit their fields.
|
||||||
|
type PromptInputConfig struct {
|
||||||
|
Name string
|
||||||
|
Required *bool
|
||||||
|
ContentType, Description *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptMessageConfig supplies message fields, including inline content or a
|
||||||
|
// content-file reference. Final content selection is validated by NewEngine.
|
||||||
|
type PromptMessageConfig struct {
|
||||||
|
Role, Content, ContentFile *string
|
||||||
|
CacheControl *PromptCacheControlConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptCacheControlConfig supplies optional cache-control source fields.
|
||||||
|
type PromptCacheControlConfig struct{ Type, TTL *string }
|
||||||
|
|
||||||
|
// PromptOutputConfig supplies optional output contract source fields.
|
||||||
|
type PromptOutputConfig struct {
|
||||||
|
Format, ValidationMode, SchemaPath *string
|
||||||
|
RepairAttempts *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptSource creates a labeled ConfigKit mapping in caller order. Nil
|
||||||
|
// top-level input supplies an empty prompts sequence. Records may be partial:
|
||||||
|
// sibling inheritance and layered sources can complete them before NewEngine
|
||||||
|
// validates the final configuration. Mapping snapshots supported values when
|
||||||
|
// constructed; the helper neither resolves inheritance nor reads files.
|
||||||
|
// Relative content files use the ConfigKit Input filesystem context of the
|
||||||
|
// contributing field. Raw mappings remain available for explicit null and
|
||||||
|
// generic ConfigKit operations.
|
||||||
|
func PromptSource(label string, prompts []PromptConfig) configkit.Source {
|
||||||
|
definitions := make([]any, 0, len(prompts))
|
||||||
|
for _, config := range prompts {
|
||||||
|
fields := map[string]any{"id": config.ID}
|
||||||
|
if config.Extends != nil {
|
||||||
|
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||||
|
}
|
||||||
|
if config.DefaultProfile != nil {
|
||||||
|
fields["default_profile"] = *config.DefaultProfile
|
||||||
|
}
|
||||||
|
if config.Description != nil {
|
||||||
|
fields["description"] = *config.Description
|
||||||
|
}
|
||||||
|
if config.SessionID != nil {
|
||||||
|
fields["session_id"] = *config.SessionID
|
||||||
|
}
|
||||||
|
if config.Inputs != nil {
|
||||||
|
inputs := make([]any, 0, len(config.Inputs))
|
||||||
|
for _, input := range config.Inputs {
|
||||||
|
item := map[string]any{"name": input.Name}
|
||||||
|
if input.Required != nil {
|
||||||
|
item["required"] = *input.Required
|
||||||
|
}
|
||||||
|
if input.ContentType != nil {
|
||||||
|
item["content_type"] = *input.ContentType
|
||||||
|
}
|
||||||
|
if input.Description != nil {
|
||||||
|
item["description"] = *input.Description
|
||||||
|
}
|
||||||
|
inputs = append(inputs, item)
|
||||||
|
}
|
||||||
|
fields["inputs"] = inputs
|
||||||
|
}
|
||||||
|
if config.Messages != nil {
|
||||||
|
messages := make([]any, 0, len(config.Messages))
|
||||||
|
for _, message := range config.Messages {
|
||||||
|
item := make(map[string]any)
|
||||||
|
if message.Role != nil {
|
||||||
|
item["role"] = *message.Role
|
||||||
|
}
|
||||||
|
if message.Content != nil {
|
||||||
|
item["content"] = *message.Content
|
||||||
|
}
|
||||||
|
if message.ContentFile != nil {
|
||||||
|
item["content_file"] = *message.ContentFile
|
||||||
|
}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
cache := make(map[string]any)
|
||||||
|
if message.CacheControl.Type != nil {
|
||||||
|
cache["type"] = *message.CacheControl.Type
|
||||||
|
}
|
||||||
|
if message.CacheControl.TTL != nil {
|
||||||
|
cache["ttl"] = *message.CacheControl.TTL
|
||||||
|
}
|
||||||
|
item["cache_control"] = cache
|
||||||
|
}
|
||||||
|
messages = append(messages, item)
|
||||||
|
}
|
||||||
|
fields["messages"] = messages
|
||||||
|
}
|
||||||
|
if config.Output != nil {
|
||||||
|
output := make(map[string]any)
|
||||||
|
if config.Output.Format != nil {
|
||||||
|
output["format"] = *config.Output.Format
|
||||||
|
}
|
||||||
|
if config.Output.ValidationMode != nil {
|
||||||
|
output["validation_mode"] = *config.Output.ValidationMode
|
||||||
|
}
|
||||||
|
if config.Output.SchemaPath != nil {
|
||||||
|
output["schema_path"] = *config.Output.SchemaPath
|
||||||
|
}
|
||||||
|
if config.Output.RepairAttempts != nil {
|
||||||
|
output["repair_attempts"] = *config.Output.RepairAttempts
|
||||||
|
}
|
||||||
|
fields["output"] = output
|
||||||
|
}
|
||||||
|
definitions = append(definitions, fields)
|
||||||
|
}
|
||||||
|
return configkit.Mapping(label, map[string]any{"prompts": definitions})
|
||||||
|
}
|
||||||
85
resource_inspection.go
Normal file
85
resource_inspection.go
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
// ProfileInspection describes a resolved profile resource, before backend
|
||||||
|
// completion or request overrides. References are not resolved. APIKeyEnv is
|
||||||
|
// an environment-variable name, never a credential value. The result and its
|
||||||
|
// mutable values are caller-owned. Nil pointers denote absent fields; non-nil
|
||||||
|
// pointers preserve explicit zero values and empty collections. JSON and YAML
|
||||||
|
// use the indicated field names and omit absent optional fields.
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string `json:"id" yaml:"id"`
|
||||||
|
Backend *string `json:"backend,omitempty" yaml:"backend,omitempty"`
|
||||||
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
|
Model *string `json:"model,omitempty" yaml:"model,omitempty"`
|
||||||
|
Temperature *float64 `json:"temperature,omitempty" yaml:"temperature,omitempty"`
|
||||||
|
TopP *float64 `json:"top_p,omitempty" yaml:"top_p,omitempty"`
|
||||||
|
MaxTokens *int `json:"max_tokens,omitempty" yaml:"max_tokens,omitempty"`
|
||||||
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty" yaml:"timeout_seconds,omitempty"`
|
||||||
|
ServiceTier *string `json:"service_tier,omitempty" yaml:"service_tier,omitempty"`
|
||||||
|
ReasoningEffort *string `json:"reasoning_effort,omitempty" yaml:"reasoning_effort,omitempty"`
|
||||||
|
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
||||||
|
APIKeyRequired *bool `json:"api_key_required,omitempty" yaml:"api_key_required,omitempty"`
|
||||||
|
ExtraParams *map[string]any `json:"extra_params,omitempty" yaml:"extra_params,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendInspection describes a resolved backend resource, not runtime capacity
|
||||||
|
// or diagnostic destinations derived during execution. APIKeyEnv names an
|
||||||
|
// environment variable, not its value. The result and its mutable values are
|
||||||
|
// caller-owned. Nil pointers denote absent fields; non-nil pointers preserve
|
||||||
|
// explicit zero values and empty collections in JSON and YAML.
|
||||||
|
type BackendInspection struct {
|
||||||
|
BackendID string `json:"id" yaml:"id"`
|
||||||
|
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
||||||
|
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
||||||
|
ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"`
|
||||||
|
QueueCapacity *int `json:"queue_capacity,omitempty" yaml:"queue_capacity,omitempty"`
|
||||||
|
ExtraParams *map[string]any `json:"extra_params,omitempty" yaml:"extra_params,omitempty"`
|
||||||
|
SmokeTests *BackendSmokeTestConfig `json:"smoke_tests,omitempty" yaml:"smoke_tests,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInspection describes a resolved prompt resource. Messages contain
|
||||||
|
// captured inline or imported text, not rendered content. References are not
|
||||||
|
// resolved by inspection; PromptHash is opaque. The result and its mutable
|
||||||
|
// values are caller-owned. Nil pointers denote absent fields; non-nil pointers
|
||||||
|
// preserve explicit zero values and empty collections in JSON and YAML.
|
||||||
|
type PromptInspection struct {
|
||||||
|
PromptID string `json:"id" yaml:"id"`
|
||||||
|
PromptHash string `json:"prompt_hash" yaml:"prompt_hash"`
|
||||||
|
DefaultProfileID *string `json:"default_profile,omitempty" yaml:"default_profile,omitempty"`
|
||||||
|
Description *string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||||
|
SessionID *string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
|
||||||
|
Inputs *[]PromptInputInspection `json:"inputs,omitempty" yaml:"inputs,omitempty"`
|
||||||
|
Messages []PromptMessageInspection `json:"messages" yaml:"messages"`
|
||||||
|
OutputContract PromptOutputInspection `json:"output" yaml:"output"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInputInspection describes a declared input and its optional metadata.
|
||||||
|
type PromptInputInspection struct {
|
||||||
|
Name string `json:"name" yaml:"name"`
|
||||||
|
Required *bool `json:"required,omitempty" yaml:"required,omitempty"`
|
||||||
|
ContentType *string `json:"content_type,omitempty" yaml:"content_type,omitempty"`
|
||||||
|
Description *string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptMessageInspection describes a captured, unrendered message.
|
||||||
|
type PromptMessageInspection struct {
|
||||||
|
Role string `json:"role" yaml:"role"`
|
||||||
|
Content string `json:"content" yaml:"content"`
|
||||||
|
CacheControl *PromptCacheControlInspection `json:"cache_control,omitempty" yaml:"cache_control,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptCacheControlInspection describes configured message cache metadata.
|
||||||
|
// A non-nil TTL preserves an explicitly empty value.
|
||||||
|
type PromptCacheControlInspection struct {
|
||||||
|
Type CacheControlType `json:"type" yaml:"type"`
|
||||||
|
TTL *string `json:"ttl,omitempty" yaml:"ttl,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptOutputInspection describes a prompt's output contract. SchemaPath is
|
||||||
|
// a reference and is not loaded by inspection.
|
||||||
|
type PromptOutputInspection struct {
|
||||||
|
Format OutputFormat `json:"format" yaml:"format"`
|
||||||
|
ValidationMode ValidationMode `json:"validation_mode" yaml:"validation_mode"`
|
||||||
|
SchemaPath *string `json:"schema_path,omitempty" yaml:"schema_path,omitempty"`
|
||||||
|
RepairAttempts *int `json:"repair_attempts,omitempty" yaml:"repair_attempts,omitempty"`
|
||||||
|
}
|
||||||
51
resource_inspection_api_test.go
Normal file
51
resource_inspection_api_test.go
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResourceInspectionSelections(t *testing.T) {
|
||||||
|
engine, err := NewEngine(Config{Sources: ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1"}},
|
||||||
|
"profiles": []any{map[string]any{"id": "partial", "backend": "local"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic"}}},
|
||||||
|
})}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
if got, err := engine.InspectBackend(ctx, "local"); err != nil || got.BackendID != "local" {
|
||||||
|
t.Fatalf("backend = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := engine.InspectProfile(ctx, "partial"); err != nil || got.Backend == nil || *got.Backend != "local" || got.Endpoint != nil {
|
||||||
|
t.Fatalf("profile = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := engine.InspectPrompt(ctx, "prompt"); err != nil || got.Messages[0].Content != "hello" {
|
||||||
|
t.Fatalf("prompt = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := engine.InspectBackend(ctx, "missing"); got != nil || !errors.Is(err, ErrBackendNotFound) || errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectBackend(ctx, "bad/id/extra"); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("invalid ID = %v", err)
|
||||||
|
}
|
||||||
|
var nilEngine *Engine
|
||||||
|
if _, err := nilEngine.InspectBackend(ctx, "local"); !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine = %v", err)
|
||||||
|
}
|
||||||
|
canceled, cancel := context.WithCancel(ctx)
|
||||||
|
cancel()
|
||||||
|
if _, err := engine.InspectBackend(canceled, "local"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectPrompt(canceled, "prompt"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled prompt = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectProfile(canceled, "partial"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled profile = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
101
resource_isolation_test.go
Normal file
101
resource_isolation_test.go
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}}}, promptkit.WithLLMClient(client), promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
if got, err := engine.InspectPrompt(ctx, "reference"); err != nil || got.DefaultProfileID == nil || *got.DefaultProfileID != "missing" || got.OutputContract.SchemaPath == nil || *got.OutputContract.SchemaPath != "missing.json" || got.Messages[0].Content != "{{.missing}}" {
|
||||||
|
t.Fatalf("reference = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := engine.InspectProfile(ctx, "partial"); err != nil || got.Endpoint != nil || got.Model != nil {
|
||||||
|
t.Fatalf("partial = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
profile, err := engine.InspectProfile(ctx, "child")
|
||||||
|
if err != nil || profile.Endpoint != nil || profile.ExtraParams != nil || profile.APIKeyEnv != nil {
|
||||||
|
t.Fatalf("child = %+v, %v", profile, err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, field := range []string{"endpoint", "extra_params", "api_key_env"} {
|
||||||
|
path := configkit.Keys("profiles").Index(1).Key(field)
|
||||||
|
explanation, ok := view.Explain(path)
|
||||||
|
if !ok || !strings.Contains(explanation.Effective.YAML, "null") || len(explanation.History) == 0 {
|
||||||
|
t.Errorf("%s provenance = %+v, %t", field, explanation, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
backend, err := engine.InspectBackend(ctx, "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "missing" {
|
||||||
|
t.Fatalf("diagnostics = %+v", backend)
|
||||||
|
}
|
||||||
|
*backend.APIKeyEnv = "MUTATED"
|
||||||
|
*backend.SmokeTests.DefaultProfile = "mutated"
|
||||||
|
(*backend.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] = "mutated"
|
||||||
|
*profile.Model = "mutated"
|
||||||
|
prompt, err := engine.InspectPrompt(ctx, "ready")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
(*prompt.Inputs)[0].Name = "changed"
|
||||||
|
*(*prompt.Inputs)[0].Required = true
|
||||||
|
prompt.Messages[0].Content = "changed"
|
||||||
|
*prompt.Messages[0].CacheControl.TTL = "changed"
|
||||||
|
again, err := engine.InspectBackend(ctx, "local")
|
||||||
|
if err != nil || *again.APIKeyEnv != "UNSET_INSPECTION_KEY" || *again.SmokeTests.DefaultProfile != "missing" || (*again.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] != "captured" {
|
||||||
|
t.Fatalf("backend mutation leaked: %+v, %v", again, err)
|
||||||
|
}
|
||||||
|
againPrompt, err := engine.InspectPrompt(ctx, "ready")
|
||||||
|
if err != nil || (*againPrompt.Inputs)[0].Name != "x" || *(*againPrompt.Inputs)[0].Required || againPrompt.Messages[0].Content != "hello" || *againPrompt.Messages[0].CacheControl.TTL != "" {
|
||||||
|
t.Fatalf("prompt mutation leaked: %+v, %v", againPrompt, err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "ready", ProfileID: "child"})
|
||||||
|
if err != nil || prepared.EffectiveModelParams.Endpoint != "https://backend.test/v1" || prepared.EffectiveModelParams.APIKeyEnv != "UNSET_INSPECTION_KEY" || prepared.EffectiveModelParams.Model != "model" || prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["array"].([]any)[0] != "captured" {
|
||||||
|
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatal("inspection invoked provider")
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for j := 0; j < 20; j++ {
|
||||||
|
b, err := engine.InspectBackend(ctx, "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
(*b.ExtraParams)["nested"].(map[string]any)["array"].([]any)[0] = "local"
|
||||||
|
p, err := engine.InspectPrompt(ctx, "ready")
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.Messages[0].Content = "local"
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
124
resource_serialization_test.go
Normal file
124
resource_serialization_test.go
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||||
|
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
backend, err := engine.InspectBackend(ctx, "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plain, err := engine.InspectBackend(ctx, "plain")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile, err := engine.InspectProfile(ctx, "profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
unset, err := engine.InspectProfile(ctx, "unset")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompt, err := engine.InspectPrompt(ctx, "prompt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
empty, err := engine.InspectPrompt(ctx, "empty-inputs")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
omitted, err := engine.InspectPrompt(ctx, "omitted-inputs")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, codec := range map[string]struct {
|
||||||
|
marshal func(any) ([]byte, error)
|
||||||
|
unmarshal func([]byte, any) error
|
||||||
|
}{
|
||||||
|
"json": {json.Marshal, json.Unmarshal}, "yaml": {yaml.Marshal, yaml.Unmarshal},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
decode := func(v any) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
data, err := codec.marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m map[string]any
|
||||||
|
if err := codec.unmarshal(data, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
absent := func(m map[string]any, keys ...string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, key := range keys {
|
||||||
|
if _, ok := m[key]; ok {
|
||||||
|
t.Errorf("unexpected %s in %#v", key, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b := decode(backend)
|
||||||
|
if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
|
||||||
|
t.Errorf("backend = %#v", b)
|
||||||
|
}
|
||||||
|
params := b["extra_params"].(map[string]any)
|
||||||
|
if v, ok := params["null"]; !ok || v != nil || params["array"].([]any)[0] != nil {
|
||||||
|
t.Errorf("provider nulls = %#v", params)
|
||||||
|
}
|
||||||
|
smoke := b["smoke_tests"].(map[string]any)
|
||||||
|
if smoke["authentication"].(map[string]any)["kind"] != "none" {
|
||||||
|
t.Errorf("authentication = %#v", smoke)
|
||||||
|
}
|
||||||
|
if reach := smoke["reachability"].(map[string]any); len(reach) != 0 {
|
||||||
|
t.Errorf("reachability = %#v", reach)
|
||||||
|
}
|
||||||
|
absent(b, "extends")
|
||||||
|
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit")
|
||||||
|
p := decode(profile)
|
||||||
|
if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 {
|
||||||
|
t.Errorf("profile = %#v", p)
|
||||||
|
}
|
||||||
|
absent(p, "extends", "effective_model_params")
|
||||||
|
absent(decode(unset), "extra_params", "endpoint")
|
||||||
|
q := decode(prompt)
|
||||||
|
inputs := q["inputs"].([]any)
|
||||||
|
a := inputs[0].(map[string]any)
|
||||||
|
bInput := inputs[1].(map[string]any)
|
||||||
|
if a["required"] != false || a["description"] != "" {
|
||||||
|
t.Errorf("input = %#v", a)
|
||||||
|
}
|
||||||
|
absent(bInput, "required", "description")
|
||||||
|
message := q["messages"].([]any)[0].(map[string]any)
|
||||||
|
if message["content"] != "" || message["cache_control"].(map[string]any)["ttl"] != "" {
|
||||||
|
t.Errorf("message = %#v", message)
|
||||||
|
}
|
||||||
|
absent(message, "content_file", "template")
|
||||||
|
if fmt.Sprint(q["output"].(map[string]any)["repair_attempts"]) != "0" {
|
||||||
|
t.Errorf("output = %#v", q["output"])
|
||||||
|
}
|
||||||
|
if inputs := decode(empty)["inputs"].([]any); len(inputs) != 0 {
|
||||||
|
t.Errorf("empty inputs = %#v", inputs)
|
||||||
|
}
|
||||||
|
absent(decode(omitted), "inputs")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -422,7 +422,7 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(profilesBefore, profilesAfter) || !reflect.DeepEqual(promptsBefore, promptsAfter) ||
|
if !reflect.DeepEqual(profilesBefore, profilesAfter) || !reflect.DeepEqual(promptsBefore, promptsAfter) ||
|
||||||
inspectionBefore.PromptHash != inspectionAfter.PromptHash || inspectionAfter.OutputContract.SchemaPath != schemaMarker {
|
inspectionBefore.PromptHash != inspectionAfter.PromptHash || inspectionAfter.OutputContract.SchemaPath == nil || *inspectionAfter.OutputContract.SchemaPath != schemaMarker {
|
||||||
t.Fatalf("diagnostic changed captured listings or application inspection: profiles %v -> %v, prompts %v -> %v, inspection %+v -> %+v",
|
t.Fatalf("diagnostic changed captured listings or application inspection: profiles %v -> %v, prompts %v -> %v, inspection %+v -> %+v",
|
||||||
profilesBefore, profilesAfter, promptsBefore, promptsAfter, inspectionBefore, inspectionAfter)
|
profilesBefore, profilesAfter, promptsBefore, promptsAfter, inspectionBefore, inspectionAfter)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ func TestProgrammaticProfileVariantInherits(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profile, err := engine.InspectProfile(context.Background(), "child")
|
profile, err := engine.InspectProfile(context.Background(), "child")
|
||||||
if err != nil || profile.EffectiveModelParams.Model != model || profile.EffectiveModelParams.Endpoint != endpoint || profile.EffectiveModelParams.TimeoutSeconds != timeout {
|
if err != nil || profile.Model == nil || *profile.Model != model || profile.Endpoint == nil || *profile.Endpoint != endpoint || profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != timeout {
|
||||||
t.Fatalf("inherited profile = %+v, %v", profile, err)
|
t.Fatalf("inherited profile = %+v, %v", profile, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
65
types.go
65
types.go
@@ -85,6 +85,20 @@ const (
|
|||||||
// own private execution snapshot for RunPrepared. Excessively deep or large
|
// own private execution snapshot for RunPrepared. Excessively deep or large
|
||||||
// JSON-shaped values are rejected for safety.
|
// JSON-shaped values are rejected for safety.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
|
// CaptureExchange requests an immutable snapshot of a completed run, even
|
||||||
|
// when its structural validation fails. Operational errors return no result.
|
||||||
|
// The snapshot is returned in RunResult.Exchange; see [Exchange] for its
|
||||||
|
// logical history contract. Prepare and ValidateRequest create no exchange.
|
||||||
|
// PrepareExecution retains this choice for its eventual RunPrepared call.
|
||||||
|
CaptureExchange bool
|
||||||
|
// ContinueFrom supplies captured history and implies capture of the new run.
|
||||||
|
// A non-nil zero-value exchange is invalid. Follow-up settings are selected
|
||||||
|
// normally and are not inherited from the exchange.
|
||||||
|
// Messages are assembled as captured history, rendered definition messages,
|
||||||
|
// then AppendedMessages. Prior templates and inputs are not rendered or read
|
||||||
|
// again. Continuation adds no history-size limit or truncation; upstream
|
||||||
|
// rejection follows the ordinary generation-error contract.
|
||||||
|
ContinueFrom *Exchange `json:"-" yaml:"-"`
|
||||||
// PromptID is the required prompt identifier. A namespace-scoped engine view
|
// PromptID is the required prompt identifier. A namespace-scoped engine view
|
||||||
// qualifies a slashless caller value; otherwise it selects the exact
|
// qualifies a slashless caller value; otherwise it selects the exact
|
||||||
// configured ID.
|
// configured ID.
|
||||||
@@ -160,6 +174,8 @@ type RunRequest struct {
|
|||||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||||
// opaque.
|
// opaque.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
|
// ParentRunID identifies the immediate source exchange, if any.
|
||||||
|
ParentRunID string `json:"parent_run_id,omitempty"`
|
||||||
// PromptID is the selected prompt identifier.
|
// PromptID is the selected prompt identifier.
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
// PromptHash is an opaque equality value for the selected definition.
|
// PromptHash is an opaque equality value for the selected definition.
|
||||||
@@ -187,8 +203,8 @@ type PreparedRun struct {
|
|||||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
// Messages are the complete effective messages that Run or RunPrepared passes
|
// Messages are the complete effective messages that Run or RunPrepared passes
|
||||||
// to the LLM client: definition messages in rendered order followed by any
|
// to the LLM client: captured history, if any, followed by definition
|
||||||
// RunRequest.AppendedMessages in caller order.
|
// messages and RunRequest.AppendedMessages in caller order.
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
// StartTime is the UTC time at which preparation began.
|
// StartTime is the UTC time at which preparation began.
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
@@ -207,6 +223,10 @@ type PreparedRun struct {
|
|||||||
// use RFC 3339 and zero timing values are omitted. Run IDs and hash formats are
|
// use RFC 3339 and zero timing values are omitted. Run IDs and hash formats are
|
||||||
// opaque.
|
// opaque.
|
||||||
type RunResult struct {
|
type RunResult struct {
|
||||||
|
// Exchange holds captured conversation history when requested.
|
||||||
|
Exchange *Exchange `json:"-" yaml:"-"`
|
||||||
|
// ParentRunID identifies the immediate source exchange, if any.
|
||||||
|
ParentRunID string `json:"parent_run_id,omitempty"`
|
||||||
// RunID is an opaque identifier for this invocation.
|
// RunID is an opaque identifier for this invocation.
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
// Artifact contains the generated output and derived metadata.
|
// Artifact contains the generated output and derived metadata.
|
||||||
@@ -356,27 +376,6 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `json:"extra_params"`
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
|
|
||||||
// It has no stable JSON representation.
|
|
||||||
//
|
|
||||||
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
|
|
||||||
// separate from that target to preserve ExecutionTarget's general execution
|
|
||||||
// and stable JSON contracts.
|
|
||||||
type ProfileInspection struct {
|
|
||||||
// ProfileID is the exact profile ID inspected by the engine.
|
|
||||||
ProfileID string
|
|
||||||
// EffectiveModelParams contains settings resolved from the framework timeout
|
|
||||||
// baseline, selected backend, and then profile, without a request override.
|
|
||||||
// Unset optional provider controls remain zero rather than reporting a
|
|
||||||
// provider default. APIKeyEnv is an environment-variable name, never its
|
|
||||||
// credential value.
|
|
||||||
EffectiveModelParams ExecutionTarget
|
|
||||||
// APIKeyRequired reports that a later execution must supply a usable direct
|
|
||||||
// key or environment credential. An explicit profile environment source can
|
|
||||||
// coexist with this requirement.
|
|
||||||
APIKeyRequired bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptInputDefinition describes one declared prompt input.
|
// PromptInputDefinition describes one declared prompt input.
|
||||||
// It has no stable JSON representation.
|
// It has no stable JSON representation.
|
||||||
type PromptInputDefinition struct {
|
type PromptInputDefinition struct {
|
||||||
@@ -392,26 +391,6 @@ type PromptInputDefinition struct {
|
|||||||
Description string
|
Description string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
|
|
||||||
// It has no stable JSON representation.
|
|
||||||
//
|
|
||||||
// Inputs contains copied declared input metadata in definition order.
|
|
||||||
// OutputContract is the normalized contract declared by the prompt definition,
|
|
||||||
// rather than a request-level effective override. PromptHash is opaque.
|
|
||||||
type PromptInspection struct {
|
|
||||||
// PromptID is the exact selected prompt ID after any namespace-view
|
|
||||||
// qualification.
|
|
||||||
PromptID string
|
|
||||||
// PromptHash is the opaque equality value for the selected definition.
|
|
||||||
PromptHash string
|
|
||||||
// DefaultProfileID is declared metadata and is not resolved by inspection.
|
|
||||||
DefaultProfileID string
|
|
||||||
// Inputs contains caller-owned declared input metadata in definition order.
|
|
||||||
Inputs []PromptInputDefinition
|
|
||||||
// OutputContract is the normalized contract declared by the definition.
|
|
||||||
OutputContract OutputContract
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
||||||
// has no stable JSON representation.
|
// has no stable JSON representation.
|
||||||
//
|
//
|
||||||
|
|||||||
Reference in New Issue
Block a user