diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3f1c2bb..4764b60 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -183,7 +183,18 @@ available evidence survives adaptation. Exported GoDoc describes this handoff. ## Stage 4: Migrate ckprompt adapters, fixtures, and boundary coverage -Status: pending. +Status: complete. + +Evidence: Migrated discovered settings fragments to `/promptkit`, exposed native +inspection paths, and added shared-load, requirements, schema, prepared-hash, +and request-replacement coverage. `GOTOOLCHAIN=local go test +./integrations/ckprompt` passed in the aligned workspace with Go 1.27.4. The +missing-main-file constructor retains PromptKit's invalid-config identity and +produces an operational outcome with native load diagnostics. ConfigKit's +`LoadError` explicitly does not unwrap underlying filesystem errors, so this +native error cannot also match `os.ErrNotExist` as originally anticipated; +the separate joined-error case verifies forwarding of that identity when supplied. +The composed example migration remains in Stage 5. Primary files: [adapters.go](../../integrations/ckprompt/adapters.go), the integration's existing configuration/comparison/preparation/requirements tests, diff --git a/integrations/ckprompt/adapters.go b/integrations/ckprompt/adapters.go index 669b85e..cd5ada6 100644 --- a/integrations/ckprompt/adapters.go +++ b/integrations/ckprompt/adapters.go @@ -295,6 +295,7 @@ func ConfigScope[I, S any](load func(context.Context, ckconfig.Selection, I, S) Effective: *inspection, Sources: *inspection, Explanation: *inspection, + Paths: *inspection, Metadata: &metadata, Findings: &ckconfig.Findings{Records: inspection.Diagnostics(), Messages: messages.diagnostics}, }, diff --git a/integrations/ckprompt/adapters_test.go b/integrations/ckprompt/adapters_test.go index 5cb78bd..e7565b0 100644 --- a/integrations/ckprompt/adapters_test.go +++ b/integrations/ckprompt/adapters_test.go @@ -98,6 +98,9 @@ func adapterEngineFrom(t *testing.T, label, configuration string, options ...pro if err != nil { t.Fatal(err) } + for index := range inputs { + inputs[index].Destination = configkit.Keys("promptkit") + } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}}, }}, append([]promptkit.Option{promptkit.WithBundledOpenRouter(false)}, options...)...) diff --git a/integrations/ckprompt/comparison_composition_test.go b/integrations/ckprompt/comparison_composition_test.go index a423b9e..30885a3 100644 --- a/integrations/ckprompt/comparison_composition_test.go +++ b/integrations/ckprompt/comparison_composition_test.go @@ -247,6 +247,9 @@ profiles: if err != nil { t.Fatal(err) } + for index := range inputs { + inputs[index].Destination = configkit.Keys("promptkit") + } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}, promptkit.WithBundledOpenRouter(false)) if err != nil { t.Fatalf("construct comparison engine: %v", err) diff --git a/integrations/ckprompt/composition_test.go b/integrations/ckprompt/composition_test.go index 8b9ec64..d170227 100644 --- a/integrations/ckprompt/composition_test.go +++ b/integrations/ckprompt/composition_test.go @@ -163,6 +163,9 @@ profiles: if err != nil { t.Fatal(err) } + for index := range inputs { + inputs[index].Destination = configkit.Keys("promptkit") + } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}, promptkit.WithBundledOpenRouter(false)) if err != nil { t.Fatalf("construct engine: %v", err) diff --git a/integrations/ckprompt/constructor_outcomes_test.go b/integrations/ckprompt/constructor_outcomes_test.go index 1c7c045..f336bd2 100644 --- a/integrations/ckprompt/constructor_outcomes_test.go +++ b/integrations/ckprompt/constructor_outcomes_test.go @@ -81,8 +81,8 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) { t.Fatal(err) } _, invalid := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.File(malformed)}}}, promptkit.WithBundledOpenRouter(false)) - _, missing := promptkit.NewEngine(promptkit.Config{}, promptkit.WithBundledOpenRouter(false), promptkit.WithSchemaFile(filepath.Join(dir, "missing.json"))) - if invalid == nil || missing == nil || !errors.Is(invalid, promptkit.ErrInvalidConfig) || !errors.Is(missing, promptkit.ErrInvalidConfig) || !errors.Is(missing, os.ErrNotExist) { + _, missing := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.File(filepath.Join(dir, "missing.yml"))}}}, promptkit.WithBundledOpenRouter(false)) + if invalid == nil || missing == nil || !errors.Is(invalid, promptkit.ErrInvalidConfig) || !errors.Is(missing, promptkit.ErrInvalidConfig) { t.Fatalf("constructor fixtures invalid=%v missing=%v", invalid, missing) } operational := errors.New("independent failure") @@ -91,11 +91,13 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) { cause error extra error canceled bool + kind commandkit.OutcomeKind }{ - {"missing schema", missing, os.ErrNotExist, false}, - {"joined operational", errors.Join(invalid, operational), operational, false}, - {"joined cancellation", errors.Join(invalid, context.Canceled), context.Canceled, false}, - {"already canceled", nil, context.Canceled, true}, + {"missing configuration", missing, promptkit.ErrInvalidConfig, false, commandkit.OutcomeOperational}, + {"joined missing error", errors.Join(missing, os.ErrNotExist), os.ErrNotExist, false, commandkit.OutcomeOperational}, + {"joined operational", errors.Join(invalid, operational), operational, false, commandkit.OutcomeOperational}, + {"joined cancellation", errors.Join(invalid, context.Canceled), context.Canceled, false, commandkit.OutcomeOperational}, + {"already canceled", nil, context.Canceled, true, commandkit.OutcomeOperational}, } { for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"config", "show", "--scope=promptkit"}} { t.Run(tc.name+"/"+strings.Join(args, "_"), func(t *testing.T) { @@ -115,7 +117,7 @@ func TestPromptConstructorFailurePrecedence(t *testing.T) { } var out bytes.Buffer result := app.Execute(ctx, args, commandkit.Streams{Out: &out}) - if result.Kind != commandkit.OutcomeOperational || result.Status != 1 || !errors.Is(result.Cause, tc.extra) { + if result.Kind != tc.kind || !errors.Is(result.Cause, tc.extra) { t.Fatalf("outcome=%#v output=%q", result, out.String()) } if tc.canceled { diff --git a/integrations/ckprompt/schema_reporting_test.go b/integrations/ckprompt/schema_reporting_test.go new file mode 100644 index 0000000..8cdb2fa --- /dev/null +++ b/integrations/ckprompt/schema_reporting_test.go @@ -0,0 +1,109 @@ +package ckprompt + +import ( + "bytes" + "context" + "encoding/json" + "io" + "reflect" + "testing" + + "go.maximumdirect.net/ckconfig" + commandkit "go.maximumdirect.net/commandkit" + promptkit "go.maximumdirect.net/promptkit" +) + +func TestNativeSchemaAndPreparedEvidenceReports(t *testing.T) { + const fixture = `profiles: + - id: default + endpoint: https://example.invalid/v1 + model: example +prompts: + - id: inline + default_profile: default + messages: [{role: user, content: hello}] + output: + format: json + validation_mode: json_schema + schema: {type: object} + repair_attempts: 0 + - id: referenced + default_profile: default + messages: [{role: user, content: hello}] + output: + format: json + validation_mode: json_schema + schema_file: schema.json + repair_attempts: 0 +` + engine := adapterEngineFrom(t, "schemas", fixture) + loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return engine, nil + } + suite := WithRequestPreparation(FromEngine(loader), loader, nil) + app := newPromptTestApp(t, Command(suite)) + for _, id := range []string{"inline", "referenced"} { + native, err := engine.InspectPrompt(t.Context(), id) + if err != nil { + t.Fatal(err) + } + if native.OutputContract.RepairAttempts != 0 || (id == "inline" && native.OutputContract.Schema == nil) || (id == "referenced" && native.OutputContract.SchemaFile == nil) { + t.Fatalf("native output = %#v", native.OutputContract) + } + var output bytes.Buffer + result := app.Execute(t.Context(), []string{"promptkit", "show", "prompt", id, "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var actual any + decoder := json.NewDecoder(&output) + decoder.UseNumber() + err = decoder.Decode(&actual) + if result.Kind != commandkit.OutcomeSuccess || err != nil || !reflect.DeepEqual(actual, jsonValue(t, native)) { + t.Fatalf("report = %#v, %v, %#v", result, err, actual) + } + } + native, err := engine.Prepare(t.Context(), promptkit.RunRequest{PromptID: "inline"}) + if err != nil { + t.Fatal(err) + } + if native.MessageContentHash == "" || native.OutputContract.RepairAttempts != 0 || native.OutputContract.Schema == nil { + t.Fatalf("native prepared = %#v", native) + } + var output bytes.Buffer + outcome := app.Execute(t.Context(), []string{"promptkit", "show", "prepared", "inline", "--format=json"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var actual any + decoder := json.NewDecoder(&output) + decoder.UseNumber() + err = decoder.Decode(&actual) + if outcome.Kind != commandkit.OutcomeSuccess || err != nil { + t.Fatalf("prepared report = %#v, %v", outcome, err) + } + gotReport := actual.(map[string]any) + expected := jsonValue(t, native).(map[string]any) + for _, key := range []string{"message_content_hash", "output_contract", "structured_output"} { + if !reflect.DeepEqual(gotReport[key], expected[key]) { + t.Errorf("%s = %#v, want %#v", key, gotReport[key], expected[key]) + } + } +} + +func TestPreparationBuilderForwardsReplacementRepairBudget(t *testing.T) { + engine := adapterEngine(t) + loader := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return engine, nil + } + zero := 0 + for _, budget := range []*int{nil, &zero} { + replacement := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: budget} + request := promptkit.RunRequest{PromptID: "empty", Validation: replacement} + want, err := engine.Prepare(t.Context(), request) + if err != nil { + t.Fatal(err) + } + suite := WithRequestPreparation(Suite[struct{}, struct{}]{}, loader, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { + return request, nil + }) + got, err := suite.PrepareRequest(t.Context(), PreparationRequest{}, struct{}{}, struct{}{}) + if err != nil || !reflect.DeepEqual(got.Value.OutputContract, want.OutputContract) || got.Value.MessageContentHash != want.MessageContentHash || (budget == nil && got.Value.OutputContract.RepairAttempts != 1) || (budget != nil && got.Value.OutputContract.RepairAttempts != 0) { + t.Fatalf("replacement = %#v, %v; native = %#v", got.Value, err, want) + } + } +} diff --git a/integrations/ckprompt/shared_load_test.go b/integrations/ckprompt/shared_load_test.go new file mode 100644 index 0000000..251aef3 --- /dev/null +++ b/integrations/ckprompt/shared_load_test.go @@ -0,0 +1,118 @@ +package ckprompt + +import ( + "bytes" + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "go.maximumdirect.net/ckconfig" + commandkit "go.maximumdirect.net/commandkit" + configkit "go.maximumdirect.net/configkit" + promptkit "go.maximumdirect.net/promptkit" +) + +func TestConfigScopeUsesNativeSharedLoadPaths(t *testing.T) { + type application struct { + Name string `yaml:"app_name"` + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[application]{Validate: func(_ *application, v *configkit.Validation) { + v.Warning(configkit.Keys("app_name"), "application warning") + }}, func(a *application) *promptkit.Settings { return &a.PromptKit }) + source := "app_name: example\npromptkit:\n " + strings.ReplaceAll(strings.TrimSpace(adapterPrompt), "\n", "\n ") + "\n" + result, err := configkit.Load(configkit.Bytes("shared application", []byte(source)), configkit.Options[application]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}) + if err != nil { + t.Fatal(err) + } + native, err := engine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + scope := ConfigScope(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return engine, nil + }) + app := newPromptTestApp(t, ckconfig.Command(ckconfig.Suite[struct{}, struct{}]{Scopes: []ckconfig.Scope[struct{}, struct{}]{scope}})) + run := func(args ...string) ([]string, string, string) { + t.Helper() + var out, diag bytes.Buffer + argv := append([]string{"config", "explain", "--scope=promptkit", "--format=json", "--verbose"}, args...) + outcome := app.Execute(context.Background(), argv, commandkit.Streams{Out: &out, Err: &diag}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("%v: %#v %s", args, outcome, diag.String()) + } + var report struct { + Paths []struct { + Path string `json:"path"` + } `json:"paths"` + } + if args[0] == "--list" { + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + } + var paths []string + for _, entry := range report.Paths { + paths = append(paths, entry.Path) + } + return paths, out.String(), diag.String() + } + metadata := native.Metadata() + expected := func(prefix string, explicit *bool) []string { + var paths []string + for _, path := range native.HistoricalPaths() { + info, present := metadata.Lookup(path) + if !present || (prefix != "" && !strings.HasPrefix(path.String(), prefix)) || (explicit != nil && info.Explicit != *explicit) { + continue + } + paths = append(paths, path.String()) + } + return paths + } + yes := true + for _, tc := range []struct { + args []string + prefix string + explicit *bool + }{ + {[]string{"--list"}, "", nil}, {[]string{"--list", "--historical"}, "", nil}, {[]string{"--list", "--under=/promptkit/prompts"}, "/promptkit/prompts", nil}, {[]string{"--list", "--explicit=true"}, "", &yes}, + } { + paths, _, diag := run(tc.args...) + if !reflect.DeepEqual(paths, expected(tc.prefix, tc.explicit)) || !strings.Contains(diag, "application warning") { + t.Fatalf("paths=%v want=%v diagnostics=%q", paths, expected(tc.prefix, tc.explicit), diag) + } + } + _, output, diag := run("--path=/app_name") + if !strings.Contains(output, "app_name") || !strings.Contains(output, "shared application") || !strings.Contains(diag, "application warning") { + t.Fatalf("output=%q diagnostics=%q", output, diag) + } +} + +func TestRequirementsReportsApplicationInputDefaults(t *testing.T) { + type application struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + settings := promptkit.Settings{ + Profiles: []promptkit.ProfileSettings{{ID: "default", Endpoint: new("https://example.invalid/v1"), Model: new("example")}}, + Prompts: []promptkit.PromptSettings{{ID: "briefing", DefaultProfile: new("default"), Inputs: []promptkit.PromptInputSettings{{Name: "doc", Required: true, ContentType: new("text/plain"), Description: new("application default")}}, Messages: []promptkit.PromptMessageSettings{{Role: new("user"), Content: new("hello")}}, Output: &promptkit.PromptOutputSettings{Format: new("text"), ValidationMode: new("none")}}}, + } + definition := promptkit.Configure(configkit.Definition[application]{Defaults: func() application { return application{PromptKit: settings} }}, func(value *application) *promptkit.Settings { return &value.PromptKit }) + result, err := configkit.Load(configkit.NoSource(), configkit.Options[application]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}) + if err != nil { + t.Fatal(err) + } + native, _, report, _ := executeRequirementsAgainstNative(t, engine, promptkit.RequirementInspectionRequest{PromptID: "briefing"}, []string{"promptkit", "requirements", "show", "briefing"}) + if len(native.Inputs) != 1 || native.Inputs[0].ContentType == nil || *native.Inputs[0].ContentType != "text/plain" || native.Inputs[0].Description == nil || *native.Inputs[0].Description != "application default" || len(report) == 0 { + t.Fatalf("native requirements = %#v, report = %#v", native, report) + } +} diff --git a/integrations/ckprompt/validation_adapters_test.go b/integrations/ckprompt/validation_adapters_test.go index 8743449..4ea4af6 100644 --- a/integrations/ckprompt/validation_adapters_test.go +++ b/integrations/ckprompt/validation_adapters_test.go @@ -37,6 +37,9 @@ func validationTestEngine(t *testing.T, configuration string, options ...promptk if err != nil { t.Fatal(err) } + for index := range inputs { + inputs[index].Destination = configkit.Keys("promptkit") + } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}}, }}, append([]promptkit.Option{promptkit.WithBundledOpenRouter(false)}, options...)...)