diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 0d74e9e..8d80fe3 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -1,13 +1,22 @@ # Configuration outcome review remediation -Status — two open findings from the stage-by-stage review of +Status — R1 complete, R2 open. Findings from the stage-by-stage review of [the implementation plan](implementation.md) at commit `8a22dd6`. Review findings concern the accepted [feature target](invalid-configuration-outcomes.md). ## R1 — Present ConfigKit diagnostics returned by request builders -Priority — P2. Status — open. Affects Stages 3 and 4. +Priority — P2. Status — complete. Affects Stages 3 and 4. + +Completed by sharing ConfigKit loading-error diagnostic adaptation between engine +loaders and all three request builders. Public-command regression coverage now +checks malformed configuration, warnings and semantic rejection, ordinary builder +failures, cancellation, preserved causes, and skipped engine loading across +validation, preparation, individual smoke, and aggregate smoke. The regression +fails against the original adapters using a temporary Go overlay and passes with +the fix. All three aligned-workspace module suites and maintained examples passed +with Go 1.27.0, offline dependencies, and read-only module metadata. ### Relevant implementation diff --git a/integrations/ckprompt/adapters.go b/integrations/ckprompt/adapters.go index fd3ecc9..669b85e 100644 --- a/integrations/ckprompt/adapters.go +++ b/integrations/ckprompt/adapters.go @@ -196,7 +196,8 @@ func WithRequestValidation[I, S any]( if build != nil { built, err := build(ctx, request, input, state) if err != nil { - return commandkit.Acquired[struct{}]{}, err + messages := promptKitLoadErrorDiagnostics(err, "") + return commandkit.Acquired[struct{}]{Diagnostics: messages.diagnostics, Explanation: messages.explanation}, err } concreteRequest = built } @@ -476,12 +477,7 @@ func loadPromptKitEngine[I, S any]( ) (*promptkit.Engine, *promptkit.ConfigurationInspection, promptKitAcquisition, error) { engine, err := load(ctx, selection, input, state) if err != nil { - var loadErr *configkit.LoadError - if errors.As(err, &loadErr) { - messages := promptKitDiagnostics("promptkit", side, loadErr.Sources(), loadErr.Diagnostics()) - return nil, nil, messages, err - } - return nil, nil, promptKitAcquisition{}, err + return nil, nil, promptKitLoadErrorDiagnostics(err, side), err } if engine == nil { return nil, nil, promptKitAcquisition{}, errors.New("PromptKit configuration loader returned a nil engine") @@ -509,6 +505,14 @@ func appendPromptKitAcquisition(left, right promptKitAcquisition) promptKitAcqui return left } +func promptKitLoadErrorDiagnostics(err error, side string) promptKitAcquisition { + var loadErr *configkit.LoadError + if errors.As(err, &loadErr) && loadErr != nil { + return promptKitDiagnostics("promptkit", side, loadErr.Sources(), loadErr.Diagnostics()) + } + return promptKitAcquisition{} +} + func promptKitDiagnostics(scope, side string, sources []configkit.SourceInfo, records []configkit.Diagnostic) promptKitAcquisition { heading := "Configuration scope " + strconv.Quote(scope) if side != "" { diff --git a/integrations/ckprompt/configuration_outcomes_test.go b/integrations/ckprompt/configuration_outcomes_test.go index f9fef86..21b5ce8 100644 --- a/integrations/ckprompt/configuration_outcomes_test.go +++ b/integrations/ckprompt/configuration_outcomes_test.go @@ -90,21 +90,96 @@ func TestPromptNativeConfigurationFailures(t *testing.T) { } }) } - engine := adapterEngine(t) - good := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { - return engine, nil - } - validation := WithRequestValidation(Suite[struct{}, struct{}]{}, good, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { - return promptkit.RunRequest{}, loadErr + +} + +func TestBuilderConfigurationFailureDiagnostics(t *testing.T) { + _, malformed := configkit.Load[map[string]any](configkit.Bytes("builder-config", []byte("broken: ["))) + path := configkit.Keys("version") + _, rejected := configkit.Load(configkit.Bytes("builder-config", []byte("version: 1\n")), configkit.Options[map[string]any]{ + Definition: configkit.Definition[map[string]any]{Version: &configkit.VersionRule{ + Path: path, + Check: func(_ configkit.Version, validation *configkit.Validation) error { + validation.Warning(path, "builder warning") + validation.Error(path, "builder rejection") + return nil + }, + }}, }) - preparation := WithRequestPreparation(validation, good, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { - return promptkit.RunRequest{}, loadErr - }) - app = newPromptTestApp(t, Command(preparation)) - for _, args := range [][]string{{"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} { - result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) - if result.Status != 2 || !errors.Is(result.Cause, loadErr) { - t.Fatalf("builder %v: %#v", args, result) + for _, fixture := range []struct { + name string + cause error + code string + warning string + status int + }{ + {"malformed", malformed, "yaml.invalid", "", 2}, + {"rejected", rejected, "version.error", "builder warning", 2}, + {"application failure", errors.New("builder unavailable"), "", "", 1}, + } { + if fixture.cause == nil { + t.Fatalf("%s fixture succeeded", fixture.name) + } + for _, args := range [][]string{ + {"promptkit", "requirements", "validate", "example"}, + {"promptkit", "show", "prepared", "example"}, + {"promptkit", "test", "reachability"}, + {"promptkit", "test", "all"}, + } { + for _, canceled := range []bool{false, true} { + name := fmt.Sprintf("%s/%s/canceled=%t", fixture.name, strings.Join(args, "_"), canceled) + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + loaderCalls, builderCalls := 0, 0 + load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loaderCalls++ + return nil, errors.New("unexpected engine load") + } + wrapped := fmt.Errorf("builder load: %w", fixture.cause) + buildError := func() error { + builderCalls++ + if canceled { + cancel() + } + return wrapped + } + suite := WithRequestValidation(Suite[struct{}, struct{}]{}, load, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { + return promptkit.RunRequest{}, buildError() + }) + suite = WithRequestPreparation(suite, load, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { + return promptkit.RunRequest{}, buildError() + }) + suite = WithSmokeTesting(suite, load, func(context.Context, SmokeRequest, struct{}, struct{}) (promptkit.SmokeTestRequest, error) { + return promptkit.SmokeTestRequest{}, buildError() + }) + app := newPromptTestApp(t, Command(suite)) + var out, diagnostic bytes.Buffer + result := app.Execute(ctx, append(append([]string{}, args...), "--verbose"), commandkit.Streams{Out: &out, Err: &diagnostic}) + wantStatus, wantCalls := fixture.status, 1 + if args[len(args)-1] == "all" && !canceled { + wantCalls = 4 + } + if canceled { + wantStatus = 1 + if !errors.Is(result.Cause, context.Canceled) { + t.Fatalf("lost cancellation: %#v", result) + } + } + if result.Status != wantStatus || !errors.Is(result.Cause, wrapped) || loaderCalls != 0 || builderCalls != wantCalls { + t.Fatalf("result=%#v loader calls=%d builder calls=%d", result, loaderCalls, builderCalls) + } + if fixture.code != "" { + var expected, retained *configkit.LoadError + if !errors.As(fixture.cause, &expected) || !errors.As(result.Cause, &retained) || retained != expected { + t.Fatalf("lost loading error: %#v", result) + } + if !strings.Contains(diagnostic.String(), fixture.code) || !strings.Contains(diagnostic.String(), "builder-config") || !strings.Contains(diagnostic.String(), fixture.warning) { + t.Fatalf("diagnostics: %q", diagnostic.String()) + } + } + }) + } } } } diff --git a/integrations/ckprompt/preparation.go b/integrations/ckprompt/preparation.go index 430b9aa..82972c7 100644 --- a/integrations/ckprompt/preparation.go +++ b/integrations/ckprompt/preparation.go @@ -43,7 +43,8 @@ func WithRequestPreparation[I, S any]( if build != nil { built, err := build(ctx, request, input, state) if err != nil { - return commandkit.Acquired[*promptkit.PreparedRun]{}, err + messages := promptKitLoadErrorDiagnostics(err, "") + return commandkit.Acquired[*promptkit.PreparedRun]{Diagnostics: messages.diagnostics, Explanation: messages.explanation}, err } concrete = built } diff --git a/integrations/ckprompt/smoke.go b/integrations/ckprompt/smoke.go index b36467c..d7ceb0c 100644 --- a/integrations/ckprompt/smoke.go +++ b/integrations/ckprompt/smoke.go @@ -51,7 +51,8 @@ func WithSmokeTesting[I, S any]( if build != nil { built, err := build(ctx, request, input, state) if err != nil { - return commandkit.Acquired[*promptkit.SmokeTestReport]{}, err + messages := promptKitLoadErrorDiagnostics(err, "") + return commandkit.Acquired[*promptkit.SmokeTestReport]{Diagnostics: messages.diagnostics, Explanation: messages.explanation}, err } concreteRequest = built }