base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate PromptKit inspection reports to the reporter

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:30:19 +00:00
parent 37dfea5b25
commit 7fc89768ef
3 changed files with 204 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
integrations/ckprompt/command_test.go
View File

@@ -701,41 +701,39 @@ func TestCatalogRejectsOperandsAndUnavailableSelectors(t *testing.T) {
}
}
func TestCatalogFailuresRetainDiagnosticsAndDeliveryState(t *testing.T) {
diagnostic := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "catalog warning\n")
return err
}}
hasCatalogWarning := func(diagnostics []commandkit.Diagnostic) bool {
for _, got := range diagnostics {
if got.Severity != diagnostic.Severity || got.Write == nil {
continue
}
var output bytes.Buffer
if got.Write(&output) == nil && output.String() == "catalog warning\n" {
return true
}
}
return false
}
func TestCatalogFailureReportsWarningAndNativeExplanationSeparately(t *testing.T) {
diagnostic := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "catalog warning")
providerErr := errors.New("catalog provider failed")
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
return commandkit.Acquired[[]promptkit.PromptIdentity]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, providerErr
return commandkit.Acquired[[]promptkit.PromptIdentity]{
Diagnostics: []commandkit.Diagnostic{diagnostic},
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
_, err := io.WriteString(writer, "PromptKit lookup context\n")
return err
}}},
}, providerErr
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !hasCatalogWarning(outcome.Diagnostics) {
var output, diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"--verbose", "promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) {
t.Fatalf("provider failure outcome = %#v", outcome)
}
warningAt := strings.Index(diagnosticOutput.String(), "catalog warning")
explanationAt := strings.Index(diagnosticOutput.String(), "PromptKit lookup context")
if warningAt < 0 || explanationAt < 0 || warningAt >= explanationAt || strings.Count(diagnosticOutput.String(), "catalog warning") != 1 {
t.Fatalf("warning/native terminal context were not independently ordered once: %q", diagnosticOutput.String())
}
ctx, cancel := context.WithCancel(context.Background())
canceled := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
cancel()
return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{{ID: "briefing", Version: testStringPointer("1")}}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil
}}))
var output bytes.Buffer
outcome = canceled.Execute(ctx, []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !hasCatalogWarning(outcome.Diagnostics) || output.Len() != 0 {
t.Fatalf("post-provider cancellation outcome = %#v output=%q", outcome, output.String())
output.Reset()
diagnosticOutput.Reset()
outcome = canceled.Execute(ctx, []string{"--verbose", "promptkit", "list", "prompts"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnosticOutput.String(), "catalog warning") || output.Len() != 0 {
t.Fatalf("post-provider cancellation outcome = %#v output=%q err=%q", outcome, output.String(), diagnosticOutput.String())
}
writeErr := errors.New("catalog report writer failed")
@@ -752,13 +750,96 @@ func TestCatalogFailuresRetainDiagnosticsAndDeliveryState(t *testing.T) {
return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{{ID: "briefing", Version: testStringPointer("1")}}, Diagnostics: []commandkit.Diagnostic{diagnostic}}, nil
}}))
outcome := writerApp.Execute(context.Background(), []string{"promptkit", "list", "prompts"}, commandkit.Streams{Out: test.writer, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.DeliveryError, test.want) || !errors.Is(outcome.Cause, test.want) || !outcome.ReportIncomplete || !hasCatalogWarning(outcome.Diagnostics) {
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete {
t.Fatalf("catalog delivery outcome = %#v", outcome)
}
})
}
}
func TestCatalogAndInspectionRespectSuccessfulReportingModes(t *testing.T) {
for _, mode := range []string{"ordinary", "quiet", "verbose", "debug"} {
t.Run(mode, func(t *testing.T) {
calls := 0
suite := Suite[struct{}, struct{}]{
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
calls++
return commandkit.Acquired[[]promptkit.PromptIdentity]{Value: []promptkit.PromptIdentity{{ID: "briefing"}}}, nil
},
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
calls++
return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: "briefing"}}, nil
},
}
app := newPromptTestApp(t, Command(suite))
args := []string{"promptkit", "list", "prompts"}
if mode != "ordinary" {
args = append([]string{"--" + mode}, args...)
}
var output, diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeSuccess || calls != 1 {
t.Fatalf("%s listing = %#v after %d calls", mode, outcome, calls)
}
if mode == "quiet" {
if output.Len() != 0 {
t.Fatalf("quiet listing wrote result output %q", output.String())
}
} else if !strings.Contains(output.String(), "briefing") {
t.Fatalf("%s listing omitted result: %q", mode, output.String())
}
if mode == "debug" && !strings.Contains(diagnosticOutput.String(), "DEBUG:") {
t.Fatalf("debug listing omitted core debug events: %q", diagnosticOutput.String())
}
output.Reset()
diagnosticOutput.Reset()
args = []string{"promptkit", "show", "prompt", "briefing"}
if mode != "ordinary" {
args = append([]string{"--" + mode}, args...)
}
outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeSuccess || calls != 2 {
t.Fatalf("%s inspection = %#v after %d callbacks", mode, outcome, calls)
}
if mode == "quiet" {
if output.Len() != 0 {
t.Fatalf("quiet inspection wrote result output %q", output.String())
}
} else if !strings.Contains(output.String(), "Prompt ID: \"briefing\"") {
t.Fatalf("%s inspection omitted result: %q", mode, output.String())
}
})
}
}
func TestQuietInspectionStillFindsReportEncodingFailure(t *testing.T) {
called := false
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
called = true
return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{
ProfileID: "broken",
EffectiveModelParams: promptkit.ExecutionTarget{ExtraParams: map[string]any{"unsupported": func() {}}},
}}, nil
}}))
result := &countingWriter{}
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"--quiet", "promptkit", "show", "profile", "broken"}, commandkit.Streams{Out: result, Err: &diagnosticOutput})
if !called || result.calls != 0 {
t.Fatalf("quiet rendering called=%t destination writes=%d", called, result.calls)
}
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || outcome.Presentation.Output.RenderError == nil {
t.Fatalf("quiet serialization failure outcome = %#v diagnostics=%q", outcome, diagnosticOutput.String())
}
}
type countingWriter struct{ calls int }
func (writer *countingWriter) Write(p []byte) (int, error) {
writer.calls++
return len(p), nil
}
func TestCommandUsesInstalledNamespaceCollisionChecks(t *testing.T) {
command := Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
return commandkit.Acquired[*promptkit.PromptInspection]{}, nil
@@ -1055,7 +1136,7 @@ func TestDefinitionFormatPreflightAndReportDeliveryFailure(t *testing.T) {
writeErr := errors.New("report writer failed")
outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: partialReportWriter{err: writeErr}, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, writeErr) || outcome.DeliveryError != writeErr || !outcome.ReportIncomplete || calls != 1 {
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, writeErr) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || calls != 1 {
t.Fatalf("report delivery outcome = %#v calls=%d", outcome, calls)
}
}