base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate PromptKit validation and smoke reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:35:39 +00:00
parent 4ace616bc2
commit 401232a1b0
6 changed files with 158 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
integrations/ckprompt/validation_command_test.go
View File

@@ -98,7 +98,7 @@ func TestValidationCommandIsAvailableIndependentlyAndOwnsItsFamily(t *testing.T)
}}))
var help bytes.Buffer
outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "show") {
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "\n show ") {
t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String())
}
@@ -359,41 +359,32 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T)
}
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
withWarning := func(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
var diagnostics bytes.Buffer
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "validation warning") {
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
}
var quietReport, quietDiagnostics bytes.Buffer
quietOutcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--quiet"}, commandkit.Streams{Out: &quietReport, Err: &quietDiagnostics})
if quietOutcome.Kind != commandkit.OutcomeSuccess || quietOutcome.Presentation.Output.State != commandkit.DeliverySuppressed || quietOutcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed || quietReport.Len() != 0 || quietDiagnostics.Len() != 0 {
t.Fatalf("quiet validation = %#v, report=%q diagnostics=%q", quietOutcome, quietReport.String(), quietDiagnostics.String())
}
providerErr := errors.New("validation provider failed")
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
diagnostics.Reset()
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete {
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete {
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
}
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Count(diagnostics.String(), providerErr.Error()) != 1 {
t.Fatalf("provider diagnostics were lost or duplicated: stderr=%q", diagnostics.String())
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Contains(diagnostics.String(), providerErr.Error()) || !strings.Contains(diagnostics.String(), "Unable to validate the PromptKit request") {
t.Fatalf("provider warning or terminal explanation was incorrect: stderr=%q", diagnostics.String())
}
builderErr := errors.New("request builder failed")
@@ -418,16 +409,13 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes
}, nil)))
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
var retained *configkit.LoadError
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
}
}
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
calls := 0
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
calls++
@@ -448,8 +436,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
output.Reset()
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
var diagnostics bytes.Buffer
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnostics.String(), "validation warning") || output.Len() != 0 {
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
}
@@ -459,8 +448,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
cancel()
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
var errorDiagnostics bytes.Buffer
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &errorDiagnostics})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !strings.Contains(errorDiagnostics.String(), "validation warning") {
t.Fatalf("provider error after cancellation = %#v", outcome)
}
}
@@ -484,28 +474,16 @@ func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testin
} {
t.Run(test.name, func(t *testing.T) {
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
}
})
}
diagnosticErr := errors.New("validation diagnostic writer failed")
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery = %#v", outcome)
}
}
func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}