base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Migrate PromptKit validation and smoke reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:35:39 +00:00
parent 4ace616bc2
commit 401232a1b0
6 changed files with 158 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
integrations/ckprompt/smoke_outcome_test.go
View File

@@ -105,10 +105,9 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
} {
t.Run(test.name, func(t *testing.T) {
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error())
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex {
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || warningIndex < 0 || strings.Contains(diagnosticOutput.String(), operationErr.Error()) {
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
}
})
@@ -116,8 +115,8 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
diagnosticErr := errors.New("smoke diagnostic writer failed")
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 {
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete || output.Len() == 0 {
t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
}
@@ -130,7 +129,7 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
}))
output.Reset()
outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 {
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || output.Len() != 0 {
t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
}
}
@@ -151,15 +150,24 @@ func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
case promptkit.SmokeTestAuthentication:
err = secondErr
}
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err
acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}
if err != nil {
acquired.Explanation = commandkit.Explanation{Message: string(request.Request.Kind) + " native context"}
}
return acquired, err
},
}))
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
var output, diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
var retained *smokeTypedError
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
}
firstContext := strings.Index(diagnosticOutput.String(), "reachability native context")
secondContext := strings.Index(diagnosticOutput.String(), "authentication native context")
if firstContext < 0 || secondContext < 0 || firstContext >= secondContext {
t.Fatalf("aggregate native explanations were not retained in order: %q", diagnosticOutput.String())
}
var document struct {
Tests []struct {
Kind string `yaml:"kind"`