base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate PromptKit validation and smoke reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:35:39 +00:00
parent 4ace616bc2
commit 401232a1b0
6 changed files with 158 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
integrations/ckprompt/smoke_command.go
View File

@@ -184,14 +184,14 @@ func checkSmokeFormat(format string) error {
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
if input.kind == "all" {
return runSmokeAggregate(invocation, suite, input, state)
@@ -199,6 +199,7 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
request := SmokeRequest{Selection: input.selection, Request: input.request}
acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if acquired.Value == nil && operationErr == nil {
operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
}
@@ -210,14 +211,21 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
}
}
if acquired.Value == nil {
return definitionFailure(operationErr, acquired.Diagnostics)
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
output, encodeErr := encodeSmokeReport(acquired.Value)
if encodeErr != nil {
return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false)
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
output, err := encodeSmokeReport(acquired.Value)
if err != nil {
return err
}
_, err = writer.Write(output)
return err
})
if operationErr != nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
return commandkit.Success()
}
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
@@ -228,13 +236,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
promptkit.SmokeTestStructuredOutput,
}
results := make([]smokeResult, 0, len(kinds))
var diagnostics []commandkit.Diagnostic
var operationErrors []error
var explanations []commandkit.Explanation
for _, kind := range kinds {
if err := promptKitContextError(invocation.Context); err != nil {
if len(results) == 0 {
return definitionFailure(err, diagnostics)
return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
}
operationErrors = append(operationErrors, err)
break
@@ -253,10 +261,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
}
}
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
diagnostics = append(diagnostics, acquired.Diagnostics...)
if operationErr != nil {
operationErrors = append(operationErrors, operationErr)
if acquired.Explanation.Message != "" || len(acquired.Explanation.Details) > 0 {
explanations = append(explanations, acquired.Explanation)
}
}
if err := promptKitContextError(invocation.Context); err != nil {
break
@@ -264,13 +275,57 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
}
if len(results) == 0 {
return definitionFailure(errors.Join(operationErrors...), diagnostics)
return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
output, encodeErr := encodeSmokeResults(results)
if encodeErr != nil {
return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false)
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
output, err := encodeSmokeResults(results)
if err != nil {
return err
}
_, err = writer.Write(output)
return err
})
if len(operationErrors) > 0 {
return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...))
return commandkit.Success()
}
func smokeFailureMessage(kind string) string {
return fmt.Sprintf("PromptKit %s smoke testing failed.", kind)
}
func combineSmokeExplanations(explanations []commandkit.Explanation) commandkit.Explanation {
var selected []commandkit.Explanation
for _, explanation := range explanations {
if explanation.Message != "" || len(explanation.Details) > 0 {
selected = append(selected, explanation)
}
}
if len(selected) == 0 {
return commandkit.Explanation{}
}
if len(selected) == 1 {
return selected[0]
}
combined := commandkit.Explanation{Message: selected[0].Message}
combined.Details = append(combined.Details, selected[0].Details...)
for _, explanation := range selected[1:] {
if explanation.Message != "" {
message := explanation.Message
combined.Details = append(combined.Details, func(writer io.Writer) error {
return writeSmokeExplanationMessage(writer, message)
})
}
combined.Details = append(combined.Details, explanation.Details...)
}
return combined
}
func writeSmokeExplanationMessage(writer io.Writer, message string) error {
_, err := io.WriteString(writer, message+"\n")
return err
}
func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
@@ -312,30 +367,3 @@ func copyString(value *string) *string {
copy := *value
return &copy
}
func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
written, deliveryErr := invocation.Streams.Out.Write(output)
if deliveryErr == nil && written != len(output) {
deliveryErr = io.ErrShortWrite
}
if deliveryErr != nil {
return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true)
}
if operationErr != nil {
return definitionFailure(operationErr, diagnostics)
}
outcome := commandkit.Success()
outcome.Diagnostics = diagnostics
return outcome
}
func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome {
cause := operationErr
if cause == nil {
cause = deliveryErr
}
outcome := definitionFailure(cause, diagnostics)
outcome.DeliveryError = deliveryErr
outcome.ReportIncomplete = reportStarted
return outcome
}