Migrate PromptKit validation and smoke reporting
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
# Invocation reporting implementation plan
|
# Invocation reporting implementation plan
|
||||||
|
|
||||||
Status: Stages 1–14 complete. Stage 15 is next.
|
Status: Stages 1–15 complete. Stage 16 is next.
|
||||||
|
|
||||||
## Objective and authority
|
## Objective and authority
|
||||||
|
|
||||||
@@ -569,7 +569,7 @@ Stage 20 will validate the assembled module.
|
|||||||
|
|
||||||
## Stage 15: Migrate request validation and smoke commands
|
## Stage 15: Migrate request validation and smoke commands
|
||||||
|
|
||||||
Status: pending.
|
Status: complete.
|
||||||
|
|
||||||
Primary files:
|
Primary files:
|
||||||
[validation_command.go](../../integrations/ckprompt/validation_command.go) and
|
[validation_command.go](../../integrations/ckprompt/validation_command.go) and
|
||||||
@@ -599,6 +599,15 @@ bindings, reports alongside errors, diagnostic ordering, cancellation,
|
|||||||
render/write failures, and quiet semantics. Test CommandKit coordination,
|
render/write failures, and quiet semantics. Test CommandKit coordination,
|
||||||
not whether PromptKit's validation or smoke checks implement their domain rules.
|
not whether PromptKit's validation or smoke checks implement their domain rules.
|
||||||
|
|
||||||
|
Concrete request validation and PromptKit smoke commands now submit passive
|
||||||
|
messages through the reporter and render confirmation, native, and aggregate
|
||||||
|
reports inside managed output. Smoke reports remain available alongside
|
||||||
|
operation failures, with native explanations and causes retained separately;
|
||||||
|
aggregate collection order and shared input/state lifetimes remain covered.
|
||||||
|
`GOTOOLCHAIN=local go test -count=1 .` passes in `integrations/ckprompt`. The
|
||||||
|
module-wide `go test ./...` reaches and passes that package but cannot compile
|
||||||
|
`examples/composed`, whose legacy API callers are scheduled for Stage 17.
|
||||||
|
|
||||||
## Stage 16: Maintain a core reporting consumer
|
## Stage 16: Maintain a core reporting consumer
|
||||||
|
|
||||||
Status: pending.
|
Status: pending.
|
||||||
|
|||||||
@@ -184,14 +184,14 @@ func checkSmokeFormat(format string) error {
|
|||||||
|
|
||||||
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
|
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure(err, nil)
|
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
|
||||||
}
|
}
|
||||||
var state S
|
var state S
|
||||||
if suite.NewState != nil {
|
if suite.NewState != nil {
|
||||||
state = suite.NewState()
|
state = suite.NewState()
|
||||||
}
|
}
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure(err, nil)
|
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
|
||||||
}
|
}
|
||||||
if input.kind == "all" {
|
if input.kind == "all" {
|
||||||
return runSmokeAggregate(invocation, suite, input, state)
|
return runSmokeAggregate(invocation, suite, input, state)
|
||||||
@@ -199,6 +199,7 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
|
|||||||
|
|
||||||
request := SmokeRequest{Selection: input.selection, Request: input.request}
|
request := SmokeRequest{Selection: input.selection, Request: input.request}
|
||||||
acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
|
acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
|
||||||
|
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
|
||||||
if acquired.Value == nil && operationErr == nil {
|
if acquired.Value == nil && operationErr == nil {
|
||||||
operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
|
operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
|
||||||
}
|
}
|
||||||
@@ -210,14 +211,21 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if acquired.Value == nil {
|
if acquired.Value == nil {
|
||||||
return definitionFailure(operationErr, acquired.Diagnostics)
|
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
output, encodeErr := encodeSmokeReport(acquired.Value)
|
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
|
||||||
if encodeErr != nil {
|
output, err := encodeSmokeReport(acquired.Value)
|
||||||
return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false)
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = writer.Write(output)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if operationErr != nil {
|
||||||
|
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
|
||||||
}
|
}
|
||||||
return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
|
return commandkit.Success()
|
||||||
}
|
}
|
||||||
|
|
||||||
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
|
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
|
||||||
@@ -228,13 +236,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
|
|||||||
promptkit.SmokeTestStructuredOutput,
|
promptkit.SmokeTestStructuredOutput,
|
||||||
}
|
}
|
||||||
results := make([]smokeResult, 0, len(kinds))
|
results := make([]smokeResult, 0, len(kinds))
|
||||||
var diagnostics []commandkit.Diagnostic
|
|
||||||
var operationErrors []error
|
var operationErrors []error
|
||||||
|
var explanations []commandkit.Explanation
|
||||||
|
|
||||||
for _, kind := range kinds {
|
for _, kind := range kinds {
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
return definitionFailure(err, diagnostics)
|
return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
|
||||||
}
|
}
|
||||||
operationErrors = append(operationErrors, err)
|
operationErrors = append(operationErrors, err)
|
||||||
break
|
break
|
||||||
@@ -253,10 +261,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
|
||||||
results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
|
results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
|
||||||
diagnostics = append(diagnostics, acquired.Diagnostics...)
|
|
||||||
if operationErr != nil {
|
if operationErr != nil {
|
||||||
operationErrors = append(operationErrors, operationErr)
|
operationErrors = append(operationErrors, operationErr)
|
||||||
|
if acquired.Explanation.Message != "" || len(acquired.Explanation.Details) > 0 {
|
||||||
|
explanations = append(explanations, acquired.Explanation)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
break
|
break
|
||||||
@@ -264,13 +275,57 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
return definitionFailure(errors.Join(operationErrors...), diagnostics)
|
return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
|
||||||
}
|
}
|
||||||
output, encodeErr := encodeSmokeResults(results)
|
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
|
||||||
if encodeErr != nil {
|
output, err := encodeSmokeResults(results)
|
||||||
return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false)
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = writer.Write(output)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if len(operationErrors) > 0 {
|
||||||
|
return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
|
||||||
}
|
}
|
||||||
return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...))
|
return commandkit.Success()
|
||||||
|
}
|
||||||
|
|
||||||
|
func smokeFailureMessage(kind string) string {
|
||||||
|
return fmt.Sprintf("PromptKit %s smoke testing failed.", kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
func combineSmokeExplanations(explanations []commandkit.Explanation) commandkit.Explanation {
|
||||||
|
var selected []commandkit.Explanation
|
||||||
|
for _, explanation := range explanations {
|
||||||
|
if explanation.Message != "" || len(explanation.Details) > 0 {
|
||||||
|
selected = append(selected, explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(selected) == 0 {
|
||||||
|
return commandkit.Explanation{}
|
||||||
|
}
|
||||||
|
if len(selected) == 1 {
|
||||||
|
return selected[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
combined := commandkit.Explanation{Message: selected[0].Message}
|
||||||
|
combined.Details = append(combined.Details, selected[0].Details...)
|
||||||
|
for _, explanation := range selected[1:] {
|
||||||
|
if explanation.Message != "" {
|
||||||
|
message := explanation.Message
|
||||||
|
combined.Details = append(combined.Details, func(writer io.Writer) error {
|
||||||
|
return writeSmokeExplanationMessage(writer, message)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
combined.Details = append(combined.Details, explanation.Details...)
|
||||||
|
}
|
||||||
|
return combined
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeSmokeExplanationMessage(writer io.Writer, message string) error {
|
||||||
|
_, err := io.WriteString(writer, message+"\n")
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
|
func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
|
||||||
@@ -312,30 +367,3 @@ func copyString(value *string) *string {
|
|||||||
copy := *value
|
copy := *value
|
||||||
return ©
|
return ©
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
|
|
||||||
written, deliveryErr := invocation.Streams.Out.Write(output)
|
|
||||||
if deliveryErr == nil && written != len(output) {
|
|
||||||
deliveryErr = io.ErrShortWrite
|
|
||||||
}
|
|
||||||
if deliveryErr != nil {
|
|
||||||
return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true)
|
|
||||||
}
|
|
||||||
if operationErr != nil {
|
|
||||||
return definitionFailure(operationErr, diagnostics)
|
|
||||||
}
|
|
||||||
outcome := commandkit.Success()
|
|
||||||
outcome.Diagnostics = diagnostics
|
|
||||||
return outcome
|
|
||||||
}
|
|
||||||
|
|
||||||
func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome {
|
|
||||||
cause := operationErr
|
|
||||||
if cause == nil {
|
|
||||||
cause = deliveryErr
|
|
||||||
}
|
|
||||||
outcome := definitionFailure(cause, diagnostics)
|
|
||||||
outcome.DeliveryError = deliveryErr
|
|
||||||
outcome.ReportIncomplete = reportStarted
|
|
||||||
return outcome
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -105,16 +105,36 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
|
|||||||
|
|
||||||
func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
|
func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
|
||||||
sentinel := errors.New("native smoke operation failed")
|
sentinel := errors.New("native smoke operation failed")
|
||||||
|
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")
|
||||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{
|
||||||
|
Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true},
|
||||||
|
Diagnostics: []commandkit.Diagnostic{warning},
|
||||||
|
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
|
||||||
|
_, err := io.WriteString(writer, "Native authentication context\n")
|
||||||
|
return err
|
||||||
|
}}},
|
||||||
|
}, sentinel
|
||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
var output bytes.Buffer
|
var output, diagnosticOutput bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
|
||||||
t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
|
t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
|
||||||
}
|
}
|
||||||
|
warningAt := strings.Index(diagnosticOutput.String(), "independent smoke warning")
|
||||||
|
explanationAt := strings.Index(diagnosticOutput.String(), "Native authentication context")
|
||||||
|
if warningAt < 0 || explanationAt < 0 || warningAt >= explanationAt || strings.Contains(diagnosticOutput.String(), sentinel.Error()) {
|
||||||
|
t.Fatalf("smoke warning and native terminal context were not separate and ordered: %q", diagnosticOutput.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
output.Reset()
|
||||||
|
diagnosticOutput.Reset()
|
||||||
|
outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
|
||||||
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "Native authentication context") || strings.Contains(diagnosticOutput.String(), "independent smoke warning") {
|
||||||
|
t.Fatalf("quiet report-plus-error = (%#v, out=%q, err=%q)", outcome, output.String(), diagnosticOutput.String())
|
||||||
|
}
|
||||||
|
|
||||||
app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||||
@@ -154,7 +174,7 @@ func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, io.ErrShortWrite) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete {
|
||||||
t.Fatalf("short report delivery = %#v", outcome)
|
t.Fatalf("short report delivery = %#v", outcome)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -105,10 +105,9 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
var diagnosticOutput bytes.Buffer
|
var diagnosticOutput bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
||||||
warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
|
warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
|
||||||
operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error())
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || warningIndex < 0 || strings.Contains(diagnosticOutput.String(), operationErr.Error()) {
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex {
|
|
||||||
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -116,8 +115,8 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
|
|||||||
|
|
||||||
diagnosticErr := errors.New("smoke diagnostic writer failed")
|
diagnosticErr := errors.New("smoke diagnostic writer failed")
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete || output.Len() == 0 {
|
||||||
t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
|
t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,7 +129,7 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
output.Reset()
|
output.Reset()
|
||||||
outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 {
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || output.Len() != 0 {
|
||||||
t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
|
t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -151,15 +150,24 @@ func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
|
|||||||
case promptkit.SmokeTestAuthentication:
|
case promptkit.SmokeTestAuthentication:
|
||||||
err = secondErr
|
err = secondErr
|
||||||
}
|
}
|
||||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err
|
acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}
|
||||||
|
if err != nil {
|
||||||
|
acquired.Explanation = commandkit.Explanation{Message: string(request.Request.Kind) + " native context"}
|
||||||
|
}
|
||||||
|
return acquired, err
|
||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
var output bytes.Buffer
|
var output, diagnosticOutput bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
|
||||||
var retained *smokeTypedError
|
var retained *smokeTypedError
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
|
||||||
t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
|
t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
|
||||||
}
|
}
|
||||||
|
firstContext := strings.Index(diagnosticOutput.String(), "reachability native context")
|
||||||
|
secondContext := strings.Index(diagnosticOutput.String(), "authentication native context")
|
||||||
|
if firstContext < 0 || secondContext < 0 || firstContext >= secondContext {
|
||||||
|
t.Fatalf("aggregate native explanations were not retained in order: %q", diagnosticOutput.String())
|
||||||
|
}
|
||||||
var document struct {
|
var document struct {
|
||||||
Tests []struct {
|
Tests []struct {
|
||||||
Kind string `yaml:"kind"`
|
Kind string `yaml:"kind"`
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package ckprompt
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||||
@@ -102,26 +104,26 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command {
|
|||||||
|
|
||||||
func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *validationCommandInput[I]) commandkit.Outcome {
|
func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *validationCommandInput[I]) commandkit.Outcome {
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure(err, nil)
|
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
|
||||||
}
|
}
|
||||||
var state S
|
var state S
|
||||||
if suite.NewState != nil {
|
if suite.NewState != nil {
|
||||||
state = suite.NewState()
|
state = suite.NewState()
|
||||||
}
|
}
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure(err, nil)
|
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state)
|
acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state)
|
||||||
|
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return definitionFailure(err, acquired.Diagnostics)
|
return definitionFailure(fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
|
||||||
}
|
}
|
||||||
if err := promptKitContextError(invocation.Context); err != nil {
|
if err := promptKitContextError(invocation.Context); err != nil {
|
||||||
return definitionFailure(err, acquired.Diagnostics)
|
return definitionFailure(fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
|
||||||
}
|
}
|
||||||
output, err := encodePromptYAMLBytes(requestValidationReport{PromptID: input.promptID, Valid: true})
|
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
|
||||||
if err != nil {
|
return encodePromptYAML(writer, requestValidationReport{PromptID: input.promptID, Valid: true})
|
||||||
return definitionFailure(err, acquired.Diagnostics)
|
})
|
||||||
}
|
return commandkit.Success()
|
||||||
return writePromptReport(invocation, output, acquired.Diagnostics)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ func TestValidationCommandIsAvailableIndependentlyAndOwnsItsFamily(t *testing.T)
|
|||||||
}}))
|
}}))
|
||||||
var help bytes.Buffer
|
var help bytes.Buffer
|
||||||
outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "show") {
|
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "\n show ") {
|
||||||
t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String())
|
t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -359,41 +359,32 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
|
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
|
||||||
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
|
||||||
_, err := io.WriteString(writer, "validation warning\n")
|
|
||||||
return err
|
|
||||||
}}
|
|
||||||
withWarning := func(diagnostics []commandkit.Diagnostic) bool {
|
|
||||||
for _, diagnostic := range diagnostics {
|
|
||||||
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
|
|
||||||
var output bytes.Buffer
|
|
||||||
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var diagnostics bytes.Buffer
|
var diagnostics bytes.Buffer
|
||||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
||||||
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
||||||
}}))
|
}}))
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
||||||
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "validation warning") {
|
||||||
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
|
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
|
||||||
}
|
}
|
||||||
|
var quietReport, quietDiagnostics bytes.Buffer
|
||||||
|
quietOutcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--quiet"}, commandkit.Streams{Out: &quietReport, Err: &quietDiagnostics})
|
||||||
|
if quietOutcome.Kind != commandkit.OutcomeSuccess || quietOutcome.Presentation.Output.State != commandkit.DeliverySuppressed || quietOutcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed || quietReport.Len() != 0 || quietDiagnostics.Len() != 0 {
|
||||||
|
t.Fatalf("quiet validation = %#v, report=%q diagnostics=%q", quietOutcome, quietReport.String(), quietDiagnostics.String())
|
||||||
|
}
|
||||||
|
|
||||||
providerErr := errors.New("validation provider failed")
|
providerErr := errors.New("validation provider failed")
|
||||||
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
||||||
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
||||||
}}))
|
}}))
|
||||||
diagnostics.Reset()
|
diagnostics.Reset()
|
||||||
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete {
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete {
|
||||||
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
|
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
|
||||||
}
|
}
|
||||||
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Count(diagnostics.String(), providerErr.Error()) != 1 {
|
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Contains(diagnostics.String(), providerErr.Error()) || !strings.Contains(diagnostics.String(), "Unable to validate the PromptKit request") {
|
||||||
t.Fatalf("provider diagnostics were lost or duplicated: stderr=%q", diagnostics.String())
|
t.Fatalf("provider warning or terminal explanation was incorrect: stderr=%q", diagnostics.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
builderErr := errors.New("request builder failed")
|
builderErr := errors.New("request builder failed")
|
||||||
@@ -418,16 +409,13 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes
|
|||||||
}, nil)))
|
}, nil)))
|
||||||
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
var retained *configkit.LoadError
|
var retained *configkit.LoadError
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
|
||||||
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
|
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
|
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
|
||||||
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
|
||||||
_, err := io.WriteString(writer, "validation warning\n")
|
|
||||||
return err
|
|
||||||
}}
|
|
||||||
calls := 0
|
calls := 0
|
||||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
||||||
calls++
|
calls++
|
||||||
@@ -448,8 +436,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
|
|||||||
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
||||||
}}))
|
}}))
|
||||||
output.Reset()
|
output.Reset()
|
||||||
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
var diagnostics bytes.Buffer
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
|
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||||
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnostics.String(), "validation warning") || output.Len() != 0 {
|
||||||
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
|
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -459,8 +448,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
|
|||||||
cancel()
|
cancel()
|
||||||
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
||||||
}}))
|
}}))
|
||||||
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
var errorDiagnostics bytes.Buffer
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
|
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &errorDiagnostics})
|
||||||
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !strings.Contains(errorDiagnostics.String(), "validation warning") {
|
||||||
t.Fatalf("provider error after cancellation = %#v", outcome)
|
t.Fatalf("provider error after cancellation = %#v", outcome)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -484,28 +474,16 @@ func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testin
|
|||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
var diagnosticOutput bytes.Buffer
|
var diagnosticOutput bytes.Buffer
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
|
||||||
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
diagnosticErr := errors.New("validation diagnostic writer failed")
|
diagnosticErr := errors.New("validation diagnostic writer failed")
|
||||||
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
|
||||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
|
||||||
t.Fatalf("diagnostic delivery = %#v", outcome)
|
t.Fatalf("diagnostic delivery = %#v", outcome)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
|
|
||||||
for _, diagnostic := range diagnostics {
|
|
||||||
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
|
|
||||||
var output bytes.Buffer
|
|
||||||
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user