Migrate maintained examples to invocation reporting
This commit is contained in:
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -351,6 +352,66 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) {
|
||||
wantCause := errors.New("local example inference failed")
|
||||
smokeTest := func(_ context.Context, request ckprompt.SmokeRequest, _ *inspectionInput, _ *runCache) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{
|
||||
Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true},
|
||||
Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")},
|
||||
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
|
||||
_, err := io.WriteString(writer, "Native text inference context.\n")
|
||||
return err
|
||||
}}},
|
||||
}, wantCause
|
||||
}
|
||||
app, err := newAppWithEngineLoader(&exampleLLMClient{}, loadPromptEngine, smokeTest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) {
|
||||
t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
if !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") {
|
||||
t.Fatalf("native smoke report was not delivered: %q", output.String())
|
||||
}
|
||||
warningAt := strings.Index(diagnostics.String(), "independent smoke warning")
|
||||
contextAt := strings.Index(diagnostics.String(), "Native text inference context")
|
||||
if warningAt < 0 || contextAt < 0 || warningAt >= contextAt || strings.Contains(diagnostics.String(), wantCause.Error()) {
|
||||
t.Fatalf("terminal explanation/diagnostics = %q / %q", outcome.Explanation.Message, diagnostics.String())
|
||||
}
|
||||
if outcome.Explanation.Message != "" {
|
||||
t.Errorf("unexpected synthesized terminal explanation %q", outcome.Explanation.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposedCustomRestoreUsesSharedReportingAndTerminalFailure(t *testing.T) {
|
||||
app, err := newApp()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"session", "restore", "session-42", "--campaign", "spring", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || diagnostics.Len() != 0 {
|
||||
t.Fatalf("quiet custom command outcome/output/diagnostics = %#v / %q / %q", outcome, output.String(), diagnostics.String())
|
||||
}
|
||||
|
||||
output.Reset()
|
||||
diagnostics.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"session", "restore", "missing", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Kind != commandkit.OutcomeDomain || outcome.Status != 4 || outcome.Domain != "session_not_found" || !errors.Is(outcome.Cause, errSessionNotFound) {
|
||||
t.Fatalf("missing session outcome = %#v", outcome)
|
||||
}
|
||||
if output.Len() != 0 || outcome.Explanation.Message == "" || !strings.Contains(diagnostics.String(), outcome.Explanation.Message) {
|
||||
t.Fatalf("terminal failure output/explanation/diagnostics = %q / %q / %q", output.String(), outcome.Explanation.Message, diagnostics.String())
|
||||
}
|
||||
if strings.Contains(diagnostics.String(), errSessionNotFound.Error()) {
|
||||
t.Errorf("terminal presentation exposed the underlying cause: %q", diagnostics.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
|
||||
clientCalls := 0
|
||||
engineLoads := 0
|
||||
@@ -358,7 +419,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
|
||||
engineLoads++
|
||||
return loadPromptEngine(ctx, selection, input, cache)
|
||||
}
|
||||
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader)
|
||||
app, err := newAppWithEngineLoader(&exampleLLMClient{calls: &clientCalls}, engineLoader, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -368,6 +429,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) {
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "--hlep → --help") || strings.Contains(output.String(), "field-notes") || engineLoads != 0 || clientCalls != 0 {
|
||||
t.Fatalf("group help alias outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
|
||||
}
|
||||
output.Reset()
|
||||
outcome = app.Execute(context.Background(), []string{"pk", "--hlep", "--quiet"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 {
|
||||
t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls)
|
||||
}
|
||||
|
||||
for _, args := range [][]string{
|
||||
{"list", "collections", "--campaign", "spring"},
|
||||
|
||||
Reference in New Issue
Block a user