Add PromptKit inspection commands
This commit is contained in:
357
integrations/ckprompt/command.go
Normal file
357
integrations/ckprompt/command.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type definitionInput[I any] struct {
|
||||
selection ckconfig.Selection
|
||||
id string
|
||||
version commandkit.Supplied[string]
|
||||
format string
|
||||
captured commandkit.FlagValues[I]
|
||||
selected I
|
||||
}
|
||||
|
||||
// Command returns the optional named PromptKit inspection tree. It adds no
|
||||
// configuration family; callers compose configuration scopes independently.
|
||||
func Command[I, S any](suite Suite[I, S]) commandkit.Command {
|
||||
if suite.InspectPrompt == nil && suite.InspectProfile == nil {
|
||||
command := commandkit.NoCommand()
|
||||
for _, issue := range suite.issues {
|
||||
command = commandkit.WithError(command, issue)
|
||||
}
|
||||
return command
|
||||
}
|
||||
|
||||
showCommands := make([]commandkit.Command, 0, 2)
|
||||
if suite.InspectPrompt != nil {
|
||||
showCommands = append(showCommands, newDefinitionCommand(suite, "prompt", true))
|
||||
}
|
||||
if suite.InspectProfile != nil {
|
||||
showCommands = append(showCommands, newDefinitionCommand(suite, "profile", false))
|
||||
}
|
||||
command := commandkit.RootGroup("promptkit", "Inspect named PromptKit definitions.",
|
||||
commandkit.Group("show", "Show a named PromptKit definition.", showCommands...),
|
||||
)
|
||||
for _, issue := range suite.issues {
|
||||
command = commandkit.WithError(command, issue)
|
||||
}
|
||||
return command
|
||||
}
|
||||
|
||||
func newDefinitionCommand[I, S any](suite Suite[I, S], name string, prompt bool) commandkit.Command {
|
||||
flags := commandkit.CaptureFlags(suite.Selectors, func(input **definitionInput[I]) *commandkit.FlagValues[I] {
|
||||
if input == nil || *input == nil {
|
||||
return nil
|
||||
}
|
||||
return &(*input).captured
|
||||
})
|
||||
flags = append(flags,
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Configuration input path."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).selection.Config = value }),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay path."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) {
|
||||
(*input).selection.Overlay = value
|
||||
}),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "format", Usage: "Report format: text or yaml."}, stringDefault("text"), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).format = value.Value }),
|
||||
)
|
||||
if prompt {
|
||||
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "version", Usage: "Prompt version to inspect."}, (*string)(nil), parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).version = value }))
|
||||
}
|
||||
|
||||
spec := commandkit.CommandSpec[*definitionInput[I]]{
|
||||
Name: name,
|
||||
Summary: "Inspect a named PromptKit " + name + ".",
|
||||
NewInput: func() *definitionInput[I] {
|
||||
return &definitionInput[I]{}
|
||||
},
|
||||
Flags: flags,
|
||||
Arguments: []commandkit.Argument[*definitionInput[I]]{
|
||||
commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "id", Usage: "Prompt or profile ID."}, true, parseDefinitionString,
|
||||
func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).id = value.Value }),
|
||||
},
|
||||
CheckInput: func(input *definitionInput[I]) error {
|
||||
if err := checkDefinitionFormat(input.format); err != nil {
|
||||
return err
|
||||
}
|
||||
selected, err := prepareDefinitionInput(suite, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
input.selected = selected
|
||||
return nil
|
||||
},
|
||||
Run: func(invocation commandkit.Invocation, input *definitionInput[I]) commandkit.Outcome {
|
||||
return runDefinitionInspection(invocation, suite, input, prompt)
|
||||
},
|
||||
}
|
||||
return commandkit.NewCommand(spec)
|
||||
}
|
||||
|
||||
func prepareDefinitionInput[I, S any](suite Suite[I, S], input *definitionInput[I]) (I, error) {
|
||||
var selected I
|
||||
if suite.NewInput != nil {
|
||||
selected = suite.NewInput()
|
||||
}
|
||||
if err := input.captured.ApplyTo(&selected, suite.Selectors...); err != nil {
|
||||
return selected, err
|
||||
}
|
||||
if suite.CheckInput != nil {
|
||||
if err := suite.CheckInput(input.selection, selected); err != nil {
|
||||
return selected, err
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], prompt bool) commandkit.Outcome {
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, nil)
|
||||
}
|
||||
}
|
||||
var state S
|
||||
if suite.NewState != nil {
|
||||
state = suite.NewState()
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, nil)
|
||||
}
|
||||
}
|
||||
|
||||
if prompt {
|
||||
request := PromptRequest{Selection: input.selection, ID: input.id, Version: input.version}
|
||||
acquired, err := suite.InspectPrompt(invocation.Context, request, input.selected, state)
|
||||
if err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
if acquired.Value == nil {
|
||||
return definitionFailure(errors.New("PromptKit prompt inspection returned nil"), acquired.Diagnostics)
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
}
|
||||
return renderPromptReport(invocation, promptReportFromInspection(acquired.Value), input.format, acquired.Diagnostics)
|
||||
}
|
||||
|
||||
request := ProfileRequest{Selection: input.selection, ID: input.id}
|
||||
acquired, err := suite.InspectProfile(invocation.Context, request, input.selected, state)
|
||||
if err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
if acquired.Value == nil {
|
||||
return definitionFailure(errors.New("PromptKit profile inspection returned nil"), acquired.Diagnostics)
|
||||
}
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, acquired.Diagnostics)
|
||||
}
|
||||
}
|
||||
return renderProfileReport(invocation, profileReportFromInspection(acquired.Value), input.format, acquired.Diagnostics)
|
||||
}
|
||||
|
||||
type promptInputReport struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptOutputContractReport struct {
|
||||
Format string `yaml:"format"`
|
||||
ValidationMode string `yaml:"validation_mode"`
|
||||
SchemaPath string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
type promptInspectionReport struct {
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
PromptVersion string `yaml:"prompt_version"`
|
||||
PromptHash string `yaml:"prompt_hash"`
|
||||
DefaultProfileID string `yaml:"default_profile_id"`
|
||||
Inputs []promptInputReport `yaml:"inputs"`
|
||||
OutputContract promptOutputContractReport `yaml:"output_contract"`
|
||||
}
|
||||
|
||||
type modelParamsReport struct {
|
||||
BackendID string `yaml:"backend_id"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type profileInspectionReport struct {
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
EffectiveModelParams modelParamsReport `yaml:"effective_model_params"`
|
||||
APIKeyRequired bool `yaml:"api_key_required"`
|
||||
}
|
||||
|
||||
func promptReportFromInspection(inspection *promptkit.PromptInspection) promptInspectionReport {
|
||||
report := promptInspectionReport{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptHash: inspection.PromptHash,
|
||||
DefaultProfileID: inspection.DefaultProfileID,
|
||||
Inputs: make([]promptInputReport, 0, len(inspection.Inputs)),
|
||||
OutputContract: promptOutputContractReport{
|
||||
Format: string(inspection.OutputContract.Format),
|
||||
ValidationMode: string(inspection.OutputContract.ValidationMode),
|
||||
SchemaPath: inspection.OutputContract.SchemaPath,
|
||||
RepairAttempts: inspection.OutputContract.RepairAttempts,
|
||||
},
|
||||
}
|
||||
for _, input := range inspection.Inputs {
|
||||
report.Inputs = append(report.Inputs, promptInputReport{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description})
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func profileReportFromInspection(inspection *promptkit.ProfileInspection) profileInspectionReport {
|
||||
params := inspection.EffectiveModelParams
|
||||
extra := params.ExtraParams
|
||||
if extra == nil {
|
||||
extra = map[string]any{}
|
||||
}
|
||||
return profileInspectionReport{
|
||||
ProfileID: inspection.ProfileID,
|
||||
EffectiveModelParams: modelParamsReport{
|
||||
BackendID: params.BackendID, Endpoint: params.Endpoint, Model: params.Model,
|
||||
Temperature: params.Temperature, MaxTokens: params.MaxTokens, TopP: params.TopP,
|
||||
TimeoutSeconds: params.TimeoutSeconds, ServiceTier: params.ServiceTier,
|
||||
ReasoningEffort: params.ReasoningEffort, APIKeyEnv: params.APIKeyEnv, ExtraParams: extra,
|
||||
},
|
||||
APIKeyRequired: inspection.APIKeyRequired,
|
||||
}
|
||||
}
|
||||
|
||||
func renderPromptReport(invocation commandkit.Invocation, report promptInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
var output bytes.Buffer
|
||||
if format == "yaml" {
|
||||
if err := encodePromptYAML(&output, report); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
} else {
|
||||
output.WriteString(formatPromptText(report))
|
||||
}
|
||||
return writePromptReport(invocation, output.Bytes(), diagnostics)
|
||||
}
|
||||
|
||||
func renderProfileReport(invocation commandkit.Invocation, report profileInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
var output bytes.Buffer
|
||||
if format == "yaml" {
|
||||
if err := encodePromptYAML(&output, report); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
} else {
|
||||
text, err := formatProfileText(report)
|
||||
if err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
output.WriteString(text)
|
||||
}
|
||||
return writePromptReport(invocation, output.Bytes(), diagnostics)
|
||||
}
|
||||
|
||||
func formatPromptText(report promptInspectionReport) string {
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, "Prompt ID: %s\n", strconv.Quote(report.PromptID))
|
||||
fmt.Fprintf(&output, "Prompt version: %s\n", strconv.Quote(report.PromptVersion))
|
||||
fmt.Fprintf(&output, "Prompt hash: %s\n", strconv.Quote(report.PromptHash))
|
||||
fmt.Fprintf(&output, "Default profile ID: %s\n", strconv.Quote(report.DefaultProfileID))
|
||||
output.WriteString("Inputs:\n")
|
||||
for _, input := range report.Inputs {
|
||||
fmt.Fprintf(&output, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", strconv.Quote(input.Name), input.Required, strconv.Quote(input.ContentType), strconv.Quote(input.Description))
|
||||
}
|
||||
if len(report.Inputs) == 0 {
|
||||
output.WriteString(" []\n")
|
||||
}
|
||||
fmt.Fprintf(&output, "Output contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", strconv.Quote(report.OutputContract.Format), strconv.Quote(report.OutputContract.ValidationMode), strconv.Quote(report.OutputContract.SchemaPath), report.OutputContract.RepairAttempts)
|
||||
return output.String()
|
||||
}
|
||||
|
||||
func formatProfileText(report profileInspectionReport) (string, error) {
|
||||
params := report.EffectiveModelParams
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, "Profile ID: %s\n", strconv.Quote(report.ProfileID))
|
||||
output.WriteString("Effective model params:\n")
|
||||
fmt.Fprintf(&output, " backend_id: %s\n endpoint: %s\n model: %s\n temperature: %s\n max_tokens: %d\n top_p: %s\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n", strconv.Quote(params.BackendID), strconv.Quote(params.Endpoint), strconv.Quote(params.Model), strconv.FormatFloat(params.Temperature, 'g', -1, 64), params.MaxTokens, strconv.FormatFloat(params.TopP, 'g', -1, 64), params.TimeoutSeconds, strconv.Quote(params.ServiceTier), strconv.Quote(params.ReasoningEffort), strconv.Quote(params.APIKeyEnv))
|
||||
output.WriteString(" extra_params:\n")
|
||||
var nested strings.Builder
|
||||
if err := encodePromptYAML(&nested, params.ExtraParams); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSuffix(nested.String(), "\n"), "\n") {
|
||||
output.WriteString(" ")
|
||||
output.WriteString(line)
|
||||
output.WriteByte('\n')
|
||||
}
|
||||
fmt.Fprintf(&output, "API key required: %t\n", report.APIKeyRequired)
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func encodePromptYAML(writer io.Writer, value any) error {
|
||||
encoder := yaml.NewEncoder(writer)
|
||||
encoder.SetIndent(2)
|
||||
return encoder.Encode(value)
|
||||
}
|
||||
|
||||
func writePromptReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
if invocation.Context != nil {
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return definitionFailure(err, diagnostics)
|
||||
}
|
||||
}
|
||||
written, err := invocation.Streams.Out.Write(output)
|
||||
if err == nil && written != len(output) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err != nil {
|
||||
outcome := definitionFailure(err, diagnostics)
|
||||
outcome.ReportIncomplete = true
|
||||
outcome.DeliveryError = err
|
||||
return outcome
|
||||
}
|
||||
outcome := commandkit.Success()
|
||||
outcome.Diagnostics = diagnostics
|
||||
return outcome
|
||||
}
|
||||
|
||||
func definitionFailure(cause error, diagnostics []commandkit.Diagnostic) commandkit.Outcome {
|
||||
outcome := commandkit.Failure(cause)
|
||||
outcome.Diagnostics = diagnostics
|
||||
return outcome
|
||||
}
|
||||
|
||||
func checkDefinitionFormat(format string) error {
|
||||
if format != "text" && format != "yaml" {
|
||||
return fmt.Errorf("unsupported format %q; expected text or yaml", format)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseDefinitionString(value string) (string, error) { return value, nil }
|
||||
|
||||
func stringDefault(value string) *string { return &value }
|
||||
Reference in New Issue
Block a user