Add detailed validation evidence
This commit is contained in:
@@ -335,12 +335,20 @@ with each reached scope and its outcome (`valid`, `failed`, or `canceled`).
|
||||
example config validate
|
||||
example config validate --scope promptkit --format yaml
|
||||
example config validate --scope all
|
||||
example config validate --scope all --details --severity warning --code schema.version
|
||||
```
|
||||
|
||||
Each successful scope is reported as `SCOPE: valid`. Aggregate validation stops
|
||||
at the first load failure or cancellation, then submits one report containing
|
||||
the reached scopes. Output errors occur after acquisition has stopped. Provider
|
||||
warnings go to the diagnostic stream and do not alone make validation fail.
|
||||
`--details` includes native findings and the source references needed to
|
||||
interpret them. Each scope's detail state is `unavailable`, `empty`, `filtered`,
|
||||
or `present`. Repeat `--severity error|warning` or `--code CODE` to filter
|
||||
findings: values within a filter match any, and the two filters narrow together.
|
||||
Filters require `--details` and never change the validation outcome. Detailed
|
||||
output suppresses ordinary warnings associated with its native findings; other
|
||||
messages and terminal failure explanations remain visible.
|
||||
`--config` and `--overlay` preserve whether an
|
||||
empty value was explicitly supplied, so a scope callback can map those inputs to
|
||||
its own ConfigKit load policy. Help and invalid syntax do not call scope
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Shared reporting and PromptKit v0.20.0 implementation plan
|
||||
|
||||
Status: implementation in progress; Stages 1–19 complete.
|
||||
Status: implementation in progress; Stages 1–20 complete.
|
||||
|
||||
## Scope, authority, and execution
|
||||
|
||||
@@ -709,6 +709,11 @@ add a comparison callback, reload data, or implement list matching.
|
||||
|
||||
## Stage 20: Detailed validation and reached-scope aggregation
|
||||
|
||||
**Complete.** Detailed validation filters native findings and reports reached
|
||||
scopes after acquisition stops, preserving independent diagnostics and causes.
|
||||
Root, ckconfig, and ckprompt workspace suites passed with Go 1.27.0, including
|
||||
failure, cancellation, source references, quiet mode, and output failure.
|
||||
|
||||
Complete Stage 7's validation aggregation using Stage 14's evidence. Add
|
||||
`--details`, repeatable `--severity`, and repeatable `--code` on validation;
|
||||
filters require detailed mode. Parse severity using native known severities;
|
||||
|
||||
136
integrations/ckconfig/detailed_validation_test.go
Normal file
136
integrations/ckconfig/detailed_validation_test.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package ckconfig
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func validationDiagnostic(message string) commandkit.Diagnostic {
|
||||
return commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, message+"\n"); return err }}
|
||||
}
|
||||
|
||||
func TestDetailedValidationFiltersAndRetainsFailureEvidence(t *testing.T) {
|
||||
cause := errors.New("second scope failed")
|
||||
calls := []string{}
|
||||
reader := &sourceInventory{values: []configkit.SourceInfo{{ID: "primary", Label: "main", Path: "main.yml"}, {ID: "related", Label: "included", Path: "include.yml"}, {ID: "unused", Label: "unused"}}}
|
||||
records := []configkit.Diagnostic{
|
||||
{Code: "chosen", Severity: configkit.SeverityWarning, Path: configkit.Keys("a/b"), Message: "selected warning", Primary: &configkit.Location{SourceID: "primary", Line: 2, Column: 3}, Related: []configkit.Location{{SourceID: "related", Line: 7, Column: 8}}},
|
||||
{Code: "other", Severity: configkit.SeverityWarning, Path: configkit.Keys("other"), Message: "other warning"},
|
||||
}
|
||||
scopes := []Scope[struct{}, struct{}]{
|
||||
{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls = append(calls, "app")
|
||||
return commandkit.Acquired[View]{Value: View{Sources: reader, Findings: &Findings{Records: records, Messages: []commandkit.Diagnostic{validationDiagnostic("native warnings")}}}, Diagnostics: []commandkit.Diagnostic{validationDiagnostic("application warning")}}, nil
|
||||
}},
|
||||
{Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls = append(calls, "second")
|
||||
return commandkit.Acquired[View]{Value: View{Findings: &Findings{Records: []configkit.Diagnostic{{Code: "failed", Severity: configkit.SeverityError, Message: "failed finding"}}}}, Explanation: commandkit.Explanation{Message: "terminal failure"}}, cause
|
||||
}},
|
||||
{Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls = append(calls, "third")
|
||||
return commandkit.Acquired[View]{}, nil
|
||||
}},
|
||||
}
|
||||
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes}))
|
||||
var out, diag bytes.Buffer
|
||||
result := app.Execute(context.Background(), []string{"config", "validate", "--scope=all", "--details", "--severity=warning", "--code=chosen", "--format=json", "--verbose"}, commandkit.Streams{Out: &out, Err: &diag})
|
||||
decoder := json.NewDecoder(&out)
|
||||
var report validationReport
|
||||
if err := decoder.Decode(&report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := decoder.Decode(new(any)); err != io.EOF {
|
||||
t.Fatalf("extra JSON: %v", err)
|
||||
}
|
||||
if !errors.Is(result.Cause, cause) || result.Kind != commandkit.OutcomeOperational || len(calls) != 2 || calls[0] != "app" || calls[1] != "second" || len(report.Scopes) != 2 || report.Scopes[0].Outcome != "valid" || report.Scopes[1].Outcome != "failed" {
|
||||
t.Fatalf("result=%#v calls=%v report=%#v", result, calls, report)
|
||||
}
|
||||
first, second := report.Scopes[0].Details, report.Scopes[1].Details
|
||||
if first == nil || first.State != "present" || len(first.Findings) != 1 || first.Findings[0].Path != "/a~1b" || len(first.Sources) != 2 || first.Sources[0].ID != "primary" || first.Sources[1].ID != "related" || second == nil || second.State != "filtered" || len(second.Findings) != 0 {
|
||||
t.Fatalf("details=%#v %#v", first, second)
|
||||
}
|
||||
if strings.Contains(diag.String(), "native warnings") || !strings.Contains(diag.String(), "application warning") || !strings.Contains(diag.String(), "terminal failure") {
|
||||
t.Fatalf("diagnostics=%q", diag.String())
|
||||
}
|
||||
before := len(calls)
|
||||
for _, args := range [][]string{{"--severity=warning"}, {"--code=x"}, {"--details", "--severity=unknown"}} {
|
||||
result = app.Execute(context.Background(), append([]string{"config", "validate"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeUsage || len(calls) != before {
|
||||
t.Fatalf("invalid %v: %#v", args, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationCancellationAndDeliveryRetainReachedScopes(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
scopes := []Scope[struct{}, struct{}]{
|
||||
{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls++
|
||||
return commandkit.Acquired[View]{Value: View{Findings: &Findings{}}}, nil
|
||||
}},
|
||||
{Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls++
|
||||
cancel()
|
||||
return commandkit.Acquired[View]{}, nil
|
||||
}},
|
||||
{Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
calls++
|
||||
return commandkit.Acquired[View]{}, nil
|
||||
}},
|
||||
}
|
||||
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes}))
|
||||
var out bytes.Buffer
|
||||
result := app.Execute(ctx, []string{"config", "validate", "--scope=all", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
var report validationReport
|
||||
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(result.Cause, context.Canceled) || calls != 2 || len(report.Scopes) != 2 || report.Scopes[0].Details.State != "empty" || report.Scopes[1].Details.State != "unavailable" || report.Scopes[1].Outcome != "canceled" {
|
||||
t.Fatalf("result=%#v calls=%d report=%#v", result, calls, report)
|
||||
}
|
||||
failure := errors.New("load failure")
|
||||
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
|
||||
return commandkit.Acquired[View]{}, failure
|
||||
}}
|
||||
app = newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||
result = app.Execute(context.Background(), []string{"config", "validate", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
|
||||
if !errors.Is(result.Cause, failure) || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
|
||||
t.Fatalf("failure and delivery: %#v", result)
|
||||
}
|
||||
out.Reset()
|
||||
result = app.Execute(context.Background(), []string{"config", "validate", "--details", "--quiet"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
if !errors.Is(result.Cause, failure) || out.Len() != 0 {
|
||||
t.Fatalf("quiet failure: %#v %q", result, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetailedValidationReportsNativeLoadError(t *testing.T) {
|
||||
_, failure := configkit.Load[map[string]any](configkit.Bytes("invalid source", []byte("invalid: [")))
|
||||
var native *configkit.LoadError
|
||||
if !errors.As(failure, &native) {
|
||||
t.Fatalf("fixture: %v", failure)
|
||||
}
|
||||
scope := FromConfigKit("app", "", func(context.Context, Selection, struct{}, struct{}) (*configkit.Result[map[string]any], error) {
|
||||
return nil, failure
|
||||
})
|
||||
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||
var out bytes.Buffer
|
||||
result := app.Execute(context.Background(), []string{"config", "validate", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
var report validationReport
|
||||
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(result.Cause, failure) || len(report.Scopes) != 1 || report.Scopes[0].Outcome != "failed" || report.Scopes[0].Details.State != "present" || len(report.Scopes[0].Details.Findings) != len(native.Diagnostics()) {
|
||||
t.Fatalf("native failure: %#v %q", result, out.String())
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,8 @@ type configCommandInput[I, S any] struct {
|
||||
sourceID commandkit.Supplied[string]
|
||||
details commandkit.Supplied[bool]
|
||||
check commandkit.Supplied[bool]
|
||||
severities []configkit.Severity
|
||||
codes []string
|
||||
values commandkit.FlagValues[I]
|
||||
againstValues commandkit.FlagValues[I]
|
||||
plan *configOperationPlan[I, S]
|
||||
@@ -45,6 +47,8 @@ type configOperationPlan[I, S any] struct {
|
||||
sourceID commandkit.Supplied[string]
|
||||
details bool
|
||||
check bool
|
||||
severities []configkit.Severity
|
||||
codes []string
|
||||
list bool
|
||||
help bool
|
||||
historical bool
|
||||
@@ -94,6 +98,17 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
|
||||
}),
|
||||
)
|
||||
}
|
||||
if operation == Validate {
|
||||
flags = append(flags,
|
||||
commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native validation findings."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
|
||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "severity", Usage: "Filter detailed findings by error or warning."}, []configkit.Severity(nil), parseValidationSeverity, func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Severity]) {
|
||||
(*input).severities = value.Value
|
||||
}),
|
||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "code", Usage: "Filter detailed findings by exact native code."}, []string(nil), parseConfigString, func(input **configCommandInput[I, S], value commandkit.Supplied[[]string]) {
|
||||
(*input).codes = value.Value
|
||||
}),
|
||||
)
|
||||
}
|
||||
if supportsOverrides(prepared, operation) {
|
||||
flags = append(flags, commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON (pointer keys cannot contain =)."}, []configkit.Override(nil), overrideParser("--set"),
|
||||
func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Override]) {
|
||||
@@ -179,6 +194,9 @@ func operationDeclaration(operation Operation) (string, string) {
|
||||
}
|
||||
|
||||
func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation Operation, input *configCommandInput[I, S]) error {
|
||||
if operation == Validate && !input.details.Value && (len(input.severities) > 0 || len(input.codes) > 0) {
|
||||
return fmt.Errorf("--severity and --code require --details")
|
||||
}
|
||||
if operation == Explain {
|
||||
if input.list.Value && input.path.Set {
|
||||
return fmt.Errorf("--list and --path cannot be combined")
|
||||
@@ -256,6 +274,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
|
||||
sourceID: input.sourceID,
|
||||
details: input.details.Value,
|
||||
check: input.check.Value,
|
||||
severities: append([]configkit.Severity(nil), input.severities...),
|
||||
codes: append([]string(nil), input.codes...),
|
||||
list: input.list.Value,
|
||||
historical: input.historical.Value,
|
||||
under: under,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
package ckconfig
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"text/template"
|
||||
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type validationReport struct {
|
||||
@@ -12,11 +15,48 @@ type validationReport struct {
|
||||
}
|
||||
|
||||
type validationScope struct {
|
||||
Scope string `json:"scope"`
|
||||
Outcome string `json:"outcome"`
|
||||
Scope string `json:"scope"`
|
||||
Outcome string `json:"outcome"`
|
||||
Details *validationDetails `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
type validationDetails struct {
|
||||
State string `json:"state"`
|
||||
Findings []validationFinding `json:"findings"`
|
||||
Sources []validationSource `json:"sources"`
|
||||
}
|
||||
|
||||
type validationFinding struct {
|
||||
Code string `json:"code"`
|
||||
Severity string `json:"severity"`
|
||||
Path string `json:"path"`
|
||||
Message string `json:"message"`
|
||||
Primary *locationDTO `json:"primary"`
|
||||
Related []locationDTO `json:"related"`
|
||||
}
|
||||
|
||||
type validationSource struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
var validationTextTemplate = template.Must(commandkit.ParseTextTemplate("validation", "{{range .Scopes}}{{.Scope}}: {{.Outcome}}\n{{end}}", nil))
|
||||
var validationDetailsTemplate = template.Must(commandkit.ParseTextTemplate("validation-details", `{{range .Scopes}}{{.Scope}}: {{.Outcome}} ({{.Details.State}})
|
||||
{{range .Details.Findings}} {{.Severity}} {{.Code}} {{printf "%q" .Path}}: {{.Message}}
|
||||
{{if .Primary}} {{.Primary.SourceID}}:{{.Primary.Line}}:{{.Primary.Column}}
|
||||
{{end}}{{range .Related}} {{.SourceID}}:{{.Line}}:{{.Column}}
|
||||
{{end}}{{end}}{{range .Details.Sources}} source {{printf "%q" .ID}}: {{.Label}} {{.Path}}
|
||||
{{end}}{{end}}`, nil))
|
||||
|
||||
func parseValidationSeverity(raw string) (configkit.Severity, error) {
|
||||
switch configkit.Severity(raw) {
|
||||
case configkit.SeverityError, configkit.SeverityWarning:
|
||||
return configkit.Severity(raw), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown validation severity %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *configOperationPlan[I, S], state S) commandkit.Outcome {
|
||||
reporter := invocation.Reporter()
|
||||
@@ -29,30 +69,88 @@ func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *confi
|
||||
}
|
||||
name := selected.scope.scope.Name
|
||||
if selected.scope.scope.Acquire == nil {
|
||||
report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "failed"})
|
||||
entry := validationScope{Scope: name, Outcome: "failed"}
|
||||
if plan.details {
|
||||
entry.Details = projectValidationDetails(View{}, plan.severities, plan.codes)
|
||||
}
|
||||
report.Scopes = append(report.Scopes, entry)
|
||||
outcome = commandkit.Failure(fmt.Sprintf("Configuration scope %q has no acquisition callback.", name))
|
||||
break
|
||||
}
|
||||
acquired, err := selected.scope.scope.Acquire(invocation.Context, Request{Operation: Validate, Selection: selected.selection}, selected.input, state)
|
||||
if err != nil {
|
||||
status := "failed"
|
||||
if invocation.Context.Err() != nil {
|
||||
status = "canceled"
|
||||
queueConfigDiagnostics(reporter, acquired.Diagnostics)
|
||||
if !plan.details && acquired.Value.Findings != nil {
|
||||
queueConfigDiagnostics(reporter, acquired.Value.Findings.Messages)
|
||||
}
|
||||
entry := validationScope{Scope: name, Outcome: "valid"}
|
||||
if plan.details {
|
||||
entry.Details = projectValidationDetails(acquired.Value, plan.severities, plan.codes)
|
||||
}
|
||||
if canceled := invocation.Context.Err(); canceled != nil {
|
||||
entry.Outcome = "canceled"
|
||||
if err != nil && !errors.Is(err, canceled) {
|
||||
err = errors.Join(err, canceled)
|
||||
} else if err == nil {
|
||||
err = canceled
|
||||
}
|
||||
report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: status})
|
||||
outcome = configAcquisitionFailure(reporter, "validate", name, acquired, err)
|
||||
} else if err != nil {
|
||||
entry.Outcome = "failed"
|
||||
}
|
||||
report.Scopes = append(report.Scopes, entry)
|
||||
if err != nil {
|
||||
message := fmt.Sprintf("Unable to validate configuration scope %q.", name)
|
||||
if entry.Outcome == "canceled" {
|
||||
message = "Configuration validation was canceled."
|
||||
}
|
||||
outcome = configFailure(message, acquired.Explanation, err)
|
||||
break
|
||||
}
|
||||
queueConfigAcquisition(reporter, acquired)
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "canceled"})
|
||||
outcome = configFailure("Configuration validation was canceled.", acquired.Explanation, err)
|
||||
break
|
||||
}
|
||||
report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "valid"})
|
||||
}
|
||||
if len(report.Scopes) > 0 {
|
||||
_ = reporter.Report(commandkit.Report{Value: report, TextTemplate: validationTextTemplate})
|
||||
text := validationTextTemplate
|
||||
if plan.details {
|
||||
text = validationDetailsTemplate
|
||||
}
|
||||
_ = reporter.Report(commandkit.Report{Value: report, TextTemplate: text})
|
||||
return outcome
|
||||
}
|
||||
|
||||
func projectValidationDetails(view View, severities []configkit.Severity, codes []string) *validationDetails {
|
||||
details := &validationDetails{State: "unavailable", Findings: []validationFinding{}, Sources: []validationSource{}}
|
||||
if view.Findings == nil {
|
||||
return details
|
||||
}
|
||||
if len(view.Findings.Records) == 0 {
|
||||
details.State = "empty"
|
||||
return details
|
||||
}
|
||||
details.State = "filtered"
|
||||
references := make(map[string]bool)
|
||||
for _, record := range view.Findings.Records {
|
||||
if len(severities) > 0 && !slices.Contains(severities, record.Severity) {
|
||||
continue
|
||||
}
|
||||
if len(codes) > 0 && !slices.Contains(codes, string(record.Code)) {
|
||||
continue
|
||||
}
|
||||
finding := validationFinding{Code: string(record.Code), Severity: string(record.Severity), Path: record.Path.String(), Message: record.Message, Related: locationReports(record.Related)}
|
||||
if record.Primary != nil {
|
||||
finding.Primary = &locationDTO{SourceID: record.Primary.SourceID, Line: record.Primary.Line, Column: record.Primary.Column}
|
||||
references[record.Primary.SourceID] = true
|
||||
}
|
||||
for _, location := range record.Related {
|
||||
references[location.SourceID] = true
|
||||
}
|
||||
details.Findings = append(details.Findings, finding)
|
||||
}
|
||||
if len(details.Findings) > 0 {
|
||||
details.State = "present"
|
||||
}
|
||||
if view.Sources != nil {
|
||||
for _, source := range view.Sources.Sources() {
|
||||
if references[source.ID] {
|
||||
details.Sources = append(details.Sources, validationSource{ID: source.ID, Label: source.Label, Path: source.Path})
|
||||
}
|
||||
}
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user