Add detailed validation evidence

2026-11-07 04:52:58 +00:00
parent 1986c44f49
commit 0b70f37267
5 changed files with 286 additions and 19 deletions
--- a/integrations/ckconfig/detailed_validation_test.go
+++ b/integrations/ckconfig/detailed_validation_test.go
@@ -0,0 +1,136 @@
+package ckconfig
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	commandkit "go.maximumdirect.net/commandkit"
+	configkit "go.maximumdirect.net/configkit"
+)
+
+func validationDiagnostic(message string) commandkit.Diagnostic {
+	return commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, message+"\n"); return err }}
+}
+
+func TestDetailedValidationFiltersAndRetainsFailureEvidence(t *testing.T) {
+	cause := errors.New("second scope failed")
+	calls := []string{}
+	reader := &sourceInventory{values: []configkit.SourceInfo{{ID: "primary", Label: "main", Path: "main.yml"}, {ID: "related", Label: "included", Path: "include.yml"}, {ID: "unused", Label: "unused"}}}
+	records := []configkit.Diagnostic{
+		{Code: "chosen", Severity: configkit.SeverityWarning, Path: configkit.Keys("a/b"), Message: "selected warning", Primary: &configkit.Location{SourceID: "primary", Line: 2, Column: 3}, Related: []configkit.Location{{SourceID: "related", Line: 7, Column: 8}}},
+		{Code: "other", Severity: configkit.SeverityWarning, Path: configkit.Keys("other"), Message: "other warning"},
+	}
+	scopes := []Scope[struct{}, struct{}]{
+		{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls = append(calls, "app")
+			return commandkit.Acquired[View]{Value: View{Sources: reader, Findings: &Findings{Records: records, Messages: []commandkit.Diagnostic{validationDiagnostic("native warnings")}}}, Diagnostics: []commandkit.Diagnostic{validationDiagnostic("application warning")}}, nil
+		}},
+		{Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls = append(calls, "second")
+			return commandkit.Acquired[View]{Value: View{Findings: &Findings{Records: []configkit.Diagnostic{{Code: "failed", Severity: configkit.SeverityError, Message: "failed finding"}}}}, Explanation: commandkit.Explanation{Message: "terminal failure"}}, cause
+		}},
+		{Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls = append(calls, "third")
+			return commandkit.Acquired[View]{}, nil
+		}},
+	}
+	app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes}))
+	var out, diag bytes.Buffer
+	result := app.Execute(context.Background(), []string{"config", "validate", "--scope=all", "--details", "--severity=warning", "--code=chosen", "--format=json", "--verbose"}, commandkit.Streams{Out: &out, Err: &diag})
+	decoder := json.NewDecoder(&out)
+	var report validationReport
+	if err := decoder.Decode(&report); err != nil {
+		t.Fatal(err)
+	}
+	if err := decoder.Decode(new(any)); err != io.EOF {
+		t.Fatalf("extra JSON: %v", err)
+	}
+	if !errors.Is(result.Cause, cause) || result.Kind != commandkit.OutcomeOperational || len(calls) != 2 || calls[0] != "app" || calls[1] != "second" || len(report.Scopes) != 2 || report.Scopes[0].Outcome != "valid" || report.Scopes[1].Outcome != "failed" {
+		t.Fatalf("result=%#v calls=%v report=%#v", result, calls, report)
+	}
+	first, second := report.Scopes[0].Details, report.Scopes[1].Details
+	if first == nil || first.State != "present" || len(first.Findings) != 1 || first.Findings[0].Path != "/a~1b" || len(first.Sources) != 2 || first.Sources[0].ID != "primary" || first.Sources[1].ID != "related" || second == nil || second.State != "filtered" || len(second.Findings) != 0 {
+		t.Fatalf("details=%#v %#v", first, second)
+	}
+	if strings.Contains(diag.String(), "native warnings") || !strings.Contains(diag.String(), "application warning") || !strings.Contains(diag.String(), "terminal failure") {
+		t.Fatalf("diagnostics=%q", diag.String())
+	}
+	before := len(calls)
+	for _, args := range [][]string{{"--severity=warning"}, {"--code=x"}, {"--details", "--severity=unknown"}} {
+		result = app.Execute(context.Background(), append([]string{"config", "validate"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard})
+		if result.Kind != commandkit.OutcomeUsage || len(calls) != before {
+			t.Fatalf("invalid %v: %#v", args, result)
+		}
+	}
+}
+
+func TestValidationCancellationAndDeliveryRetainReachedScopes(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	calls := 0
+	scopes := []Scope[struct{}, struct{}]{
+		{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls++
+			return commandkit.Acquired[View]{Value: View{Findings: &Findings{}}}, nil
+		}},
+		{Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls++
+			cancel()
+			return commandkit.Acquired[View]{}, nil
+		}},
+		{Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+			calls++
+			return commandkit.Acquired[View]{}, nil
+		}},
+	}
+	app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes}))
+	var out bytes.Buffer
+	result := app.Execute(ctx, []string{"config", "validate", "--scope=all", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
+	var report validationReport
+	if err := json.Unmarshal(out.Bytes(), &report); err != nil {
+		t.Fatal(err)
+	}
+	if !errors.Is(result.Cause, context.Canceled) || calls != 2 || len(report.Scopes) != 2 || report.Scopes[0].Details.State != "empty" || report.Scopes[1].Details.State != "unavailable" || report.Scopes[1].Outcome != "canceled" {
+		t.Fatalf("result=%#v calls=%d report=%#v", result, calls, report)
+	}
+	failure := errors.New("load failure")
+	scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) {
+		return commandkit.Acquired[View]{}, failure
+	}}
+	app = newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
+	result = app.Execute(context.Background(), []string{"config", "validate", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
+	if !errors.Is(result.Cause, failure) || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
+		t.Fatalf("failure and delivery: %#v", result)
+	}
+	out.Reset()
+	result = app.Execute(context.Background(), []string{"config", "validate", "--details", "--quiet"}, commandkit.Streams{Out: &out, Err: io.Discard})
+	if !errors.Is(result.Cause, failure) || out.Len() != 0 {
+		t.Fatalf("quiet failure: %#v %q", result, out.String())
+	}
+}
+
+func TestDetailedValidationReportsNativeLoadError(t *testing.T) {
+	_, failure := configkit.Load[map[string]any](configkit.Bytes("invalid source", []byte("invalid: [")))
+	var native *configkit.LoadError
+	if !errors.As(failure, &native) {
+		t.Fatalf("fixture: %v", failure)
+	}
+	scope := FromConfigKit("app", "", func(context.Context, Selection, struct{}, struct{}) (*configkit.Result[map[string]any], error) {
+		return nil, failure
+	})
+	app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
+	var out bytes.Buffer
+	result := app.Execute(context.Background(), []string{"config", "validate", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
+	var report validationReport
+	if err := json.Unmarshal(out.Bytes(), &report); err != nil {
+		t.Fatal(err)
+	}
+	if !errors.Is(result.Cause, failure) || len(report.Scopes) != 1 || report.Scopes[0].Outcome != "failed" || report.Scopes[0].Details.State != "present" || len(report.Scopes[0].Details.Findings) != len(native.Diagnostics()) {
+		t.Fatalf("native failure: %#v %q", result, out.String())
+	}
+}
--- a/integrations/ckconfig/input.go
+++ b/integrations/ckconfig/input.go
@@ -20,6 +20,8 @@ type configCommandInput[I, S any] struct {
 	sourceID      commandkit.Supplied[string]
 	details       commandkit.Supplied[bool]
 	check         commandkit.Supplied[bool]
+	severities    []configkit.Severity
+	codes         []string
 	values        commandkit.FlagValues[I]
 	againstValues commandkit.FlagValues[I]
 	plan          *configOperationPlan[I, S]
@@ -45,6 +47,8 @@ type configOperationPlan[I, S any] struct {
 	sourceID   commandkit.Supplied[string]
 	details    bool
 	check      bool
+	severities []configkit.Severity
+	codes      []string
 	list       bool
 	help       bool
 	historical bool
@@ -94,6 +98,17 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
 				}),
 		)
 	}
+	if operation == Validate {
+		flags = append(flags,
+			commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native validation findings."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
+			commandkit.RepeatFlag(commandkit.FlagMeta{Name: "severity", Usage: "Filter detailed findings by error or warning."}, []configkit.Severity(nil), parseValidationSeverity, func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Severity]) {
+				(*input).severities = value.Value
+			}),
+			commandkit.RepeatFlag(commandkit.FlagMeta{Name: "code", Usage: "Filter detailed findings by exact native code."}, []string(nil), parseConfigString, func(input **configCommandInput[I, S], value commandkit.Supplied[[]string]) {
+				(*input).codes = value.Value
+			}),
+		)
+	}
 	if supportsOverrides(prepared, operation) {
 		flags = append(flags, commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON (pointer keys cannot contain =)."}, []configkit.Override(nil), overrideParser("--set"),
 			func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Override]) {
@@ -179,6 +194,9 @@ func operationDeclaration(operation Operation) (string, string) {
 }

 func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation Operation, input *configCommandInput[I, S]) error {
+	if operation == Validate && !input.details.Value && (len(input.severities) > 0 || len(input.codes) > 0) {
+		return fmt.Errorf("--severity and --code require --details")
+	}
 	if operation == Explain {
 		if input.list.Value && input.path.Set {
 			return fmt.Errorf("--list and --path cannot be combined")
@@ -256,6 +274,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
 		sourceID:   input.sourceID,
 		details:    input.details.Value,
 		check:      input.check.Value,
+		severities: append([]configkit.Severity(nil), input.severities...),
+		codes:      append([]string(nil), input.codes...),
 		list:       input.list.Value,
 		historical: input.historical.Value,
 		under:      under,
--- a/integrations/ckconfig/validate.go
+++ b/integrations/ckconfig/validate.go
@@ -1,10 +1,13 @@
 package ckconfig

 import (
+	"errors"
 	"fmt"
+	"slices"
 	"text/template"

 	commandkit "go.maximumdirect.net/commandkit"
+	configkit "go.maximumdirect.net/configkit"
 )

 type validationReport struct {
@@ -12,11 +15,48 @@ type validationReport struct {
 }

 type validationScope struct {
-	Scope   string `json:"scope"`
-	Outcome string `json:"outcome"`
+	Scope   string             `json:"scope"`
+	Outcome string             `json:"outcome"`
+	Details *validationDetails `json:"details,omitempty"`
+}
+
+type validationDetails struct {
+	State    string              `json:"state"`
+	Findings []validationFinding `json:"findings"`
+	Sources  []validationSource  `json:"sources"`
+}
+
+type validationFinding struct {
+	Code     string        `json:"code"`
+	Severity string        `json:"severity"`
+	Path     string        `json:"path"`
+	Message  string        `json:"message"`
+	Primary  *locationDTO  `json:"primary"`
+	Related  []locationDTO `json:"related"`
+}
+
+type validationSource struct {
+	ID    string `json:"id"`
+	Label string `json:"label"`
+	Path  string `json:"path"`
 }

 var validationTextTemplate = template.Must(commandkit.ParseTextTemplate("validation", "{{range .Scopes}}{{.Scope}}: {{.Outcome}}\n{{end}}", nil))
+var validationDetailsTemplate = template.Must(commandkit.ParseTextTemplate("validation-details", `{{range .Scopes}}{{.Scope}}: {{.Outcome}} ({{.Details.State}})
+{{range .Details.Findings}}  {{.Severity}} {{.Code}} {{printf "%q" .Path}}: {{.Message}}
+{{if .Primary}}    {{.Primary.SourceID}}:{{.Primary.Line}}:{{.Primary.Column}}
+{{end}}{{range .Related}}    {{.SourceID}}:{{.Line}}:{{.Column}}
+{{end}}{{end}}{{range .Details.Sources}}  source {{printf "%q" .ID}}: {{.Label}} {{.Path}}
+{{end}}{{end}}`, nil))
+
+func parseValidationSeverity(raw string) (configkit.Severity, error) {
+	switch configkit.Severity(raw) {
+	case configkit.SeverityError, configkit.SeverityWarning:
+		return configkit.Severity(raw), nil
+	default:
+		return "", fmt.Errorf("unknown validation severity %q", raw)
+	}
+}

 func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *configOperationPlan[I, S], state S) commandkit.Outcome {
 	reporter := invocation.Reporter()
@@ -29,30 +69,88 @@ func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *confi
 		}
 		name := selected.scope.scope.Name
 		if selected.scope.scope.Acquire == nil {
-			report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "failed"})
+			entry := validationScope{Scope: name, Outcome: "failed"}
+			if plan.details {
+				entry.Details = projectValidationDetails(View{}, plan.severities, plan.codes)
+			}
+			report.Scopes = append(report.Scopes, entry)
 			outcome = commandkit.Failure(fmt.Sprintf("Configuration scope %q has no acquisition callback.", name))
 			break
 		}
 		acquired, err := selected.scope.scope.Acquire(invocation.Context, Request{Operation: Validate, Selection: selected.selection}, selected.input, state)
-		if err != nil {
-			status := "failed"
-			if invocation.Context.Err() != nil {
-				status = "canceled"
+		queueConfigDiagnostics(reporter, acquired.Diagnostics)
+		if !plan.details && acquired.Value.Findings != nil {
+			queueConfigDiagnostics(reporter, acquired.Value.Findings.Messages)
+		}
+		entry := validationScope{Scope: name, Outcome: "valid"}
+		if plan.details {
+			entry.Details = projectValidationDetails(acquired.Value, plan.severities, plan.codes)
+		}
+		if canceled := invocation.Context.Err(); canceled != nil {
+			entry.Outcome = "canceled"
+			if err != nil && !errors.Is(err, canceled) {
+				err = errors.Join(err, canceled)
+			} else if err == nil {
+				err = canceled
 			}
-			report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: status})
-			outcome = configAcquisitionFailure(reporter, "validate", name, acquired, err)
+		} else if err != nil {
+			entry.Outcome = "failed"
+		}
+		report.Scopes = append(report.Scopes, entry)
+		if err != nil {
+			message := fmt.Sprintf("Unable to validate configuration scope %q.", name)
+			if entry.Outcome == "canceled" {
+				message = "Configuration validation was canceled."
+			}
+			outcome = configFailure(message, acquired.Explanation, err)
 			break
 		}
-		queueConfigAcquisition(reporter, acquired)
-		if err := invocation.Context.Err(); err != nil {
-			report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "canceled"})
-			outcome = configFailure("Configuration validation was canceled.", acquired.Explanation, err)
-			break
-		}
-		report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "valid"})
 	}
-	if len(report.Scopes) > 0 {
-		_ = reporter.Report(commandkit.Report{Value: report, TextTemplate: validationTextTemplate})
+	text := validationTextTemplate
+	if plan.details {
+		text = validationDetailsTemplate
 	}
+	_ = reporter.Report(commandkit.Report{Value: report, TextTemplate: text})
 	return outcome
 }
+
+func projectValidationDetails(view View, severities []configkit.Severity, codes []string) *validationDetails {
+	details := &validationDetails{State: "unavailable", Findings: []validationFinding{}, Sources: []validationSource{}}
+	if view.Findings == nil {
+		return details
+	}
+	if len(view.Findings.Records) == 0 {
+		details.State = "empty"
+		return details
+	}
+	details.State = "filtered"
+	references := make(map[string]bool)
+	for _, record := range view.Findings.Records {
+		if len(severities) > 0 && !slices.Contains(severities, record.Severity) {
+			continue
+		}
+		if len(codes) > 0 && !slices.Contains(codes, string(record.Code)) {
+			continue
+		}
+		finding := validationFinding{Code: string(record.Code), Severity: string(record.Severity), Path: record.Path.String(), Message: record.Message, Related: locationReports(record.Related)}
+		if record.Primary != nil {
+			finding.Primary = &locationDTO{SourceID: record.Primary.SourceID, Line: record.Primary.Line, Column: record.Primary.Column}
+			references[record.Primary.SourceID] = true
+		}
+		for _, location := range record.Related {
+			references[location.SourceID] = true
+		}
+		details.Findings = append(details.Findings, finding)
+	}
+	if len(details.Findings) > 0 {
+		details.State = "present"
+	}
+	if view.Sources != nil {
+		for _, source := range view.Sources.Sources() {
+			if references[source.ID] {
+				details.Sources = append(details.Sources, validationSource{ID: source.ID, Label: source.Label, Path: source.Path})
+			}
+		}
+	}
+	return details
+}