Verify native workflows and explain identity mismatches

This commit is contained in:
2026-11-20 06:50:56 +00:00
parent 0e9cbc9c2a
commit 0117f3f35d
9 changed files with 184 additions and 16 deletions

View File

@@ -1,6 +1,7 @@
# CommandKit Adoption # CommandKit Adoption
Status: agreed target; implementation pending. Status: implemented; all stages in the [implementation plan](implementation.md)
are complete.
This roadmap consolidates the [migration discussion](commandkit-notepad.md). This roadmap consolidates the [migration discussion](commandkit-notepad.md).
It defines the intended end state and required work. The It defines the intended end state and required work. The

View File

@@ -837,6 +837,22 @@ polling is part of this implementation sprint.
## Stage 21: Verify the complete cutover and close implementation ## Stage 21: Verify the complete cutover and close implementation
**Completed.** The final audit covers the native tree and four scopes,
independent comparisons and invocation-local overrides, immutable configuration
handoff and selected-roster derivation, reports/visibility/progress, partial
failure and cancellation, semantic resume, publication, restore, and cleanup.
Strengthened the assembled offline family workflow with imported configuration,
an overlay, native inspection/catalog/planning, safe synthetic credentials,
a later artifact failure, durable checkpoint reuse, and unchanged loaded values.
Native inspection now explains typed identity mismatches and labels campaign
sources correctly; focused regressions preserve the original causes. Removed
stale test diagnostic names. Full tests, race tests, vet, build, module-tidiness,
documentation/examples, and diff checks pass with official Go 1.27.4. The existing
nonpublishing builder produced Linux amd64/arm64 and macOS arm64 assets and
verified the embedded test version; macOS evidence remains compilation-only.
Deferred features and separate pending roadmaps remain unchanged. No release,
tag, push, or CI polling was performed.
Read: roadmap verification/completion, this plan, the canonical updated Read: roadmap verification/completion, this plan, the canonical updated
documentation, and [Development validation](../development.md#validation). documentation, and [Development validation](../development.md#validation).

View File

@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
@@ -16,16 +17,22 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/manifest"
"go.maximumdirect.net/commandkit"
) )
func TestSelectedPartyCatalogDrivesBoundedAnalysis(t *testing.T) { func TestSelectedPartyCatalogDrivesBoundedAnalysis(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml") pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, `whisperx: const secret = "synthetic-family-workflow-credential"
t.Setenv("NARRATIO_WORKFLOW_TEST_KEY", secret)
mustWriteTestFile(t, pipelinePath, "config:\n imports:\n - {path: family.yml, required: true}\n")
mustWriteTestFile(t, filepath.Join(dir, "overlays", "testing.yml"), "scriptorium:\n artifact_families:\n character_note:\n prompt_id: note-testing\n")
mustWriteTestFile(t, filepath.Join(dir, "family.yml"), `whisperx:
transcribe_url: https://transcription.example.test/transcribe transcribe_url: https://transcription.example.test/transcribe
workspace: workspace:
root: `+filepath.Join(dir, "work")+` root: `+filepath.Join(dir, "work")+`
scriptorium: scriptorium:
credentials: [NARRATIO_WORKFLOW_TEST_KEY]
artifact_families: artifact_families:
character_meta: character_meta:
enabled: false enabled: false
@@ -50,7 +57,8 @@ scriptorium:
mustWriteTestFile(t, partyPath, "schema_version: narratio.party.v1\ncharacters:\n ivy:\n player: {name: Selected Player}\n character: {name: Ivy, classes: [{name: ranger}]}\n") mustWriteTestFile(t, partyPath, "schema_version: narratio.party.v1\ncharacters:\n ivy:\n player: {name: Selected Player}\n character: {name: Ivy, classes: [{name: ranger}]}\n")
sessionPath := filepath.Join(dir, "session.yml") sessionPath := filepath.Join(dir, "session.yml")
mustWriteTestFile(t, sessionPath, "session_id: session\ninputs: {audio_dir: audio}\n") mustWriteTestFile(t, sessionPath, "session_id: session\ninputs: {audio_dir: audio}\n")
pipeline, err := config.LoadPipeline(pipelinePath) overlay := "testing"
pipeline, err := config.LoadPipelineWithOptions(pipelinePath, config.PipelineLoadOptions{Overlay: &overlay})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -70,6 +78,45 @@ scriptorium:
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
catalogSelection := []string{"--config", pipelinePath, "--overlay", overlay, "--campaign-file", campaignPath, "--party-file", partyPath, "--secrets-dir", t.TempDir(), "--format", "json"}
common := append(append([]string{}, catalogSelection...), "--session-file", sessionPath)
for _, scope := range []string{"app", "session", "party"} {
output := nativeInspectionForTest(t, append([]string{"config", "show", "--scope", scope}, common...))
if strings.Contains(output, secret) {
t.Fatal("native inspection exposed credentials")
}
if scope == "app" && (!strings.Contains(output, "artifact_families") || strings.Contains(output, "character_meta_ivy")) {
t.Fatal("native app inspection substituted derived configuration")
}
if scope == "session" {
var effective map[string]any
if err := json.Unmarshal([]byte(output), &effective); err != nil {
t.Fatal(err)
}
if effective["campaign"] != "campaign" || effective["session_id"] != "session" {
t.Fatalf("inspection lost resolved session defaults: %#v", effective)
}
}
if scope == "party" && !strings.Contains(output, "Selected Player") {
t.Fatal("party inspection used a different roster")
}
}
diff := nativeInspectionForTest(t, []string{"config", "diff", "--config", pipelinePath, "--against-config", pipelinePath, "--against-overlay", overlay, "--details", "--format", "json"})
if !strings.Contains(diff, "/scriptorium/artifact_families/character_note/prompt_id") || strings.Contains(diff, secret) {
t.Fatalf("unexpected independent comparison: %s", diff)
}
catalog := nativeInspectionForTest(t, append([]string{"artifacts", "list"}, catalogSelection...))
if !strings.Contains(catalog, "character_note_ivy") {
t.Fatal("catalog omitted selected member")
}
selected := []string{"--stage", "analyze", "--artifacts", "character_note"}
planned := nativeInspectionForTest(t, append(append([]string{"session", "plan", "session"}, common...), "--stage", "prepare"))
if !strings.Contains(planned, "prepare") || strings.Contains(planned, secret) {
t.Fatal("initial plan lost selection or exposed credentials")
}
if _, err := os.Stat(cfg.Pipeline.Workspace.Root); !os.IsNotExist(err) {
t.Fatalf("inspection or planning created workspace: %v", err)
}
plan, err := BuildBoundedPlan("analyze", "analyze") plan, err := BuildBoundedPlan("analyze", "analyze")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -93,16 +140,71 @@ scriptorium:
} }
seed.Inputs = append(seed.Inputs, manifest.InputRecord{Kind: descriptor.ManifestKind, Path: playersPath, Checksum: checksum}) seed.Inputs = append(seed.Inputs, manifest.InputRecord{Kind: descriptor.ManifestKind, Path: playersPath, Checksum: checksum})
saveBoundedManifest(t, cfg, seed) saveBoundedManifest(t, cfg, seed)
runner := &scriptorium.FakeRunner{} persisted, err := os.ReadFile(manifestPathFor(cfg))
summary, err := executePlan(context.Background(), cfg, plan, RunOptions{ if err != nil {
SelectedArtifacts: []string{"character_note"}, t.Fatal(err)
Env: &Env{Scriptorium: runner}, }
planned = nativeInspectionForTest(t, append(append([]string{"session", "plan", "session"}, common...), selected...))
if !strings.Contains(planned, "character_meta_ivy") || strings.Contains(planned, secret) {
t.Fatal("plan lost prerequisites or exposed credentials")
}
afterPlan, err := os.ReadFile(manifestPathFor(cfg))
if err != nil || !bytes.Equal(persisted, afterPlan) {
t.Fatal("planning changed durable state")
}
failure := errors.New("synthetic later artifact failure")
runner := &familyWorkflowRunner{failure: failure}
var captured *config.Config
application, err := newApplication(commandDependencies{}, func(ctx context.Context, acquired *config.Config, selected BoundedPlan, options RunOptions) (*RunSummary, error) {
captured = acquired
before, err := json.Marshal(acquired.Pipeline)
if err != nil {
return nil, err
}
options.Env = &Env{Scriptorium: runner}
summary, runErr := executePlan(ctx, acquired, selected, options)
after, err := json.Marshal(acquired.Pipeline)
if err != nil || !bytes.Equal(before, after) {
t.Fatal("native execution mutated completed configuration")
}
return summary, runErr
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(summary.Executed, []string{"analyze"}) || len(runner.RunRequests) != 2 || runner.RunRequests[0].PromptID != "meta" || runner.RunRequests[1].PromptID != "note" { args := append(append([]string{"run", "session"}, common...), selected...)
t.Fatalf("bounded family execution: summary=%+v requests=%+v", summary, runner.RunRequests) var output, diagnostics bytes.Buffer
outcome := application.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, failure) {
t.Fatalf("failed outcome=%#v diagnostics=%s", outcome, diagnostics.String())
}
partial := decodeNativeReport[runReport](t, output.Bytes())
if partial.Complete || len(partial.StageResults) != 1 || partial.StageResults[0].Outcome != "failed" || len(partial.AvailableArtifacts) == 0 {
t.Fatalf("partial report=%#v", partial)
}
failed, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatal(err)
}
if failed.Stages["analyze"].AnalyzeArtifacts["character_meta_ivy"].Status != manifest.AnalyzeArtifactCurrent {
t.Fatal("earlier family checkpoint was lost")
}
if strings.Contains(output.String()+diagnostics.String(), secret) {
t.Fatal("failure report exposed credentials")
}
runner.failure = nil
output.Reset()
diagnostics.Reset()
outcome = application.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 {
t.Fatalf("retry status=%d diagnostics=%s", outcome.Status, diagnostics.String())
}
summary := decodeNativeReport[runReport](t, output.Bytes())
if !summary.Complete || !reflect.DeepEqual(summary.Executed, []string{"analyze"}) || len(runner.RunRequests) != 2 || runner.RunRequests[0].PromptID != "meta" || runner.RunRequests[1].PromptID != "note-testing" {
t.Fatalf("bounded family retry: summary=%+v requests=%+v", summary, runner.RunRequests)
}
if captured == nil || len(captured.Pipeline.Scriptorium.Artifacts) != 0 || strings.Contains(output.String()+diagnostics.String(), secret) {
t.Fatal("retry lost the configuration/report boundary")
} }
players, err := os.ReadFile(runner.RunRequests[0].InputPaths["players"]) players, err := os.ReadFile(runner.RunRequests[0].InputPaths["players"])
if err != nil || !strings.Contains(string(players), "Selected Player") || runner.RunRequests[1].Vars["character_name"] != "Ivy" { if err != nil || !strings.Contains(string(players), "Selected Player") || runner.RunRequests[1].Vars["character_name"] != "Ivy" {
@@ -126,3 +228,17 @@ scriptorium:
t.Fatal("execution changed completed pipeline configuration") t.Fatal("execution changed completed pipeline configuration")
} }
} }
// familyWorkflowRunner replaces only the downstream process, retaining real
// acquisition, catalog derivation, execution, persistence, and reporting.
type familyWorkflowRunner struct {
scriptorium.FakeRunner
failure error
}
func (r *familyWorkflowRunner) RunArtifact(ctx context.Context, request scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
if request.PromptID == "note-testing" && r.failure != nil {
return scriptorium.ArtifactResult{}, r.failure
}
return r.FakeRunner.RunArtifact(ctx, request)
}

View File

@@ -38,7 +38,7 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), SessionFile: suppliedSelection(sessionPath), Session: suppliedSelection("2026-05-03")}, deps) loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), SessionFile: suppliedSelection(sessionPath), Session: suppliedSelection("2026-05-03")}, deps)
resolved, err := loaded.Domain(context.Background()) resolved, err := loaded.Domain(context.Background())
if err != nil { if err != nil {
t.Fatalf("loadCommandConfig() error = %v", err) t.Fatalf("configuration acquisition error = %v", err)
} }
defer func() { _ = loaded.Close() }() defer func() { _ = loaded.Close() }()

View File

@@ -77,6 +77,10 @@ func configurationCommand(deps commandDependencies) commandkit.Command {
scopes[index].Acquire = func(ctx context.Context, request ckconfig.Request, input configurationSelection, state *configurationInspectionState) (commandkit.Acquired[ckconfig.View], error) { scopes[index].Acquire = func(ctx context.Context, request ckconfig.Request, input configurationSelection, state *configurationInspectionState) (commandkit.Acquired[ckconfig.View], error) {
result, err := acquire(ctx, request, input, state) result, err := acquire(ctx, request, input, state)
result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...) result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...)
var identity *config.SessionIdentityError
if errors.As(err, &identity) {
result.Explanation.Message = identity.Error()
}
return result, err return result, err
} }
compare := scopes[index].Compare compare := scopes[index].Compare
@@ -85,6 +89,10 @@ func configurationCommand(deps commandDependencies) commandkit.Command {
separate.independent = true separate.independent = true
result, err := compare(ctx, request, &separate) result, err := compare(ctx, request, &separate)
result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...) result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...)
var identity *config.SessionIdentityError
if errors.As(err, &identity) {
result.Explanation.Message = identity.Error()
}
return result, err return result, err
} }
} }

View File

@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -167,3 +168,25 @@ func TestConfigurationAggregateDoesNotOmitUnavailableContext(t *testing.T) {
t.Fatalf("missing aggregate scope report: %s", out.String()) t.Fatalf("missing aggregate scope report: %s", out.String())
} }
} }
func TestConfigurationInspectionExplainsIdentityMismatches(t *testing.T) {
_, campaign, session := writeValidConfigFiles(t, t.TempDir())
for _, args := range [][]string{
{"config", "show", "--scope", "campaign", "--campaign-file", campaign, "--campaign", "other-campaign"},
{"config", "show", "--scope", "session", "--campaign-file", campaign, "--session-file", session, "--session", "other-session"},
{"config", "diff", "--scope", "campaign", "--campaign-file", campaign, "--against-campaign-file", campaign, "--against-campaign", "other-campaign"},
} {
var output, diagnostics bytes.Buffer
result := Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
var identity *config.SessionIdentityError
if result.Status == 0 || !errors.As(result.Cause, &identity) {
t.Fatalf("%v: identity cause missing: %#v", args, result)
}
if output.Len() != 0 || !strings.Contains(diagnostics.String(), identity.Field) || !strings.Contains(diagnostics.String(), identity.Expected) || !strings.Contains(diagnostics.String(), identity.Actual) {
t.Fatalf("%v: assertion not explained: %s", args, diagnostics.String())
}
if identity.Field == "campaign_id" && strings.Contains(diagnostics.String(), "session file") {
t.Fatalf("campaign assertion mislabeled: %s", diagnostics.String())
}
}
}

View File

@@ -100,7 +100,7 @@ inputs:
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps) loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps)
resolved, err := loaded.Domain(context.Background()) resolved, err := loaded.Domain(context.Background())
if err != nil { if err != nil {
t.Fatalf("loadCommandConfig() error = %v", err) t.Fatalf("configuration acquisition error = %v", err)
} }
defer func() { _ = loaded.Close() }() defer func() { _ = loaded.Close() }()
if resolved.Party.Canonical == nil { if resolved.Party.Canonical == nil {
@@ -222,7 +222,7 @@ inputs:
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps) loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps)
resolved, err := loaded.Domain(context.Background()) resolved, err := loaded.Domain(context.Background())
if err != nil { if err != nil {
t.Fatalf("loadCommandConfig() error = %v", err) t.Fatalf("configuration acquisition error = %v", err)
} }
downloadedPath := resolved.SessionSource.LocalPath downloadedPath := resolved.SessionSource.LocalPath
if err := loaded.Close(); err != nil { if err := loaded.Close(); err != nil {

View File

@@ -230,7 +230,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
} }
} }
func TestExecuteRestoreNonDryRunConflictDoesNotexecuteForTest(t *testing.T) { func TestExecuteRestoreNonDryRunConflictDoesNotWrite(t *testing.T) {
deps := restoreCommandDependencies(&storage.FakeBackend{}) deps := restoreCommandDependencies(&storage.FakeBackend{})
deps.discoverRemoteCurrentState = func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error) { deps.discoverRemoteCurrentState = func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error) {
return &RemoteCurrentState{ return &RemoteCurrentState{

View File

@@ -82,10 +82,14 @@ type SessionIdentityError struct {
} }
func (e *SessionIdentityError) Error() string { func (e *SessionIdentityError) Error() string {
if e.Actual == "" { sourceKind := "session"
return fmt.Sprintf("load session config: session file %q: %s is required when %q is selected", e.Source, e.Field, e.Expected) if e.Field == "campaign_id" {
sourceKind = "campaign"
} }
return fmt.Sprintf("load session config: session file %q: %s mismatch: selected %q does not match %q", e.Source, e.Field, e.Expected, e.Actual) if e.Actual == "" {
return fmt.Sprintf("load %s config: %s file %q: %s is required when %q is selected", sourceKind, sourceKind, e.Source, e.Field, e.Expected)
}
return fmt.Sprintf("load %s config: %s file %q: %s mismatch: selected %q does not match %q", sourceKind, sourceKind, e.Source, e.Field, e.Expected, e.Actual)
} }
func validateSessionIdentity(label string, cfg *SessionConfig, opts SessionLoadOptions) error { func validateSessionIdentity(label string, cfg *SessionConfig, opts SessionLoadOptions) error {