notarius/internal/cli/run_contract_test.go

package cli

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)

func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
	tests := []struct {
		name string
		args func(stateTestRoots) []string
	}{
		{name: "missing pipeline", args: func(roots stateTestRoots) []string {
			return []string{"run", "--config", roots.config, "--input", roots.input}
		}},
		{name: "missing input", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config}
		}},
		{name: "unknown flag", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"}
		}},
		{name: "blank output directory", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""}
		}},
		{name: "blank debug directory", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
		}},
		{name: "missing configuration overlay", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-overlay"}
		}},
		{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
		}},
		{name: "blank session ID", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
		}},
		{name: "blank LLM profile", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", ""}
		}},
		{name: "whitespace LLM profile", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", " \t "}
		}},
		{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
		}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			var stdout, stderr bytes.Buffer
			code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
			if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			assertAbsent(t, roots.output)
			assertAbsent(t, roots.debug)
		})
	}
}

func TestRecomputeStepCLIContract(t *testing.T) {
	tests := []struct {
		name       string
		configure  func(*testing.T, stateTestRoots)
		flags      []string
		wantCode   int
		wantOutput string
		wantError  string
	}{
		{
			name: "explicit step",
			configure: func(t *testing.T, roots stateTestRoots) {
				replaceStateTestConfigLine(t, roots.config, "    artifacts:\n      items:\n        extract:\n          module: test/extract\n        merge:\n          module: test/merge\n        normalize:\n          module: test/normalize\n", "    steps:\n      - id: chosen\n        artifacts:\n          items:\n            extract:\n              module: test/extract\n            merge:\n              module: test/merge\n            normalize:\n              module: test/normalize\n")
			},
			flags:      []string{"--resume", "--recompute-step", "chosen"},
			wantCode:   0,
			wantOutput: "outputs=1",
		},
		{name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"},
		{name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"},
		{name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"},
		{name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"},
		{name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"},
		{
			name: "checkpoint recording disabled",
			configure: func(t *testing.T, roots stateTestRoots) {
				replaceStateTestConfigLine(t, roots.config, "    enabled: true\n", "    enabled: false\n")
			},
			flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled",
		},
		{name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			if tt.configure != nil {
				tt.configure(t, roots)
			}
			args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
			args = append(args, tt.flags...)
			var stdout, stderr bytes.Buffer
			code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
			if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
		})
	}
}

func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) {
	tests := []struct {
		name      string
		args      func(stateTestRoots) []string
		wantError string
		wantDebug bool
	}{
		{name: "unknown pipeline", args: func(roots stateTestRoots) []string {
			return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
		}, wantError: `pipeline "missing"`},
		{name: "unknown lane", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"}
		}, wantError: `lane "missing"`, wantDebug: true},
		{name: "unreadable input", args: func(roots stateTestRoots) []string {
			return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"}
		}, wantError: "read input", wantDebug: true},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			var stdout, stderr bytes.Buffer
			code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
			if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			if tt.wantDebug {
				if !strings.Contains(stderr.String(), "debug=") {
					t.Fatalf("stderr=%q, want debug path", stderr.String())
				}
				onlyChildDir(t, roots.debug)
			} else {
				assertAbsent(t, roots.debug)
			}
			assertAbsent(t, roots.output)
		})
	}
}

func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
	roots := newStateTestRoots(t)
	data, err := os.ReadFile(roots.config)
	if err != nil {
		t.Fatal(err)
	}
	data = []byte(replaceRequiredOnce(t, string(data), "    output:\n      module: test/output\n", "      other:\n        extract:\n          module: test/extract\n    output:\n      module: test/output\n"))
	if err := os.WriteFile(roots.config, data, 0o600); err != nil {
		t.Fatal(err)
	}
	harness := newStateTestHarness()
	var stdout, stderr bytes.Buffer
	code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options())
	if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 {
		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
	harness.mu.Lock()
	extractCalls := harness.extractCalls
	harness.mu.Unlock()
	if extractCalls != 1 {
		t.Fatalf("extract calls = %d, want only the selected lane", extractCalls)
	}
}

func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
	t.Run("retired environment roots have no effect", func(t *testing.T) {
		roots := newStateTestRoots(t)
		environmentOutput := filepath.Join(t.TempDir(), "environment-output")
		environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
		opts := newStateTestHarness().options()
		opts.LookupEnv = lookupRunContractEnv(map[string]string{
			"NOTARIUS_OUTPUT_DIR": environmentOutput,
			"NOTARIUS_DEBUG_DIR":  environmentDebug,
		})
		result := runWithStateRoots(t, roots, opts, nil)
		if result.code != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
		}
		assertFile(t, filepath.Join(roots.output, filepath.Base(onlyChildDir(t, roots.output)), "result.json"))
		onlyChildDir(t, roots.debug)
		assertAbsent(t, environmentOutput)
		assertAbsent(t, environmentDebug)
	})

	t.Run("command flags override environment", func(t *testing.T) {
		roots := newStateTestRoots(t)
		environmentOutput := filepath.Join(t.TempDir(), "environment-output")
		environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
		flagOutput := filepath.Join(t.TempDir(), "flag-output")
		flagDebug := filepath.Join(t.TempDir(), "flag-debug")
		opts := newStateTestHarness().options()
		opts.LookupEnv = lookupRunContractEnv(map[string]string{
			"NOTARIUS_OUTPUT_DIR": environmentOutput,
			"NOTARIUS_DEBUG_DIR":  environmentDebug,
		})
		result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug})
		if result.code != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
		}
		assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json"))
		onlyChildDir(t, flagDebug)
		assertAbsent(t, environmentOutput)
		assertAbsent(t, environmentDebug)
	})

	t.Run("built-in roots", func(t *testing.T) {
		roots := newStateTestRoots(t)
		data, err := os.ReadFile(roots.config)
		if err != nil {
			t.Fatal(err)
		}
		text := string(data)
		text = replaceRequiredOnce(t, text, fmt.Sprintf("output:\n  directory: %q\n", roots.output), "")
		text = replaceRequiredOnce(t, text, fmt.Sprintf("debug:\n  directory: %q\n", roots.debug), "")
		if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
			t.Fatal(err)
		}
		opts := newStateTestHarness().options()
		result := runWithStateRoots(t, roots, opts, nil)
		if result.code != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
		}
		configDirectory := filepath.Dir(roots.config)
		assertFile(t, filepath.Join(configDirectory, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(configDirectory, "notarius-output"))), "result.json"))
		onlyChildDir(t, filepath.Join(configDirectory, "notarius-debug"))
	})
}

func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
	roots := newStateTestRoots(t)
	overlayDirectory := filepath.Join(filepath.Dir(roots.config), "overlays")
	overlayOutput := filepath.Join(overlayDirectory, "overlay-output")
	overlayDebug := filepath.Join(overlayDirectory, "overlay-debug")
	writeConfigKitFile(t, filepath.Join(overlayDirectory, "development.yml"), "output:\n  directory: ./overlay-output\ndebug:\n  directory: ./overlay-debug\n")

	var stdout, stderr bytes.Buffer
	code := RunWithOptions([]string{
		"run", "sample", "--config", roots.config, "--config-overlay", "development",
		"--input", roots.input, "--chunk_cache", "bypass", "--debug",
	}, &stdout, &stderr, newStateTestHarness().options())
	if code != 0 || stderr.Len() != 0 {
		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
	assertFile(t, filepath.Join(overlayOutput, filepath.Base(onlyChildDir(t, overlayOutput)), "result.json"))
	bundle := onlyChildDir(t, overlayDebug)
	var invocation debugbundle.Invocation
	readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation)
	if invocation.ConfigOverlay != "development" {
		t.Fatalf("config overlay = %q, want development", invocation.ConfigOverlay)
	}
}

func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
	for _, overlay := range []string{"", "default"} {
		if got := effectiveConfigOverlay(overlay); got != "" {
			t.Fatalf("effectiveConfigOverlay(%q) = %q, want empty", overlay, got)
		}
	}
	if got := effectiveConfigOverlay("development"); got != "development" {
		t.Fatalf("effectiveConfigOverlay(development) = %q", got)
	}
}

func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
	t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
		roots := newStateTestRoots(t)
		profileDir := writeRunContractProfiles(t, "override-profile")
		prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n  profile_dir: %q\n", profileDir))
		harness := newStateTestHarness()
		var factoryProfiles []string
		opts := harness.options()
		var factoryOverrides []LLMRuntimeOverrides
		opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
			factoryProfiles = append(factoryProfiles, profileID)
			factoryOverrides = append(factoryOverrides, overrides)
			return nil, nil, nil
		}
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", " override-profile "}, &stdout, &stderr, opts)
		if code != 0 || stderr.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
		}
		if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
			t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
		}
		if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
			t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
		}
		harness.mu.Lock()
		profiles := append([]string(nil), harness.moduleProfiles...)
		harness.mu.Unlock()
		if len(profiles) < 4 {
			t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles)
		}
		for _, profile := range profiles {
			if profile != "override-profile" {
				t.Fatalf("module profiles = %#v, want override on every request", profiles)
			}
		}
	})

	t.Run("runtime override applies to validators", func(t *testing.T) {
		roots := newStateTestRoots(t)
		profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
		prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n  profile_dir: %q\n", profileDir))
		harness := newStateTestHarness()
		var validatorProfiles []string
		opts := harness.options()
		registerRunContractValidator(t, &opts, &validatorProfiles)
		factoryProfiles := []string{}
		opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
			factoryProfiles = append(factoryProfiles, profileID)
			return nil, nil, nil
		}
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
		if code != 0 || stderr.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
		}
		if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
			t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
		}
		if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" {
			t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles)
		}
	})

	t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
		roots := newStateTestRoots(t)
		profileDir := writeRunContractProfiles(t, "override-profile")
		prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n  profile_dir: %q\n", profileDir))
		factoryCalls := 0
		opts := newStateTestHarness().options()
		opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
			factoryCalls++
			return nil, nil, nil
		}
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts)
		if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
		}
	})

	t.Run("pipeline default is rejected before factory access", func(t *testing.T) {
		roots := newStateTestRoots(t)
		profileDir := writeRunContractProfiles(t, "configured-profile")
		prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n  profile_dir: %q\n", profileDir))
		replaceStateTestConfigLine(t, roots.config, "  sample:\n", "  sample:\n    llm_profile: missing-profile\n")
		factoryCalls := 0
		opts := newStateTestHarness().options()
		opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
			factoryCalls++
			return nil, nil, nil
		}
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
		if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
		}
	})
}

func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
	tests := []struct {
		name      string
		flags     []string
		wantValue string
		wantSet   bool
	}{
		{name: "inherit"},
		{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
		{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			opts := newStateTestHarness().options()
			var got []LLMRuntimeOverrides
			opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
				got = append(got, overrides)
				return nil, nil, nil
			}
			args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
			var stdout, stderr bytes.Buffer
			if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			if len(got) != 1 {
				t.Fatalf("factory overrides = %#v, want one call", got)
			}
			if !tt.wantSet {
				if got[0].ReasoningEffort != nil {
					t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
				}
				return
			}
			if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
				t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
			}
		})
	}
}

func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
	tests := []struct {
		name      string
		flags     []string
		wantError string
	}{
		{
			name:      "mutually exclusive controls",
			flags:     []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
			wantError: "cannot be combined",
		},
		{
			name:      "empty replacement",
			flags:     []string{"--reasoning-effort", "  "},
			wantError: "must not be empty",
		},
		{
			name:      "duplicate replacement",
			flags:     []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
			wantError: "may be specified only once",
		},
		{
			name:      "missing replacement",
			flags:     []string{"--reasoning-effort"},
			wantError: "flag needs an argument",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
			var stdout, stderr bytes.Buffer
			code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
			if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			assertNoRunState(t, roots)
		})
	}
}

func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
	replacement := " focused "
	cleared := ""
	states := []struct {
		name      string
		overrides LLMRuntimeOverrides
		wantValue string
		wantSet   bool
	}{
		{name: "inherit"},
		{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
		{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
	}
	digests := make(map[string]string, len(states))
	for _, state := range states {
		fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
		var value string
		var found bool
		for _, fingerprint := range fingerprints {
			if fingerprint.Name == "reasoning_effort_override" {
				value, found = fingerprint.Value, true
			}
		}
		if found != state.wantSet || (found && value != state.wantValue) {
			t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
		}
		identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
			Pipeline:         pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
			RawInputDigest:   "sha256:input",
			RuntimeOverrides: fingerprints,
		})
		if err != nil {
			t.Fatal(err)
		}
		digests[state.name] = identity.Digest
	}
	if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
		t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
	}
}

func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
	resolved := pipeline.ResolvedPipeline{
		Input:               pipeline.ModuleBinding{LLMProfile: "input-profile"},
		InputExecutionClass: contracts.ExecutionClassLLMBacked,
		Chunk:               pipeline.ModuleBinding{LLMProfile: " zeta "},
		ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
		Steps: []pipeline.ResolvedPipelineStep{{
			ID: "default",
			ArtifactLanes: []pipeline.ResolvedArtifactLane{{
				Extract:                 pipeline.ModuleBinding{LLMProfile: "alpha"},
				ExtractExecutionClass:   contracts.ExecutionClassLLMBacked,
				Merge:                   pipeline.ModuleBinding{LLMProfile: "deterministic-merge"},
				MergeExecutionClass:     contracts.ExecutionClassDeterministic,
				Normalize:               pipeline.ModuleBinding{LLMProfile: " gamma "},
				NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
			}},
		}},
		ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
			{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
			{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
		}}},
		Output:               pipeline.ModuleBinding{LLMProfile: "output-profile"},
		OutputExecutionClass: contracts.ExecutionClassLLMBacked,
	}
	got := effectiveLLMProfileIDs(resolved)
	want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"}
	if strings.Join(got, ",") != strings.Join(want, ",") {
		t.Fatalf("effective profiles = %#v, want %#v", got, want)
	}
}

func TestRunSessionIDUsesEffectiveValueForPromptRequests(t *testing.T) {
	for _, tt := range []struct {
		name string
		args []string
		want string
	}{
		{name: "derived default"},
		{name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"},
	} {
		t.Run(tt.name, func(t *testing.T) {
			roots := newStateTestRoots(t)
			want := tt.want
			if want == "" {
				rawInput, err := os.ReadFile(roots.input)
				if err != nil {
					t.Fatal(err)
				}
				want, err = resolvePromptSessionID("", "test/input", rawInput)
				if err != nil {
					t.Fatal(err)
				}
			}
			harness := newStateTestHarness()
			args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, tt.args...)
			var stdout, stderr bytes.Buffer
			code := RunWithOptions(args, &stdout, &stderr, harness.options())
			if code != 0 || stderr.Len() != 0 {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			harness.mu.Lock()
			sessions := append([]string(nil), harness.sessionIDs...)
			harness.mu.Unlock()
			if len(sessions) < 4 {
				t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions)
			}
			for _, session := range sessions {
				if session != want {
					t.Fatalf("session IDs = %#v, want %q", sessions, want)
				}
			}
			var manifest artifacts.RunManifest
			readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "run-manifest.json", &manifest)
			if session, ok := manifest.Metadata["session_id"]; !ok || session != want {
				t.Fatalf("manifest session = %#v, want %q; metadata = %#v", session, want, manifest.Metadata)
			}
		})
	}
}

func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
	t.Run("LLM factory", func(t *testing.T) {
		roots := newStateTestRoots(t)
		opts := newStateTestHarness().options()
		opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
			return nil, nil, errors.New("injected LLM factory failure")
		}
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
		if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
		}
	})

	t.Run("pipeline preparation", func(t *testing.T) {
		roots := newStateTestRoots(t)
		data, err := os.ReadFile(roots.config)
		if err != nil {
			t.Fatal(err)
		}
		data = []byte(replaceRequiredOnce(t, string(data), "module: test/extract", "module: test/failing-extract"))
		if err := os.WriteFile(roots.config, data, 0o600); err != nil {
			t.Fatal(err)
		}
		opts := newStateTestHarness().options()
		if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
			return nil, errors.New("injected extractor construction failure")
		}); err != nil {
			t.Fatal(err)
		}
		opts.Catalog = catalogFromRegistries(opts.Registries)
		var stdout, stderr bytes.Buffer
		code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
		if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 {
			t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
		}
	})
}

func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
	roots := newStateTestRoots(t)
	harness := newStateTestHarness()
	harness.includeWarnings = true
	harness.includeWarningFile = true
	harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
	var stdout, stderr bytes.Buffer
	code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
	if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
	outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
	output, err := os.ReadFile(outputPath)
	if err != nil || !strings.Contains(string(output), "contract-warning") {
		t.Fatalf("durable output = %q, %v", output, err)
	}
	assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
	bundle := onlyChildDir(t, roots.debug)
	var diagnostics contracts.DiagnosticCollection
	readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
	if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
		t.Fatalf("debug diagnostics = %#v", diagnostics)
	}
}

func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult {
	t.Helper()
	args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
	args = append(args, extra...)
	var stdout, stderr bytes.Buffer
	return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
}

func lookupRunContractEnv(values map[string]string) func(string) (string, bool) {
	return func(name string) (string, bool) {
		value, ok := values[name]
		return value, ok
	}
}

func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) {
	t.Helper()
	data, err := os.ReadFile(roots.config)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil {
		t.Fatal(err)
	}
}

func writeRunContractProfiles(t *testing.T, ids ...string) string {
	t.Helper()
	dir := t.TempDir()
	for _, id := range ids {
		profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
		if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) {
	t.Helper()
	if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) {
		return runContractValidator{profiles: profiles}, nil
	}); err != nil {
		t.Fatal(err)
	}
	if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil {
		t.Fatal(err)
	}
	opts.Catalog = catalogFromRegistries(opts.Registries)
}

type runContractValidator struct {
	profiles *[]string
}

func (v runContractValidator) Name() string { return "run-contract-validator" }

func (v runContractValidator) ExecutionClass() contracts.ExecutionClass {
	return contracts.ExecutionClassLLMBacked
}

func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) {
	*v.profiles = append(*v.profiles, req.LLMProfile)
	return contracts.ValidationResult{Approved: true}, nil
}