weatherreporter/internal/cli/comparison_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)

func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) {
	opts, err := parseComparisonFlags(app.ReportDaily, []string{
		"--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep",
		"--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet",
	})
	if err != nil {
		t.Fatalf("parseComparisonFlags() error = %v", err)
	}
	wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"}
	if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet {
		t.Fatalf("options = %#v", opts)
	}
}

func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n  units: metric\n  timezone: America/Chicago\noutput:\n  directory: configured-reports\npromptkit:\n  profile: configured-default\n")
	clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}
	var factoryConfig PromptExecutorConfig
	factoryCalls := 0
	executor := &factoryExecutor{}
	runner := Runner{
		Clock: clock, WorkingDir: workingDir,
		ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) {
			factoryCalls++
			factoryConfig = value
			return executor, nil
		},
	}
	req, opts, err := runner.resolveComparisonAction([]string{
		"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep",
		"--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath,
	})
	if err != nil {
		t.Fatalf("resolveComparisonAction() error = %v", err)
	}
	if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" {
		t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls)
	}
	if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) {
		t.Fatalf("profile IDs = %#v, want %#v", got, want)
	}
	if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet {
		t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig)
	}
	location, loadErr := time.LoadLocation("America/New_York")
	if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" {
		t.Fatalf("date/location = %v/%v", req.Date, loadErr)
	}
}

func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\noutput:\n  directory: configured/../reports\n")
	req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{
		"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath,
	})
	if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" {
		t.Fatalf("request/error = %#v/%v", req, err)
	}
}

func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) {
	for _, test := range []struct {
		name string
		args []string
	}{
		{name: "missing report", args: nil},
		{name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}},
		{name: "single profile", args: []string{"today", "--profile", "one"}},
		{name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}},
		{name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}},
		{name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}},
		{name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}},
		{name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}},
	} {
		t.Run(test.name, func(t *testing.T) {
			factoryCalls, applicationCalls := 0, 0
			runner := Runner{
				Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)},
				ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
					factoryCalls++
					return &factoryExecutor{}, nil
				},
				compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
					applicationCalls++
					return nil, nil
				},
			}
			var stdout, stderr bytes.Buffer
			if err := runner.Run(context.Background(), append([]string{"compare"}, test.args...), &stdout, &stderr); err == nil {
				t.Fatal("Run() error = nil")
			}
			if factoryCalls != 0 || applicationCalls != 0 || stdout.Len() != 0 || stderr.Len() != 0 {
				t.Fatalf("factory/application calls/output = %d/%d/%q/%q", factoryCalls, applicationCalls, stdout.String(), stderr.String())
			}
		})
	}
}

func TestCompareCommandUsesOneExecutorAndInjectedApplication(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n")
	executor := &factoryExecutor{}
	factoryCalls, applicationCalls := 0, 0
	var received app.ComparisonRequest
	wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"}
	comparisonErr := errors.New("comparison completed with 1 failed profiles")
	runner := Runner{
		Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
		ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
			factoryCalls++
			return executor, nil
		},
		compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
			applicationCalls++
			received = req
			return wantResult, comparisonErr
		},
	}
	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{
		"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath,
	}, &stdout, &stderr)
	if !errors.Is(err, comparisonErr) || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace || stdout.Len() == 0 || stderr.Len() != 0 {
		t.Fatalf("error/calls/request/output = %v/%d/%d/%#v/%q/%q", err, factoryCalls, applicationCalls, received, stdout.String(), stderr.String())
	}
	if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) {
		t.Fatalf("profile IDs = %#v", received.ProfileIDs)
	}
}

func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n")
	profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
	result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
		{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
		{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
	})
	partialErr := errors.New("comparison completed with 1 failed profiles")
	factoryCalls, applicationCalls := 0, 0
	runner := Runner{
		Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
		ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
			factoryCalls++
			return &factoryExecutor{}, nil
		},
		compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
			applicationCalls++
			return result, partialErr
		},
	}
	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
	if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
		t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
	}
	var summary comparisonSummary
	if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
		t.Fatalf("decode summary: %v\n%s", err, stdout.String())
	}
	if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
		t.Fatalf("summary = %#v", summary)
	}
}

func TestCompareCommandPreservesMixedCancellationOutcomes(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n")
	profileFailure := comparison.NewSafeError("validation_rejected", "validate prompt execution failed")
	canceledProfile := comparison.NewSafeError("canceled", "profile execution canceled")
	result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
		{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, ValidationStatus: promptexec.ValidationFailed, Error: &profileFailure},
		{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, ValidationStatus: promptexec.ValidationSkipped, Error: &canceledProfile},
	})
	runner := comparisonRunner(t, workingDir)
	runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
		return result, context.Canceled
	}

	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
	if !errors.Is(err, context.Canceled) || stderr.Len() != 0 {
		t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
	}
	var summary comparisonSummary
	if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
		t.Fatalf("decode summary: %v\n%s", err, stdout.String())
	}
	if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "canceled" || len(summary.Results) != 2 {
		t.Fatalf("summary = %#v", summary)
	}
	if first := summary.Results[0]; first.Error == nil || first.Error.Category != "validation_rejected" || first.ValidationStatus != string(promptexec.ValidationFailed) {
		t.Fatalf("completed profile summary = %#v", first)
	}
	if second := summary.Results[1]; second.Error == nil || second.Error.Category != "canceled" || second.ValidationStatus != string(promptexec.ValidationSkipped) {
		t.Fatalf("canceled profile summary = %#v", second)
	}
}

func TestCompareCommandReportsCommittedBundleWhenCleanupFails(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n")
	outputDirectory := filepath.Join(workingDir, "comparison-daily")
	result := comparisonResult(outputDirectory, []app.ComparisonProfileResult{
		{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "01-weather-light.md")},
		{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "02-weather-deep.md")},
	})
	recoveryPath := filepath.Join(workingDir, ".comparison-daily.backup-retained")
	cleanupCause := errors.New("filesystem cleanup detail")
	cleanupErr := &comparison.PublicationCleanupError{RecoveryState: comparison.BackupRecoveryComplete, RecoveryPath: recoveryPath, Err: cleanupCause}
	runner := comparisonRunner(t, workingDir)
	runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
		return result, cleanupErr
	}

	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
	if !errors.Is(err, cleanupCause) || stderr.Len() != 0 {
		t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
	}
	var summary comparisonSummary
	if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
		t.Fatalf("decode summary: %v\n%s", err, stdout.String())
	}
	if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "publication_cleanup" || summary.Error.Message != "comparison published but a complete prior bundle remains" || summary.ManifestPath != result.ManifestPath || summary.DataPackagePath != result.DataPackagePath || summary.Results[0].ReportPath == "" {
		t.Fatalf("summary = %#v", summary)
	}
	for _, unsafe := range []string{cleanupCause.Error(), recoveryPath} {
		if strings.Contains(stdout.String(), unsafe) {
			t.Fatalf("summary contains unsafe recovery detail %q: %s", unsafe, stdout.String())
		}
	}
}

func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
	workingDir := t.TempDir()
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\noutput:\n  directory: configured-reports\n")
	for _, test := range []struct {
		name            string
		outputArgument  []string
		outputDirectory string
		wantRequestPath string
	}{
		{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
		{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
	} {
		t.Run(test.name, func(t *testing.T) {
			var received app.ComparisonRequest
			runner := comparisonRunner(t, workingDir)
			runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
				received = req
				return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
					{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
					{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
				}), nil
			}
			args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
			args = append(args, test.outputArgument...)
			var stdout, stderr bytes.Buffer
			if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
				t.Fatalf("Run() error = %v", err)
			}
			var summary comparisonSummary
			if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
				t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
			}
		})
	}
}

func TestCompareCommandQuietPreservesFailure(t *testing.T) {
	configPath := comparisonConfigPath(t, "weather_api:\n  base_url: https://weather.api.example.com/\n")
	failure := errors.New("comparison completed with 2 failed profiles")
	calls := 0
	runner := comparisonRunner(t, t.TempDir())
	runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
		calls++
		return comparisonResult("/reports/comparison-daily", nil), failure
	}
	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
	if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
		t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
	}
}

func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
	runner := comparisonRunner(t, t.TempDir())
	called := false
	runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
		called = true
		return nil, nil
	}
	var stdout, stderr bytes.Buffer
	err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
	if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
		t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
	}
}

func TestCompareHelpIncludesCommand(t *testing.T) {
	var stdout, stderr bytes.Buffer
	if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") || !strings.Contains(stdout.String(), "--date YYYY-MM-DD") || !strings.Contains(stdout.String(), "Suppress action summaries and routine batch status output.") {
		t.Fatalf("help/error = %q/%v", stdout.String(), err)
	}
}

func TestCompareHelpDoesNotRequireConfiguration(t *testing.T) {
	var stdout, stderr bytes.Buffer
	if err := (Runner{}).Run(context.Background(), []string{"compare", "--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || stderr.Len() != 0 {
		t.Fatalf("help/error/stderr = %q/%v/%q", stdout.String(), err, stderr.String())
	}
}

func comparisonRunner(t *testing.T, workingDir string) Runner {
	t.Helper()
	return Runner{
		Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
		ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil },
	}
}

func comparisonConfigPath(t *testing.T, contents string) string {
	t.Helper()
	path := filepath.Join(t.TempDir(), "config.yml")
	if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
		t.Fatal(err)
	}
	return path
}

func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
	started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
	result := &app.ComparisonResult{
		ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
		PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0", PromptHash: strings.Repeat("a", 64),
		StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
		ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
		ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
		Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
	}
	for _, profile := range results {
		if profile.Status == "succeeded" {
			result.Succeeded++
		} else {
			result.Failed++
		}
	}
	return result
}