Files
notarius/internal/cli/pipeline_run.go

168 lines
8.1 KiB
Go
Raw Normal View History

package cli
import (
"fmt"
"os"
"path/filepath"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func pipelineRunCommand(version string, newState func() *nativeLoaders, modules func() (pipeline.Registries, error), options Options) commandkit.Command {
options = defaultOptions(options)
return commandkit.NewCommand(commandkit.CommandSpec[pipelineRunInput]{Name: "run", Summary: "Execute a configured pipeline and publish its artifacts.",
Flags: runFlags(), Arguments: []commandkit.Argument[pipelineRunInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, parseRequiredRunValue, func(input *pipelineRunInput, value commandkit.Supplied[string]) { input.PipelineID = value.Value })},
CheckInput: checkPipelineRunInput,
Run: func(invocation commandkit.Invocation, input pipelineRunInput) commandkit.Outcome {
state := newState()
selection := captureSelection(input.Selection)
if input.OutputDirectory.Set {
selection.Overrides = append(selection.Overrides, configkit.Override{Path: configkit.Keys("output", "directory"), Value: input.OutputDirectory.Value, Flag: "--output-dir"})
}
result, err := state.application(invocation.Context, selection)
if err != nil {
return pipelineConfigurationFailure(err)
}
diagnostics, _ := ckconfig.AdaptDiagnostics("Configuration\n", result.Sources(), result.Diagnostics())
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning {
_ = invocation.Reporter().AtEnd().Warn("", diagnostic.Write)
} else {
_ = invocation.Reporter().AtEnd().Info("", diagnostic.Write)
}
}
loaded, err := config.FromResult(result)
if err != nil {
return pipelineInvalidConfiguration(err)
}
cfg := loaded.RuntimeConfig()
startedAt := options.Now().UTC()
runID, err := options.RunIDGenerator(startedAt)
if err != nil {
return commandkit.Failure("Generate run identity.").WithCause(err)
}
if err := validateRunID(runID); err != nil {
return commandkit.Failure(err.Error()).WithCause(err)
}
outputDirectory := filepath.Join(cfg.Output.Directory, runID)
var capture *frameworkdebug.Recorder
var output pipeline.RunOutput
published := false
captureOutcome := func() any {
return map[string]any{"pipeline": input.PipelineID, "output_directory": outputDirectory, "artifact_count": output.ArtifactCount(), "fallback_count": output.FallbackCount, "published": published}
}
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
if outcome.Status == 0 {
outcome = commandkit.Failure(err.Error())
} else {
outcome = outcome.WithExplanation(err.Error(), outcome.Explanation.Details...)
}
outcome = outcome.WithCause(err)
}
if outcome.Status != 0 {
availableOutput := ""
if published {
availableOutput = outputDirectory
}
outcome = outcome.WithExplanation(outcome.Explanation.Message+runLocations(availableOutput, capture.Path()), outcome.Explanation.Details...)
}
return outcome
}
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": input.PipelineID, "input_path": input.InputFile.Value, "run_id": runID, "started_at": startedAt, "lanes": input.Lanes, "resources": input.Overrides.Resources, "execution_overrides": input.Overrides.Execution}); err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
if err := captureRunConfiguration(capture, result); err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
registries, err := modules()
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
catalog := catalogFromRegistries(registries)
binds, unbinds, err := resolveCLIReferenceRequests(cfg, input.PipelineID, input.Lanes.Value, catalog, input.References.Value, input.Unbinds.Value)
if err != nil {
return finish(pipelineInvalidConfiguration(err))
}
resolve := config.ResolveInput{PipelineID: input.PipelineID, Only: input.Lanes.Value, Catalog: catalog, ReferenceOverrides: binds, ReferenceUnbinds: unbinds}
if input.Cache.Set {
resolve.CacheOverride = &input.Cache.Value
}
if input.Overrides.Resources.Profile.Set {
resolve.LLMProfileOverride = &input.Overrides.Resources.Profile.Value
}
effective, err := cfg.Resolve(resolve)
if err != nil {
return finish(pipelineInvalidConfiguration(err))
}
var engine *promptkit.Engine
if preflight.RequiresEngine(effective.ResolvedPipeline) {
engine, err = state.engine(invocation.Context, selection)
if err != nil {
return finish(pipelineConfigurationFailure(err))
}
}
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
return finish(pipelineResourceFailure(err))
}
workingDirectory, err := os.Getwd()
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
materialized, referenceDiagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: engine})
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
raw, err := os.ReadFile(input.InputFile.Value)
if err != nil {
return finish(commandkit.Failure(fmt.Sprintf("Read input %q: %v", input.InputFile.Value, err)).WithCause(err))
}
sessionID, err := runSessionID(input.Overrides.Resources.SessionID, materialized.Input.Module, raw)
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
cache, err := artifactcache.NewFilesystemStore(cfg.CacheDirectory)
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
var recorder pipeline.DebugRecorder
if capture.Enabled() {
recorder = newRunEvidenceRecorder(capture, result)
}
group := &promptkit.ExecutionGroup{}
_ = invocation.Reporter().Progress(fmt.Sprintf("Running pipeline %q.", effective.ResolvedPipeline.ID))
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: referenceDiagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
reportRunDiagnostics(invocation.Reporter(), output.Diagnostics)
if err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
if err := writeOutputFiles(outputDirectory, output.OutputFiles); err != nil {
return finish(commandkit.Failure(err.Error()).WithCause(err))
}
published = true
outcome := finish(commandkit.Success())
if outcome.Status != 0 {
return outcome
}
return reportRunResult(invocation, runResult{Pipeline: effective.ResolvedPipeline.ID, ArtifactCount: output.ArtifactCount(), OutputDirectory: outputDirectory, DebugDirectory: capture.Path(), FallbackCount: output.FallbackCount})
},
})
}