Wire native pipeline run commands and invocation-wide overrides
This commit is contained in:
152
internal/cli/pipeline_run.go
Normal file
152
internal/cli/pipeline_run.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func pipelineRunCommand(version string, newState func() *nativeLoaders, modules func() (pipeline.Registries, error), options Options) commandkit.Command {
|
||||
options = defaultOptions(options)
|
||||
return commandkit.NewCommand(commandkit.CommandSpec[pipelineRunInput]{Name: "run", Summary: "Execute a configured pipeline and publish its artifacts.",
|
||||
Flags: runFlags(), Arguments: []commandkit.Argument[pipelineRunInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, parseRequiredRunValue, func(input *pipelineRunInput, value commandkit.Supplied[string]) { input.PipelineID = value.Value })},
|
||||
CheckInput: checkPipelineRunInput,
|
||||
Run: func(invocation commandkit.Invocation, input pipelineRunInput) commandkit.Outcome {
|
||||
state := newState()
|
||||
selection := captureSelection(input.Selection)
|
||||
if input.OutputDirectory.Set {
|
||||
selection.Overrides = append(selection.Overrides, configkit.Override{Path: configkit.Keys("output", "directory"), Value: input.OutputDirectory.Value, Flag: "--output-dir"})
|
||||
}
|
||||
result, err := state.application(invocation.Context, selection)
|
||||
if err != nil {
|
||||
return pipelineConfigurationFailure(err)
|
||||
}
|
||||
loaded, err := config.FromResult(result)
|
||||
if err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
}
|
||||
cfg := loaded.RuntimeConfig()
|
||||
startedAt := options.Now().UTC()
|
||||
runID, err := options.RunIDGenerator(startedAt)
|
||||
if err != nil {
|
||||
return commandkit.Failure("Generate run identity.").WithCause(err)
|
||||
}
|
||||
if err := validateRunID(runID); err != nil {
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
}
|
||||
outputDirectory := filepath.Join(cfg.Output.Directory, runID)
|
||||
var capture *frameworkdebug.Recorder
|
||||
var output pipeline.RunOutput
|
||||
published := false
|
||||
captureOutcome := func() any {
|
||||
return map[string]any{"pipeline": input.PipelineID, "output_directory": outputDirectory, "artifact_count": output.ArtifactCount(), "fallback_count": output.FallbackCount, "published": published}
|
||||
}
|
||||
finish := func(outcome commandkit.Outcome) commandkit.Outcome {
|
||||
if err := capture.Finalize(captureOutcome(), outcome.Cause); err != nil {
|
||||
message := err.Error()
|
||||
if published {
|
||||
message += fmt.Sprintf("\nArtifacts: %s", outputDirectory)
|
||||
}
|
||||
if capture.Path() != "" {
|
||||
message += fmt.Sprintf("\nDebug capture: %s", capture.Path())
|
||||
}
|
||||
if outcome.Status == 0 {
|
||||
outcome = commandkit.Failure(message)
|
||||
} else {
|
||||
outcome = outcome.WithExplanation(message)
|
||||
}
|
||||
outcome = outcome.WithCause(err)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
capture, err = startRunDebug(input.Debug, cfg.Debug.Directory, runID, startedAt)
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": input.PipelineID, "input_path": input.InputFile.Value, "run_id": runID, "started_at": startedAt, "lanes": input.Lanes, "resources": input.Overrides.Resources, "execution_overrides": input.Overrides.Execution}); err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
if err := captureRunConfiguration(capture, result); err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
registries, err := modules()
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
catalog := catalogFromRegistries(registries)
|
||||
binds, unbinds, err := resolveCLIReferenceRequests(cfg, input.PipelineID, input.Lanes.Value, catalog, input.References.Value, input.Unbinds.Value)
|
||||
if err != nil {
|
||||
return finish(pipelineInvalidConfiguration(err))
|
||||
}
|
||||
resolve := config.ResolveInput{PipelineID: input.PipelineID, Only: input.Lanes.Value, Catalog: catalog, ReferenceOverrides: binds, ReferenceUnbinds: unbinds}
|
||||
if input.Cache.Set {
|
||||
resolve.CacheOverride = &input.Cache.Value
|
||||
}
|
||||
if input.Overrides.Resources.Profile.Set {
|
||||
resolve.LLMProfileOverride = &input.Overrides.Resources.Profile.Value
|
||||
}
|
||||
effective, err := cfg.Resolve(resolve)
|
||||
if err != nil {
|
||||
return finish(pipelineInvalidConfiguration(err))
|
||||
}
|
||||
var engine *promptkit.Engine
|
||||
if preflight.RequiresEngine(effective.ResolvedPipeline) {
|
||||
engine, err = state.engine(invocation.Context, selection)
|
||||
if err != nil {
|
||||
return finish(pipelineConfigurationFailure(err))
|
||||
}
|
||||
}
|
||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, catalog, engine); err != nil {
|
||||
return finish(pipelineInvalidConfiguration(err))
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
materialized, diagnostics, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: loaded.MainSourcePath(), WorkingDir: workingDirectory})
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
raw, err := os.ReadFile(input.InputFile.Value)
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(fmt.Sprintf("Read input %q: %v", input.InputFile.Value, err)).WithCause(err))
|
||||
}
|
||||
sessionID, err := runSessionID(input.Overrides.Resources.SessionID, materialized.Input.Module, raw)
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
cache, err := artifactcache.NewFilesystemStore(cfg.CacheDirectory)
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
var recorder pipeline.DebugRecorder
|
||||
if capture.Enabled() {
|
||||
recorder = newRunEvidenceRecorder(capture, result)
|
||||
}
|
||||
group := &promptkit.ExecutionGroup{}
|
||||
output, err = pipeline.New().Run(invocation.Context, pipeline.RunInput{Prepared: prepared, Path: input.InputFile.Value, RawInput: raw, SessionID: sessionID, RunID: runID, StartedAt: startedAt, Diagnostics: diagnostics, Cache: cache, Debug: recorder, Workers: cfg.Concurrency.Workers, Executions: group, Execution: contracts.ExecutionOptions{Execution: input.Overrides.Execution.Execution, TransientRetries: input.Overrides.Execution.TransientRetries}})
|
||||
if err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
if err := writeOutputFiles(outputDirectory, output.OutputFiles); err != nil {
|
||||
return finish(commandkit.Failure(err.Error()).WithCause(err))
|
||||
}
|
||||
published = true
|
||||
return finish(commandkit.Success())
|
||||
},
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user