2026-05-02 11:05:28 -05:00
package app
import (
"context"
2026-08-10 18:14:09 +00:00
"errors"
2026-05-02 11:05:28 -05:00
"fmt"
"os"
2026-05-03 17:49:26 -05:00
"strings"
2026-09-04 01:11:11 +00:00
"time"
2026-05-02 11:05:28 -05:00
2026-05-02 11:16:27 -05:00
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
2026-05-25 00:40:06 +00:00
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
2026-05-02 11:05:28 -05:00
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
2026-11-20 05:59:09 +00:00
"gitea.maximumdirect.net/eric/narratio/internal/logging"
2026-05-02 11:05:28 -05:00
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type RunOptions struct {
2026-11-20 05:59:09 +00:00
Reporter logging . Reporter
2026-08-10 22:27:06 +00:00
Force bool
SelectedArtifacts [ ] string
EffectiveArtifacts artifacts . EffectiveArtifactSet
Env * Env
2026-05-02 11:05:28 -05:00
}
type RunSummary struct {
2026-11-20 05:47:41 +00:00
SessionID string
RunID string
ManifestPath string
StageNames [ ] string
Executed [ ] string
Skipped [ ] string
StageResults [ ] RunStageResult
AvailableArtifacts [ ] RunArtifact
Complete bool
2026-05-02 11:05:28 -05:00
}
2026-09-04 01:21:37 +00:00
type pipelineExecutor func ( context . Context , * config . Config , BoundedPlan , RunOptions ) ( * RunSummary , error )
2026-05-19 22:23:25 -05:00
2026-08-29 20:49:43 +00:00
func executePlan ( ctx context . Context , cfg * config . Config , plan BoundedPlan , opts RunOptions ) ( summary * RunSummary , resultErr error ) {
2026-09-04 01:11:11 +00:00
return executePlanWithDependencies ( ctx , cfg , plan , opts , invocationDependencies { } )
}
func executePlanWithDependencies ( ctx context . Context , cfg * config . Config , plan BoundedPlan , opts RunOptions , dependencies invocationDependencies ) ( summary * RunSummary , resultErr error ) {
dependencies = dependencies . normalized ( )
runtime , err := prepareInvocation ( ctx , cfg , plan , opts , dependencies )
2026-08-10 19:25:03 +00:00
if err != nil {
2026-09-04 01:05:44 +00:00
return nil , err
2026-05-02 11:16:27 -05:00
}
2026-09-04 01:05:44 +00:00
if err := prepareInvocationLayout ( runtime ) ; err != nil {
return nil , err
2026-05-02 11:05:28 -05:00
}
2026-09-04 01:05:44 +00:00
lock , err := runtime . artifactStore . AcquireSessionLockForContext ( ctx , runtime . identity . Campaign , runtime . identity . SessionID )
2026-05-02 11:05:28 -05:00
if err != nil {
return nil , fmt . Errorf ( "acquire session lock: %w" , err )
}
defer func ( ) {
2026-09-04 01:05:44 +00:00
if releaseErr := runtime . artifactStore . ReleaseSessionLock ( lock ) ; releaseErr != nil {
2026-08-10 18:14:09 +00:00
if resultErr == nil {
resultErr = fmt . Errorf ( "release session lock: %w" , releaseErr )
} else {
resultErr = errors . Join ( resultErr , fmt . Errorf ( "release session lock: %w" , releaseErr ) )
}
}
2026-11-20 05:47:41 +00:00
if summary != nil {
summary . Complete = resultErr == nil
}
} ( )
defer func ( ) {
if runtime . started {
summary , resultErr = finalizeInvocation ( runtime , resultErr )
}
2026-05-02 11:05:28 -05:00
} ( )
2026-11-20 05:47:41 +00:00
for _ , selected := range runtime . stages {
runtime . stageNames = append ( runtime . stageNames , selected . Name ( ) )
runtime . stageResults = append ( runtime . stageResults , RunStageResult { Name : selected . Name ( ) , Outcome : "not_attempted" } )
}
2026-09-04 01:05:44 +00:00
if err := initializeLockedInvocation ( runtime ) ; err != nil {
2026-08-10 21:01:28 +00:00
return nil , err
}
2026-09-04 01:11:11 +00:00
if err := composeInvocation ( runtime ) ; err != nil {
return nil , err
}
2026-09-04 01:05:44 +00:00
2026-11-20 05:59:09 +00:00
runtime . env . Info ( "selected run destinations" , "session" , runtime . identity . SessionID , "run" , runtime . identity . RunID , "manifest" , runtime . manifestPath , "workspace" , runtime . identity . LocalWorkDir )
2026-11-20 05:47:41 +00:00
for index , selected := range runtime . stages {
2026-11-20 05:53:49 +00:00
if err := ctx . Err ( ) ; err != nil {
return nil , err
}
2026-09-04 01:14:09 +00:00
outcome , err := executeSelectedStage ( runtime , selected )
2026-11-20 05:47:41 +00:00
result := & runtime . stageResults [ index ]
result . Attempted = outcome . executed
result . Reason = outcome . reason
switch {
case err != nil :
result . Outcome = "failed"
case outcome . reused :
result . Outcome = "reused"
case outcome . skipped :
result . Outcome = "skipped"
default :
result . Outcome = "completed"
2026-05-02 11:05:28 -05:00
}
2026-09-04 01:14:09 +00:00
if outcome . executed {
runtime . executed = append ( runtime . executed , selected . Name ( ) )
2026-08-29 18:36:21 +00:00
}
2026-11-20 05:47:41 +00:00
if outcome . skipped && err == nil {
2026-09-04 01:14:09 +00:00
runtime . skipped = append ( runtime . skipped , selected . Name ( ) )
2026-05-02 11:05:28 -05:00
}
2026-11-20 05:47:41 +00:00
if err != nil {
return nil , err
}
2026-05-02 11:05:28 -05:00
}
2026-11-20 05:53:49 +00:00
return nil , ctx . Err ( )
2026-05-02 11:05:28 -05:00
}
2026-08-29 18:17:05 +00:00
func stagesContainAny ( stages [ ] stage . Stage , names ... string ) bool {
wanted := make ( map [ string ] struct { } , len ( names ) )
for _ , name := range names {
wanted [ name ] = struct { } { }
}
for _ , candidate := range stages {
if candidate == nil {
continue
}
if _ , ok := wanted [ candidate . Name ( ) ] ; ok {
return true
}
}
return false
}
2026-09-03 04:09:12 +00:00
func persistSessionTerminalFailure (
2026-08-10 19:34:46 +00:00
ctx context . Context ,
sessionStore manifest . Store ,
sessionManifestPath string ,
sessionManifest * manifest . Manifest ,
operationErr error ,
2026-09-04 01:11:11 +00:00
at time . Time ,
2026-08-10 19:34:46 +00:00
) error {
if operationErr == nil {
return nil
}
if sessionManifest != nil {
sessionManifest . RecordFailure ( at , operationErr . Error ( ) )
}
2026-11-20 05:53:49 +00:00
// Only terminal local persistence is detached; adapters and checkpoints keep
// the canceled operation context.
sessionErr := sessionStore . Save ( context . WithoutCancel ( ctx ) , sessionManifestPath , sessionManifest )
2026-08-10 19:34:46 +00:00
return errors . Join (
operationErr ,
wrapTerminalPersistenceError ( "save terminal session manifest" , sessionErr ) ,
)
}
func wrapTerminalPersistenceError ( operation string , err error ) error {
if err == nil {
return nil
}
return fmt . Errorf ( "%s: %w" , operation , err )
}
2026-05-03 17:49:26 -05:00
func buildDefaultWhisperXClient ( cfg * config . Config ) ( whisperx . Client , error ) {
if cfg == nil || cfg . Pipeline == nil {
return & whisperx . NoopClient { } , nil
}
wx := cfg . Pipeline . WhisperX
if strings . TrimSpace ( wx . TranscribeURL ) == "" {
// Compatibility fallback for tests or internal call paths that bypass config validation.
return & whisperx . NoopClient { } , nil
}
retries := 0
if wx . Retries != nil {
retries = * wx . Retries
}
client , err := whisperx . NewHTTPClientFromConfigValues (
wx . TranscribeURL ,
wx . Language ,
wx . Timeout ,
wx . RetryDelay ,
retries ,
)
if err != nil {
return nil , fmt . Errorf ( "from pipeline.whisperx: %w" , err )
}
return client , nil
}
2026-05-03 22:17:17 -05:00
func buildDefaultSeriatimRunner ( cfg * config . Config ) ( seriatim . Runner , error ) {
if cfg == nil || cfg . Pipeline == nil {
return & seriatim . NoopRunner { } , nil
}
s := cfg . Pipeline . Seriatim
if strings . TrimSpace ( s . Binary ) == "" || strings . TrimSpace ( s . Timeout ) == "" || strings . TrimSpace ( s . OutputSchema ) == "" {
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
return & seriatim . NoopRunner { } , nil
}
report := false
if s . Report != nil {
report = * s . Report
}
runner , err := seriatim . NewSubprocessRunnerFromConfigValues (
s . Binary ,
s . Timeout ,
s . OutputSchema ,
s . CoalesceGap ,
report ,
seriatim . EnvConfig {
OverlapWordRunGap : s . Env . OverlapWordRunGap ,
OverlapWordRunReorderWindow : s . Env . OverlapWordRunReorderWindow ,
BackchannelMaxDuration : s . Env . BackchannelMaxDuration ,
FillerMaxDuration : s . Env . FillerMaxDuration ,
} ,
)
if err != nil {
return nil , fmt . Errorf ( "from pipeline.seriatim: %w" , err )
}
return runner , nil
}
2026-05-04 08:05:27 -05:00
func buildDefaultAuditaRunner ( cfg * config . Config ) ( audita . Runner , error ) {
if cfg == nil || cfg . Pipeline == nil {
return & audita . NoopRunner { } , nil
}
a := cfg . Pipeline . Audita
2026-05-16 23:30:40 +00:00
if strings . TrimSpace ( a . Binary ) == "" || strings . TrimSpace ( a . Timeout ) == "" {
2026-05-04 08:05:27 -05:00
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
return & audita . NoopRunner { } , nil
}
2026-09-25 15:45:02 +00:00
timeout , err := time . ParseDuration ( a . Timeout )
if err != nil {
return nil , fmt . Errorf ( "parse pipeline.audita.timeout %q: %w" , a . Timeout , err )
}
runner , err := audita . NewSubprocessRunner ( audita . SubprocessRunnerConfig {
Binary : a . Binary , Timeout : timeout , Credentials : a . Credentials ,
Modules : append ( a . Modules [ : 0 : 0 ] , a . Modules ... ) ,
Overlay : a . Overlay , ProposalProfile : a . ProposalProfile ,
ValidationProfile : a . ValidationProfile ,
TranscriptDescription : a . TranscriptDescription , ConfigPath : a . ConfigPath ,
OutputSchema : a . OutputSchema ,
} )
2026-05-04 08:05:27 -05:00
if err != nil {
return nil , fmt . Errorf ( "from pipeline.audita: %w" , err )
}
return runner , nil
}
2026-08-10 19:25:03 +00:00
func loadManifestAtPathIfPresent ( ctx context . Context , store manifest . Store , path string ) ( * manifest . Manifest , bool , error ) {
2026-05-02 11:05:28 -05:00
exists , err := fileExists ( path )
if err != nil {
2026-08-10 19:25:03 +00:00
return nil , false , fmt . Errorf ( "check manifest existence %q: %w" , path , err )
2026-05-02 11:05:28 -05:00
}
if exists {
m , err := store . Load ( ctx , path )
if err != nil {
2026-08-10 19:25:03 +00:00
return nil , false , fmt . Errorf ( "load manifest %q: %w" , path , err )
2026-05-02 11:05:28 -05:00
}
2026-08-10 19:25:03 +00:00
return m , true , nil
2026-05-02 11:05:28 -05:00
}
2026-08-10 19:25:03 +00:00
return nil , false , nil
2026-05-02 11:05:28 -05:00
}
func fileExists ( path string ) ( bool , error ) {
_ , err := os . Stat ( path )
if err == nil {
return true , nil
}
if os . IsNotExist ( err ) {
return false , nil
}
return false , err
}
2026-05-19 19:07:44 -05:00
func mapResultOutputs ( stageName string , result * stage . StageResult , runID string ) [ ] manifest . ArtifactRecord {
2026-05-02 11:05:28 -05:00
if result == nil || len ( result . Outputs ) == 0 {
return nil
}
2026-05-18 01:13:51 +00:00
runID = strings . TrimSpace ( runID )
2026-05-02 11:05:28 -05:00
out := make ( [ ] manifest . ArtifactRecord , 0 , len ( result . Outputs ) )
for _ , ref := range result . Outputs {
localPath := ref . AbsolutePath
if localPath == "" {
localPath = ref . RelativePath
}
2026-05-19 19:07:44 -05:00
kind := ref . Kind
2026-08-09 23:20:32 +00:00
sourceID := strings . TrimSpace ( ref . SourceID )
if sourceID == "" {
if stageName == "analyze" {
sourceID = artifacts . ConfiguredArtifactSourceID ( ref . Kind )
kind = "scriptorium_artifact"
} else {
sourceID = sourceIDForOutputKind ( kind )
}
2026-05-19 19:07:44 -05:00
}
2026-09-04 15:57:54 +00:00
producerRunID := strings . TrimSpace ( ref . ProducerRunID )
if producerRunID == "" {
producerRunID = runID
}
2026-05-02 11:05:28 -05:00
out = append ( out , manifest . ArtifactRecord {
2026-08-09 23:20:32 +00:00
Kind : kind ,
SourceID : sourceID ,
LocalPath : localPath ,
2026-08-10 19:16:18 +00:00
ArchivePath : ref . ArchivePath ,
2026-08-09 23:20:32 +00:00
Contract : cloneContractMetadata ( ref . Contract ) ,
ExternalProvenance : cloneExternalProvenance ( ref . ExternalProvenance ) ,
2026-09-04 15:57:54 +00:00
ProducerRunID : producerRunID ,
2026-08-09 23:20:32 +00:00
RemoteKey : ref . RemoteKey ,
Checksum : ref . Checksum ,
2026-05-02 11:05:28 -05:00
} )
}
return out
}
2026-08-09 23:20:32 +00:00
func validateStageResult ( result * stage . StageResult ) error {
if result == nil {
return nil
}
switch result . Disposition {
case stage . StageDispositionSucceeded :
if strings . TrimSpace ( result . SkipReason ) != "" {
return fmt . Errorf ( "successful result contains a skip reason" )
}
return nil
case stage . StageDispositionSkipped :
if strings . TrimSpace ( result . SkipReason ) == "" {
return fmt . Errorf ( "skipped result requires a skip reason" )
}
if len ( result . Outputs ) != 0 {
return fmt . Errorf ( "skipped result contains %d output(s)" , len ( result . Outputs ) )
}
return nil
default :
return fmt . Errorf ( "unsupported stage result disposition %q" , result . Disposition )
}
}
func cloneContractMetadata ( value * artifactmodel . ContractMetadata ) * artifactmodel . ContractMetadata {
if value == nil {
return nil
}
cloned := * value
return & cloned
}
func cloneExternalProvenance ( value * artifactmodel . ExternalProvenance ) * artifactmodel . ExternalProvenance {
if value == nil {
return nil
}
cloned := * value
return & cloned
}
2026-05-25 00:40:06 +00:00
func sourceIDForOutputKind ( kind string ) string {
trimmed := strings . TrimSpace ( kind )
if trimmed == "" {
return ""
}
if trimmed == "session_bounds" {
return artifacts . ArtifactBoundsSession
}
for _ , spec := range artifactmodel . RuntimeTranscriptArtifacts ( ) {
if spec . OutputKind == trimmed {
return spec . SourceID
}
}
return ""
}
2026-05-02 11:05:28 -05:00
func applyStageResultToManifest ( m * manifest . Manifest , stageName string , result * stage . StageResult ) {
if m == nil || result == nil {
return
}
sr := m . Stages [ stageName ]
if sr == nil {
return
}
if len ( result . Logs ) > 0 {
sr . Logs = append ( [ ] string ( nil ) , result . Logs ... )
}
if len ( result . GeneratedConfigs ) > 0 {
sr . GeneratedConfigs = append ( [ ] string ( nil ) , result . GeneratedConfigs ... )
}
if len ( result . Metadata ) > 0 {
sr . Metadata = result . Metadata
}
}
2026-08-10 19:25:03 +00:00
type invocationIdentity struct {
Campaign string
SessionID string
RunID string
LocalWorkDir string
LocalSpoolDir string
S3Bucket string
S3SessionPrefix string
S3RunPrefix string
}
func resolveInvocationIdentity ( cfg * config . Config , runID string ) ( invocationIdentity , error ) {
if cfg == nil || cfg . Pipeline == nil || cfg . Campaign == nil || cfg . Session == nil {
return invocationIdentity { } , fmt . Errorf ( "resolved configuration requires pipeline, campaign, and session" )
2026-05-16 14:11:59 +00:00
}
campaign := strings . TrimSpace ( cfg . Session . Campaign )
2026-08-10 19:25:03 +00:00
configuredCampaign := strings . TrimSpace ( config . CampaignID ( cfg . Campaign ) )
if campaign == "" {
return invocationIdentity { } , fmt . Errorf ( "configured session campaign is required" )
}
if configuredCampaign == "" {
return invocationIdentity { } , fmt . Errorf ( "configured campaign is required" )
}
if campaign != configuredCampaign {
return invocationIdentity { } , fmt . Errorf ( "configured session campaign %q does not match configured campaign %q" , campaign , configuredCampaign )
}
2026-05-16 14:11:59 +00:00
sessionID := strings . TrimSpace ( cfg . Session . SessionID )
if sessionID == "" {
2026-08-10 19:25:03 +00:00
return invocationIdentity { } , fmt . Errorf ( "configured session_id is required" )
}
runID = strings . TrimSpace ( runID )
if runID == "" {
return invocationIdentity { } , fmt . Errorf ( "run id is required" )
2026-05-16 14:11:59 +00:00
}
2026-08-10 19:25:03 +00:00
identity := invocationIdentity {
Campaign : campaign ,
SessionID : sessionID ,
RunID : runID ,
LocalWorkDir : artifacts . SessionRunRootForCampaign ( cfg . Pipeline . Workspace . Root , campaign , sessionID , runID ) ,
2026-05-16 14:11:59 +00:00
}
2026-08-10 19:25:03 +00:00
if spoolRoot := strings . TrimSpace ( cfg . Pipeline . Spool . Root ) ; spoolRoot != "" {
identity . LocalSpoolDir = artifacts . SessionSpoolAudioDir ( spoolRoot , campaign , sessionID , runID )
}
if s3 := cfg . Pipeline . Storage . S3 ; s3 != nil {
identity . S3Bucket = strings . TrimSpace ( s3 . Bucket )
identity . S3SessionPrefix = artifacts . S3SessionPrefix ( s3 . RootPrefix , campaign , sessionID )
identity . S3RunPrefix = artifacts . S3RunPrefix ( identity . S3SessionPrefix , runID )
}
return identity , nil
}
func ( identity invocationIdentity ) validateSessionManifest ( m * manifest . Manifest ) error {
if m == nil {
return fmt . Errorf ( "persisted manifest is nil" )
}
if sessionID := strings . TrimSpace ( m . SessionID ) ; sessionID != "" && sessionID != identity . SessionID {
return fmt . Errorf ( "persisted manifest session_id %q does not match configured session_id %q" , sessionID , identity . SessionID )
}
if campaign := strings . TrimSpace ( m . Campaign ) ; campaign != "" && campaign != identity . Campaign {
return fmt . Errorf ( "persisted manifest campaign %q does not match configured campaign %q" , campaign , identity . Campaign )
}
return nil
}
func ( identity invocationIdentity ) applyToSessionManifest ( m * manifest . Manifest ) {
if m == nil {
return
2026-05-16 14:11:59 +00:00
}
2026-08-10 19:25:03 +00:00
m . SessionID = identity . SessionID
m . Campaign = identity . Campaign
m . RunID = identity . RunID
m . LocalWorkDir = identity . LocalWorkDir
m . LocalSpoolDir = identity . LocalSpoolDir
m . S3Bucket = identity . S3Bucket
m . S3SessionPrefix = identity . S3SessionPrefix
m . S3RunPrefix = identity . S3RunPrefix
}
2026-05-16 14:11:59 +00:00
2026-05-02 11:05:28 -05:00
func manifestPathFor ( cfg * config . Config ) string {
2026-05-17 20:57:27 +00:00
return artifacts . SessionManifestPathForCampaign (
cfg . Pipeline . Workspace . Root ,
cfg . Session . Campaign ,
cfg . Session . SessionID ,
)
2026-05-02 11:05:28 -05:00
}
2026-05-16 14:43:29 +00:00
2026-08-10 22:27:06 +00:00
func needsObjectStoreForRun ( cfg * config . Config , stages [ ] stage . Stage , effectiveSets ... artifacts . EffectiveArtifactSet ) bool {
2026-05-16 14:43:29 +00:00
if cfg == nil || cfg . Pipeline == nil || cfg . Session == nil {
return false
}
2026-08-10 22:27:06 +00:00
effective := artifacts . EffectiveArtifactSet { }
if len ( effectiveSets ) > 0 {
effective = effectiveSets [ 0 ]
}
if ! effective . Resolved ( ) && cfg . Pipeline . Scriptorium != nil {
var err error
effective , err = artifacts . ResolveEffectiveArtifactSet (
2026-11-20 04:42:23 +00:00
artifacts . ConfiguredArtifactDefinitions ( cfg . ConcreteArtifacts ( ) ) ,
2026-08-10 22:27:06 +00:00
nil ,
)
if err != nil {
return false
}
}
2026-05-16 14:43:29 +00:00
stageRequested := func ( name string ) bool {
for _ , s := range stages {
if s != nil && s . Name ( ) == name {
return true
}
}
return false
}
if cfg . Session . Inputs . AudioS3 != nil && stageRequested ( "prepare" ) {
return true
}
2026-05-20 14:53:22 +00:00
if stageRequested ( "prepare" ) {
2026-08-10 22:27:06 +00:00
requirements := artifacts . CollectPreviousArtifactRequirements ( configuredScriptoriumArtifacts ( cfg ) , effective )
2026-05-20 14:53:22 +00:00
if len ( requirements ) > 0 && strings . TrimSpace ( cfg . Session . PreviousSessionID ) != "" {
return true
}
}
2026-05-23 04:30:45 +00:00
if ! stageRequested ( "publish" ) {
2026-05-16 14:43:29 +00:00
return false
}
2026-05-23 04:30:45 +00:00
if cfg . Pipeline . Publish == nil {
2026-05-16 14:43:29 +00:00
return false
}
2026-05-23 04:30:45 +00:00
if cfg . Pipeline . Publish . Enabled != nil && ! * cfg . Pipeline . Publish . Enabled {
2026-05-16 14:43:29 +00:00
return false
}
2026-05-23 04:30:45 +00:00
if cfg . Pipeline . Publish . UploadRun != nil && ! * cfg . Pipeline . Publish . UploadRun {
2026-05-16 14:43:29 +00:00
return false
}
return true
}
2026-05-20 14:53:22 +00:00
2026-08-09 23:59:50 +00:00
func needsNotariusForRun ( cfg * config . Config , stages [ ] stage . Stage ) bool {
if cfg == nil || cfg . Pipeline == nil || cfg . Pipeline . Notarius == nil || ! cfg . Pipeline . Notarius . Enabled {
return false
}
for _ , candidate := range stages {
if candidate != nil && candidate . Name ( ) == "extract" {
return true
}
}
return false
}
2026-05-20 14:53:22 +00:00
func configuredScriptoriumArtifacts ( cfg * config . Config ) map [ string ] config . ScriptoriumArtifactConfig {
if cfg == nil || cfg . Pipeline == nil || cfg . Pipeline . Scriptorium == nil {
return nil
}
2026-11-20 04:42:23 +00:00
return cfg . ConcreteArtifacts ( )
2026-05-20 14:53:22 +00:00
}