eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Identify pipeline entries in reports and diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 17:37:41 +00:00
parent 7efa07ee9d
commit 926fa26a31
18 changed files with 176 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/cli/process_inputs.go
View File

@@ -88,8 +88,8 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
}
started := time.Now().UTC()
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
for _, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, item.Module)
for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
}
secrets := loaded.Runtime.ConfiguredSecrets(ctx)
initial, err := renderReportBundle(snapshot, secrets...)

45
internal/cli/process_ordered_integration_test.go
View File

@@ -15,8 +15,11 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
@@ -28,6 +31,7 @@ func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
var mu sync.Mutex
var models []string
var prompts []string
proposalCalls := 0
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
var request struct {
Model string `json:"model"`
@@ -45,11 +49,18 @@ func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
mu.Lock()
models = append(models, request.Model)
prompts = append(prompts, contents)
if request.Model == "model-one" {
proposalCalls++
}
firstProposalCall := proposalCalls
mu.Unlock()
result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}`
switch request.Model {
case "model-one":
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}`
if firstProposalCall == 1 {
result = `{"invalid":true}`
}
case "model-two":
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}`
case "model-three":
@@ -73,9 +84,11 @@ llm: {validation_profile: review}
pipeline:
stages:
- module: glossary
id: "party/../|<tag>"
glossaries: [party]
llm: {proposal_profile: one}
- module: glossary
id: spell
glossaries: [spells]
llm: {proposal_profile: two}
- module: grammar
@@ -103,7 +116,8 @@ pipeline:
t.Fatal(err)
}
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
loaded.Config.Diagnostics.WorkDir = dir
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
@@ -115,10 +129,29 @@ pipeline:
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
}
if len(prompts) > 3 && !strings.Contains(prompts[3], "Alice casts Sheeld.") {
t.Fatalf("second proposal missed earlier edit: %s", prompts[3])
if len(prompts) > 4 && !strings.Contains(prompts[4], "Alice casts Sheeld.") {
t.Fatalf("second proposal missed earlier edit: %s", prompts[4])
}
if !reflect.DeepEqual(models, []string{"model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
if len(result.Snapshot.WorkUnits) != 3 || result.Snapshot.WorkUnits[0].StageID == nil || *result.Snapshot.WorkUnits[0].StageID != "party/../|<tag>" || result.Snapshot.WorkUnits[1].StageIndex != 1 || result.Snapshot.WorkUnits[2].StageID != nil {
t.Fatalf("work unit identities: %+v", result.Snapshot.WorkUnits)
}
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
}
markdown := string(reporting.Markdown(result.Snapshot))
if !strings.Contains(markdown, `1: glossary (party/../\|&lt;tag&gt;)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
t.Fatalf("report labels: %s", markdown)
}
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
if readErr != nil {
t.Fatal(readErr)
}
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw)
}
}
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
t.Fatalf("models=%v", models)
}
}
@@ -126,7 +159,7 @@ pipeline:
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar}]\n",
"config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
}
@@ -160,7 +193,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" {
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" || result.Snapshot.Errors[0].StageID == nil || *result.Snapshot.Errors[0].StageID != "failing" {
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
}
}

20
internal/cli/process_pipeline.go
View File

@@ -46,7 +46,25 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
}
var failures []processreport.RunFailure
if runErr != nil {
failures = append(failures, processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure})
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
if ctx.Err() != nil {
failure.Reason = outcomes.ReasonCanceled
}
if len(output.ModuleResults) > 0 {
last := output.ModuleResults[len(output.ModuleResults)-1]
index := last.StageIndex
failure.StageIndex = &index
origin := outcomes.Reference{StageIndex: &index, StageID: last.StageID, ModuleKey: last.ModuleKey, ModuleInstance: last.ModuleInstance}
for _, operation := range last.Operations {
if operation.FailureReason != "" {
origin = operation.Reference
failure.Reason = operation.FailureReason
break
}
}
failure.Origin = &origin
}
failures = append(failures, failure)
}
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, ModuleSpecs: specs, RunOutput: &output, RunFailures: failures})
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr