eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Identify pipeline entries in reports and diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 17:37:41 +00:00
parent 7efa07ee9d
commit 926fa26a31
18 changed files with 176 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/framework/contracts/contracts.go
View File

@@ -70,6 +70,8 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
// ExecutionContext carries shared execution state for proposal generation and validation.
type ExecutionContext struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Settings config.Stage `json:"-"`

8
internal/framework/llm/observations.go
View File

@@ -92,6 +92,14 @@ func (c *ObservationCollector) Hooks() CallHooks {
}
func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext {
if value.StageIndex != nil {
index := *value.StageIndex
value.StageIndex = &index
}
if value.StageID != nil {
id := *value.StageID
value.StageID = &id
}
if value.SectionIndex != nil {
index := *value.SectionIndex
value.SectionIndex = &index

2
internal/framework/outcomes/outcomes.go
View File

@@ -16,6 +16,8 @@ const (
// are scoped to a module instance, while batch indices are local to a validator
// invocation within one section.
type Reference struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
SectionIndex int `json:"section_index"`

36
internal/framework/processreport/attribution_test.go Normal file
View File

@@ -0,0 +1,36 @@
package processreport
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
func TestUnavailableCandidateAndFailureKeepIdentity(t *testing.T) {
id := "review/name"
index := 1
output := &runner.RunOutput{InventoryAvailable: true, ModuleResults: []runner.ModuleResult{
{StageIndex: 1, StageID: &id, ModuleKey: "grammar", ModuleInstance: "grammar_2", ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}},
}, WorkUnits: []runner.WorkUnitOutcome{{ModuleIndex: 1, StageIndex: 1, StageID: &id, ModuleKey: "grammar", ModuleInstance: "grammar_2", State: outcomes.WorkCompleted, Operations: []outcomes.Operation{{Reference: outcomes.Reference{StageIndex: &index, StageID: &id, Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed}}}}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 1 || ledger[0].Disposition != "unavailable" || ledger[0].StageIndex != 1 || ledger[0].StageID == nil || *ledger[0].StageID != id {
t.Fatalf("ledger=%+v", ledger)
}
snapshot := Build(BuildInput{StartedAt: time.Now(), Status: reporting.StatusCompleted, ModuleSpecs: []contracts.ModuleRunSpec{{Index: 0, ModuleKey: "glossary"}, {Index: 1, ID: &id, ModuleKey: "grammar"}}, RunOutput: output})
if len(snapshot.Errors) != 1 || snapshot.Errors[0].StageID == nil || *snapshot.Errors[0].StageID != id || snapshot.Errors[0].StageIndex == nil || *snapshot.Errors[0].StageIndex != 1 {
t.Fatalf("errors=%+v", snapshot.Errors)
}
raw, err := reporting.YAML(snapshot.ErrorsDocument())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), "stage_id: review/name") || !strings.Contains(string(raw), "stage_index: 1") {
t.Fatalf("error report: %s", raw)
}
}

15
internal/framework/processreport/correction_ledger.go
View File

@@ -25,6 +25,8 @@ type CorrectionLedgerInput struct {
}
type CorrectionLedgerEntry struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
RunID string `json:"run_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
@@ -71,6 +73,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
for _, change := range module.AppliedChanges {
seen[identity(module.ModuleInstance, change.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
@@ -88,6 +91,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
for _, change := range module.SkippedChanges {
seen[identity(module.ModuleInstance, change.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
@@ -110,6 +114,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
seen[identity(module.ModuleInstance, rejection.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
@@ -127,6 +132,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
if module.Status == runner.ModuleStatusFailed {
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
@@ -141,11 +147,14 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
continue
}
var decisions []runner.ValidatorDecisionRecord
var stageIndex int
var stageID *string
policy := ""
for _, module := range runOutput.ModuleResults {
if module.ModuleInstance != candidate.ModuleInstance {
continue
}
stageIndex, stageID = module.StageIndex, module.StageID
policy = string(module.ReplacementPolicy)
for _, decision := range module.ValidatorDecisions {
if decision.ProposalIndex == candidate.ProposalIndex {
@@ -155,7 +164,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
break
}
entries = append(entries, CorrectionLedgerEntry{
RunID: runID, ModuleKey: candidate.ModuleKey, ModuleInstance: candidate.ModuleInstance,
StageIndex: stageIndex, StageID: stageID, RunID: runID, ModuleKey: candidate.ModuleKey, ModuleInstance: candidate.ModuleInstance,
ProposalIndex: candidate.ProposalIndex, SegmentID: candidate.TargetSegmentID,
OriginalText: candidate.OriginalText, ProposedCorrectedText: &candidate.CorrectedText,
ReplacementPolicy: policy, Disposition: correctionDispositionInterrupted,
@@ -166,8 +175,8 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
sort.SliceStable(entries, func(i, j int) bool {
if entries[i].ModuleInstance != entries[j].ModuleInstance {
return entries[i].ModuleInstance < entries[j].ModuleInstance
if entries[i].StageIndex != entries[j].StageIndex {
return entries[i].StageIndex < entries[j].StageIndex
}
if entries[i].ProposalIndex != entries[j].ProposalIndex {
return entries[i].ProposalIndex < entries[j].ProposalIndex

12
internal/framework/processreport/report.go
View File

@@ -41,7 +41,7 @@ func Build(input BuildInput) reporting.Snapshot {
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
}
for i, spec := range input.ModuleSpecs {
snapshot.StageNames[i] = spec.InstanceName
snapshot.StageNames[i] = reporting.StageLabel(spec.Index, spec.ModuleKey, spec.ID)
}
if input.RunOutput != nil && input.RunOutput.InventoryAvailable {
units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...)
@@ -66,7 +66,7 @@ func Build(input BuildInput) reporting.Snapshot {
interrupted++
}
corrections += unit.CorrectionsApplied
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied}
operations := append([]outcomes.Operation(nil), unit.Operations...)
@@ -95,7 +95,7 @@ func Build(input BuildInput) reporting.Snapshot {
withErrors++
row.ErrorCount = 1
stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage,
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage, StageID: unit.StageID,
ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: &chunk,
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
}
@@ -118,6 +118,10 @@ func Build(input BuildInput) reporting.Snapshot {
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
if failure.Origin != nil {
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
record.StageID = failure.Origin.StageID
if record.StageIndex == nil {
record.StageIndex = failure.Origin.StageIndex
}
chunk := failure.Origin.SectionIndex
record.ChunkIndex = &chunk
record.Operation = string(failure.Origin.Kind)
@@ -148,7 +152,7 @@ func mapRecovery(unit runner.WorkUnitOutcome, operation outcomes.Operation) repo
filtered[i] = reporting.FilteredProposal{ProposalIndex: item.ProposalIndex, Reason: string(item.Reason), Count: item.Count}
}
sort.Slice(filtered, func(i, j int) bool { return filtered[i].ProposalIndex < filtered[j].ProposalIndex })
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName, BatchIndex: operation.Reference.BatchIndex,
Reasons: reasons, Message: "Model operation succeeded after recovery or response filtering.",

6
internal/framework/proposal_generation/generate.go
View File

@@ -49,6 +49,8 @@ type StructuredCorrectionSet struct {
// Request captures reusable proposal-generation inputs for future modules.
type Request struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
ModuleKey string `json:"module_key"`
@@ -124,7 +126,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
}, &response)
sectionIndex := 0
if req.Section != nil {
@@ -132,6 +134,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
operation := outcomes.Operation{
Reference: outcomes.Reference{
StageIndex: req.StageIndex, StageID: req.StageID,
ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance,
SectionIndex: sectionIndex, Kind: outcomes.OperationProposal,
},
@@ -139,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
requestMetadata := map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID,
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,

1
internal/framework/proposal_generation/module_proposal.go
View File

@@ -53,6 +53,7 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
}
generated, err := GenerateCandidates(ctx, Request{
StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,

13
internal/framework/runner/runner.go
View File

@@ -38,6 +38,8 @@ type Runner struct {
// ModuleResult captures deterministic per-module execution output.
type ModuleResult struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
@@ -56,6 +58,8 @@ type ModuleResult struct {
// WorkUnitOutcome keeps initial section identity while text evolves between
// module instances. Detailed candidate and decision records are debug evidence.
type WorkUnitOutcome struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
@@ -149,7 +153,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
for moduleIndex, spec := range input.ModuleSpecs {
for _, section := range inventory {
workUnits = append(workUnits, WorkUnitOutcome{
ModuleIndex: moduleIndex, ModuleKey: spec.ModuleKey,
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section,
State: outcomes.WorkNotStarted,
})
@@ -167,6 +171,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
Status: ModuleStatusFailed,
@@ -196,6 +201,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
if pipelineErr != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
@@ -235,6 +241,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
results = append(results, ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
@@ -358,6 +365,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
@@ -399,7 +407,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
for i, section := range input.Sections {
out.WorkUnits[i] = WorkUnitOutcome{
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName,
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID,
Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted,
}
}
@@ -631,6 +639,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
validatorStarted := time.Now()
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
ModuleKey: input.Spec.ModuleKey,

14
internal/framework/structuredoutput/call.go
View File

@@ -11,12 +11,14 @@ import (
// DiagnosticContext identifies the logical call without examining rendered text.
type DiagnosticContext struct {
Role string `json:"role"`
ModuleKey string `json:"module_key,omitempty"`
ModuleInstance string `json:"module_instance,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
Role string `json:"role"`
ModuleKey string `json:"module_key,omitempty"`
ModuleInstance string `json:"module_instance,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
}
// Request names an embedded prompt and response schema. Profile selection and

4
internal/framework/validators/llm_validators.go
View File

@@ -94,6 +94,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
operationRef := func(kind outcomes.OperationKind, batchIndex *int) outcomes.Reference {
return outcomes.Reference{
StageIndex: req.StageIndex, StageID: req.StageID,
ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance,
SectionIndex: sectionIndex, Kind: kind, ValidatorName: v.name,
ValidatorOrder: req.ValidatorOrder, BatchIndex: batchIndex,
@@ -149,7 +150,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
@@ -165,6 +166,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,

2
internal/framework/validators/models.go
View File

@@ -32,6 +32,8 @@ const (
// Request is the runtime input shared by deterministic validators.
type Request struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
SectionIndex *int `json:"section_index,omitempty"`