eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Skip report and debug work for disabled outputs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 20:48:32 +00:00
parent a9b7c37a7e
commit 71a9b1f560
11 changed files with 161 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/framework/llm/adapter.go
View File

@@ -69,26 +69,29 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
}
terminalMu.Unlock()
}()
record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()}
record.Context.Role = c.role
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
if c.hooks.OnCall != nil {
c.hooks.OnCall(record)
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
record.Context = copyContext(req.Context)
record.Context.Role = c.role
}
defer func() {
record.Finished = time.Now().UTC()
if err != nil {
var failure *structuredoutput.Failure
if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
record.Failure = structuredoutput.FailureMalformedOutput
} else {
record.Failure = structuredoutput.FailureRuntime
if onCall := c.hooks.OnCall; onCall != nil {
record.Started = time.Now().UTC()
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
onCall(record)
defer func() {
record.Finished = time.Now().UTC()
if err != nil {
var failure *structuredoutput.Failure
if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
record.Failure = structuredoutput.FailureMalformedOutput
} else {
record.Failure = structuredoutput.FailureRuntime
}
}
}
if c.hooks.OnCall != nil {
c.hooks.OnCall(record)
}
}()
onCall(record)
}()
}
if parent.Err() != nil {
return response, classifiedFailure(parent, ctx, parent.Err(), nil)
}

66
internal/framework/processreport/report.go
View File

@@ -29,6 +29,7 @@ type BuildInput struct {
ModuleSpecs []contracts.ModuleRunSpec
RunOutput *runner.RunOutput
RunFailures []RunFailure
SummaryOnly bool
}
// Build maps processing facts once into an immutable presentation snapshot.
@@ -36,20 +37,28 @@ func Build(input BuildInput) reporting.Snapshot {
snapshot := reporting.Snapshot{
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
TranscriptPath: input.TranscriptPath, OutputPath: input.OutputPath,
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
}
for i, spec := range input.ModuleSpecs {
snapshot.StageNames[i] = reporting.StageLabel(spec.Index, spec.ModuleKey, spec.ID)
details := !input.SummaryOnly
if details {
snapshot.StageNames = make([]string, len(input.ModuleSpecs))
snapshot.Errors = make([]reporting.ErrorRecord, 0)
snapshot.Recoveries = make([]reporting.RecoveryRecord, 0)
snapshot.WorkUnits = make([]reporting.WorkUnit, 0)
for i, spec := range input.ModuleSpecs {
snapshot.StageNames[i] = reporting.StageLabel(spec.Index, spec.ModuleKey, spec.ID)
}
}
if input.RunOutput != nil && input.RunOutput.InventoryAvailable {
units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...)
sort.SliceStable(units, func(i, j int) bool {
if units[i].ModuleIndex != units[j].ModuleIndex {
return units[i].ModuleIndex < units[j].ModuleIndex
}
return units[i].Section.Index < units[j].Section.Index
})
units := input.RunOutput.WorkUnits
if details {
units = append([]runner.WorkUnitOutcome(nil), units...)
sort.SliceStable(units, func(i, j int) bool {
if units[i].ModuleIndex != units[j].ModuleIndex {
return units[i].ModuleIndex < units[j].ModuleIndex
}
return units[i].Section.Index < units[j].Section.Index
})
}
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
for _, unit := range units {
@@ -65,12 +74,14 @@ func Build(input BuildInput) reporting.Snapshot {
interrupted++
}
corrections += unit.CorrectionsApplied
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied}
operations := append([]outcomes.Operation(nil), unit.Operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
failures := make([]reporting.Failure, 0)
operations := unit.Operations
if details {
operations = append([]outcomes.Operation(nil), operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
}
var failures []reporting.Failure
unitHasError := false
unitRecoveries := 0
for _, operation := range operations {
if operation.Reference.Kind != outcomes.OperationValidationInput {
llmOperations++
@@ -80,18 +91,31 @@ func Build(input BuildInput) reporting.Snapshot {
}
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
recoveryCount++
row.RecoveryCount++
unitRecoveries++
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
filteringResponses++
}
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
if details {
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
}
if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) {
failures = append(failures, mapFailure(operation))
unitHasError = true
if details {
failures = append(failures, mapFailure(operation))
}
}
}
if len(failures) > 0 {
if unitHasError {
withErrors++
}
if !details {
continue
}
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
if unitHasError {
row.ErrorCount = 1
stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied
snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage, StageID: unit.StageID,

10
internal/framework/processreport/report_test.go
View File

@@ -27,8 +27,14 @@ func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
}}
snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output})
input := BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output}
snapshot := Build(input)
input.SummaryOnly = true
summary := Build(input)
if !reflect.DeepEqual(snapshot.Summary, summary.Summary) || len(summary.Errors)+len(summary.Recoveries)+len(summary.WorkUnits) != 0 {
t.Fatalf("summary-only snapshot changed counts or retained details: %+v", summary)
}
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
t.Fatalf("summary: %+v", snapshot.Summary)
}

23
internal/framework/proposal_generation/generate.go
View File

@@ -123,18 +123,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
Counts: completion.Execution,
}
requestMetadata := map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID,
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
@@ -154,6 +142,17 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if writer != nil {
requestMetadata := map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID,
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction(
stage,

22
internal/framework/validators/llm_validators.go
View File

@@ -124,13 +124,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
var response LLMValidationResponse
var completion StructuredCompletionResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
promptMetadata := validatorPromptMetadata(v.validatorType)
stageInstance := req.ModuleInstance
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
@@ -142,10 +136,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Counts: completion.Execution,
}
writeInteraction := func(responsePayload any, callErr error) InteractionArtifacts {
writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts {
if req.DiagnosticsWriter == nil {
return InteractionArtifacts{}
}
stageInstance := req.ModuleInstance
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
if reconciled != nil {
responsePayload["reconciliation"] = *reconciled
}
var diagnosticErr error
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
@@ -168,7 +172,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
return artifacts
}
if err != nil {
writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, err)
writeInteraction(nil, err)
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
@@ -186,7 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
reconciled := reconcileLLMResponse(batch.Items, response)
artifacts := writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model, "reconciliation": reconciled}, nil)
artifacts := writeInteraction(&reconciled, nil)
if len(reconciled.UnavailableIndices) > 0 {
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = reconciled.UnavailableIndices