Files
audita/internal/framework/processreport/report_test.go

255 lines
18 KiB
Go

package processreport
import (
"context"
"errors"
"reflect"
"sort"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gopkg.in/yaml.v3"
)
func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
started := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
completed := started.Add(time.Minute)
proposal := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, TransientRetries: 1, StructuralRepairs: 1},
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
batch := 1
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
output := &runner.RunOutput{InventoryAvailable: true, WorkUnits: []runner.WorkUnitOutcome{
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
}}
input := BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output}
snapshot := Build(input)
input.SummaryOnly = true
summary := Build(input)
if !reflect.DeepEqual(snapshot.Summary, summary.Summary) || len(summary.Errors)+len(summary.Recoveries)+len(summary.WorkUnits) != 0 {
t.Fatalf("summary-only snapshot changed counts or retained details: %+v", summary)
}
if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 2 || snapshot.Errors[0].Failures[0].Operation != "validation_input" || !reflect.DeepEqual(snapshot.Errors[0].Failures[1].ProposalIndices, []int{3, 9}) {
t.Fatalf("failure ordering: %+v", snapshot.Errors)
}
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"output_repaired", "proposals_filtered", "service_recovered"}) || snapshot.Recoveries[0].FilteredProposals[0].ProposalIndex != 2 {
t.Fatalf("recovery: %+v", snapshot.Recoveries)
}
if snapshot.WorkUnits[0].ModuleInstance != "grammar_1" || snapshot.WorkUnits[1].State != "not_started" {
t.Fatalf("unit order: %+v", snapshot.WorkUnits)
}
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument())
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := yaml.Unmarshal(errorsYAML, &decoded); err != nil {
t.Fatal(err)
}
if decoded["version"] != 1 || decoded["schema"] != "audita-errors" || decoded["status"] != "completed" {
t.Fatalf("envelope: %#v", decoded)
}
if strings.Contains(string(errorsYAML), "provider_attempts: null") || !strings.Contains(string(errorsYAML), "errors:") {
t.Fatal(string(errorsYAML))
}
}
func TestBuildReportsUnavailableDecisionsAndFilteredExtras(t *testing.T) {
started := time.Date(2026, 9, 25, 12, 0, 0, 0, time.UTC)
completed := started.Add(time.Second)
failedBatch, cleanBatch := 0, 1
output := &runner.RunOutput{InventoryAvailable: true, WorkUnits: []runner.WorkUnitOutcome{{
ModuleKey: "grammar", ModuleInstance: "grammar", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 1},
State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &failedBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{3, 1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &cleanBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryValidationDecisionsFiltered}},
},
}}}
snapshot := Build(BuildInput{RunID: "partial", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, RunOutput: output})
if snapshot.Summary.WorkUnitsWithErrors == nil || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 0 || *snapshot.Summary.LLMOperations != 3 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 1 || !reflect.DeepEqual(snapshot.Errors[0].Failures[0].ProposalIndices, []int{1, 3}) || snapshot.Errors[0].Failures[0].Reason != string(outcomes.ReasonValidationDecisionsInvalid) || snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs == nil || *snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs != 1 {
t.Fatalf("failure attribution: %+v", snapshot.Errors)
}
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{string(outcomes.RecoveryValidationDecisionsFiltered)}) || len(snapshot.Recoveries[0].FilteredProposals) != 0 {
t.Fatalf("recovery attribution: %+v", snapshot.Recoveries)
}
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
encoded, err := reporting.YAML(document)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := yaml.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if decoded["version"] != 1 || decoded["status"] != "completed" {
t.Fatalf("YAML envelope: %#v", decoded)
}
summary := decoded["summary"].(map[string]any)
if summary["work_units_with_errors"] != 1 || summary["recoveries"] != 1 || summary["proposal_responses_with_filtering"] != 0 {
t.Fatalf("YAML summary: %#v", summary)
}
if errors, ok := decoded["errors"].([]any); ok {
failure := errors[0].(map[string]any)["failures"].([]any)[0].(map[string]any)
if !reflect.DeepEqual(failure["proposal_indices"], []any{1, 3}) {
t.Fatalf("YAML failure indices: %#v", failure)
}
}
if recoveries, ok := decoded["recoveries"].([]any); ok {
recovery := recoveries[0].(map[string]any)
filtered, hasFiltered := recovery["filtered_proposals"]
if !reflect.DeepEqual(recovery["reasons"], []any{string(outcomes.RecoveryValidationDecisionsFiltered)}) || hasFiltered && len(filtered.([]any)) != 0 {
t.Fatalf("YAML recovery: %#v", recovery)
}
}
}
}
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
start := time.Now().UTC()
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})
initialDoc := initial.ErrorsDocument()
if initialDoc.CompletedAt != nil || initialDoc.Summary.WorkUnitsTotal != nil || initialDoc.Errors == nil {
t.Fatalf("initial: %+v", initialDoc)
}
finished := Build(BuildInput{RunID: "r", Status: reporting.StatusCompleted, StartedAt: start, RunOutput: &runner.RunOutput{InventoryAvailable: true}})
if finished.Summary.WorkUnitsTotal == nil || *finished.Summary.WorkUnitsTotal != 0 || finished.RecoveriesDocument().Recoveries == nil {
t.Fatalf("empty: %+v", finished)
}
failed := Build(BuildInput{RunID: "r", Status: reporting.StatusFailed, StartedAt: start,
RunFailures: []RunFailure{{Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "z.yml"}, {Phase: "processing", Reason: outcomes.ReasonCanceled}, {Phase: "report_write", Reason: outcomes.ReasonInternalFailure, OutputPath: "a.yml"}}})
if len(failed.Errors) != 3 || failed.Errors[0].Phase != "processing" || failed.Errors[1].OutputPath != "a.yml" || failed.Errors[2].OutputPath != "z.yml" || failed.Summary.WorkUnitsTotal != nil || *failed.Summary.RunErrors != 3 {
t.Fatalf("run failures: %+v", failed)
}
}
func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, AttemptCount: 1})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}
func TestReportsDistinguishFatalPeerAndDeliveryFailures(t *testing.T) {
defect := errors.New("private provider detail")
origin := outcomes.Reference{StageIndex: intPtr(1), ModuleKey: "grammar", ModuleInstance: "grammar_2",
SectionIndex: 2, Kind: outcomes.OperationValidation, ValidatorName: "review", BatchIndex: intPtr(0),
ProposalIndex: intPtr(3), RepairAttempt: intPtr(1)}
primary := &runner.ExecutionFailure{Phase: "validation", Origin: &origin, HasSection: true, Cause: defect}
peerOrigin := origin
peerOrigin.SectionIndex, peerOrigin.ProposalIndex, peerOrigin.RepairAttempt = 3, nil, nil
peer := &runner.ExecutionFailure{Phase: "validation", Origin: &peerOrigin, HasSection: true, Cause: context.Canceled}
work := []runner.WorkUnitOutcome{
{ModuleIndex: 0, ModuleInstance: "grammar_1", State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryProposalsFiltered},
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureClass: outcomes.FailureRecoverable,
FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{4},
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}},
}},
{ModuleIndex: 1, ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 2}, State: outcomes.WorkInterrupted, Failure: primary, Operations: []outcomes.Operation{
{Reference: origin, FailureClass: outcomes.FailureFatal, Cause: defect, FailureReason: outcomes.ReasonProviderFailed},
}},
{ModuleIndex: 1, ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 3}, State: outcomes.WorkInterrupted, Failure: peer},
}
snapshot := Build(BuildInput{
RunID: "mixed", StartedAt: time.Unix(1, 0),
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}, {Index: 1, ModuleKey: "grammar"}},
RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: work}, ProcessingError: primary,
})
snapshot = Failed(snapshot, primary, "processing", []diagnostics.FileFailure{
{Phase: diagnostics.OutputReportWrite, Path: "recoveries.yml", Err: errors.New("disk failure")},
})
raw, err := reporting.YAML(snapshot.ErrorsDocument())
if err != nil {
t.Fatal(err)
}
var doc reporting.ErrorsDocument
if err := yaml.Unmarshal(raw, &doc); err != nil {
t.Fatal(err)
}
if doc.Status != reporting.StatusFailed || *doc.Summary.WorkUnitsWithErrors != 1 || *doc.Summary.RunErrors != 2 ||
*doc.Summary.WorkUnitsInterrupted != 2 || *doc.Summary.Recoveries != 1 || len(doc.Errors) != 4 {
t.Fatalf("unexpected report: %s", raw)
}
unit, secondary, initiating, delivery := doc.Errors[0], doc.Errors[1], doc.Errors[2], doc.Errors[3]
if unit.Scope != "work_unit" || len(unit.Failures) != 1 || !reflect.DeepEqual(unit.Failures[0].ProposalIndices, []int{4}) ||
unit.Failures[0].ProviderAttempts == nil || *unit.Failures[0].ProviderAttempts != 1 ||
secondary.Scope != "interrupted" || !reflect.DeepEqual(secondary.ChunkIndex, intPtr(3)) || secondary.Reason != "canceled" ||
initiating.Scope != "run" || initiating.Phase != "validation" || initiating.Validator != "review" ||
!reflect.DeepEqual(initiating.ChunkIndex, intPtr(2)) || !reflect.DeepEqual(initiating.ProposalIndex, intPtr(3)) ||
!reflect.DeepEqual(initiating.RepairAttempt, intPtr(1)) ||
delivery.Scope != "run" || delivery.Phase != "report_write" || delivery.OutputPath != "recoveries.yml" ||
strings.Contains(string(raw), defect.Error()) {
t.Fatalf("failure facts were merged or lost: %s", raw)
}
if markdown := string(reporting.Markdown(snapshot)); !strings.Contains(markdown, "Errors: 1 work units; 2 run errors") ||
!strings.Contains(markdown, "Recoveries: 1") {
t.Fatalf("Markdown totals differ from YAML: %s", markdown)
}
}