Refresh reports when delivery failures accumulate
This commit is contained in:
@@ -29,10 +29,10 @@ files and supplies those source paths to CLI;
|
|||||||
no second filename-based discovery rule is used.
|
no second filename-based discovery rule is used.
|
||||||
CLI composes it after configuration, transcript acquisition, and preflight
|
CLI composes it after configuration, transcript acquisition, and preflight
|
||||||
readiness and before dispatching model work. It does not create ordinary report parents.
|
readiness and before dispatching model work. It does not create ordinary report parents.
|
||||||
Final report rendering uses the delivery and cleanup errors already observed.
|
Final report rendering includes delivery and cleanup errors observed so far.
|
||||||
If a report write first fails, the remaining writes use the failed bundle and
|
When another failure becomes known, later writes use updated failed reports.
|
||||||
one pass attempts to update the reports still missing failed evidence. A
|
One best-effort pass refreshes earlier reports with stale evidence. A published
|
||||||
published transcript remains available after a later report failure.
|
transcript remains available after a later report failure.
|
||||||
|
|
||||||
Debug is independent. The shared `--debug` flag enables a unique retained child
|
Debug is independent. The shared `--debug` flag enables a unique retained child
|
||||||
under `diagnostics.work_dir`. The CLI creates one interaction sink rooted in
|
under `diagnostics.work_dir`. The CLI creates one interaction sink rooted in
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Reviewed revision:** `e8c0708`
|
**Reviewed revision:** `e8c0708`
|
||||||
|
|
||||||
**Status:** Fourteen open findings from review of the 20
|
**Status:** Findings and remediation progress from review of the 20
|
||||||
[implementation stages](implementation.md) against the
|
[implementation stages](implementation.md) against the
|
||||||
[target roadmap](reliability-and-maintainability.md), including follow-up
|
[target roadmap](reliability-and-maintainability.md), including follow-up
|
||||||
audits of component interactions and algorithmic costs.
|
audits of component interactions and algorithmic costs.
|
||||||
@@ -48,6 +48,11 @@ normalization, chunking, preflight, validators, the runner, and reports.
|
|||||||
|
|
||||||
### R1 — Include Later Delivery Failures in Writable Reports (P2)
|
### R1 — Include Later Delivery Failures in Writable Reports (P2)
|
||||||
|
|
||||||
|
**Complete.** Later report writes include newly observed delivery failures.
|
||||||
|
One best-effort pass updates earlier reports with stale evidence. Diagnostics
|
||||||
|
tests cover both initially failed runs and failures that arise during
|
||||||
|
publication of a completed run.
|
||||||
|
|
||||||
**Affected plan:** Stage 4, item 3, and Stage 17, item 1.
|
**Affected plan:** Stage 4, item 3, and Stage 17, item 1.
|
||||||
|
|
||||||
**Evidence:** [Report publication](../../internal/core/diagnostics/output_session.go#L311)
|
**Evidence:** [Report publication](../../internal/core/diagnostics/output_session.go#L311)
|
||||||
|
|||||||
@@ -314,31 +314,31 @@ func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle
|
|||||||
result = append(result, FileFailure{Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
result = append(result, FileFailure{Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
primary = includeCancellation(ctx, primary)
|
|
||||||
phase := OutputReportWrite
|
phase := OutputReportWrite
|
||||||
if action == "report initialization" {
|
if action == "report initialization" {
|
||||||
phase = OutputInitialization
|
phase = OutputInitialization
|
||||||
}
|
}
|
||||||
failed := false
|
type reportState struct {
|
||||||
var failedWritten [3]bool
|
failed, canceled bool
|
||||||
write := func(i int, payload []byte) {
|
failureCount int
|
||||||
writeCtx := ctx
|
|
||||||
kind := action
|
|
||||||
if failed {
|
|
||||||
writeCtx = context.WithoutCancel(ctx)
|
|
||||||
kind = "failed " + action
|
|
||||||
}
|
|
||||||
if err := s.replace(writeCtx, s.reportDestinations[i], payload); err != nil {
|
|
||||||
failures = append(failures, FileFailure{Path: s.reports[i], Phase: phase, Kind: kind, Err: err})
|
|
||||||
} else if failed {
|
|
||||||
failedWritten[i] = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
renderFailed := func() bool {
|
observed := func() reportState {
|
||||||
if action == "report initialization" {
|
canceled := ctx.Err() != nil
|
||||||
|
return reportState{primary != nil || canceled || len(failures) > 0, canceled, len(failures)}
|
||||||
|
}
|
||||||
|
var rendered reportState
|
||||||
|
var written [3]reportState
|
||||||
|
var attemptedFailed [3]bool
|
||||||
|
refresh := func() bool {
|
||||||
|
state := observed()
|
||||||
|
if state == rendered {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if phase == OutputInitialization && !rendered.failed {
|
||||||
if err := s.Close(); err != nil {
|
if err := s.Close(); err != nil {
|
||||||
failures = append(failures, FileFailure{Path: s.transcript, Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
failures = append(failures, FileFailure{Path: s.transcript, Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
||||||
}
|
}
|
||||||
|
state = observed()
|
||||||
}
|
}
|
||||||
if render == nil {
|
if render == nil {
|
||||||
return false
|
return false
|
||||||
@@ -349,28 +349,38 @@ func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle
|
|||||||
failures = append(failures, FileFailure{Phase: phase, Kind: "failed report rendering", Err: err})
|
failures = append(failures, FileFailure{Phase: phase, Kind: "failed report rendering", Err: err})
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
failed = true
|
rendered = state
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
knownFailure := primary != nil || len(failures) != 0
|
write := func(i int) {
|
||||||
if knownFailure && !renderFailed() {
|
writeCtx, kind := ctx, action
|
||||||
return failures
|
if rendered.failed {
|
||||||
|
writeCtx, kind = context.WithoutCancel(ctx), "failed "+action
|
||||||
|
attemptedFailed[i] = true
|
||||||
|
}
|
||||||
|
if err := s.replace(writeCtx, s.reportDestinations[i], bundleParts(reports)[i]); err != nil {
|
||||||
|
failures = append(failures, FileFailure{Path: s.reports[i], Phase: phase, Kind: kind, Err: err})
|
||||||
|
} else {
|
||||||
|
written[i] = rendered
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for i := range s.reports {
|
for i := range s.reports {
|
||||||
if !failed && (len(failures) != 0 || ctx.Err() != nil) && !renderFailed() {
|
if !refresh() {
|
||||||
return failures
|
return failures
|
||||||
}
|
}
|
||||||
write(i, bundleParts(reports)[i])
|
write(i)
|
||||||
}
|
}
|
||||||
if !knownFailure && (failed || len(failures) != 0 || ctx.Err() != nil) {
|
// Refresh stale publications once and give any report that failed before
|
||||||
if !failed && !renderFailed() {
|
// the switch to failed status an attempt with the failed bundle.
|
||||||
|
for i := range s.reports {
|
||||||
|
state := observed()
|
||||||
|
if !state.failed || written[i] == state || (attemptedFailed[i] && !written[i].failed) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !refresh() {
|
||||||
return failures
|
return failures
|
||||||
}
|
}
|
||||||
for i, payload := range bundleParts(reports) {
|
write(i)
|
||||||
if !failedWritten[i] {
|
|
||||||
write(i, payload)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return failures
|
return failures
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -361,6 +362,71 @@ func TestFailedRenderingKeepsInitializedEvidenceAfterStreamError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFailedReportsIncludeLaterDeliveryFailures(t *testing.T) {
|
||||||
|
primary := errors.New("processing failed")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
primary error
|
||||||
|
failFiles []string
|
||||||
|
checkFiles []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "later writable report", primary: primary,
|
||||||
|
failFiles: []string{ErrorsFile, RecoveriesFile}, checkFiles: []string{MarkdownFile},
|
||||||
|
want: "processing failed,errors.yml,recoveries.yml",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "previously written reports", primary: primary,
|
||||||
|
failFiles: []string{MarkdownFile}, checkFiles: []string{ErrorsFile, RecoveriesFile},
|
||||||
|
want: "processing failed,report.md",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "completion switches to failed status",
|
||||||
|
failFiles: []string{RecoveriesFile, MarkdownFile}, checkFiles: []string{ErrorsFile},
|
||||||
|
want: "recoveries.yml,report.md",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
session, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
actual := session.replace
|
||||||
|
attempts := make(map[string]int)
|
||||||
|
session.replace = func(ctx context.Context, path string, payload []byte) error {
|
||||||
|
name := filepath.Base(path)
|
||||||
|
attempts[name]++
|
||||||
|
if slices.Contains(tc.failFiles, name) && attempts[name] == 1 {
|
||||||
|
return os.ErrPermission
|
||||||
|
}
|
||||||
|
return actual(ctx, path, payload)
|
||||||
|
}
|
||||||
|
failures := session.finalize(context.Background(), tc.primary, nil, func(cause error, failures []FileFailure) (ReportBundle, error) {
|
||||||
|
var facts []string
|
||||||
|
if cause != nil {
|
||||||
|
facts = append(facts, cause.Error())
|
||||||
|
}
|
||||||
|
for _, failure := range failures {
|
||||||
|
facts = append(facts, filepath.Base(failure.Path))
|
||||||
|
}
|
||||||
|
payload := []byte(strings.Join(facts, ","))
|
||||||
|
return ReportBundle{Errors: payload, Recoveries: payload, Markdown: payload}, nil
|
||||||
|
})
|
||||||
|
if len(failures) != len(tc.failFiles) {
|
||||||
|
t.Fatalf("file failures = %v", failures)
|
||||||
|
}
|
||||||
|
for _, name := range tc.checkFiles {
|
||||||
|
got, err := os.ReadFile(filepath.Join(dir, name))
|
||||||
|
if err != nil || string(got) != tc.want {
|
||||||
|
t.Errorf("%s = %q, %v, want %q", name, got, err, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type writerFunc func([]byte) (int, error)
|
type writerFunc func([]byte) (int, error)
|
||||||
|
|
||||||
func (write writerFunc) Write(payload []byte) (int, error) { return write(payload) }
|
func (write writerFunc) Write(payload []byte) (int, error) { return write(payload) }
|
||||||
|
|||||||
Reference in New Issue
Block a user