Record debug evidence in one run-scoped capture session
This commit is contained in:
@@ -1,38 +1,167 @@
|
||||
// Package debug records explicitly requested application evidence.
|
||||
package debug
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type FilesystemRecorder struct {
|
||||
root string
|
||||
// Recorder writes evidence incrementally. It retains only lifecycle information
|
||||
// and the first recording error, not the pipeline's content.
|
||||
type Recorder struct {
|
||||
bundle *debugbundle.Bundle
|
||||
mu sync.Mutex
|
||||
recordingErr error
|
||||
finalized bool
|
||||
finalErr error
|
||||
}
|
||||
|
||||
func NewFilesystemRecorder(root string) (pipeline.DebugRecorder, error) {
|
||||
root = strings.TrimSpace(root)
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopDebugRecorder(), nil
|
||||
func Start(parent, runID string, startedAt time.Time) (*Recorder, error) {
|
||||
bundle, err := debugbundle.Allocate(parent, runID, startedAt)
|
||||
if bundle == nil {
|
||||
return nil, err
|
||||
}
|
||||
return &FilesystemRecorder{root: root}, nil
|
||||
recorder := &Recorder{bundle: bundle}
|
||||
if err != nil {
|
||||
recorder.RecordError(err)
|
||||
return recorder, err
|
||||
}
|
||||
if err := recorder.WriteJSON("run.json", struct {
|
||||
RunID string `json:"run_id"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}{runID, startedAt}); err != nil {
|
||||
return recorder, err
|
||||
}
|
||||
if err := recorder.WriteBytes("overview.txt", []byte(fmt.Sprintf("Run: %s\nStarted: %s\nCapture: %s\nStatus: running\n", runID, startedAt.Format(time.RFC3339Nano), recorder.Path()))); err != nil {
|
||||
return recorder, err
|
||||
}
|
||||
return recorder, nil
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) Enabled() bool {
|
||||
return r != nil && strings.TrimSpace(r.root) != ""
|
||||
func (r *Recorder) Enabled() bool { return r != nil && r.bundle != nil }
|
||||
func (r *Recorder) Path() string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return r.bundle.Path()
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) WriteJSON(name string, payload any) error {
|
||||
// RecordError retains callback failures for a boundary that can return errors.
|
||||
func (r *Recorder) RecordError(err error) {
|
||||
if !r.Enabled() || err == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.remember(err)
|
||||
}
|
||||
func (r *Recorder) remember(err error) {
|
||||
if r.recordingErr == nil {
|
||||
r.recordingErr = err
|
||||
}
|
||||
}
|
||||
func (r *Recorder) Err() error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
return fileio.WriteJSON(r.root, name, payload, 0o700, 0o600)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.recordingErr
|
||||
}
|
||||
|
||||
func (r *FilesystemRecorder) WriteBytes(name string, data []byte) error {
|
||||
func (r *Recorder) WriteJSON(name string, payload any) error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
return fileio.WriteBytes(r.root, name, data, 0o700, 0o600)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.finalized {
|
||||
return fmt.Errorf("debug capture %q is finalized", r.Path())
|
||||
}
|
||||
if err := fileio.WriteJSON(r.Path(), name, payload, 0o700, 0o600); err != nil {
|
||||
r.remember(fmt.Errorf("record debug %q in %q: %w", name, r.Path(), err))
|
||||
}
|
||||
return r.recordingErr
|
||||
}
|
||||
func (r *Recorder) WriteBytes(name string, content []byte) error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.finalized {
|
||||
return fmt.Errorf("debug capture %q is finalized", r.Path())
|
||||
}
|
||||
if err := fileio.WriteBytes(r.Path(), name, content, 0o700, 0o600); err != nil {
|
||||
r.remember(fmt.Errorf("record debug %q in %q: %w", name, r.Path(), err))
|
||||
}
|
||||
return r.recordingErr
|
||||
}
|
||||
|
||||
// Finalize writes one terminal outcome, retaining a primary operation failure
|
||||
// before additional capture failures. Repeated calls neither write nor retarget
|
||||
// latest. An additional primary error is preserved alongside the original
|
||||
// finalization result, including when presentation fails after capture finishes.
|
||||
func (r *Recorder) Finalize(outcome any, primaryErr error) error {
|
||||
if !r.Enabled() {
|
||||
return primaryErr
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.finalized {
|
||||
if primaryErr == nil {
|
||||
return r.finalErr
|
||||
}
|
||||
if errors.Is(primaryErr, r.finalErr) {
|
||||
return primaryErr
|
||||
}
|
||||
return errors.Join(primaryErr, r.finalErr)
|
||||
}
|
||||
r.finalized = true
|
||||
completedAt := time.Now().UTC()
|
||||
terminal := struct {
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
Outcome any `json:"outcome,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CaptureError string `json:"capture_error,omitempty"`
|
||||
}{CompletedAt: completedAt, Outcome: outcome}
|
||||
if primaryErr != nil {
|
||||
terminal.Error = primaryErr.Error()
|
||||
}
|
||||
if r.recordingErr != nil {
|
||||
terminal.CaptureError = r.recordingErr.Error()
|
||||
}
|
||||
outcomeErr := fileio.WriteJSON(r.Path(), "outcome.json", terminal, 0o700, 0o600)
|
||||
if outcomeErr != nil {
|
||||
outcomeErr = fmt.Errorf("finalize debug outcome in %q: %w", r.Path(), outcomeErr)
|
||||
}
|
||||
status := "completed"
|
||||
if primaryErr != nil || r.recordingErr != nil || outcomeErr != nil {
|
||||
status = "failed"
|
||||
}
|
||||
overview := fmt.Sprintf("Run: %s\nStarted: %s\nCompleted: %s\nCapture: %s\nStatus: %s\n", r.bundle.RunID(), r.bundle.CreatedAt().Format(time.RFC3339Nano), completedAt.Format(time.RFC3339Nano), r.Path(), status)
|
||||
if primaryErr != nil {
|
||||
overview += "Error: " + primaryErr.Error() + "\n"
|
||||
}
|
||||
if r.recordingErr != nil {
|
||||
overview += "Capture error: " + r.recordingErr.Error() + "\n"
|
||||
}
|
||||
if outcomeErr != nil {
|
||||
overview += "Capture error: " + outcomeErr.Error() + "\n"
|
||||
}
|
||||
overviewErr := fileio.WriteBytes(r.Path(), "overview.txt", []byte(overview), 0o700, 0o600)
|
||||
if overviewErr != nil {
|
||||
overviewErr = fmt.Errorf("finalize debug overview in %q: %w", r.Path(), overviewErr)
|
||||
}
|
||||
captureErr := r.recordingErr
|
||||
if primaryErr != nil && errors.Is(primaryErr, captureErr) {
|
||||
captureErr = nil
|
||||
}
|
||||
r.finalErr = errors.Join(primaryErr, captureErr, outcomeErr, overviewErr)
|
||||
return r.finalErr
|
||||
}
|
||||
|
||||
@@ -1,79 +1,210 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestFilesystemRecorderWritesWithinTraceRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder, err := NewFilesystemRecorder(root)
|
||||
func startTestCapture(t *testing.T, parent, id string) *Recorder {
|
||||
t.Helper()
|
||||
recorder, err := Start(parent, id, time.Unix(42, 0).UTC())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.WriteBytes("attempt/data", []byte("payload")); err != nil {
|
||||
return recorder
|
||||
}
|
||||
|
||||
func TestRecorderPersistsIncrementallyAndFinalizesOnce(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
recorder := startTestCapture(t, parent, "first")
|
||||
var _ pipeline.DebugRecorder = recorder
|
||||
source := strings.Repeat("Source text and sk-ordinary-application-content. ", 400)
|
||||
if err := recorder.WriteBytes("operations/extract/source.txt", []byte(source)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(root, "attempt", "data"))
|
||||
if err != nil {
|
||||
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "operations/extract/source.txt")); err != nil || string(got) != source {
|
||||
t.Fatalf("incremental evidence=%q %v", got, err)
|
||||
}
|
||||
second := startTestCapture(t, parent, "second")
|
||||
primary := errors.New("pipeline failed")
|
||||
result := struct {
|
||||
OutputDirectory string
|
||||
Published bool
|
||||
}{"/output/already-published", true}
|
||||
if err := recorder.Finalize(result, primary); !errors.Is(err, primary) {
|
||||
t.Fatalf("primary failure lost: %v", err)
|
||||
}
|
||||
before, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
|
||||
if err != nil || !strings.Contains(string(before), result.OutputDirectory) || !strings.Contains(string(before), primary.Error()) {
|
||||
t.Fatalf("terminal evidence=%s %v", before, err)
|
||||
}
|
||||
if err := recorder.Finalize("replacement", nil); !errors.Is(err, primary) {
|
||||
t.Fatalf("repeat finalization=%v", err)
|
||||
}
|
||||
after, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
|
||||
if err != nil || string(after) != string(before) {
|
||||
t.Fatalf("terminal outcome rewritten: %q %v", after, err)
|
||||
}
|
||||
if err := second.Finalize(map[string]any{"success": true}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("mode=%#o", info.Mode().Perm())
|
||||
deliveryErr := errors.New("stdout delivery failed after publication")
|
||||
if err := second.Finalize(nil, deliveryErr); !errors.Is(err, deliveryErr) {
|
||||
t.Fatalf("post-capture delivery failure lost: %v", err)
|
||||
}
|
||||
if err := recorder.WriteBytes("../outside", nil); err == nil {
|
||||
t.Fatal("accepted traversal")
|
||||
target, err := os.Readlink(filepath.Join(parent, "latest"))
|
||||
if err != nil || target != "second" {
|
||||
t.Fatalf("completion retargeted latest: %q %v", target, err)
|
||||
}
|
||||
if err := recorder.WriteBytes("late.txt", nil); err == nil {
|
||||
t.Fatal("write after finalization succeeded")
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json"))
|
||||
if err != nil || info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("capture file=%v %v", info, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemRecorderPropagatesWriteFailures(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "trace")
|
||||
if err := os.WriteFile(root, []byte("not a directory"), 0o600); err != nil {
|
||||
func TestRecorderRetainsCallbackErrorAndExposesItAtBoundary(t *testing.T) {
|
||||
recorder := startTestCapture(t, t.TempDir(), "run")
|
||||
callbackErr := errors.New("observer recording failed")
|
||||
recorder.RecordError(callbackErr)
|
||||
if !errors.Is(recorder.Err(), callbackErr) {
|
||||
t.Fatal("callback failure not retained")
|
||||
}
|
||||
if err := recorder.WriteJSON("available.json", map[string]string{"available": "evidence"}); !errors.Is(err, callbackErr) {
|
||||
t.Fatalf("boundary hid callback error: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(recorder.Path(), "available.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder, err := NewFilesystemRecorder(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.WriteBytes("attempt/data", []byte("payload")); err == nil {
|
||||
t.Fatal("WriteBytes concealed a trace-root write failure")
|
||||
primary := errors.New("primary operation failed")
|
||||
err := recorder.Finalize(map[string]string{"output_directory": "/published"}, primary)
|
||||
if !errors.Is(err, primary) || !errors.Is(err, callbackErr) || !strings.HasPrefix(err.Error(), primary.Error()) {
|
||||
t.Fatalf("failure causes/order lost: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynchronizedFilesystemRecorderSupportsConcurrentWrites(t *testing.T) {
|
||||
func TestRecorderWriteAndFinalizationFailuresPreserveEvidence(t *testing.T) {
|
||||
for _, defect := range []string{"incremental write", "outcome write", "overview write", "outcome encoding"} {
|
||||
t.Run(defect, func(t *testing.T) {
|
||||
recorder := startTestCapture(t, t.TempDir(), "run")
|
||||
if err := recorder.WriteBytes("available.txt", []byte("retained")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result any = map[string]any{"output_directory": "/published", "artifact_count": 2}
|
||||
switch defect {
|
||||
case "incremental write":
|
||||
if err := os.WriteFile(filepath.Join(recorder.Path(), "operations"), nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.WriteJSON("operations/result.json", result); err == nil {
|
||||
t.Fatal("write failure hidden")
|
||||
}
|
||||
case "outcome write":
|
||||
if err := os.Mkdir(filepath.Join(recorder.Path(), "outcome.json"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "overview write":
|
||||
// Replace only the initial overview file with a directory to make the
|
||||
// terminal write fail, without relying on effective UID permissions.
|
||||
filename := filepath.Join(recorder.Path(), "overview.txt")
|
||||
if err := os.Rename(filename, filepath.Join(recorder.Path(), "initial-overview.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filename, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "outcome encoding":
|
||||
result = make(chan int)
|
||||
}
|
||||
finalErr := recorder.Finalize(result, nil)
|
||||
if finalErr == nil {
|
||||
t.Fatal("capture failure reported success")
|
||||
}
|
||||
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "available.txt")); err != nil || string(got) != "retained" {
|
||||
t.Fatalf("evidence removed: %q %v", got, err)
|
||||
}
|
||||
primary := errors.New("new failure")
|
||||
if err := recorder.Finalize(nil, primary); !errors.Is(err, primary) || !errors.Is(err, finalErr) {
|
||||
t.Fatalf("additional primary error or original capture failure lost: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecorderStartFailureProvidesAvailableDirectory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder, err := NewFilesystemRecorder(root)
|
||||
if err != nil {
|
||||
if err := os.Mkdir(filepath.Join(root, "latest"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder = pipeline.SynchronizedDebugRecorder(recorder)
|
||||
recorder, err := Start(root, "run", time.Now())
|
||||
if err == nil || recorder == nil || recorder.Path() != filepath.Join(root, "run") {
|
||||
t.Fatalf("start=%#v %v", recorder, err)
|
||||
}
|
||||
if err := recorder.Finalize(nil, err); err == nil {
|
||||
t.Fatal("start failure concealed")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
const writes = 16
|
||||
func TestRecorderConcurrentWritesAndBoundaryChecks(t *testing.T) {
|
||||
recorder := startTestCapture(t, t.TempDir(), "run")
|
||||
const count = 16
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, writes)
|
||||
for i := 0; i < writes; i++ {
|
||||
i := i
|
||||
failures := make(chan error, count)
|
||||
for i := range count {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs <- recorder.WriteBytes(fmt.Sprintf("attempt/%02d/data", i), []byte("payload"))
|
||||
}()
|
||||
failures <- recorder.WriteJSON(fmt.Sprintf("operations/%02d/result.json", i), map[string]int{"value": i})
|
||||
_ = recorder.Err()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
close(failures)
|
||||
for err := range failures {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < writes; i++ {
|
||||
if _, err := os.Stat(filepath.Join(root, "attempt", fmt.Sprintf("%02d", i), "data")); err != nil {
|
||||
t.Fatalf("trace artifact %d: %v", i, err)
|
||||
if err := recorder.Finalize(map[string]int{"operations": count}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range count {
|
||||
content, err := os.ReadFile(filepath.Join(recorder.Path(), fmt.Sprintf("operations/%02d/result.json", i)))
|
||||
var decoded map[string]int
|
||||
if err != nil || json.Unmarshal(content, &decoded) != nil || decoded["value"] != i {
|
||||
t.Fatalf("operation %d: %s %v", i, content, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledRecorderIsANoopAndPreservesPrimaryFailure(t *testing.T) {
|
||||
var recorder *Recorder
|
||||
primary := errors.New("operation failed")
|
||||
if recorder.Enabled() || recorder.Path() != "" || recorder.Err() != nil {
|
||||
t.Fatal("nil recorder enabled")
|
||||
}
|
||||
if err := recorder.WriteJSON("../ignored", make(chan int)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.WriteBytes("../ignored", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder.RecordError(errors.New("ignored"))
|
||||
if err := recorder.Finalize(nil, primary); err != primary {
|
||||
t.Fatalf("primary=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user