2026-07-18 04:57:50 +00:00
|
|
|
package debug
|
|
|
|
|
|
|
|
|
|
import (
|
2026-11-22 02:49:48 +00:00
|
|
|
"encoding/json"
|
|
|
|
|
"errors"
|
2026-07-18 13:42:17 +00:00
|
|
|
"fmt"
|
2026-07-18 04:57:50 +00:00
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
2026-11-22 02:49:48 +00:00
|
|
|
"strings"
|
2026-07-18 13:42:17 +00:00
|
|
|
"sync"
|
2026-07-18 04:57:50 +00:00
|
|
|
"testing"
|
2026-11-22 02:49:48 +00:00
|
|
|
"time"
|
2026-07-18 13:42:17 +00:00
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
2026-07-18 04:57:50 +00:00
|
|
|
)
|
|
|
|
|
|
2026-11-22 02:49:48 +00:00
|
|
|
func startTestCapture(t *testing.T, parent, id string) *Recorder {
|
|
|
|
|
t.Helper()
|
|
|
|
|
recorder, err := Start(parent, id, time.Unix(42, 0).UTC())
|
2026-07-18 04:57:50 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
return recorder
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRecorderPersistsIncrementallyAndFinalizesOnce(t *testing.T) {
|
|
|
|
|
parent := t.TempDir()
|
|
|
|
|
recorder := startTestCapture(t, parent, "first")
|
|
|
|
|
var _ pipeline.DebugRecorder = recorder
|
|
|
|
|
source := strings.Repeat("Source text and sk-ordinary-application-content. ", 400)
|
|
|
|
|
if err := recorder.WriteBytes("operations/extract/source.txt", []byte(source)); err != nil {
|
2026-07-18 04:57:50 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "operations/extract/source.txt")); err != nil || string(got) != source {
|
|
|
|
|
t.Fatalf("incremental evidence=%q %v", got, err)
|
|
|
|
|
}
|
|
|
|
|
second := startTestCapture(t, parent, "second")
|
|
|
|
|
primary := errors.New("pipeline failed")
|
|
|
|
|
result := struct {
|
|
|
|
|
OutputDirectory string
|
|
|
|
|
Published bool
|
|
|
|
|
}{"/output/already-published", true}
|
|
|
|
|
if err := recorder.Finalize(result, primary); !errors.Is(err, primary) {
|
|
|
|
|
t.Fatalf("primary failure lost: %v", err)
|
|
|
|
|
}
|
|
|
|
|
before, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
|
|
|
|
|
if err != nil || !strings.Contains(string(before), result.OutputDirectory) || !strings.Contains(string(before), primary.Error()) {
|
|
|
|
|
t.Fatalf("terminal evidence=%s %v", before, err)
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.Finalize("replacement", nil); !errors.Is(err, primary) {
|
|
|
|
|
t.Fatalf("repeat finalization=%v", err)
|
|
|
|
|
}
|
|
|
|
|
after, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
|
|
|
|
|
if err != nil || string(after) != string(before) {
|
|
|
|
|
t.Fatalf("terminal outcome rewritten: %q %v", after, err)
|
|
|
|
|
}
|
|
|
|
|
if err := second.Finalize(map[string]any{"success": true}, nil); err != nil {
|
2026-07-18 04:57:50 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
deliveryErr := errors.New("stdout delivery failed after publication")
|
|
|
|
|
if err := second.Finalize(nil, deliveryErr); !errors.Is(err, deliveryErr) {
|
|
|
|
|
t.Fatalf("post-capture delivery failure lost: %v", err)
|
2026-07-18 04:57:50 +00:00
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
target, err := os.Readlink(filepath.Join(parent, "latest"))
|
|
|
|
|
if err != nil || target != "second" {
|
|
|
|
|
t.Fatalf("completion retargeted latest: %q %v", target, err)
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.WriteBytes("late.txt", nil); err == nil {
|
|
|
|
|
t.Fatal("write after finalization succeeded")
|
|
|
|
|
}
|
|
|
|
|
info, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json"))
|
|
|
|
|
if err != nil || info.Mode().Perm() != 0o600 {
|
|
|
|
|
t.Fatalf("capture file=%v %v", info, err)
|
2026-07-18 04:57:50 +00:00
|
|
|
}
|
|
|
|
|
}
|
2026-07-18 13:42:17 +00:00
|
|
|
|
2026-11-22 02:49:48 +00:00
|
|
|
func TestRecorderRetainsCallbackErrorAndExposesItAtBoundary(t *testing.T) {
|
|
|
|
|
recorder := startTestCapture(t, t.TempDir(), "run")
|
|
|
|
|
callbackErr := errors.New("observer recording failed")
|
|
|
|
|
recorder.RecordError(callbackErr)
|
|
|
|
|
if !errors.Is(recorder.Err(), callbackErr) {
|
|
|
|
|
t.Fatal("callback failure not retained")
|
2026-07-18 13:42:17 +00:00
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
if err := recorder.WriteJSON("available.json", map[string]string{"available": "evidence"}); !errors.Is(err, callbackErr) {
|
|
|
|
|
t.Fatalf("boundary hid callback error: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := os.Stat(filepath.Join(recorder.Path(), "available.json")); err != nil {
|
2026-07-18 13:42:17 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
primary := errors.New("primary operation failed")
|
|
|
|
|
err := recorder.Finalize(map[string]string{"output_directory": "/published"}, primary)
|
|
|
|
|
if !errors.Is(err, primary) || !errors.Is(err, callbackErr) || !strings.HasPrefix(err.Error(), primary.Error()) {
|
|
|
|
|
t.Fatalf("failure causes/order lost: %v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRecorderWriteAndFinalizationFailuresPreserveEvidence(t *testing.T) {
|
|
|
|
|
for _, defect := range []string{"incremental write", "outcome write", "overview write", "outcome encoding"} {
|
|
|
|
|
t.Run(defect, func(t *testing.T) {
|
|
|
|
|
recorder := startTestCapture(t, t.TempDir(), "run")
|
|
|
|
|
if err := recorder.WriteBytes("available.txt", []byte("retained")); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var result any = map[string]any{"output_directory": "/published", "artifact_count": 2}
|
|
|
|
|
switch defect {
|
|
|
|
|
case "incremental write":
|
|
|
|
|
if err := os.WriteFile(filepath.Join(recorder.Path(), "operations"), nil, 0o600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.WriteJSON("operations/result.json", result); err == nil {
|
|
|
|
|
t.Fatal("write failure hidden")
|
|
|
|
|
}
|
|
|
|
|
case "outcome write":
|
|
|
|
|
if err := os.Mkdir(filepath.Join(recorder.Path(), "outcome.json"), 0o700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
case "overview write":
|
|
|
|
|
// Replace only the initial overview file with a directory to make the
|
|
|
|
|
// terminal write fail, without relying on effective UID permissions.
|
|
|
|
|
filename := filepath.Join(recorder.Path(), "overview.txt")
|
|
|
|
|
if err := os.Rename(filename, filepath.Join(recorder.Path(), "initial-overview.txt")); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := os.Mkdir(filename, 0o700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
case "outcome encoding":
|
|
|
|
|
result = make(chan int)
|
|
|
|
|
}
|
|
|
|
|
finalErr := recorder.Finalize(result, nil)
|
|
|
|
|
if finalErr == nil {
|
|
|
|
|
t.Fatal("capture failure reported success")
|
|
|
|
|
}
|
|
|
|
|
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "available.txt")); err != nil || string(got) != "retained" {
|
|
|
|
|
t.Fatalf("evidence removed: %q %v", got, err)
|
|
|
|
|
}
|
|
|
|
|
primary := errors.New("new failure")
|
|
|
|
|
if err := recorder.Finalize(nil, primary); !errors.Is(err, primary) || !errors.Is(err, finalErr) {
|
|
|
|
|
t.Fatalf("additional primary error or original capture failure lost: %v", err)
|
|
|
|
|
}
|
|
|
|
|
})
|
2026-07-18 13:42:17 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:49:48 +00:00
|
|
|
func TestRecorderStartFailureProvidesAvailableDirectory(t *testing.T) {
|
2026-07-18 13:42:17 +00:00
|
|
|
root := t.TempDir()
|
2026-11-22 02:49:48 +00:00
|
|
|
if err := os.Mkdir(filepath.Join(root, "latest"), 0o700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
recorder, err := Start(root, "run", time.Now())
|
|
|
|
|
if err == nil || recorder == nil || recorder.Path() != filepath.Join(root, "run") {
|
|
|
|
|
t.Fatalf("start=%#v %v", recorder, err)
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.Finalize(nil, err); err == nil {
|
|
|
|
|
t.Fatal("start failure concealed")
|
|
|
|
|
}
|
|
|
|
|
if _, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json")); err != nil {
|
2026-07-18 13:42:17 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
}
|
2026-07-18 13:42:17 +00:00
|
|
|
|
2026-11-22 02:49:48 +00:00
|
|
|
func TestRecorderConcurrentWritesAndBoundaryChecks(t *testing.T) {
|
|
|
|
|
recorder := startTestCapture(t, t.TempDir(), "run")
|
|
|
|
|
const count = 16
|
2026-07-18 13:42:17 +00:00
|
|
|
var wg sync.WaitGroup
|
2026-11-22 02:49:48 +00:00
|
|
|
failures := make(chan error, count)
|
|
|
|
|
for i := range count {
|
2026-07-18 13:42:17 +00:00
|
|
|
wg.Add(1)
|
2026-11-22 02:49:48 +00:00
|
|
|
go func(i int) {
|
2026-07-18 13:42:17 +00:00
|
|
|
defer wg.Done()
|
2026-11-22 02:49:48 +00:00
|
|
|
failures <- recorder.WriteJSON(fmt.Sprintf("operations/%02d/result.json", i), map[string]int{"value": i})
|
|
|
|
|
_ = recorder.Err()
|
|
|
|
|
}(i)
|
2026-07-18 13:42:17 +00:00
|
|
|
}
|
|
|
|
|
wg.Wait()
|
2026-11-22 02:49:48 +00:00
|
|
|
close(failures)
|
|
|
|
|
for err := range failures {
|
2026-07-18 13:42:17 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
if err := recorder.Finalize(map[string]int{"operations": count}, nil); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
for i := range count {
|
|
|
|
|
content, err := os.ReadFile(filepath.Join(recorder.Path(), fmt.Sprintf("operations/%02d/result.json", i)))
|
|
|
|
|
var decoded map[string]int
|
|
|
|
|
if err != nil || json.Unmarshal(content, &decoded) != nil || decoded["value"] != i {
|
|
|
|
|
t.Fatalf("operation %d: %s %v", i, content, err)
|
2026-07-18 13:42:17 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-22 02:49:48 +00:00
|
|
|
|
|
|
|
|
func TestDisabledRecorderIsANoopAndPreservesPrimaryFailure(t *testing.T) {
|
|
|
|
|
var recorder *Recorder
|
|
|
|
|
primary := errors.New("operation failed")
|
|
|
|
|
if recorder.Enabled() || recorder.Path() != "" || recorder.Err() != nil {
|
|
|
|
|
t.Fatal("nil recorder enabled")
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.WriteJSON("../ignored", make(chan int)); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := recorder.WriteBytes("../ignored", nil); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
recorder.RecordError(errors.New("ignored"))
|
|
|
|
|
if err := recorder.Finalize(nil, primary); err != primary {
|
|
|
|
|
t.Fatalf("primary=%v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|