From 3ef55b1cf00021d755ef6c19de008cdbb1434537 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 01:52:21 +0000 Subject: [PATCH] Add a best-effort accepted-artifact filesystem cache --- docs/roadmap/implementation.md | 16 ++ .../framework/artifactcache/filesystem.go | 111 ++++++++ .../filesystem_permissions_unix_test.go | 99 +++++++ .../artifactcache/filesystem_test.go | 242 ++++++++++++++++++ internal/framework/contracts/producer.go | 9 + 5 files changed, 477 insertions(+) create mode 100644 internal/framework/artifactcache/filesystem.go create mode 100644 internal/framework/artifactcache/filesystem_permissions_unix_test.go create mode 100644 internal/framework/artifactcache/filesystem_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 572bc720..0f4c3ec7 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1184,6 +1184,22 @@ testing when supported; it is a migration check, not a new release requirement. ## Stage 26: Implement The Small Accepted-Artifact Filesystem Cache +**Status:** Complete. Added a narrow accepted-artifact cache collaborator and +filesystem implementation storing unchanged JSON bytes at the selected root +under the opaque native hash plus `.json`. Construction/lookup create no cache +directories. Auto reads/writes, bypass performs no cache I/O, and refresh skips +lookup and replaces only a newly supplied valid result; failed writes preserve +previous acceptance. Missing entries are ordinary misses, while malformed, +unreadable, non-file, and unsafe entries return diagnostic misses. Caller-owned +typed decoding/usability and best-effort warning handling remain in the next +integration stage. Reused confined atomic process-permission writes without new +permission policy, envelopes, fingerprints, expiry, or legacy data migration. +Real-filesystem tests cover raw-byte identity, exact root/hash placement, modes, +failed refresh, cancellation, concurrent complete replacement, temporary-file +cleanup, narrow paths/symlinks, unchanged legacy records, umask/setgid inheritance, +and unchanged provisioned root permissions. Cache/fileio/contracts tests, vet, +race checks, focused pipeline tests, and `git diff --check` pass. + **Goal:** one best-effort cache facility replaces complex persistent records. **Inspect/change:** existing `internal/core/fileio` and cache permission/path tests; diff --git a/internal/framework/artifactcache/filesystem.go b/internal/framework/artifactcache/filesystem.go new file mode 100644 index 00000000..5ba9f17d --- /dev/null +++ b/internal/framework/artifactcache/filesystem.go @@ -0,0 +1,111 @@ +// Package artifactcache stores accepted producer artifacts without execution or +// provenance envelopes. Cache I/O is best effort at the calling operation. +package artifactcache + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +// FilesystemStore is safe for concurrent operations. Construction and lookup do +// not create directories; only storing a newly accepted result can do so. +type FilesystemStore struct { + root string +} + +var _ contracts.ArtifactCache = (*FilesystemStore)(nil) + +func NewFilesystemStore(root string) (*FilesystemStore, error) { + if strings.TrimSpace(root) == "" || strings.ContainsRune(root, '\x00') { + return nil, fmt.Errorf("artifact cache root must be a nonempty path without NUL") + } + absolute, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("resolve artifact cache root: %w", err) + } + // Retain the exact selected directory, including literal trailing spaces, + // when passing it through the common confined-path helper. + return &FilesystemStore{root: absolute + string(filepath.Separator)}, nil +} + +func (store *FilesystemStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + switch mode { + case contracts.CacheBypass, contracts.CacheRefresh: + return nil, nil + case contracts.CacheAuto, "": + default: + return nil, fmt.Errorf("unknown artifact cache mode %q", mode) + } + name, err := cacheName(hash) + if err != nil { + return nil, err + } + target, err := fileio.SafePath(store.root, name) + if err != nil { + return nil, fmt.Errorf("locate artifact cache entry %q: %w", hash, err) + } + info, err := os.Lstat(target) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("inspect artifact cache entry %q: %w", hash, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("artifact cache entry %q is not a regular file", hash) + } + content, err := os.ReadFile(target) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read artifact cache entry %q: %w", hash, err) + } + if !json.Valid(content) { + return nil, fmt.Errorf("artifact cache entry %q contains malformed JSON", hash) + } + return content, nil +} + +func (store *FilesystemStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error { + if err := ctx.Err(); err != nil { + return err + } + switch mode { + case contracts.CacheBypass: + return nil + case contracts.CacheAuto, contracts.CacheRefresh, "": + default: + return fmt.Errorf("unknown artifact cache mode %q", mode) + } + name, err := cacheName(hash) + if err != nil { + return err + } + if !json.Valid(content) { + return fmt.Errorf("artifact cache entry %q is not valid JSON", hash) + } + if err := fileio.WriteBytesWithProcessPermissions(store.root, name, content); err != nil { + return fmt.Errorf("store artifact cache entry %q: %w", hash, err) + } + return nil +} + +func cacheName(hash string) (string, error) { + // The native hash is opaque: do not strip a prefix, recalculate it, encode + // it, or enforce a particular digest format. It must identify one filename. + if hash == "" || strings.ContainsAny(hash, "/\\") { + return "", fmt.Errorf("artifact cache hash must be a nonempty filename component") + } + return hash + ".json", nil +} diff --git a/internal/framework/artifactcache/filesystem_permissions_unix_test.go b/internal/framework/artifactcache/filesystem_permissions_unix_test.go new file mode 100644 index 00000000..7c26afbf --- /dev/null +++ b/internal/framework/artifactcache/filesystem_permissions_unix_test.go @@ -0,0 +1,99 @@ +//go:build linux || darwin + +package artifactcache + +import ( + "context" + "errors" + "os" + "path/filepath" + "syscall" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func TestFilesystemCacheUsesDeploymentCreationPermissions(t *testing.T) { + for _, test := range []struct { + name string + umask int + directory, file os.FileMode + }{ + {"private", 0o077, 0o700, 0o600}, + {"group", 0o007, 0o770, 0o660}, + } { + t.Run(test.name, func(t *testing.T) { + previous := syscall.Umask(test.umask) + defer syscall.Umask(previous) + root := filepath.Join(t.TempDir(), "cache") + if err := newStore(t, root).Store(context.Background(), contracts.CacheAuto, "task", []byte("{}")); err != nil { + t.Fatal(err) + } + for path, expected := range map[string]os.FileMode{root: test.directory, filepath.Join(root, "task.json"): test.file} { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != expected { + t.Fatalf("%s: info %v, error %v; want mode %o", path, info, err, expected) + } + } + }) + } +} + +func TestFilesystemCacheDoesNotConvergeProvisionedRootPermissions(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, os.ModeSetgid|0o751); err != nil { + t.Fatal(err) + } + before, err := os.Stat(root) + if err != nil { + t.Fatal(err) + } + previous := syscall.Umask(0o007) + defer syscall.Umask(previous) + if err := newStore(t, root).Store(context.Background(), contracts.CacheRefresh, "task", []byte("{}")); err != nil { + t.Fatal(err) + } + after, err := os.Stat(root) + if err != nil || before.Mode() != after.Mode() { + t.Fatalf("provisioned root changed: before %v, after %v, error %v", before, after, err) + } + file, err := os.Stat(filepath.Join(root, "task.json")) + if err != nil || file.Mode().Perm() != 0o660 || file.Sys().(*syscall.Stat_t).Gid != after.Sys().(*syscall.Stat_t).Gid { + t.Fatalf("new artifact did not inherit deployment policy: %v, %v", file, err) + } +} + +func TestFilesystemCachePermissionFailuresPreservePriorAcceptance(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses ordinary Unix permission failures") + } + ctx := context.Background() + root := t.TempDir() + store := newStore(t, root) + want := []byte(`{"accepted":"before"}`) + if err := store.Store(ctx, contracts.CacheAuto, "task", want); err != nil { + t.Fatal(err) + } + entry := filepath.Join(root, "task.json") + if err := os.Chmod(entry, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(entry, 0o600) }) + if got, err := store.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || !errors.Is(err, os.ErrPermission) { + t.Fatalf("unreadable artifact = %q, %v", got, err) + } + if err := os.Chmod(entry, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(root, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(root, 0o700) }) + if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte(`{"accepted":"after"}`)); !errors.Is(err, os.ErrPermission) { + t.Fatalf("unwritable cache = %v", err) + } + got, err := store.Lookup(ctx, contracts.CacheAuto, "task") + if err != nil || string(got) != string(want) { + t.Fatalf("failed refresh changed acceptance: %q, %v", got, err) + } +} diff --git a/internal/framework/artifactcache/filesystem_test.go b/internal/framework/artifactcache/filesystem_test.go new file mode 100644 index 00000000..c5d11a5e --- /dev/null +++ b/internal/framework/artifactcache/filesystem_test.go @@ -0,0 +1,242 @@ +package artifactcache + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func newStore(t *testing.T, root string) *FilesystemStore { + t.Helper() + store, err := NewFilesystemStore(root) + if err != nil { + t.Fatal(err) + } + return store +} + +func TestFilesystemCacheUsesUnchangedHashAndRawArtifact(t *testing.T) { + root := filepath.Join(t.TempDir(), "selected cache ") + store := newStore(t, root) + const hash = "opaque-native:ABC_def-123" + ctx := context.Background() + content, err := store.Lookup(ctx, contracts.CacheAuto, hash) + if err != nil || content != nil { + t.Fatalf("missing lookup = %q, %v", content, err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("lookup created the cache root: %v", err) + } + want := []byte("{\n \"items\": [\"accepted\"]\n}\n") + if err := store.Store(ctx, contracts.CacheAuto, hash, want); err != nil { + t.Fatal(err) + } + onDisk, err := os.ReadFile(filepath.Join(root, hash+".json")) + if err != nil || !bytes.Equal(onDisk, want) { + t.Fatalf("stored bytes = %q, %v; want unchanged artifact bytes", onDisk, err) + } + got, err := store.Lookup(ctx, contracts.CacheAuto, hash) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("lookup = %q, %v", got, err) + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 1 || entries[0].Name() != hash+".json" { + t.Fatalf("cache contains other than the raw artifact: %v, %v", entries, err) + } +} + +func TestFilesystemCacheModesPreservePriorAcceptedResultUntilReplacement(t *testing.T) { + ctx := context.Background() + for _, mode := range []contracts.CacheMode{contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh} { + t.Run(string(mode), func(t *testing.T) { + root := t.TempDir() + store := newStore(t, root) + before, after := []byte(`{"items":["before"]}`), []byte(`{"items":["after"]}`) + if err := store.Store(ctx, contracts.CacheAuto, "task", before); err != nil { + t.Fatal(err) + } + found, err := store.Lookup(ctx, mode, "task") + if err != nil || (found != nil) != (mode == contracts.CacheAuto) { + t.Fatalf("%s lookup = %q, %v", mode, found, err) + } + if err := store.Store(ctx, mode, "task", []byte("{")); mode != contracts.CacheBypass && err == nil { + t.Fatal("malformed replacement was accepted") + } + found, err = store.Lookup(ctx, contracts.CacheAuto, "task") + if err != nil || !bytes.Equal(found, before) { + t.Fatalf("failed replacement changed prior acceptance: %q, %v", found, err) + } + if err := store.Store(ctx, mode, "task", after); err != nil { + t.Fatal(err) + } + found, err = store.Lookup(ctx, contracts.CacheAuto, "task") + want := after + if mode == contracts.CacheBypass { + want = before + } + if err != nil || !bytes.Equal(found, want) { + t.Fatalf("replacement = %q, %v; want %q", found, err, want) + } + }) + } +} + +func TestFilesystemCacheBypassDoesNotTouchStorage(t *testing.T) { + root := filepath.Join(t.TempDir(), "absent") + store := newStore(t, root) + if got, err := store.Lookup(context.Background(), contracts.CacheBypass, "task"); err != nil || got != nil { + t.Fatalf("bypassed lookup = %q, %v", got, err) + } + if err := store.Store(context.Background(), contracts.CacheBypass, "task", []byte(`{"accepted":true}`)); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("bypass created storage: %v", err) + } +} + +func TestFilesystemCacheMissesReturnUsefulReadDiagnostics(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + store := newStore(t, root) + if err := os.WriteFile(filepath.Join(root, "malformed.json"), []byte("not JSON"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "directory.json"), 0o700); err != nil { + t.Fatal(err) + } + for _, hash := range []string{"malformed", "directory"} { + if got, err := store.Lookup(ctx, contracts.CacheAuto, hash); got != nil || err == nil || !strings.Contains(err.Error(), hash) { + t.Fatalf("%s lookup = %q, %v; want diagnostic miss", hash, got, err) + } + } + blockedRoot := filepath.Join(t.TempDir(), "regular-file") + if err := os.WriteFile(blockedRoot, []byte("unchanged"), 0o600); err != nil { + t.Fatal(err) + } + blocked := newStore(t, blockedRoot) + if got, err := blocked.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || err == nil { + t.Fatalf("unreadable root lookup = %q, %v", got, err) + } + if err := blocked.Store(ctx, contracts.CacheAuto, "task", []byte("{}")); err == nil { + t.Fatal("write failure did not return its diagnostic") + } + if content, err := os.ReadFile(blockedRoot); err != nil || string(content) != "unchanged" { + t.Fatalf("failed cache operation changed its root: %q, %v", content, err) + } +} + +func TestFilesystemCacheConfinesPathsAndLeavesLegacyRecordsAlone(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + store := newStore(t, root) + for _, name := range []string{"checkpoints", "chunk-plans"} { + if err := os.Mkdir(filepath.Join(root, name), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, name, "legacy.json"), []byte("old-format-record"), 0o600); err != nil { + t.Fatal(err) + } + } + for _, hash := range []string{"", "../outside", "nested/entry", "/absolute", "nested\\entry"} { + if got, err := store.Lookup(ctx, contracts.CacheAuto, hash); got != nil || err == nil { + t.Fatalf("unsafe lookup %q = %q, %v", hash, got, err) + } + if err := store.Store(ctx, contracts.CacheAuto, hash, []byte("{}")); err == nil { + t.Fatalf("unsafe store %q succeeded", hash) + } + } + if err := store.Store(ctx, contracts.CacheAuto, "new-task", []byte("{}")); err != nil { + t.Fatal(err) + } + for _, name := range []string{"checkpoints", "chunk-plans"} { + content, err := os.ReadFile(filepath.Join(root, name, "legacy.json")) + if err != nil || string(content) != "old-format-record" { + t.Fatalf("legacy %s changed: %q, %v", name, content, err) + } + } +} + +func TestFilesystemCacheRejectsSymlinkEntryWithoutChangingTarget(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.json") + if err := os.WriteFile(outside, []byte(`{"original":true}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "task.json")); err != nil { + t.Fatal(err) + } + store := newStore(t, root) + if got, err := store.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || err == nil { + t.Fatalf("symlink lookup = %q, %v", got, err) + } + if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte("{}")); err == nil { + t.Fatal("symlink entry was replaced") + } + content, err := os.ReadFile(outside) + if err != nil || string(content) != `{"original":true}` { + t.Fatalf("symlink target changed: %q, %v", content, err) + } +} + +func TestFilesystemCacheConcurrentReplacementPublishesWholeArtifacts(t *testing.T) { + store := newStore(t, t.TempDir()) + ctx := context.Background() + values := [][]byte{[]byte(`{"items":["` + strings.Repeat("a", 8192) + `"]}`), []byte(`{"items":["` + strings.Repeat("b", 8192) + `"]}`)} + if err := store.Store(ctx, contracts.CacheAuto, "shared", values[0]); err != nil { + t.Fatal(err) + } + var writers sync.WaitGroup + for _, content := range values { + writers.Go(func() { + for range 20 { + if err := store.Store(ctx, contracts.CacheAuto, "shared", content); err != nil { + t.Error(err) + return + } + } + }) + } + for range 100 { + got, err := store.Lookup(ctx, contracts.CacheAuto, "shared") + if err != nil || (!bytes.Equal(got, values[0]) && !bytes.Equal(got, values[1])) { + t.Errorf("replacement exposed partial content: %d bytes, %v", len(got), err) + break + } + } + writers.Wait() +} + +func TestFilesystemCacheCancellationAndFailedPublicationLeaveNoEntry(t *testing.T) { + root := filepath.Join(t.TempDir(), "cache") + store := newStore(t, root) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := store.Lookup(ctx, contracts.CacheAuto, "task"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled lookup = %v", err) + } + if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte("{}")); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled store = %v", err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled cache operation created storage: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, "directory.json"), 0o700); err != nil { + t.Fatal(err) + } + if err := store.Store(context.Background(), contracts.CacheRefresh, "directory", []byte("{}")); err == nil { + t.Fatal("publication over a directory unexpectedly succeeded") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 1 || entries[0].Name() != "directory.json" { + t.Fatalf("failed publication left temporary entries: %v, %v", entries, err) + } +} diff --git a/internal/framework/contracts/producer.go b/internal/framework/contracts/producer.go index 55ef1ec4..4c5a4273 100644 --- a/internal/framework/contracts/producer.go +++ b/internal/framework/contracts/producer.go @@ -21,6 +21,15 @@ const ( CacheRefresh CacheMode = "refresh" ) +// ArtifactCache stores accepted artifact bytes under an opaque native message +// hash. A nil lookup is a miss. Other lookup/write errors are diagnostic data, +// not reasons to fail otherwise successful production. Callers own typed codec +// and source-usability checks and must never store rejected or fallback output. +type ArtifactCache interface { + Lookup(context.Context, CacheMode, string) ([]byte, error) + Store(context.Context, CacheMode, string, []byte) error +} + // ProducerControls is resolved before execution; modules do not inherit settings. type ProducerControls struct { Mode ExecutionMode `json:"mode"`