Add a best-effort accepted-artifact filesystem cache

This commit is contained in:
2026-11-22 01:52:21 +00:00
parent 37e0753472
commit 3ef55b1cf0
5 changed files with 477 additions and 0 deletions

View File

@@ -1184,6 +1184,22 @@ testing when supported; it is a migration check, not a new release requirement.
## Stage 26: Implement The Small Accepted-Artifact Filesystem Cache
**Status:** Complete. Added a narrow accepted-artifact cache collaborator and
filesystem implementation storing unchanged JSON bytes at the selected root
under the opaque native hash plus `.json`. Construction/lookup create no cache
directories. Auto reads/writes, bypass performs no cache I/O, and refresh skips
lookup and replaces only a newly supplied valid result; failed writes preserve
previous acceptance. Missing entries are ordinary misses, while malformed,
unreadable, non-file, and unsafe entries return diagnostic misses. Caller-owned
typed decoding/usability and best-effort warning handling remain in the next
integration stage. Reused confined atomic process-permission writes without new
permission policy, envelopes, fingerprints, expiry, or legacy data migration.
Real-filesystem tests cover raw-byte identity, exact root/hash placement, modes,
failed refresh, cancellation, concurrent complete replacement, temporary-file
cleanup, narrow paths/symlinks, unchanged legacy records, umask/setgid inheritance,
and unchanged provisioned root permissions. Cache/fileio/contracts tests, vet,
race checks, focused pipeline tests, and `git diff --check` pass.
**Goal:** one best-effort cache facility replaces complex persistent records.
**Inspect/change:** existing `internal/core/fileio` and cache permission/path tests;

View File

@@ -0,0 +1,111 @@
// Package artifactcache stores accepted producer artifacts without execution or
// provenance envelopes. Cache I/O is best effort at the calling operation.
package artifactcache
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// FilesystemStore is safe for concurrent operations. Construction and lookup do
// not create directories; only storing a newly accepted result can do so.
type FilesystemStore struct {
root string
}
var _ contracts.ArtifactCache = (*FilesystemStore)(nil)
func NewFilesystemStore(root string) (*FilesystemStore, error) {
if strings.TrimSpace(root) == "" || strings.ContainsRune(root, '\x00') {
return nil, fmt.Errorf("artifact cache root must be a nonempty path without NUL")
}
absolute, err := filepath.Abs(root)
if err != nil {
return nil, fmt.Errorf("resolve artifact cache root: %w", err)
}
// Retain the exact selected directory, including literal trailing spaces,
// when passing it through the common confined-path helper.
return &FilesystemStore{root: absolute + string(filepath.Separator)}, nil
}
func (store *FilesystemStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
switch mode {
case contracts.CacheBypass, contracts.CacheRefresh:
return nil, nil
case contracts.CacheAuto, "":
default:
return nil, fmt.Errorf("unknown artifact cache mode %q", mode)
}
name, err := cacheName(hash)
if err != nil {
return nil, err
}
target, err := fileio.SafePath(store.root, name)
if err != nil {
return nil, fmt.Errorf("locate artifact cache entry %q: %w", hash, err)
}
info, err := os.Lstat(target)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("inspect artifact cache entry %q: %w", hash, err)
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("artifact cache entry %q is not a regular file", hash)
}
content, err := os.ReadFile(target)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("read artifact cache entry %q: %w", hash, err)
}
if !json.Valid(content) {
return nil, fmt.Errorf("artifact cache entry %q contains malformed JSON", hash)
}
return content, nil
}
func (store *FilesystemStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error {
if err := ctx.Err(); err != nil {
return err
}
switch mode {
case contracts.CacheBypass:
return nil
case contracts.CacheAuto, contracts.CacheRefresh, "":
default:
return fmt.Errorf("unknown artifact cache mode %q", mode)
}
name, err := cacheName(hash)
if err != nil {
return err
}
if !json.Valid(content) {
return fmt.Errorf("artifact cache entry %q is not valid JSON", hash)
}
if err := fileio.WriteBytesWithProcessPermissions(store.root, name, content); err != nil {
return fmt.Errorf("store artifact cache entry %q: %w", hash, err)
}
return nil
}
func cacheName(hash string) (string, error) {
// The native hash is opaque: do not strip a prefix, recalculate it, encode
// it, or enforce a particular digest format. It must identify one filename.
if hash == "" || strings.ContainsAny(hash, "/\\") {
return "", fmt.Errorf("artifact cache hash must be a nonempty filename component")
}
return hash + ".json", nil
}

View File

@@ -0,0 +1,99 @@
//go:build linux || darwin
package artifactcache
import (
"context"
"errors"
"os"
"path/filepath"
"syscall"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestFilesystemCacheUsesDeploymentCreationPermissions(t *testing.T) {
for _, test := range []struct {
name string
umask int
directory, file os.FileMode
}{
{"private", 0o077, 0o700, 0o600},
{"group", 0o007, 0o770, 0o660},
} {
t.Run(test.name, func(t *testing.T) {
previous := syscall.Umask(test.umask)
defer syscall.Umask(previous)
root := filepath.Join(t.TempDir(), "cache")
if err := newStore(t, root).Store(context.Background(), contracts.CacheAuto, "task", []byte("{}")); err != nil {
t.Fatal(err)
}
for path, expected := range map[string]os.FileMode{root: test.directory, filepath.Join(root, "task.json"): test.file} {
info, err := os.Stat(path)
if err != nil || info.Mode().Perm() != expected {
t.Fatalf("%s: info %v, error %v; want mode %o", path, info, err, expected)
}
}
})
}
}
func TestFilesystemCacheDoesNotConvergeProvisionedRootPermissions(t *testing.T) {
root := t.TempDir()
if err := os.Chmod(root, os.ModeSetgid|0o751); err != nil {
t.Fatal(err)
}
before, err := os.Stat(root)
if err != nil {
t.Fatal(err)
}
previous := syscall.Umask(0o007)
defer syscall.Umask(previous)
if err := newStore(t, root).Store(context.Background(), contracts.CacheRefresh, "task", []byte("{}")); err != nil {
t.Fatal(err)
}
after, err := os.Stat(root)
if err != nil || before.Mode() != after.Mode() {
t.Fatalf("provisioned root changed: before %v, after %v, error %v", before, after, err)
}
file, err := os.Stat(filepath.Join(root, "task.json"))
if err != nil || file.Mode().Perm() != 0o660 || file.Sys().(*syscall.Stat_t).Gid != after.Sys().(*syscall.Stat_t).Gid {
t.Fatalf("new artifact did not inherit deployment policy: %v, %v", file, err)
}
}
func TestFilesystemCachePermissionFailuresPreservePriorAcceptance(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root bypasses ordinary Unix permission failures")
}
ctx := context.Background()
root := t.TempDir()
store := newStore(t, root)
want := []byte(`{"accepted":"before"}`)
if err := store.Store(ctx, contracts.CacheAuto, "task", want); err != nil {
t.Fatal(err)
}
entry := filepath.Join(root, "task.json")
if err := os.Chmod(entry, 0); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(entry, 0o600) })
if got, err := store.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || !errors.Is(err, os.ErrPermission) {
t.Fatalf("unreadable artifact = %q, %v", got, err)
}
if err := os.Chmod(entry, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chmod(root, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(root, 0o700) })
if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte(`{"accepted":"after"}`)); !errors.Is(err, os.ErrPermission) {
t.Fatalf("unwritable cache = %v", err)
}
got, err := store.Lookup(ctx, contracts.CacheAuto, "task")
if err != nil || string(got) != string(want) {
t.Fatalf("failed refresh changed acceptance: %q, %v", got, err)
}
}

View File

@@ -0,0 +1,242 @@
package artifactcache
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func newStore(t *testing.T, root string) *FilesystemStore {
t.Helper()
store, err := NewFilesystemStore(root)
if err != nil {
t.Fatal(err)
}
return store
}
func TestFilesystemCacheUsesUnchangedHashAndRawArtifact(t *testing.T) {
root := filepath.Join(t.TempDir(), "selected cache ")
store := newStore(t, root)
const hash = "opaque-native:ABC_def-123"
ctx := context.Background()
content, err := store.Lookup(ctx, contracts.CacheAuto, hash)
if err != nil || content != nil {
t.Fatalf("missing lookup = %q, %v", content, err)
}
if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("lookup created the cache root: %v", err)
}
want := []byte("{\n \"items\": [\"accepted\"]\n}\n")
if err := store.Store(ctx, contracts.CacheAuto, hash, want); err != nil {
t.Fatal(err)
}
onDisk, err := os.ReadFile(filepath.Join(root, hash+".json"))
if err != nil || !bytes.Equal(onDisk, want) {
t.Fatalf("stored bytes = %q, %v; want unchanged artifact bytes", onDisk, err)
}
got, err := store.Lookup(ctx, contracts.CacheAuto, hash)
if err != nil || !bytes.Equal(got, want) {
t.Fatalf("lookup = %q, %v", got, err)
}
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 1 || entries[0].Name() != hash+".json" {
t.Fatalf("cache contains other than the raw artifact: %v, %v", entries, err)
}
}
func TestFilesystemCacheModesPreservePriorAcceptedResultUntilReplacement(t *testing.T) {
ctx := context.Background()
for _, mode := range []contracts.CacheMode{contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh} {
t.Run(string(mode), func(t *testing.T) {
root := t.TempDir()
store := newStore(t, root)
before, after := []byte(`{"items":["before"]}`), []byte(`{"items":["after"]}`)
if err := store.Store(ctx, contracts.CacheAuto, "task", before); err != nil {
t.Fatal(err)
}
found, err := store.Lookup(ctx, mode, "task")
if err != nil || (found != nil) != (mode == contracts.CacheAuto) {
t.Fatalf("%s lookup = %q, %v", mode, found, err)
}
if err := store.Store(ctx, mode, "task", []byte("{")); mode != contracts.CacheBypass && err == nil {
t.Fatal("malformed replacement was accepted")
}
found, err = store.Lookup(ctx, contracts.CacheAuto, "task")
if err != nil || !bytes.Equal(found, before) {
t.Fatalf("failed replacement changed prior acceptance: %q, %v", found, err)
}
if err := store.Store(ctx, mode, "task", after); err != nil {
t.Fatal(err)
}
found, err = store.Lookup(ctx, contracts.CacheAuto, "task")
want := after
if mode == contracts.CacheBypass {
want = before
}
if err != nil || !bytes.Equal(found, want) {
t.Fatalf("replacement = %q, %v; want %q", found, err, want)
}
})
}
}
func TestFilesystemCacheBypassDoesNotTouchStorage(t *testing.T) {
root := filepath.Join(t.TempDir(), "absent")
store := newStore(t, root)
if got, err := store.Lookup(context.Background(), contracts.CacheBypass, "task"); err != nil || got != nil {
t.Fatalf("bypassed lookup = %q, %v", got, err)
}
if err := store.Store(context.Background(), contracts.CacheBypass, "task", []byte(`{"accepted":true}`)); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("bypass created storage: %v", err)
}
}
func TestFilesystemCacheMissesReturnUsefulReadDiagnostics(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
store := newStore(t, root)
if err := os.WriteFile(filepath.Join(root, "malformed.json"), []byte("not JSON"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(root, "directory.json"), 0o700); err != nil {
t.Fatal(err)
}
for _, hash := range []string{"malformed", "directory"} {
if got, err := store.Lookup(ctx, contracts.CacheAuto, hash); got != nil || err == nil || !strings.Contains(err.Error(), hash) {
t.Fatalf("%s lookup = %q, %v; want diagnostic miss", hash, got, err)
}
}
blockedRoot := filepath.Join(t.TempDir(), "regular-file")
if err := os.WriteFile(blockedRoot, []byte("unchanged"), 0o600); err != nil {
t.Fatal(err)
}
blocked := newStore(t, blockedRoot)
if got, err := blocked.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || err == nil {
t.Fatalf("unreadable root lookup = %q, %v", got, err)
}
if err := blocked.Store(ctx, contracts.CacheAuto, "task", []byte("{}")); err == nil {
t.Fatal("write failure did not return its diagnostic")
}
if content, err := os.ReadFile(blockedRoot); err != nil || string(content) != "unchanged" {
t.Fatalf("failed cache operation changed its root: %q, %v", content, err)
}
}
func TestFilesystemCacheConfinesPathsAndLeavesLegacyRecordsAlone(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
store := newStore(t, root)
for _, name := range []string{"checkpoints", "chunk-plans"} {
if err := os.Mkdir(filepath.Join(root, name), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, name, "legacy.json"), []byte("old-format-record"), 0o600); err != nil {
t.Fatal(err)
}
}
for _, hash := range []string{"", "../outside", "nested/entry", "/absolute", "nested\\entry"} {
if got, err := store.Lookup(ctx, contracts.CacheAuto, hash); got != nil || err == nil {
t.Fatalf("unsafe lookup %q = %q, %v", hash, got, err)
}
if err := store.Store(ctx, contracts.CacheAuto, hash, []byte("{}")); err == nil {
t.Fatalf("unsafe store %q succeeded", hash)
}
}
if err := store.Store(ctx, contracts.CacheAuto, "new-task", []byte("{}")); err != nil {
t.Fatal(err)
}
for _, name := range []string{"checkpoints", "chunk-plans"} {
content, err := os.ReadFile(filepath.Join(root, name, "legacy.json"))
if err != nil || string(content) != "old-format-record" {
t.Fatalf("legacy %s changed: %q, %v", name, content, err)
}
}
}
func TestFilesystemCacheRejectsSymlinkEntryWithoutChangingTarget(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
outside := filepath.Join(t.TempDir(), "outside.json")
if err := os.WriteFile(outside, []byte(`{"original":true}`), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "task.json")); err != nil {
t.Fatal(err)
}
store := newStore(t, root)
if got, err := store.Lookup(ctx, contracts.CacheAuto, "task"); got != nil || err == nil {
t.Fatalf("symlink lookup = %q, %v", got, err)
}
if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte("{}")); err == nil {
t.Fatal("symlink entry was replaced")
}
content, err := os.ReadFile(outside)
if err != nil || string(content) != `{"original":true}` {
t.Fatalf("symlink target changed: %q, %v", content, err)
}
}
func TestFilesystemCacheConcurrentReplacementPublishesWholeArtifacts(t *testing.T) {
store := newStore(t, t.TempDir())
ctx := context.Background()
values := [][]byte{[]byte(`{"items":["` + strings.Repeat("a", 8192) + `"]}`), []byte(`{"items":["` + strings.Repeat("b", 8192) + `"]}`)}
if err := store.Store(ctx, contracts.CacheAuto, "shared", values[0]); err != nil {
t.Fatal(err)
}
var writers sync.WaitGroup
for _, content := range values {
writers.Go(func() {
for range 20 {
if err := store.Store(ctx, contracts.CacheAuto, "shared", content); err != nil {
t.Error(err)
return
}
}
})
}
for range 100 {
got, err := store.Lookup(ctx, contracts.CacheAuto, "shared")
if err != nil || (!bytes.Equal(got, values[0]) && !bytes.Equal(got, values[1])) {
t.Errorf("replacement exposed partial content: %d bytes, %v", len(got), err)
break
}
}
writers.Wait()
}
func TestFilesystemCacheCancellationAndFailedPublicationLeaveNoEntry(t *testing.T) {
root := filepath.Join(t.TempDir(), "cache")
store := newStore(t, root)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := store.Lookup(ctx, contracts.CacheAuto, "task"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled lookup = %v", err)
}
if err := store.Store(ctx, contracts.CacheRefresh, "task", []byte("{}")); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled store = %v", err)
}
if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("canceled cache operation created storage: %v", err)
}
if err := os.MkdirAll(filepath.Join(root, "directory.json"), 0o700); err != nil {
t.Fatal(err)
}
if err := store.Store(context.Background(), contracts.CacheRefresh, "directory", []byte("{}")); err == nil {
t.Fatal("publication over a directory unexpectedly succeeded")
}
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 1 || entries[0].Name() != "directory.json" {
t.Fatalf("failed publication left temporary entries: %v, %v", entries, err)
}
}

View File

@@ -21,6 +21,15 @@ const (
CacheRefresh CacheMode = "refresh"
)
// ArtifactCache stores accepted artifact bytes under an opaque native message
// hash. A nil lookup is a miss. Other lookup/write errors are diagnostic data,
// not reasons to fail otherwise successful production. Callers own typed codec
// and source-usability checks and must never store rejected or fallback output.
type ArtifactCache interface {
Lookup(context.Context, CacheMode, string) ([]byte, error)
Store(context.Context, CacheMode, string, []byte) error
}
// ProducerControls is resolved before execution; modules do not inherit settings.
type ProducerControls struct {
Mode ExecutionMode `json:"mode"`