Move asset reconciliation into shared delivery
This commit is contained in:
394
internal/delivery/remote_assets_test.go
Normal file
394
internal/delivery/remote_assets_test.go
Normal file
@@ -0,0 +1,394 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/releasekit/internal/artifact"
|
||||
"go.maximumdirect.net/releasekit/internal/forge"
|
||||
)
|
||||
|
||||
func TestInspectRemoteAssetsComparesContentAndPreservesUnrelatedAssets(t *testing.T) {
|
||||
backend := &assetBackend{
|
||||
lists: []assetListResult{{assets: []forge.Asset{
|
||||
{Filename: "same.bin", Size: 4, Handle: "same"},
|
||||
{Filename: "different.bin", Size: 4, Handle: "different"},
|
||||
{Filename: "duplicate.bin", Size: 4, Handle: "first"},
|
||||
{Filename: "duplicate.bin", Size: 4, Handle: "second"},
|
||||
{Filename: "unrelated.txt", Size: 3, Handle: "unrelated"},
|
||||
}}},
|
||||
content: map[string][]byte{
|
||||
"same": []byte("same"),
|
||||
"different": []byte("diff"),
|
||||
},
|
||||
}
|
||||
expected := []artifact.ExpectedAsset{
|
||||
expectedAsset("same.bin", []byte("same")),
|
||||
expectedAsset("different.bin", []byte("else")),
|
||||
expectedAsset("duplicate.bin", []byte("dupe")),
|
||||
expectedAsset("missing.bin", []byte("gone")),
|
||||
}
|
||||
|
||||
inspection, err := inspectRemoteAssets(context.Background(), backend, "release", expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantStates := []AssetState{AssetMatching, AssetConflicting, AssetConflicting, AssetMissing}
|
||||
for i, observation := range inspection.Assets {
|
||||
if observation.State != wantStates[i] {
|
||||
t.Errorf("asset %q state = %s, want %s", observation.Expected.Filename, observation.State, wantStates[i])
|
||||
}
|
||||
}
|
||||
if len(inspection.Unrelated) != 1 || inspection.Unrelated[0].Filename != "unrelated.txt" {
|
||||
t.Fatalf("unrelated assets = %#v", inspection.Unrelated)
|
||||
}
|
||||
if len(backend.opened) != 2 || !backend.opened[0].closed || !backend.opened[1].closed {
|
||||
t.Fatalf("opened asset streams = %#v", backend.opened)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRemoteAssetsListingFailureDoesNotBecomeUnknownWrite(t *testing.T) {
|
||||
assets := []artifact.ExpectedAsset{expectedAsset("a.bin", []byte("a")), expectedAsset("b.bin", []byte("b"))}
|
||||
listErr := &forge.Failure{Kind: forge.FailureTransport, Err: errors.New("service unavailable")}
|
||||
backend := &assetBackend{lists: []assetListResult{{err: listErr}}}
|
||||
inspection, err := inspectRemoteAssets(context.Background(), backend, "release", assets)
|
||||
if !errors.Is(err, listErr) || len(inspection.Assets) != len(assets) {
|
||||
t.Fatalf("inspection = %#v, error %v", inspection, err)
|
||||
}
|
||||
for _, observation := range inspection.Assets {
|
||||
if observation.State != AssetUnverifiable || !errors.Is(observation.Err, listErr) {
|
||||
t.Fatalf("listing failure observation = %#v", observation)
|
||||
}
|
||||
}
|
||||
result, err := uploadMissingAssets(context.Background(), backend, "release", inspection)
|
||||
if !isForgeFailure(err, forge.FailureUnverifiable) || result.State != NotAttempted || len(result.Uncertain) != len(assets) || len(backend.uploadNames) != 0 {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v, uploads %v", result, err, backend.uploadNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRemoteAssetsKeepsMatchingManifestWhenBinariesAreMissing(t *testing.T) {
|
||||
binary := expectedAsset("app-linux-amd64", []byte("binary"))
|
||||
manifest, err := artifact.BuildManifest([]artifact.ExpectedAsset{binary})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := &assetBackend{
|
||||
lists: []assetListResult{{assets: []forge.Asset{{Filename: manifest.Filename, Size: int64(len(manifest.Content)), Handle: "manifest"}}}},
|
||||
content: map[string][]byte{"manifest": manifest.Content},
|
||||
}
|
||||
inspection, err := inspectRemoteAssets(context.Background(), backend, "release", []artifact.ExpectedAsset{binary, manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.Assets[0].State != AssetMissing || inspection.Assets[1].State != AssetMatching {
|
||||
t.Fatalf("manifest inspection = %#v", inspection.Assets)
|
||||
}
|
||||
result, err := uploadMissingAssets(context.Background(), backend, "release", inspection)
|
||||
if err != nil || result.State != Complete || !reflect.DeepEqual(result.Uploaded, []AssetIdentity{identity(binary)}) {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v", result, err)
|
||||
}
|
||||
if !reflect.DeepEqual(backend.uploadNames, []string{binary.Filename}) {
|
||||
t.Fatalf("uploaded names = %v", backend.uploadNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRemoteAssetsRejectsSubsetOrDifferentManifestContent(t *testing.T) {
|
||||
binaryA := expectedAsset("a", []byte("a"))
|
||||
binaryB := expectedAsset("b", []byte("b"))
|
||||
manifest, err := artifact.BuildManifest([]artifact.ExpectedAsset{binaryA, binaryB})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
content []byte
|
||||
}{
|
||||
{name: "subset", content: []byte("a a\n")},
|
||||
{name: "different", content: []byte("different manifest\n")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
backend := &assetBackend{
|
||||
lists: []assetListResult{{assets: []forge.Asset{{Filename: artifact.ManifestFilename, Handle: "manifest"}}}},
|
||||
content: map[string][]byte{"manifest": test.content},
|
||||
}
|
||||
inspection, err := inspectRemoteAssets(context.Background(), backend, "release", []artifact.ExpectedAsset{binaryA, binaryB, manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.Assets[2].State != AssetConflicting {
|
||||
t.Fatalf("manifest state = %s, want conflicting", inspection.Assets[2].State)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMissingAssetsPreflightsAndOrdersWrites(t *testing.T) {
|
||||
assets := []artifact.ExpectedAsset{
|
||||
expectedAsset("z.bin", []byte("z")),
|
||||
expectedAsset(artifact.ManifestFilename, []byte("manifest")),
|
||||
expectedAsset("a.bin", []byte("a")),
|
||||
}
|
||||
inspection := inspectionFor(assets, map[string]AssetState{
|
||||
"z.bin": AssetMissing,
|
||||
artifact.ManifestFilename: AssetMissing,
|
||||
"a.bin": AssetMissing,
|
||||
})
|
||||
backend := &assetBackend{uploads: []assetUploadResult{
|
||||
{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}},
|
||||
{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}},
|
||||
{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}},
|
||||
}}
|
||||
result, err := uploadMissingAssets(context.Background(), backend, "release", inspection)
|
||||
if err != nil || result.State != Complete {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v", result, err)
|
||||
}
|
||||
want := []string{"a.bin", "z.bin", artifact.ManifestFilename}
|
||||
if !reflect.DeepEqual(backend.uploadNames, want) {
|
||||
t.Fatalf("upload order = %v, want %v", backend.uploadNames, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMissingAssetsStopsBeforeWritesForConflictOrUnverifiableContent(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
state AssetState
|
||||
want State
|
||||
}{
|
||||
{name: "conflict", state: AssetConflicting, want: Conflict},
|
||||
{name: "unverifiable", state: AssetUnverifiable, want: NotAttempted},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assets := []artifact.ExpectedAsset{expectedAsset("existing", []byte("old")), expectedAsset("missing", []byte("new"))}
|
||||
inspection := inspectionFor(assets, map[string]AssetState{"existing": test.state, "missing": AssetMissing})
|
||||
result, err := uploadMissingAssets(context.Background(), &assetBackend{}, "release", inspection)
|
||||
if err == nil || result.State != test.want || len(result.Pending) != 1 || len(result.Conflicting)+len(result.Uncertain) != 1 {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMissingAssetsReconcilesOnceAndKeepsLaterAssetsPending(t *testing.T) {
|
||||
assets := []artifact.ExpectedAsset{
|
||||
expectedAsset("a.bin", []byte("a")),
|
||||
expectedAsset("b.bin", []byte("b")),
|
||||
expectedAsset("c.bin", []byte("c")),
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
disposition forge.WriteDisposition
|
||||
followupAssets []forge.Asset
|
||||
followupData map[string][]byte
|
||||
openErrors map[string]error
|
||||
followupErr error
|
||||
wantState State
|
||||
wantCategory string
|
||||
wantContinue bool
|
||||
}{
|
||||
{
|
||||
name: "matching bytes resolve uncertain upload", disposition: forge.WriteUncertain,
|
||||
followupAssets: []forge.Asset{{Filename: "a.bin", Handle: "a"}}, followupData: map[string][]byte{"a": []byte("a")},
|
||||
wantState: Complete, wantCategory: "matching", wantContinue: true,
|
||||
},
|
||||
{
|
||||
name: "different bytes conflict", disposition: forge.WriteUncertain,
|
||||
followupAssets: []forge.Asset{{Filename: "a.bin", Handle: "a"}}, followupData: map[string][]byte{"a": []byte("x")},
|
||||
wantState: Conflict, wantCategory: "conflicting",
|
||||
},
|
||||
{
|
||||
name: "missing after conflict remains conflict", disposition: forge.WriteConflict,
|
||||
wantState: Conflict, wantCategory: "conflicting",
|
||||
},
|
||||
{
|
||||
name: "missing after uncertain upload is unknown", disposition: forge.WriteUncertain,
|
||||
wantState: Unknown, wantCategory: "uncertain",
|
||||
},
|
||||
{
|
||||
name: "unreadable follow-up is unknown", disposition: forge.WriteUncertain,
|
||||
followupAssets: []forge.Asset{{Filename: "a.bin", Handle: "a"}}, openErrors: map[string]error{"a": errors.New("download failed")},
|
||||
wantState: Unknown, wantCategory: "uncertain",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
first := assetListResult{assets: []forge.Asset{}}
|
||||
followup := assetListResult{assets: test.followupAssets, err: test.followupErr}
|
||||
backend := &assetBackend{
|
||||
lists: []assetListResult{first, followup},
|
||||
content: test.followupData,
|
||||
openErrors: test.openErrors,
|
||||
uploads: []assetUploadResult{{
|
||||
evidence: forge.WriteEvidence{Attempted: true, Disposition: test.disposition},
|
||||
err: &forge.Failure{Kind: failureKindForDisposition(test.disposition), Err: errors.New("upload outcome unresolved")},
|
||||
}},
|
||||
}
|
||||
inspection, err := inspectRemoteAssets(context.Background(), backend, "release", assets)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, uploadErr := uploadMissingAssets(context.Background(), backend, "release", inspection)
|
||||
if result.State != test.wantState || backend.listCalls != 2 {
|
||||
t.Fatalf("result = %#v, error %v, list calls %d, uploads %v", result, uploadErr, backend.listCalls, backend.uploadNames)
|
||||
}
|
||||
if test.wantContinue {
|
||||
if uploadErr != nil || !reflect.DeepEqual(backend.uploadNames, []string{"a.bin", "b.bin", "c.bin"}) || len(result.Pending) != 0 {
|
||||
t.Fatalf("continued result = %#v, error %v, uploads %v", result, uploadErr, backend.uploadNames)
|
||||
}
|
||||
} else {
|
||||
if uploadErr == nil || !reflect.DeepEqual(backend.uploadNames, []string{"a.bin"}) {
|
||||
t.Fatalf("stopped result = %#v, error %v, uploads %v", result, uploadErr, backend.uploadNames)
|
||||
}
|
||||
if !reflect.DeepEqual(result.Pending, []AssetIdentity{identity(assets[1]), identity(assets[2])}) {
|
||||
t.Fatalf("pending = %#v", result.Pending)
|
||||
}
|
||||
}
|
||||
var category []AssetIdentity
|
||||
switch test.wantCategory {
|
||||
case "matching":
|
||||
category = result.Matching
|
||||
case "conflicting":
|
||||
category = result.Conflicting
|
||||
case "uncertain":
|
||||
category = result.Uncertain
|
||||
}
|
||||
if len(category) != 1 || category[0].Filename != "a.bin" {
|
||||
t.Fatalf("%s assets = %#v", test.wantCategory, category)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMissingAssetsPreservesAcknowledgedProgressOnDefinitiveFailure(t *testing.T) {
|
||||
assets := []artifact.ExpectedAsset{expectedAsset("a.bin", []byte("a")), expectedAsset("b.bin", []byte("b")), expectedAsset("c.bin", []byte("c"))}
|
||||
backend := &assetBackend{uploads: []assetUploadResult{
|
||||
{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}},
|
||||
{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteRejected}, err: &forge.Failure{Kind: forge.FailureRejected, Err: errors.New("denied")}},
|
||||
}}
|
||||
result, err := uploadMissingAssets(context.Background(), backend, "release", inspectionFor(assets, map[string]AssetState{
|
||||
"a.bin": AssetMissing, "b.bin": AssetMissing, "c.bin": AssetMissing,
|
||||
}))
|
||||
if err == nil || result.State != Partial || !reflect.DeepEqual(result.Uploaded, []AssetIdentity{identity(assets[0])}) || !reflect.DeepEqual(result.Pending, []AssetIdentity{identity(assets[1]), identity(assets[2])}) {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMissingAssetsBorrowsReaderAndClosesItAfterCall(t *testing.T) {
|
||||
filePath := t.TempDir() + "/asset.bin"
|
||||
if err := os.WriteFile(filePath, []byte("asset"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asset := artifact.ExpectedAsset{Filename: "asset.bin", SHA256: assetDigest([]byte("asset")), Size: 5, Path: filePath}
|
||||
backend := &assetBackend{uploads: []assetUploadResult{{evidence: forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}}}}
|
||||
result, err := uploadMissingAssets(context.Background(), backend, "release", inspectionFor([]artifact.ExpectedAsset{asset}, map[string]AssetState{"asset.bin": AssetMissing}))
|
||||
if err != nil || result.State != Complete || !backend.readerWasOpen {
|
||||
t.Fatalf("uploadMissingAssets() = %#v, error %v, reader open during call %v", result, err, backend.readerWasOpen)
|
||||
}
|
||||
if backend.borrowed == nil {
|
||||
t.Fatal("backend did not receive the file reader")
|
||||
}
|
||||
if _, err := backend.borrowed.Stat(); !errors.Is(err, os.ErrClosed) {
|
||||
t.Fatalf("borrowed file stat error = %v, want closed file", err)
|
||||
}
|
||||
}
|
||||
|
||||
type assetListResult struct {
|
||||
assets []forge.Asset
|
||||
err error
|
||||
}
|
||||
|
||||
type assetUploadResult struct {
|
||||
evidence forge.WriteEvidence
|
||||
err error
|
||||
}
|
||||
|
||||
type assetBackend struct {
|
||||
pageBackend
|
||||
lists []assetListResult
|
||||
listCalls int
|
||||
content map[string][]byte
|
||||
openErrors map[string]error
|
||||
opened []*trackedAssetStream
|
||||
uploads []assetUploadResult
|
||||
uploadNames []string
|
||||
uploadContent [][]byte
|
||||
borrowed *os.File
|
||||
readerWasOpen bool
|
||||
}
|
||||
|
||||
func (backend *assetBackend) ListAssets(context.Context, any) ([]forge.Asset, error) {
|
||||
index := backend.listCalls
|
||||
backend.listCalls++
|
||||
if index >= len(backend.lists) {
|
||||
return nil, errors.New("unexpected asset listing")
|
||||
}
|
||||
result := backend.lists[index]
|
||||
return result.assets, result.err
|
||||
}
|
||||
|
||||
func (backend *assetBackend) OpenAsset(_ context.Context, handle any) (io.ReadCloser, error) {
|
||||
key, ok := handle.(string)
|
||||
if !ok {
|
||||
return nil, errors.New("unexpected asset handle")
|
||||
}
|
||||
if err := backend.openErrors[key]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream := &trackedAssetStream{Reader: bytes.NewReader(backend.content[key])}
|
||||
backend.opened = append(backend.opened, stream)
|
||||
return stream, nil
|
||||
}
|
||||
|
||||
func (backend *assetBackend) UploadAsset(_ context.Context, _ any, filename string, content io.Reader) (forge.WriteEvidence, error) {
|
||||
backend.uploadNames = append(backend.uploadNames, filename)
|
||||
if file, ok := content.(*os.File); ok {
|
||||
backend.borrowed = file
|
||||
}
|
||||
data, err := io.ReadAll(content)
|
||||
backend.uploadContent = append(backend.uploadContent, data)
|
||||
if backend.borrowed != nil {
|
||||
_, statErr := backend.borrowed.Stat()
|
||||
backend.readerWasOpen = statErr == nil
|
||||
}
|
||||
index := len(backend.uploadNames) - 1
|
||||
if err != nil {
|
||||
return forge.WriteEvidence{Attempted: true, Disposition: forge.WriteUncertain}, err
|
||||
}
|
||||
if index >= len(backend.uploads) {
|
||||
return forge.WriteEvidence{Attempted: true, Disposition: forge.WriteAccepted}, nil
|
||||
}
|
||||
result := backend.uploads[index]
|
||||
return result.evidence, result.err
|
||||
}
|
||||
|
||||
type trackedAssetStream struct {
|
||||
*bytes.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (stream *trackedAssetStream) Close() error {
|
||||
stream.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func expectedAsset(filename string, content []byte) artifact.ExpectedAsset {
|
||||
return artifact.ExpectedAsset{Filename: filename, SHA256: assetDigest(content), Size: int64(len(content)), Content: content}
|
||||
}
|
||||
|
||||
func inspectionFor(assets []artifact.ExpectedAsset, states map[string]AssetState) Inspection {
|
||||
inspection := Inspection{Assets: make([]AssetInspection, 0, len(assets))}
|
||||
for _, item := range assets {
|
||||
inspection.Assets = append(inspection.Assets, AssetInspection{Expected: item, State: states[item.Filename]})
|
||||
}
|
||||
return inspection
|
||||
}
|
||||
|
||||
func failureKindForDisposition(disposition forge.WriteDisposition) forge.FailureKind {
|
||||
if disposition == forge.WriteConflict {
|
||||
return forge.FailureConflict
|
||||
}
|
||||
return forge.FailureTransport
|
||||
}
|
||||
Reference in New Issue
Block a user