Skip to content

Commit

Permalink
refactor: remove AutoRemoveReferrers flag (#683)
Browse files Browse the repository at this point in the history 
following #680 and addressing this comment:
#680 (comment)

Signed-off-by: Xiaoxuan Wang <wangxiaoxuan119@gmail.com>
  • Loading branch information
@wangxiaoxuan273
wangxiaoxuan273 committed Jan 26, 2024
1 parent fa548ab commit 850a247
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 331 deletions.
31 changes: 12 additions & 19 deletions content/oci/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,19 +56,13 @@ type Store struct {
// - Default value: true.
AutoSaveIndex bool

// AutoGC controls if the OCI store will automatically clean newly produced
// dangling (unreferenced) blobs during Delete() operation. For example the
// blobs whose manifests have been deleted. Tagged manifests will not be
// deleted.
// AutoGC controls if the OCI store will automatically clean dangling
// (unreferenced) blobs created by the Delete() operation. This includes the
// referrers and the unreferenced successor blobs of the deleted content.
// Tagged manifests will not be deleted.
// - Default value: true.
AutoGC bool

// AutoDeleteReferrers controls if the OCI store will automatically delete its
// referrers when a manifest is deleted. When set to true, the referrers will
// be deleted even if they exist in index.json.
// - Default value: true.
AutoDeleteReferrers bool

root string
indexPath string
index *ocispec.Index
Expand Down Expand Up @@ -102,14 +96,13 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) {
}

store := &Store{
AutoSaveIndex: true,
AutoGC: true,
AutoDeleteReferrers: true,
root: rootAbs,
indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile),
storage: storage,
tagResolver: resolver.NewMemory(),
graph: graph.NewMemory(),
AutoSaveIndex: true,
AutoGC: true,
root: rootAbs,
indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile),
storage: storage,
tagResolver: resolver.NewMemory(),
graph: graph.NewMemory(),
}

if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil {
Expand Down Expand Up @@ -177,7 +170,7 @@ func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error {
deleteQueue = deleteQueue[1:]

// get referrers if applicable
if s.AutoDeleteReferrers && descriptor.IsManifest(head) {
if s.AutoGC && descriptor.IsManifest(head) {
referrers, err := registry.Referrers(ctx, &unsafeStore{s}, head, "")
if err != nil {
return err
Expand Down
313 changes: 1 addition & 312 deletions content/oci/oci_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2263,7 +2263,7 @@ func TestStore_PredecessorsAndDelete(t *testing.T) {
}
}

func TestStore_DeleteWithAutoGCAndAutoRemoveReferrers(t *testing.T) {
func TestStore_DeleteWithAutoGC(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
Expand Down Expand Up @@ -2417,317 +2417,6 @@ func TestStore_DeleteWithAutoGCAndAutoRemoveReferrers(t *testing.T) {
}
}

func TestStore_DeleteDisableAutoRemoveReferrers(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
t.Fatal("New() error =", err)
}
s.AutoDeleteReferrers = false
ctx := context.Background()

// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateManifest := func(config ocispec.Descriptor, subject *ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Config: config,
Subject: subject,
Layers: layers,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
indexJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2
appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3
generateManifest(descs[0], nil, descs[1]) // Blob 4
generateManifest(descs[0], nil, descs[2]) // Blob 5
generateManifest(descs[0], nil, descs[3]) // Blob 6
generateIndex(descs[4:6]...) // Blob 7
generateIndex(descs[6]) // Blob 8
appendBlob(ocispec.MediaTypeImageLayer, []byte("world")) // Blob 9
generateManifest(descs[0], &descs[6], descs[9]) // Blob 10
generateManifest(descs[0], &descs[10], descs[2]) // Blob 11

eg, egCtx := errgroup.WithContext(ctx)
for i := range blobs {
eg.Go(func(i int) func() error {
return func() error {
err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
if err != nil {
return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
}
return nil
}
}(i))
}
if err := eg.Wait(); err != nil {
t.Fatal(err)
}

// delete blob 4 and verify the result
if err := s.Delete(egCtx, descs[4]); err != nil {
t.Fatal(err)
}

// blob 1 and 4 are now deleted, and other blobs are still present
notPresent := []ocispec.Descriptor{descs[1], descs[4]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent := []ocispec.Descriptor{descs[0], descs[2], descs[3], descs[5], descs[6], descs[7], descs[8], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 8 and verify the result
if err := s.Delete(egCtx, descs[8]); err != nil {
t.Fatal(err)
}

// blob 1, 4 and 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[1], descs[4], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[2], descs[3], descs[5], descs[6], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 6 and verify the result
if err := s.Delete(egCtx, descs[6]); err != nil {
t.Fatal(err)
}

// blob 1, 3, 4, 6, 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[1], descs[3], descs[4], descs[6], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[2], descs[5], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// verify predecessors information
wants := [][]ocispec.Descriptor{
{descs[5], descs[10], descs[11]}, // Blob 0
nil, // Blob 1
{descs[5], descs[11]}, // Blob 2
nil, // Blob 3
{descs[7]}, // Blob 4's predecessor is descs[7], even though blob 4 no longer exist
{descs[7]}, // Blob 5
{descs[10]}, // Blob 6's predecessor is descs[10], even though blob 6 no longer exist
nil, // Blob 7
nil, // Blob 8
{descs[10]}, // Blob 9
{descs[11]}, // Blob 10
nil, // Blob 11
}
for i, want := range wants {
predecessors, err := s.Predecessors(ctx, descs[i])
if err != nil {
t.Errorf("Store.Predecessors(%d) error = %v", i, err)
}
if !equalDescriptorSet(predecessors, want) {
t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want)
}
}
}

func TestStore_DeleteDisableAutoGCAndAutoRemoveReferrers(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
t.Fatal("New() error =", err)
}
s.AutoDeleteReferrers = false
s.AutoGC = false
ctx := context.Background()

// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateManifest := func(config ocispec.Descriptor, subject *ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Config: config,
Subject: subject,
Layers: layers,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
indexJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2
appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3
generateManifest(descs[0], nil, descs[1]) // Blob 4
generateManifest(descs[0], nil, descs[2]) // Blob 5
generateManifest(descs[0], nil, descs[3]) // Blob 6
generateIndex(descs[4:6]...) // Blob 7
generateIndex(descs[6]) // Blob 8
appendBlob(ocispec.MediaTypeImageLayer, []byte("world")) // Blob 9
generateManifest(descs[0], &descs[6], descs[9]) // Blob 10
generateManifest(descs[0], &descs[10], descs[2]) // Blob 11

eg, egCtx := errgroup.WithContext(ctx)
for i := range blobs {
eg.Go(func(i int) func() error {
return func() error {
err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
if err != nil {
return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
}
return nil
}
}(i))
}
if err := eg.Wait(); err != nil {
t.Fatal(err)
}

// delete blob 4 and verify the result
if err := s.Delete(egCtx, descs[4]); err != nil {
t.Fatal(err)
}

// blob 4 is now deleted, and other blobs are still present
notPresent := []ocispec.Descriptor{descs[4]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent := []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[6], descs[7], descs[8], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 8 and verify the result
if err := s.Delete(egCtx, descs[8]); err != nil {
t.Fatal(err)
}

// blob 4 and 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[4], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[6], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 6 and verify the result
if err := s.Delete(egCtx, descs[6]); err != nil {
t.Fatal(err)
}

// blob 4, 6, 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[4], descs[6], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// verify predecessors information
wants := [][]ocispec.Descriptor{
{descs[5], descs[10], descs[11]}, // Blob 0
nil, // Blob 1
{descs[5], descs[11]}, // Blob 2
nil, // Blob 3
{descs[7]}, // Blob 4's predecessor is descs[7], even though blob 4 no longer exist
{descs[7]}, // Blob 5
{descs[10]}, // Blob 6's predecessor is descs[10], even though blob 6 no longer exist
nil, // Blob 7
nil, // Blob 8
{descs[10]}, // Blob 9
{descs[11]}, // Blob 10
nil, // Blob 11
}
for i, want := range wants {
predecessors, err := s.Predecessors(ctx, descs[i])
if err != nil {
t.Errorf("Store.Predecessors(%d) error = %v", i, err)
}
if !equalDescriptorSet(predecessors, want) {
t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want)
}
}
}

func TestStore_Untag(t *testing.T) {
content := []byte("test delete")
desc := ocispec.Descriptor{
Expand Down

0 comments on commit 850a247

Please sign in to comment.