test(e2e): update e2e specs for oras attach (#1081)

Signed-off-by: Billy Zha <jinzha1@microsoft.com>

.gitignore

@@ -49,3 +49,10 @@ _dist/
 # Distribution storage files for local E2E testing
 test/e2e/testdata/distribution/mount/docker/
 test/e2e/testdata/distribution/mount_fallback/docker/
+
+
+# OCI Layout Files ZOT storage files for local E2E testing
+test/e2e/testdata/zot/
+!test/e2e/testdata/zot/command/images
+!test/e2e/testdata/zot/config.json
+!test/e2e/testdata/zot/passwd_bcrypt

test/e2e/README.md

@@ -38,7 +38,7 @@ This is super handy when you want to do step-by-step debugging from command-line
 
 ### 5. Testing Registry Services
 The backend of E2E tests are three registry services: 
-- [oras-distribution](https://github.com/oras-project/distribution): registry service supports artifact and image types in [image-spec 1.1.0 rc2](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc2) and referrer API
+- [oras-distribution](https://github.com/oras-project/distribution): registry service supports artifact and image types in [image-spec 1.1.0 rc2](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc2) and referrer API. Will be deprecated when [image-spec 1.1.0 rc2](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc2) is not supported by oras CLI.
 - [upstream distribution](https://github.com/distribution/distribution): registry service supports image media type with subject and provide referrers via [tag schema](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#referrers-tag-schema). 
 - [zot](https://github.com/project-zot/zot): registry service supports artifact and image types in [image-spec 1.1.0 rc4](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc4) and referrer API
 
@@ -72,12 +72,13 @@ Describe: <Role>
 ### 9. Adding New Test Data
 
 #### 9.1 Command Suite
-Command suite uses pre-baked test data, which is a bunch of layered archive files compressed from registry storage. Test data are all stored in `$REPO_ROOT/test/e2e/testdata/distribution/` but separated in different sub-folders: oras distribution uses `mount` and upstream distribution uses `mount_fallback`.
+Command suite uses two kinds of pre-baked test data:
+- Layered distribution archive files: test data compressed from registry runtime storage directly and stored in `$REPO_ROOT/test/e2e/testdata/distribution/`. ORAS distribution uses sub-folder `mount` and upstream distribution uses sub-folder `mount_fallback`. For both registries, the repository name should follow the convention of `command/$repo_suffix`. To add a new layer to the test data, use the below command to compress the `docker` folder from the root directory of the registry storage and copy it to the corresponding subfolder in `$REPO_ROOT/test/e2e/testdata/distribution/mount`.
+    ```shell
+    tar -cvzf ${repo_suffix}.tar.gz --owner=0 --group=0 docker/
+    ```
+- OCI layout files: test data stored in `$REPO_ROOT/test/e2e/testdata/zot/` and used by ZOT registry service. You may use stable release of ORAS CLI to build it. When adding new artifacts in, please make sure the repository folder is excluded in `$REPO_ROOT/.gitignore`.
 
-For both registries, the repository name should follow the convention of `command/$repo_suffix`. To add a new layer to the test data, use the below command to compress the `docker` folder from the root directory of the registry storage and copy it to the corresponding subfolder in `$REPO_ROOT/test/e2e/testdata/distribution/mount`.
-```shell
-tar -cvzf ${repo_suffix}.tar.gz --owner=0 --group=0 docker/
-```
 
 ##### Test Data for ORAS-Distribution
 ```mermaid
@@ -145,5 +146,9 @@ graph TD;
         end
     end
 ```
+
+##### Test Data for ZOT
+Still WIP. Currently test data is partial, will update this section after all the specs in command suite are migrated to ZOT.
+
 #### 9.2 Scenario Suite
 Test files used by scenario-based specs are placed in `$REPO_ROOT/test/e2e/testdata/files`.

test/e2e/internal/utils/prepare.go

@@ -28,6 +28,22 @@ import (
 	"github.com/onsi/gomega/gbytes"
 )
 
+// CopyZotRepo copies oci layout data between repostories.
+func CopyZotRepo(fromRepo string, toRepo string) {
+	zotRoot := filepath.Join(TestDataRoot, "zot")
+	fromRepo = filepath.Join(zotRoot, fromRepo)
+	toRepo = filepath.Join(zotRoot, toRepo)
+	Expect(CopyFiles(fromRepo, toRepo)).ShouldNot(HaveOccurred())
+}
+
+// PrepareTempOCI prepares an OCI layout root via copying from an ZOT repo and
+// return the path.
+func PrepareTempOCI(fromZotRepo string) string {
+	root := PrepareTempFiles()
+	Expect(CopyFiles(filepath.Join(TestDataRoot, "zot", fromZotRepo), root)).ShouldNot(HaveOccurred())
+	return root
+}
+
 // PrepareTempFiles copies test data into a temp folder and return it.
 func PrepareTempFiles() string {
 	tempDir := GinkgoT().TempDir()

test/e2e/suite/command/attach.go

@@ -46,22 +46,22 @@ var _ = Describe("ORAS beginners:", func() {
 		})
 
 		It("should fail when no subject reference provided", func() {
-			ORAS("attach", "--artifact-type", "oras.test").ExpectFailure().MatchErrKeyWords("Error:").Exec()
+			ORAS("attach", "--artifact-type", "oras/test").ExpectFailure().MatchErrKeyWords("Error:").Exec()
 		})
 
 		It("should fail if no file reference or manifest annotation provided for registry", func() {
-			ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag)).
+			ORAS("attach", "--artifact-type", "oras/test", RegistryRef(ZOTHost, ImageRepo, foobar.Tag)).
 				ExpectFailure().MatchErrKeyWords("Error: no blob or manifest annotation are provided").Exec()
 		})
 
 		It("should fail if no file reference or manifest annotation provided for OCI layout", func() {
 			root := GinkgoT().TempDir()
-			ORAS("attach", "--artifact-type", "oras.test", LayoutRef(root, foobar.Tag)).
+			ORAS("attach", "--artifact-type", "oras/test", LayoutRef(root, foobar.Tag)).
 				ExpectFailure().MatchErrKeyWords("Error: no blob or manifest annotation are provided").Exec()
 		})
 
 		It("should fail if distribution spec is unkown", func() {
-			ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag), "--distribution-spec", "???").
+			ORAS("attach", "--artifact-type", "oras/test", RegistryRef(ZOTHost, ImageRepo, foobar.Tag), "--distribution-spec", "???").
 				ExpectFailure().MatchErrKeyWords("unknown distribution specification flag").Exec()
 		})
 	})
@@ -71,11 +71,10 @@ var _ = Describe("1.1 registry users:", func() {
 	When("running attach command", func() {
 		It("should attach a file to a subject", func() {
 			testRepo := attachTestRepo("simple")
-			tempDir := PrepareTempFiles()
-			subjectRef := RegistryRef(Host, testRepo, foobar.Tag)
-			prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef)
+			CopyZotRepo(ImageRepo, testRepo)
+			subjectRef := RegistryRef(ZOTHost, testRepo, foobar.Tag)
 			ORAS("attach", "--artifact-type", "test/attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)).
-				WithWorkDir(tempDir).
+				WithWorkDir(PrepareTempFiles()).
 				MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec()
 		})
 
@@ -84,8 +83,8 @@ var _ = Describe("1.1 registry users:", func() {
 			testRepo := attachTestRepo("export-manifest")
 			tempDir := PrepareTempFiles()
 			exportName := "manifest.json"
-			subjectRef := RegistryRef(Host, testRepo, foobar.Tag)
-			prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef)
+			subjectRef := RegistryRef(ZOTHost, testRepo, foobar.Tag)
+			CopyZotRepo(ImageRepo, testRepo)
 			// test
 			ORAS("attach", "--artifact-type", "test/attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--export-manifest", exportName).
 				WithWorkDir(tempDir).
@@ -95,15 +94,15 @@ var _ = Describe("1.1 registry users:", func() {
 			bytes := ORAS("discover", subjectRef, "-o", "json").Exec().Out.Contents()
 			Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred())
 			Expect(len(index.Manifests)).To(Equal(1))
-			fetched := ORAS("manifest", "fetch", RegistryRef(Host, testRepo, index.Manifests[0].Digest.String())).Exec().Out.Contents()
+			fetched := ORAS("manifest", "fetch", RegistryRef(ZOTHost, testRepo, index.Manifests[0].Digest.String())).Exec().Out.Contents()
 			MatchFile(filepath.Join(tempDir, exportName), string(fetched), DefaultTimeout)
 		})
 
 		It("should attach a file via a OCI Image", func() {
 			testRepo := attachTestRepo("image")
 			tempDir := PrepareTempFiles()
-			subjectRef := RegistryRef(Host, testRepo, foobar.Tag)
-			prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef)
+			subjectRef := RegistryRef(ZOTHost, testRepo, foobar.Tag)
+			CopyZotRepo(ImageRepo, testRepo)
 			// test
 			ORAS("attach", "--artifact-type", "test/attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)).
 				WithWorkDir(tempDir).
@@ -120,8 +119,8 @@ var _ = Describe("1.1 registry users:", func() {
 			testRepo := attachTestRepo("simple")
 			absAttachFileName := filepath.Join(PrepareTempFiles(), foobar.AttachFileName)
 
-			subjectRef := RegistryRef(Host, testRepo, foobar.Tag)
-			prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef)
+			subjectRef := RegistryRef(ZOTHost, testRepo, foobar.Tag)
+			CopyZotRepo(ImageRepo, testRepo)
 			statusKey := foobar.AttachFileStateKey
 			statusKey.Name = absAttachFileName
 			ORAS("attach", "--artifact-type", "test/attach", subjectRef, fmt.Sprintf("%s:%s", absAttachFileName, foobar.AttachFileMedia), "--disable-path-validation").
@@ -133,8 +132,8 @@ var _ = Describe("1.1 registry users:", func() {
 			testRepo := attachTestRepo("simple")
 			absAttachFileName := filepath.Join(PrepareTempFiles(), foobar.AttachFileName)
 
-			subjectRef := RegistryRef(Host, testRepo, foobar.Tag)
-			prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef)
+			subjectRef := RegistryRef(ZOTHost, testRepo, foobar.Tag)
+			CopyZotRepo(ImageRepo, testRepo)
 			statusKey := foobar.AttachFileStateKey
 			statusKey.Name = absAttachFileName
 			ORAS("attach", "--artifact-type", "test/attach", subjectRef, fmt.Sprintf("%s:%s", absAttachFileName, foobar.AttachFileMedia)).
@@ -204,24 +203,19 @@ var _ = Describe("1.0 registry users:", func() {
 
 var _ = Describe("OCI image layout users:", func() {
 	When("running attach command", func() {
-		prepare := func(root string) {
-			ORAS("cp", RegistryRef(Host, ImageRepo, foobar.Tag), Flags.ToLayout, LayoutRef(root, foobar.Tag)).Exec()
-		}
 		It("should attach a file to a subject", func() {
-			root := PrepareTempFiles()
+			root := PrepareTempOCI(ImageRepo)
 			subjectRef := LayoutRef(root, foobar.Tag)
-			prepare(root)
 			ORAS("attach", "--artifact-type", "test/attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)).
 				WithWorkDir(root).
 				MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec()
 		})
 
 		It("should attach a file to a subject and export the built manifest", func() {
 			// prepare
-			root := PrepareTempFiles()
 			exportName := "manifest.json"
+			root := PrepareTempOCI(ImageRepo)
 			subjectRef := LayoutRef(root, foobar.Tag)
-			prepare(root)
 			// test
 			ORAS("attach", "--artifact-type", "test/attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--export-manifest", exportName).
 				WithWorkDir(root).
@@ -236,9 +230,8 @@ var _ = Describe("OCI image layout users:", func() {
 			MatchFile(filepath.Join(root, exportName), string(fetched), DefaultTimeout)
 		})
 		It("should attach a file via a OCI Image", func() {
-			root := PrepareTempFiles()
+			root := PrepareTempOCI(ImageRepo)
 			subjectRef := LayoutRef(root, foobar.Tag)
-			prepare(root)
 			// test
 			ORAS("attach", "--artifact-type", "test/attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)).
 				WithWorkDir(root).

test/e2e/testdata/zot/command/images/blobs/sha256/1fd9a5fc54b634130102861815e2881f1eec22958d604301904c5353041794c1

@@ -0,0 +1,4 @@
+{
+    "architecture": "arm64",
+    "os": "linux"
+}

test/e2e/testdata/zot/command/images/blobs/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae

@@ -0,0 +1 @@
+foo

test/e2e/testdata/zot/command/images/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a

@@ -0,0 +1 @@
+{}

test/e2e/testdata/zot/command/images/blobs/sha256/4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1fd9a5fc54b634130102861815e2881f1eec22958d604301904c5353041794c1","size":53},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:2ef548696ac7dd66ef38aab5cc8fc5cc1fb637dfaedb3a9afc89bf16db9277e1","size":10240,"annotations":{"org.opencontainers.image.title":"hello.tar"}}]}

test/e2e/testdata/zot/command/images/blobs/sha256/58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:a5dfce5d28768e985f91f610591f0203bb6e07f11e2fb0ce1b8508e9aac4ebe2","size":73},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:2ef548696ac7dd66ef38aab5cc8fc5cc1fb637dfaedb3a9afc89bf16db9277e1","size":10240,"annotations":{"org.opencontainers.image.title":"hello.tar"}}]}

test/e2e/testdata/zot/command/images/blobs/sha256/9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:fe9dbc99451d0517d65e048c309f0b5afb2cc513b7a3d456b6cc29fe641386c5","size":53},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:2ef548696ac7dd66ef38aab5cc8fc5cc1fb637dfaedb3a9afc89bf16db9277e1","size":10240,"annotations":{"org.opencontainers.image.title":"hello.tar"}}]}

test/e2e/testdata/zot/command/images/blobs/sha256/a5dfce5d28768e985f91f610591f0203bb6e07f11e2fb0ce1b8508e9aac4ebe2

@@ -0,0 +1,5 @@
+{
+    "architecture": "arm",
+    "os": "linux",
+    "variant": "v7"
+}

test/e2e/testdata/zot/command/images/blobs/sha256/e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f

@@ -0,0 +1 @@
+{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1","size":458,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c","size":458,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255","size":458,"platform":{"architecture":"arm","os":"linux","variant":"v7"}}]}

test/e2e/testdata/zot/command/images/blobs/sha256/fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9

@@ -0,0 +1 @@
+bar

test/e2e/testdata/zot/command/images/blobs/sha256/fd6ed2f36b5465244d5dc86cb4e7df0ab8a9d24adc57825099f522fe009a22bb

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.unknown.config.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae","size":3,"annotations":{"org.opencontainers.image.title":"foo1"}},{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae","size":3,"annotations":{"org.opencontainers.image.title":"foo2"}},{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9","size":3,"annotations":{"org.opencontainers.image.title":"bar"}}]}

test/e2e/testdata/zot/command/images/blobs/sha256/fe9dbc99451d0517d65e048c309f0b5afb2cc513b7a3d456b6cc29fe641386c5

@@ -0,0 +1,4 @@
+{
+    "architecture": "amd64",
+    "os": "linux"
+}

test/e2e/testdata/zot/command/images/index.json

@@ -0,0 +1,49 @@
+{
+	"schemaVersion": 2,
+	"manifests": [
+		{
+			"mediaType": "application/vnd.oci.image.index.v1+json",
+			"digest": "sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f",
+			"size": 706,
+			"annotations": {
+				"org.opencontainers.image.ref.name": "multi"
+			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:fd6ed2f36b5465244d5dc86cb4e7df0ab8a9d24adc57825099f522fe009a22bb",
+			"size": 851,
+			"annotations": {
+				"org.opencontainers.image.ref.name": "foobar"
+			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255",
+			"size": 458,
+			"platform": {
+				"architecture": "arm",
+				"os": "linux",
+				"variant": "v7"
+			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c",
+			"size": 458,
+			"platform": {
+				"architecture": "arm64",
+				"os": "linux"
+			}
+		},
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1",
+			"size": 458,
+			"platform": {
+				"architecture": "amd64",
+				"os": "linux"
+			}
+		}
+	]
+}

test/e2e/testdata/zot/command/images/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}