feat: add local sign/list/verification for OCI layout directory (#595)

This PR adds local sign/list/verification for OCI image layout
directory.
For RC.4:
1. It only supports storing the generated signature into the target OCI
layout directory.
2. It supports listing signatures within the OCI layout directory.
3. It only supports verifying signatures within the target OCI layout
directory.

This PR is based on spec PR:
#601 (Merged).

This PR is dependent on the corresponding notation-go PR:
notaryproject/notation-go#288.
Please review the notation-go PR first.

Resolves #283.

Both remote registry and oci-layout scenario are tested. E2E tests are
also included.

---------

Signed-off-by: Patrick Zheng <patrickzheng@microsoft.com>

cmd/notation/inspect.go

@@ -7,8 +7,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"strings"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/notaryproject/notation-core-go/signature"
@@ -98,22 +98,16 @@ func runInspect(command *cobra.Command, opts *inspectOpts) error {
 
 	// initialize
 	reference := opts.reference
-	sigRepo, err := getSignatureRepository(ctx, &opts.SecureFlagOpts, reference)
+	sigRepo, err := getRemoteRepository(ctx, &opts.SecureFlagOpts, reference)
 	if err != nil {
 		return err
 	}
-
-	manifestDesc, ref, err := getManifestDescriptor(ctx, &opts.SecureFlagOpts, reference, sigRepo)
+	manifestDesc, resolvedRef, err := resolveReference(ctx, inputTypeRegistry, reference, sigRepo, func(ref string, manifestDesc ocispec.Descriptor) {
+		fmt.Fprintf(os.Stderr, "Warning: Always inspect the artifact using digest(@sha256:...) rather than a tag(:%s) because resolved digest may not point to the same signed artifact, as tags are mutable.\n", ref)
+	})
 	if err != nil {
 		return err
 	}
-
-	// reference is a digest reference
-	if err := ref.ValidateReferenceAsDigest(); err != nil {
-		fmt.Fprintf(os.Stderr, "Warning: Always inspect the artifact using digest(@sha256:...) rather than a tag(:%s) because resolved digest may not point to the same signed artifact, as tags are mutable.\n", ref.Reference)
-		ref.Reference = manifestDesc.Digest.String()
-	}
-
 	output := inspectOutput{MediaType: manifestDesc.MediaType, Signatures: []signatureOutput{}}
 	skippedSignatures := false
 	err = sigRepo.ListSignatures(ctx, manifestDesc, func(signatureManifests []ocispec.Descriptor) error {
@@ -177,7 +171,7 @@ func runInspect(command *cobra.Command, opts *inspectOpts) error {
 		return err
 	}
 
-	err = printOutput(opts.outputFormat, ref.String(), output)
+	err = printOutput(opts.outputFormat, resolvedRef, output)
 	if err != nil {
 		return err
 	}

cmd/notation/internal/errors/errors.go

@@ -12,3 +12,16 @@ func (e ErrorReferrersAPINotSupported) Error() string {
 	}
 	return "referrers API not supported"
 }
+
+// ErrorOCILayoutMissingReference is used when signing local content in oci
+// layout folder but missing input tag or digest.
+type ErrorOCILayoutMissingReference struct {
+	Msg string
+}
+
+func (e ErrorOCILayoutMissingReference) Error() string {
+	if e.Msg != "" {
+		return e.Msg
+	}
+	return "reference is missing either digest or tag"
+}

cmd/notation/list.go

@@ -7,21 +7,25 @@ import (
 
 	notationregistry "github.com/notaryproject/notation-go/registry"
 	"github.com/notaryproject/notation/internal/cmd"
+	"github.com/notaryproject/notation/internal/experimental"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
-	"oras.land/oras-go/v2/registry"
 )
 
 type listOpts struct {
 	cmd.LoggingFlagOpts
 	SecureFlagOpts
 	reference string
+	ociLayout bool
+	inputType inputType
 }
 
 func listCommand(opts *listOpts) *cobra.Command {
 	if opts == nil {
-		opts = &listOpts{}
+		opts = &listOpts{
+			inputType: inputTypeRegistry, // remote registry by default
+		}
 	}
 	cmd := &cobra.Command{
 		Use:     "list [flags] <reference>",
@@ -35,12 +39,20 @@ func listCommand(opts *listOpts) *cobra.Command {
 			opts.reference = args[0]
 			return nil
 		},
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if opts.ociLayout {
+				opts.inputType = inputTypeOCILayout
+			}
+			return experimental.CheckFlagsAndWarn(cmd, "oci-layout")
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runList(cmd.Context(), opts)
 		},
 	}
 	opts.LoggingFlagOpts.ApplyFlags(cmd.Flags())
 	opts.SecureFlagOpts.ApplyFlags(cmd.Flags())
+	cmd.Flags().BoolVar(&opts.ociLayout, "oci-layout", false, "[Experimental] list signatures stored in OCI image layout")
+	experimental.HideFlags(cmd, "oci-layout")
 	return cmd
 }
 
@@ -50,23 +62,21 @@ func runList(ctx context.Context, opts *listOpts) error {
 
 	// initialize
 	reference := opts.reference
-	sigRepo, err := getSignatureRepository(ctx, &opts.SecureFlagOpts, reference)
+	sigRepo, err := getRepository(ctx, opts.inputType, reference, &opts.SecureFlagOpts)
 	if err != nil {
 		return err
 	}
-	manifestDesc, ref, err := getManifestDescriptor(ctx, &opts.SecureFlagOpts, reference, sigRepo)
+	targetDesc, resolvedRef, err := resolveReference(ctx, opts.inputType, reference, sigRepo, nil)
 	if err != nil {
 		return err
 	}
-
 	// print all signature manifest digests
-	return printSignatureManifestDigests(ctx, manifestDesc, sigRepo, ref)
+	return printSignatureManifestDigests(ctx, targetDesc, sigRepo, resolvedRef)
 }
 
 // printSignatureManifestDigests returns the signature manifest digests of
 // the subject manifest.
-func printSignatureManifestDigests(ctx context.Context, manifestDesc ocispec.Descriptor, sigRepo notationregistry.Repository, ref registry.Reference) error {
-	ref.Reference = manifestDesc.Digest.String()
+func printSignatureManifestDigests(ctx context.Context, targetDesc ocispec.Descriptor, sigRepo notationregistry.Repository, ref string) error {
 	titlePrinted := false
 	printTitle := func() {
 		if !titlePrinted {
@@ -77,7 +87,7 @@ func printSignatureManifestDigests(ctx context.Context, manifestDesc ocispec.Des
 	}
 
 	var prevDigest digest.Digest
-	err := sigRepo.ListSignatures(ctx, manifestDesc, func(signatureManifests []ocispec.Descriptor) error {
+	err := sigRepo.ListSignatures(ctx, targetDesc, func(signatureManifests []ocispec.Descriptor) error {
 		for _, sigManifestDesc := range signatureManifests {
 			if prevDigest != "" {
 				// check and print title

cmd/notation/manifest.go

@@ -3,34 +3,119 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"unicode"
 
 	"github.com/notaryproject/notation-go/log"
 	notationregistry "github.com/notaryproject/notation-go/registry"
+	notationerrors "github.com/notaryproject/notation/cmd/notation/internal/errors"
+	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"oras.land/oras-go/v2/registry"
 )
 
-// getManifestDescriptor returns target artifact manifest descriptor and
-// registry.Reference given user input reference.
-func getManifestDescriptor(ctx context.Context, opts *SecureFlagOpts, reference string, sigRepo notationregistry.Repository) (ocispec.Descriptor, registry.Reference, error) {
-	logger := log.GetLogger(ctx)
-
+// resolveReference resolves user input reference based on user input type.
+// Returns the resolved manifest descriptor and resolvedRef in digest
+func resolveReference(ctx context.Context, inputType inputType, reference string, sigRepo notationregistry.Repository, fn func(string, ocispec.Descriptor)) (ocispec.Descriptor, string, error) {
+	// sanity check
 	if reference == "" {
-		return ocispec.Descriptor{}, registry.Reference{}, errors.New("missing reference")
+		return ocispec.Descriptor{}, "", errors.New("missing user input reference")
+	}
+	var tagOrDigestRef string
+	var resolvedRef string
+	switch inputType {
+	case inputTypeRegistry:
+		ref, err := registry.ParseReference(reference)
+		if err != nil {
+			return ocispec.Descriptor{}, "", fmt.Errorf("failed to resolve user input reference: %w", err)
+		}
+		tagOrDigestRef = ref.Reference
+		resolvedRef = ref.Registry + "/" + ref.Repository
+	case inputTypeOCILayout:
+		layoutPath, layoutReference, err := parseOCILayoutReference(reference)
+		if err != nil {
+			return ocispec.Descriptor{}, "", fmt.Errorf("failed to resolve user input reference: %w", err)
+		}
+		layoutPathInfo, err := os.Stat(layoutPath)
+		if err != nil {
+			return ocispec.Descriptor{}, "", fmt.Errorf("failed to resolve user input reference: %w", err)
+		}
+		if !layoutPathInfo.IsDir() {
+			return ocispec.Descriptor{}, "", errors.New("failed to resolve user input reference: input path is not a dir")
+		}
+		tagOrDigestRef = layoutReference
+		resolvedRef = layoutPath
+	default:
+		return ocispec.Descriptor{}, "", fmt.Errorf("unsupported user inputType: %d", inputType)
 	}
-	ref, err := registry.ParseReference(reference)
+
+	manifestDesc, err := getManifestDescriptor(ctx, tagOrDigestRef, sigRepo)
 	if err != nil {
-		return ocispec.Descriptor{}, registry.Reference{}, err
+		return ocispec.Descriptor{}, "", fmt.Errorf("failed to get manifest descriptor: %w", err)
+	}
+	resolvedRef = resolvedRef + "@" + manifestDesc.Digest.String()
+	if _, err := digest.Parse(tagOrDigestRef); err == nil {
+		// tagOrDigestRef is a digest reference
+		return manifestDesc, resolvedRef, nil
 	}
-	if ref.Reference == "" {
-		return ocispec.Descriptor{}, registry.Reference{}, errors.New("reference is missing digest or tag")
+	// tagOrDigestRef is a tag reference
+	if fn != nil {
+		fn(tagOrDigestRef, manifestDesc)
 	}
+	return manifestDesc, resolvedRef, nil
+}
 
-	manifestDesc, err := sigRepo.Resolve(ctx, ref.Reference)
-	if err != nil {
-		return ocispec.Descriptor{}, registry.Reference{}, err
+// resolveArtifactDigestReference creates reference in Verification given user input
+// trust policy scope
+func resolveArtifactDigestReference(reference, policyScope string) string {
+	if policyScope != "" {
+		if _, digest, ok := strings.Cut(reference, "@"); ok {
+			return policyScope + "@" + digest
+		}
 	}
+	return reference
+}
 
-	logger.Infof("Reference %s resolved to manifest descriptor: %+v", ref.Reference, manifestDesc)
-	return manifestDesc, ref, nil
+// parseOCILayoutReference parses the raw in format of <path>[:<tag>|@<digest>].
+// Returns the path to the OCI layout and the reference (tag or digest).
+func parseOCILayoutReference(raw string) (string, string, error) {
+	var path string
+	var ref string
+	if idx := strings.LastIndex(raw, "@"); idx != -1 {
+		// `digest` found
+		path, ref = raw[:idx], raw[idx+1:]
+	} else {
+		// find `tag`
+		idx := strings.LastIndex(raw, ":")
+		if idx == -1 || (idx == 1 && len(raw) > 2 && unicode.IsLetter(rune(raw[0])) && raw[2] == '\\') {
+			return "", "", notationerrors.ErrorOCILayoutMissingReference{}
+		} else {
+			path, ref = raw[:idx], raw[idx+1:]
+		}
+	}
+	if path == "" {
+		return "", "", fmt.Errorf("found empty file path in %q", raw)
+	}
+	if ref == "" {
+		return "", "", fmt.Errorf("found empty reference in %q", raw)
+	}
+	return path, ref, nil
+}
+
+// getManifestDescriptor returns target artifact manifest descriptor given
+// reference (digest or tag) and Repository.
+func getManifestDescriptor(ctx context.Context, reference string, sigRepo notationregistry.Repository) (ocispec.Descriptor, error) {
+	logger := log.GetLogger(ctx)
+
+	if reference == "" {
+		return ocispec.Descriptor{}, errors.New("reference cannot be empty")
+	}
+	manifestDesc, err := sigRepo.Resolve(ctx, reference)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+	logger.Infof("Reference %s resolved to manifest descriptor: %+v", reference, manifestDesc)
+	return manifestDesc, nil
 }

cmd/notation/registry.go

@@ -21,9 +21,53 @@ import (
 	"oras.land/oras-go/v2/registry/remote/errcode"
 )
 
-const zeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+// inputType denotes the user input type
+type inputType int
 
-func getSignatureRepository(ctx context.Context, opts *SecureFlagOpts, reference string) (notationregistry.Repository, error) {
+const (
+	inputTypeRegistry  inputType = 1 + iota // inputType remote registry
+	inputTypeOCILayout                      // inputType oci-layout
+)
+
+const (
+	zeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+)
+
+// getRepository returns a notationregistry.Repository given user input type and
+// user input reference
+func getRepository(ctx context.Context, inputType inputType, reference string, opts *SecureFlagOpts) (notationregistry.Repository, error) {
+	switch inputType {
+	case inputTypeRegistry:
+		return getRemoteRepository(ctx, opts, reference)
+	case inputTypeOCILayout:
+		layoutPath, _, err := parseOCILayoutReference(reference)
+		if err != nil {
+			return nil, err
+		}
+		return notationregistry.NewOCIRepository(layoutPath, notationregistry.RepositoryOptions{})
+	default:
+		return nil, errors.New("unsupported input type")
+	}
+}
+
+// getRepositoryForSign returns a notationregistry.Repository given user input
+// type and user input reference during Sign process
+func getRepositoryForSign(ctx context.Context, inputType inputType, reference string, opts *SecureFlagOpts, ociImageManifest bool) (notationregistry.Repository, error) {
+	switch inputType {
+	case inputTypeRegistry:
+		return getRemoteRepositoryForSign(ctx, opts, reference, ociImageManifest)
+	case inputTypeOCILayout:
+		layoutPath, _, err := parseOCILayoutReference(reference)
+		if err != nil {
+			return nil, err
+		}
+		return notationregistry.NewOCIRepository(layoutPath, notationregistry.RepositoryOptions{OCIImageManifest: ociImageManifest})
+	default:
+		return nil, errors.New("unsupported input type")
+	}
+}
+
+func getRemoteRepository(ctx context.Context, opts *SecureFlagOpts, reference string) (notationregistry.Repository, error) {
 	ref, err := registry.ParseReference(reference)
 	if err != nil {
 		return nil, err
@@ -37,13 +81,13 @@ func getSignatureRepository(ctx context.Context, opts *SecureFlagOpts, reference
 	return notationregistry.NewRepository(remoteRepo), nil
 }
 
-// getSignatureRepositoryForSign returns a registry.Repository for Sign.
+// getRemoteRepositoryForSign returns a registry.Repository for Sign.
 // ociImageManifest denotes the type of manifest used to store signatures during
 // Sign process.
 // Setting ociImageManifest to true means using OCI image manifest and the
 // Referrers tag schema.
 // Otherwise, use OCI artifact manifest and requires the Referrers API.
-func getSignatureRepositoryForSign(ctx context.Context, opts *SecureFlagOpts, reference string, ociImageManifest bool) (notationregistry.Repository, error) {
+func getRemoteRepositoryForSign(ctx context.Context, opts *SecureFlagOpts, reference string, ociImageManifest bool) (notationregistry.Repository, error) {
 	logger := log.GetLogger(ctx)
 	ref, err := registry.ParseReference(reference)
 	if err != nil {