Skip to content

Commit

Permalink
feat: add a PackOption to support packing image manifests that confor…
Browse files Browse the repository at this point in the history 
…m to image-spec v1.1.0-rc4 (#550)

1. Introduce `PackOptions.PackManifestType`
2. Introduce `DefaultPackOptions`

Resolves: #532
Signed-off-by: Lixia (Sylvia) Lei <lixlei@microsoft.com>
  • Loading branch information
@Wwwsylvia
Wwwsylvia committed Jul 21, 2023
1 parent f0a9c59 commit 9b52269
Show file tree
Hide file tree
Showing 3 changed files with 655 additions and 73 deletions.
2 changes: 1 addition & 1 deletion content/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type Fetcher interface {
// Pusher pushes content.
type Pusher interface {
// Push pushes the content, matching the expected descriptor.
// Reader is perferred to Writer so that the suitable buffer size can be
// Reader is preferred to Writer so that the suitable buffer size can be
// chosen by the underlying implementation. Furthermore, the implementation
// can also do reflection on the Reader for more advanced I/O optimization.
Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error
Expand Down
218 changes: 173 additions & 45 deletions pack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,55 +31,117 @@ import (
)

const (
// MediaTypeUnknownConfig is the default mediaType used when no
// config media type is specified.
// MediaTypeUnknownConfig is the default mediaType used for [Pack] when
// PackOptions.PackImageManifest is true and PackOptions.PackManifestType
// is PackManifestTypeImageV1_1_0_RC2 and PackOptions.ConfigDescriptor
// is not specified.
MediaTypeUnknownConfig = "application/vnd.unknown.config.v1+json"
// MediaTypeUnknownArtifact is the default artifactType used when no
// artifact type is specified.

// MediaTypeUnknownArtifact is the default artifactType used for [Pack]
// when PackOptions.PackImageManifest is false and artifactType is
// not specified.
MediaTypeUnknownArtifact = "application/vnd.unknown.artifact.v1"
)

// ErrInvalidDateTimeFormat is returned by Pack() when
// AnnotationArtifactCreated or AnnotationCreated is provided, but its value
// is not in RFC 3339 format.
// Reference: https://www.rfc-editor.org/rfc/rfc3339#section-5.6
var ErrInvalidDateTimeFormat = errors.New("invalid date and time format")
// PackManifestType represents the manifest type used for [Pack].
type PackManifestType int

const (
// PackManifestTypeImageV1_1_0_RC2 represents the OCI Image Manifest type
// defined in image-spec v1.1.0-rc2.
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc2/manifest.md
//
// Deprecated: This type is deprecated and not recommended for future use.
// Use PackManifestTypeImageV1_1_0_RC4 instead.
PackManifestTypeImageV1_1_0_RC2 PackManifestType = 0

// PackManifestTypeImageV1_1_0_RC4 represents the OCI Image Manifest type
// defined since image-spec v1.1.0-rc4.
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/manifest.md
PackManifestTypeImageV1_1_0_RC4 PackManifestType = 1
)

var (
// ErrInvalidDateTimeFormat is returned by [Pack] when
// AnnotationArtifactCreated or AnnotationCreated is provided, but its value
// is not in RFC 3339 format.
// Reference: https://www.rfc-editor.org/rfc/rfc3339#section-5.6
ErrInvalidDateTimeFormat = errors.New("invalid date and time format")

// ErrMissingArtifactType is returned by [Pack] when artifactType is not
// specified and the config media type is set to
// "application/vnd.oci.empty.v1+json".
ErrMissingArtifactType = errors.New("missing artifact type")
)

// PackOptions contains parameters for [oras.Pack].
// PackOptions contains parameters for [Pack].
type PackOptions struct {
// Subject is the subject of the manifest.
Subject *ocispec.Descriptor

// ManifestAnnotations is the annotation map of the manifest.
ManifestAnnotations map[string]string

// PackImageManifest controls whether to pack an image manifest or not.
// - If true, pack an image manifest; artifactType will be used as the
// the config descriptor mediaType of the image manifest.
// - If false, pack an artifact manifest.
// Default: false.
// PackImageManifest controls whether to pack an OCI Image Manifest or not.
// - If true, pack an OCI Image Manifest.
// - If false, pack an OCI Artifact Manifest (deprecated).
//
// Default value: false.
// Recommended value: true (See DefaultPackOptions).
PackImageManifest bool

// PackManifestType controls which type of manifest to pack.
// This option is valid only when PackImageManifest is true.
//
// Default value: PackManifestTypeImageV1_1_0_RC2 (deprecated).
// Recommended value: PackManifestTypeImageV1_1_0_RC4 (See DefaultPackOptions).
PackManifestType PackManifestType

// ConfigDescriptor is a pointer to the descriptor of the config blob.
// If not nil, artifactType will be implied by the mediaType of the
// specified ConfigDescriptor, and ConfigAnnotations will be ignored.
// This option is valid only when PackImageManifest is true.
ConfigDescriptor *ocispec.Descriptor

// ConfigAnnotations is the annotation map of the config descriptor.
// This option is valid only when PackImageManifest is true
// and ConfigDescriptor is nil.
ConfigAnnotations map[string]string
}

// DefaultPackOptions provides the default PackOptions.
// Note that the default options are subject to change in the future.
var DefaultPackOptions PackOptions = PackOptions{
PackImageManifest: true,
PackManifestType: PackManifestTypeImageV1_1_0_RC4,
}

// Pack packs the given blobs, generates a manifest for the pack,
// and pushes it to a content storage.
//
// When opts.PackImageManifest is true, artifactType will be used as the
// the config descriptor mediaType of the image manifest.
// - If opts.PackImageManifest is true and opts.PackManifestType is
// [PackManifestTypeImageV1_1_0_RC2],
// artifactType will be used as the the config media type of the image
// manifest when opts.ConfigDescriptor is not specified.
// - If opts.PackImageManifest is true and opts.PackManifestType is
// [PackManifestTypeImageV1_1_0_RC4],
// [ErrMissingArtifactType] will be returned when none of artifactType and
// opts.ConfigDescriptor is specified.
//
// If succeeded, returns a descriptor of the manifest.
func Pack(ctx context.Context, pusher content.Pusher, artifactType string, blobs []ocispec.Descriptor, opts PackOptions) (ocispec.Descriptor, error) {
if opts.PackImageManifest {
return packImage(ctx, pusher, artifactType, blobs, opts)
if !opts.PackImageManifest {
return packArtifact(ctx, pusher, artifactType, blobs, opts)
}

switch opts.PackManifestType {
case PackManifestTypeImageV1_1_0_RC2:
return packImageRC2(ctx, pusher, artifactType, blobs, opts)
case PackManifestTypeImageV1_1_0_RC4:
return packImageRC4(ctx, pusher, artifactType, blobs, opts)
default:
return ocispec.Descriptor{}, fmt.Errorf("PackManifestType(%v): %w", opts.PackManifestType, errdef.ErrUnsupported)
}
return packArtifact(ctx, pusher, artifactType, blobs, opts)
}

// packArtifact packs the given blobs, generates an artifact manifest for the
Expand All @@ -101,28 +163,14 @@ func packArtifact(ctx context.Context, pusher content.Pusher, artifactType strin
Subject: opts.Subject,
Annotations: annotations,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err)
}
manifestDesc := content.NewDescriptorFromBytes(spec.MediaTypeArtifactManifest, manifestJSON)
// populate ArtifactType and Annotations of the manifest into manifestDesc
manifestDesc.ArtifactType = manifest.ArtifactType
manifestDesc.Annotations = manifest.Annotations

// push manifest
if err := pusher.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) {
return ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err)
}

return manifestDesc, nil
return pushManifest(ctx, pusher, manifest, manifest.MediaType, manifest.ArtifactType, manifest.Annotations)
}

// packImage packs the given blobs, generates an image manifest for the pack,
// and pushes it to a content storage. artifactType will be used as the config
// descriptor mediaType of the image manifest.
// packImageRC2 packs the given blobs, generates an image manifest for the
// pack, and pushes it to a content storage.
// If succeeded, returns a descriptor of the manifest.
func packImage(ctx context.Context, pusher content.Pusher, configMediaType string, layers []ocispec.Descriptor, opts PackOptions) (ocispec.Descriptor, error) {
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc2/manifest.md
func packImageRC2(ctx context.Context, pusher content.Pusher, configMediaType string, layers []ocispec.Descriptor, opts PackOptions) (ocispec.Descriptor, error) {
if configMediaType == "" {
configMediaType = MediaTypeUnknownConfig
}
Expand All @@ -139,7 +187,7 @@ func packImage(ctx context.Context, pusher content.Pusher, configMediaType strin
configDesc = content.NewDescriptorFromBytes(configMediaType, configBytes)
configDesc.Annotations = opts.ConfigAnnotations
// push config
if err := pusher.Push(ctx, configDesc, bytes.NewReader(configBytes)); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) {
if err := pushIfNotExist(ctx, pusher, configDesc, configBytes); err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to push config: %w", err)
}
}
Expand All @@ -161,20 +209,100 @@ func packImage(ctx context.Context, pusher content.Pusher, configMediaType strin
Subject: opts.Subject,
Annotations: annotations,
}
return pushManifest(ctx, pusher, manifest, manifest.MediaType, manifest.Config.MediaType, manifest.Annotations)
}

// packImageRC4 packs the given blobs, generates an image manifest for the pack,
// and pushes it to a content storage.
// If succeeded, returns a descriptor of the manifest.
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/manifest.md#guidelines-for-artifact-usage
func packImageRC4(ctx context.Context, pusher content.Pusher, artifactType string, layers []ocispec.Descriptor, opts PackOptions) (ocispec.Descriptor, error) {
var emptyBlobExists bool
var configDesc ocispec.Descriptor
if opts.ConfigDescriptor != nil {
configDesc = *opts.ConfigDescriptor
} else {
// use the empty descriptor for config
configDesc = ocispec.DescriptorEmptyJSON
configDesc.Annotations = opts.ConfigAnnotations
configBytes := ocispec.DescriptorEmptyJSON.Data
// push config
if err := pushIfNotExist(ctx, pusher, configDesc, configBytes); err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to push config: %w", err)
}
emptyBlobExists = true
}
if artifactType == "" {
if configDesc.MediaType == ocispec.MediaTypeEmptyJSON {
// artifactType MUST be set when config.mediaType is set to the empty value
return ocispec.Descriptor{}, ErrMissingArtifactType
}
artifactType = configDesc.MediaType
}

annotations, err := ensureAnnotationCreated(opts.ManifestAnnotations, ocispec.AnnotationCreated)
if err != nil {
return ocispec.Descriptor{}, err
}
if len(layers) == 0 {
// use the empty descriptor as the single layer
layerDesc := ocispec.DescriptorEmptyJSON
layerData := ocispec.DescriptorEmptyJSON.Data
if !emptyBlobExists {
if err := pushIfNotExist(ctx, pusher, layerDesc, layerData); err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to push layer: %w", err)
}
}
layers = []ocispec.Descriptor{layerDesc}
}

manifest := ocispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2, // historical value. does not pertain to OCI or docker version
},
Config: configDesc,
MediaType: ocispec.MediaTypeImageManifest,
Layers: layers,
Subject: opts.Subject,
ArtifactType: artifactType,
Annotations: annotations,
}
return pushManifest(ctx, pusher, manifest, manifest.MediaType, manifest.ArtifactType, manifest.Annotations)
}

// pushIfNotExist pushes data described by desc if it does not exist in the
// target.
func pushIfNotExist(ctx context.Context, pusher content.Pusher, desc ocispec.Descriptor, data []byte) error {
if ros, ok := pusher.(content.ReadOnlyStorage); ok {
exists, err := ros.Exists(ctx, desc)
if err != nil {
return fmt.Errorf("failed to check existence: %s: %s: %w", desc.Digest.String(), desc.MediaType, err)
}
if exists {
return nil
}
}

if err := pusher.Push(ctx, desc, bytes.NewReader(data)); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) {
return fmt.Errorf("failed to push: %s: %s: %w", desc.Digest.String(), desc.MediaType, err)
}
return nil
}

// pushManifest marshals manifest into JSON bytes and pushes it.
func pushManifest(ctx context.Context, pusher content.Pusher, manifest any, mediaType string, artifactType string, annotations map[string]string) (ocispec.Descriptor, error) {
manifestJSON, err := json.Marshal(manifest)
if err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err)
}
manifestDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, manifestJSON)
manifestDesc := content.NewDescriptorFromBytes(mediaType, manifestJSON)
// populate ArtifactType and Annotations of the manifest into manifestDesc
manifestDesc.ArtifactType = manifest.Config.MediaType
manifestDesc.Annotations = manifest.Annotations

manifestDesc.ArtifactType = artifactType
manifestDesc.Annotations = annotations
// push manifest
if err := pusher.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) {
return ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err)
}

return manifestDesc, nil
}

Expand Down
Loading

0 comments on commit 9b52269

Please sign in to comment.