copy: set media types

When copying an image, record the compression in the BlobInfo and use
the information when updating the manifest's layer infos to set the
layers' media types correctly.

Note that consumers of the containers/image library need to update
opencontainers/image-spec to commit 775207bd45b6cb8153ce218cc59351799217451f.

Fixes: github.com/containers/podman/issues/2013
Fixes: github.com/containers/buildah/issues/1589

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

copy/copy.go

@@ -185,8 +185,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 		blobInfoCache: blobinfocache.DefaultCache(options.DestinationCtx),
 	}
 	// Default to using gzip compression unless specified otherwise.
+<<<<<<< HEAD
 	if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil {
 		algo, err := compression.AlgorithmByName("gzip")
+=======
+	if options.DestinationCtx.CompressionFormat == nil {
+		algo, err := compression.AlgorithmByName(compression.Gzip)
+>>>>>>> copy: set media types
 		if err != nil {
 			return nil, err
 		}
@@ -911,6 +916,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 		return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
 	}
 
+	uploadedInfo.CompressionOperation = compressionOperation
+	// If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest.
+	if canModifyBlob && !isConfig {
+		uploadedInfo.CompressionAlgorithm = &desiredCompressionFormat
+	}
+
 	// This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer
 	// all of the input (to compute DiffIDs), even if dest.PutBlob does not need it.
 	// So, read everything from originalLayerReader, which will cause the rest to be

image/docker_schema2.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strings"
 
@@ -207,12 +208,17 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag
 	layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors))
 	for idx := range layers {
 		layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx])
-		if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType {
+		switch m.m.LayersDescriptors[idx].MediaType {
+		case manifest.DockerV2Schema2ForeignLayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable
-		} else {
-			// we assume layers are gzip'ed because docker v2s2 only deals with
-			// gzip'ed layers. However, OCI has non-gzip'ed layers as well.
+		case manifest.DockerV2SchemaLayerMediaTypeUncompressed:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayer
+		case manifest.DockerV2Schema2LayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip
+		case manifest.DockerV2Schema2LayerMediaTypeZstd:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType)
 		}
 	}
 

image/oci.go

@@ -3,6 +3,7 @@ package image
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 
 	"github.com/containers/image/docker/reference"
@@ -187,7 +188,18 @@ func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) {
 	layers := make([]manifest.Schema2Descriptor, len(m.m.Layers))
 	for idx := range layers {
 		layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx])
-		layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		switch layers[idx].MediaType {
+		case imgspecv1.MediaTypeImageLayerNonDistributable:
+			layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaType
+		case imgspecv1.MediaTypeImageLayer:
+			layers[idx].MediaType = manifest.DockerV2SchemaLayerMediaTypeUncompressed
+		case imgspecv1.MediaTypeImageLayerGzip:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		case imgspecv1.MediaTypeImageLayerZstd:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaTypeZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType)
+		}
 	}
 
 	// Rather than copying the ConfigBlob now, we just pass m.src to the

image/sourced.go

@@ -5,6 +5,7 @@ package image
 
 import (
 	"context"
+
 	"github.com/containers/image/types"
 )
 

manifest/docker_schema2.go

@@ -2,12 +2,15 @@ package manifest
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/pkg/strslice"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // Schema2Descriptor is a “descriptor” in docker/distribution schema 2.
@@ -207,7 +210,28 @@ func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
 	original := m.LayersDescriptors
 	m.LayersDescriptors = make([]Schema2Descriptor, len(layerInfos))
 	for i, info := range layerInfos {
-		m.LayersDescriptors[i].MediaType = original[i].MediaType
+		switch info.CompressionOperation {
+		case types.PreserveOriginal:
+			m.LayersDescriptors[i].MediaType = original[i].MediaType
+		case types.Decompress:
+			m.LayersDescriptors[i].MediaType = DockerV2SchemaLayerMediaTypeUncompressed
+		case types.Compress:
+			if info.CompressionAlgorithm == nil {
+				logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest)
+				m.LayersDescriptors[i].MediaType = original[i].MediaType
+				break
+			}
+			switch info.CompressionAlgorithm.Name() {
+			case compression.Gzip:
+				m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaType
+			case compression.Zstd:
+				m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaTypeZstd
+			default:
+				return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q fo layer %q", info.CompressionAlgorithm.Name(), info.Digest)
+			}
+		default:
+			return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest)
+		}
 		m.LayersDescriptors[i].Digest = info.Digest
 		m.LayersDescriptors[i].Size = info.Size
 		m.LayersDescriptors[i].URLs = info.URLs

manifest/manifest.go

@@ -24,6 +24,10 @@ const (
 	DockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json"
 	// DockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers.
 	DockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
+	// DockerV2Schema2LayerMediaTypeZstd is the MIME type used for schema 2 layers compressed with zstd.
+	DockerV2Schema2LayerMediaTypeZstd = "application/vnd.docker.image.rootfs.diff.tar.zstd"
+	// DockerV2SchemaLayerMediaTypeUncompressed is the mediaType used for uncompressed layers.
+	DockerV2SchemaLayerMediaTypeUncompressed = "application/vnd.docker.image.rootfs.diff.tar"
 	// DockerV2ListMediaType MIME type represents Docker manifest schema 2 list
 	DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json"
 	// DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers.

manifest/oci.go

@@ -2,12 +2,15 @@ package manifest
 
 import (
 	"encoding/json"
+	"fmt"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // BlobInfoFromOCI1Descriptor returns a types.BlobInfo based on the input OCI1 descriptor.
@@ -81,7 +84,28 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
 	original := m.Layers
 	m.Layers = make([]imgspecv1.Descriptor, len(layerInfos))
 	for i, info := range layerInfos {
-		m.Layers[i].MediaType = original[i].MediaType
+		switch info.CompressionOperation {
+		case types.PreserveOriginal:
+			m.Layers[i].MediaType = original[i].MediaType
+		case types.Decompress:
+			m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayer
+		case types.Compress:
+			if info.CompressionAlgorithm == nil {
+				logrus.Debugf("Preparing updated manifest: blob %q was compressed but does not specify by which algorithm: falling back to use the original blob", info.Digest)
+				m.Layers[i].MediaType = original[i].MediaType
+				break
+			}
+			switch info.CompressionAlgorithm.Name() {
+			case compression.Gzip:
+				m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerGzip
+			case compression.Zstd:
+				m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerZstd
+			default:
+				return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q for layer %q", info.CompressionAlgorithm.Name(), info.Digest)
+			}
+		default:
+			return fmt.Errorf("Error preparing updated manifest: unknown compression operation (%d) for layer %q", info.CompressionOperation, info.Digest)
+		}
 		m.Layers[i].Digest = info.Digest
 		m.Layers[i].Size = info.Size
 		m.Layers[i].Annotations = info.Annotations

pkg/compression/compression.go

@@ -13,6 +13,17 @@ import (
 	"github.com/ulikunitz/xz"
 )
 
+const (
+	// Gzip compression.
+	Gzip = "gzip"
+	// Bzip2 compression.
+	Bzip2 = "bzip2"
+	// Xz compression.
+	Xz = "xz"
+	// Zstd compression.
+	Zstd = "zstd"
+)
+
 // DecompressorFunc returns the decompressed stream, given a compressed stream.
 // The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
 type DecompressorFunc func(io.Reader) (io.ReadCloser, error)
@@ -73,10 +84,10 @@ func (c Algorithm) Name() string {
 
 // compressionAlgos is an internal implementation detail of DetectCompression
 var compressionAlgos = []Algorithm{
-	{"gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor},             // gzip (RFC 1952)
-	{"bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor},          // bzip2 (decompress.c:BZ2_decompress)
-	{"xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt)
-	{"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor},       // zstd (http://www.zstd.net)
+	{Gzip, []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor},             // gzip (RFC 1952)
+	{Bzip2, []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor},          // bzip2 (decompress.c:BZ2_decompress)
+	{Xz, []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt)
+	{Zstd, []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor},       // zstd (http://www.zstd.net)
 }
 
 // AlgorithmByName returns the compressor by its name

storage/storage_image.go

@@ -345,9 +345,9 @@ func (s *storageImageDestination) Close() error {
 }
 
 func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression {
-	// We ultimately have to decompress layers to populate trees on disk,
-	// so callers shouldn't bother compressing them before handing them to
-	// us, if they're not already compressed.
+	// We ultimately have to decompress layers to populate trees on disk
+	// and need to explicitly ask for it here, so that the layers' MIME
+	// types can be set accordingly.
 	return types.PreserveOriginal
 }
 

types/types.go

@@ -8,7 +8,7 @@ import (
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/pkg/compression"
 	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageTransport is a top-level namespace for ways to to store/load an image.
@@ -91,14 +91,29 @@ type ImageReference interface {
 	DeleteImage(ctx context.Context, sys *SystemContext) error
 }
 
+// LayerCompression indicates if layers must be compressed, decompressed or preserved
+type LayerCompression int
+
+const (
+	// PreserveOriginal indicates the layer must be preserved, ie
+	// no compression or decompression.
+	PreserveOriginal LayerCompression = iota
+	// Decompress indicates the layer must be decompressed
+	Decompress
+	// Compress indicates the layer must be compressed
+	Compress
+)
+
 // BlobInfo collects known information about a blob (layer/config).
 // In some situations, some fields may be unknown, in others they may be mandatory; documenting an “unknown” value here does not override that.
 type BlobInfo struct {
-	Digest      digest.Digest // "" if unknown.
-	Size        int64         // -1 if unknown
-	URLs        []string
-	Annotations map[string]string
-	MediaType   string
+	Digest               digest.Digest // "" if unknown.
+	Size                 int64         // -1 if unknown
+	URLs                 []string
+	Annotations          map[string]string
+	MediaType            string
+	CompressionOperation LayerCompression
+	CompressionAlgorithm *compression.Algorithm
 }
 
 // BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present.
@@ -212,19 +227,6 @@ type ImageSource interface {
 	LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error)
 }
 
-// LayerCompression indicates if layers must be compressed, decompressed or preserved
-type LayerCompression int
-
-const (
-	// PreserveOriginal indicates the layer must be preserved, ie
-	// no compression or decompression.
-	PreserveOriginal LayerCompression = iota
-	// Decompress indicates the layer must be decompressed
-	Decompress
-	// Compress indicates the layer must be compressed
-	Compress
-)
-
 // ImageDestination is a service, possibly remote (= slow), to store components of a single image.
 //
 // There is a specific required order for some of the calls:

vendor.conf

@@ -16,7 +16,7 @@ github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade
 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9
 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7
-github.com/opencontainers/image-spec v1.0.0
+github.com/opencontainers/image-spec 775207bd45b6cb8153ce218cc59351799217451f
 github.com/opencontainers/runc 6b1d0e76f239ffb435445e5ae316d2676c07c6e3
 github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9
 github.com/pkg/errors 248dadf4e9068a0b3e79f02ed0a610d935de5302