diff --git a/pkg/image/api/helper.go b/pkg/image/api/helper.go index d985e99ddb6b..3a7cc5e9f760 100644 --- a/pkg/image/api/helper.go +++ b/pkg/image/api/helper.go @@ -474,6 +474,9 @@ func ImageWithMetadata(image *Image) error { image.DockerImageLayers[i].Name = layer.DockerBlobSum } if len(manifest.History) == len(image.DockerImageLayers) { + // This code does not work for images imported from v2 schema + // registries or converted from v2 to v1, since schema v2 does not + // contain size information. image.DockerImageLayers[0].LayerSize = v1Metadata.Size var size = DockerV1CompatibilityImageSize{} for i, obj := range manifest.History[1:] { diff --git a/pkg/image/importer/importer.go b/pkg/image/importer/importer.go index c715d822a4ab..8ebd6647eeec 100644 --- a/pkg/image/importer/importer.go +++ b/pkg/image/importer/importer.go @@ -50,6 +50,7 @@ type ImageStreamImporter struct { limiter flowcontrol.RateLimiter digestToRepositoryCache map[gocontext.Context]map[manifestKey]*api.Image + digestToLayerSizeCache map[gocontext.Context]map[string]int64 } // NewImageStreamImport creates an importer that will load images from a remote Docker registry into an @@ -65,6 +66,7 @@ func NewImageStreamImporter(retriever RepositoryRetriever, maximumTagsPerRepo in limiter: limiter, digestToRepositoryCache: make(map[gocontext.Context]map[manifestKey]*api.Image), + digestToLayerSizeCache: make(map[gocontext.Context]map[string]int64), } } @@ -78,9 +80,27 @@ func (i *ImageStreamImporter) contextImageCache(ctx gocontext.Context) map[manif return cache } +// contextLayerSizeCache returns the image layers cache entry for a context. +func (i *ImageStreamImporter) contextLayerSizeCache(ctx gocontext.Context) map[string]int64 { + cache := i.digestToLayerSizeCache[ctx] + if cache == nil { + cache = make(map[string]int64) + i.digestToLayerSizeCache[ctx] = cache + } + return cache +} + +type imageStreamImporterCache struct { + ImageCache map[manifestKey]*api.Image + LayerSizeCache map[string]int64 +} + // Import tries to complete the provided isi object with images loaded from remote registries. func (i *ImageStreamImporter) Import(ctx gocontext.Context, isi *api.ImageStreamImport) error { - cache := i.contextImageCache(ctx) + cache := &imageStreamImporterCache{ + ImageCache: i.contextImageCache(ctx), + LayerSizeCache: i.contextLayerSizeCache(ctx), + } importImages(ctx, i.retriever, isi, cache, i.limiter) importFromRepository(ctx, i.retriever, isi, i.maximumTagsPerRepo, cache, i.limiter) return nil @@ -88,7 +108,7 @@ func (i *ImageStreamImporter) Import(ctx gocontext.Context, isi *api.ImageStream // importImages updates the passed ImageStreamImport object and sets Status for each image based on whether the import // succeeded or failed. Cache is updated with any loaded images. Limiter is optional and controls how fast images are updated. -func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, cache map[manifestKey]*api.Image, limiter flowcontrol.RateLimiter) { +func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, cache *imageStreamImporterCache, limiter flowcontrol.RateLimiter) { tags := make(map[manifestKey][]int) ids := make(map[manifestKey][]int) repositories := make(map[repositoryKey]*importRepository) @@ -128,7 +148,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api if len(ids[id]) == 1 { repo.Digests = append(repo.Digests, importDigest{ Name: defaultRef.ID, - Image: cache[id], + Image: cache.ImageCache[id], }) } } else { @@ -138,7 +158,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api if len(tags[tag]) == 1 { repo.Tags = append(repo.Tags, importTag{ Name: defaultRef.Tag, - Image: cache[tag], + Image: cache.ImageCache[tag], }) } } @@ -146,12 +166,12 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api // for each repository we found, import all tags and digests for key, repo := range repositories { - importRepositoryFromDocker(ctx, retriever, repo, limiter) + importRepositoryFromDocker(ctx, retriever, repo, limiter, cache) for _, tag := range repo.Tags { j := manifestKey{repositoryKey: key} j.value = tag.Name if tag.Image != nil { - cache[j] = tag.Image + cache.ImageCache[j] = tag.Image } for _, index := range tags[j] { if tag.Err != nil { @@ -172,7 +192,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api j := manifestKey{repositoryKey: key} j.value = digest.Name if digest.Image != nil { - cache[j] = digest.Image + cache.ImageCache[j] = digest.Image } for _, index := range ids[j] { if digest.Err != nil { @@ -194,7 +214,7 @@ func importImages(ctx gocontext.Context, retriever RepositoryRetriever, isi *api // importFromRepository imports the repository named on the ImageStreamImport, if any, importing up to maximumTags, and reporting // status on each image that is attempted to be imported. If the repository cannot be found or tags cannot be retrieved, the repository // status field is set. -func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, maximumTags int, cache map[manifestKey]*api.Image, limiter flowcontrol.RateLimiter) { +func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, isi *api.ImageStreamImport, maximumTags int, cache *imageStreamImporterCache, limiter flowcontrol.RateLimiter) { if isi.Spec.Repository == nil { return } @@ -223,7 +243,7 @@ func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, Insecure: spec.ImportPolicy.Insecure, MaximumTags: maximumTags, } - importRepositoryFromDocker(ctx, retriever, repo, limiter) + importRepositoryFromDocker(ctx, retriever, repo, limiter, cache) if repo.Err != nil { status.Status = imageImportStatus(repo.Err, "", "repository") @@ -234,7 +254,7 @@ func importFromRepository(ctx gocontext.Context, retriever RepositoryRetriever, tagKey := manifestKey{repositoryKey: key} for _, s := range repo.AdditionalTags { tagKey.value = s - if image, ok := cache[tagKey]; ok { + if image, ok := cache.ImageCache[tagKey]; ok { repo.Tags = append(repo.Tags, importTag{ Name: s, Image: image, @@ -299,9 +319,35 @@ func formatRepositoryError(repository *importRepository, refName string, refID s return } +func calculateImageSize(ctx gocontext.Context, repo distribution.Repository, image *api.Image, cache *imageStreamImporterCache) error { + bs := repo.Blobs(ctx) + + size := int64(0) + for i := range image.DockerImageLayers { + layer := &image.DockerImageLayers[i] + + if layerSize, ok := cache.LayerSizeCache[layer.Name]; ok { + size += layerSize + continue + } + + desc, err := bs.Stat(ctx, digest.Digest(layer.Name)) + if err != nil { + return err + } + + cache.LayerSizeCache[layer.Name] = desc.Size + layer.LayerSize = desc.Size + size += desc.Size + } + + image.DockerImageMetadata.Size = size + return nil +} + // importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the // optional rate limiter. Errors are set onto the individual tags and digest objects. -func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) { +func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter, cache *imageStreamImporterCache) { glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure) // retrieve the repository repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure) @@ -430,6 +476,12 @@ func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetri importDigest.Err = err continue } + if importDigest.Image.DockerImageMetadata.Size == 0 { + if err := calculateImageSize(ctx, repo, importDigest.Image, cache); err != nil { + importDigest.Err = err + continue + } + } } for i := range repository.Tags { @@ -474,6 +526,12 @@ func importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetri importTag.Err = err continue } + if importTag.Image.DockerImageMetadata.Size == 0 { + if err := calculateImageSize(ctx, repo, importTag.Image, cache); err != nil { + importTag.Err = err + continue + } + } } }