diff --git a/pkg/cmd/experimental/imageprune/imageprune.go b/pkg/cmd/experimental/imageprune/imageprune.go index ddce28cad2ce..91a8ab750c2d 100644 --- a/pkg/cmd/experimental/imageprune/imageprune.go +++ b/pkg/cmd/experimental/imageprune/imageprune.go @@ -5,7 +5,6 @@ import ( "io" "net/http" - kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/golang/glog" imageapi "github.com/openshift/origin/pkg/image/api" "github.com/openshift/origin/pkg/image/prune" @@ -18,32 +17,15 @@ import ( const longDesc = ` ` -type registryURLs []string - -func (u *registryURLs) Type() string { - return "string" -} - -func (u *registryURLs) String() string { - return fmt.Sprintf("%v", *u) -} - -func (u *registryURLs) Set(value string) error { - *u = append(*u, value) - return nil -} - type config struct { DryRun bool - RegistryURLs registryURLs MinimumResourcePruningAge int TagRevisionsToKeep int } func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Writer) *cobra.Command { cfg := &config{ - DryRun: true, - RegistryURLs: registryURLs{"docker-registry.default.local"}, + DryRun: true, MinimumResourcePruningAge: 60, TagRevisionsToKeep: 3, } @@ -63,13 +45,7 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri glog.Fatalf("Error getting client: %v", err) } - if registryService, err := kClient.Services(kapi.NamespaceDefault).Get("docker-registry"); err != nil { - glog.Errorf("Error getting docker-registry service: %v", err) - } else { - cfg.RegistryURLs = append(cfg.RegistryURLs, fmt.Sprintf("%s:%d", registryService.Spec.PortalIP, registryService.Spec.Ports[0].Port)) - } - - pruner, err := prune.NewImagePruner(cfg.RegistryURLs, cfg.MinimumResourcePruningAge, cfg.TagRevisionsToKeep, osClient, osClient, kClient, kClient, osClient, osClient, osClient) + pruner, err := prune.NewImagePruner(cfg.MinimumResourcePruningAge, cfg.TagRevisionsToKeep, osClient, osClient, kClient, kClient, osClient, osClient, osClient) if err != nil { glog.Fatalf("Error creating image pruner: %v", err) } @@ -103,9 +79,8 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri } cmd.Flags().BoolVar(&cfg.DryRun, "dry-run", cfg.DryRun, "Perform an image pruning dry-run, displaying what would be deleted but not actually deleting anything (default=true).") - cmd.Flags().Var(&cfg.RegistryURLs, "registry-urls", "TODO") - cmd.Flags().IntVar(&cfg.MinimumResourcePruningAge, "older-than", cfg.MinimumResourcePruningAge, "TODO") - cmd.Flags().IntVar(&cfg.TagRevisionsToKeep, "keep-tag-revisions", cfg.TagRevisionsToKeep, "TODO") + cmd.Flags().IntVar(&cfg.MinimumResourcePruningAge, "older-than", cfg.MinimumResourcePruningAge, "Specify the minimum age for an image to be prunable, as well as the minimum age for an image stream or pod that references an image to be prunable.") + cmd.Flags().IntVar(&cfg.TagRevisionsToKeep, "keep-tag-revisions", cfg.TagRevisionsToKeep, "Specify the number of image revisions for a tag in an image stream that will be preserved.") return cmd } diff --git a/pkg/dockerregistry/server/repositorymiddleware.go b/pkg/dockerregistry/server/repositorymiddleware.go index 34b4f4a0357f..9ef1b6023cdd 100644 --- a/pkg/dockerregistry/server/repositorymiddleware.go +++ b/pkg/dockerregistry/server/repositorymiddleware.go @@ -161,6 +161,9 @@ func (r *repository) Put(ctx context.Context, manifest *manifest.SignedManifest) Image: imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: dgst.String(), + Annotations: map[string]string{ + imageapi.ManagedByOpenShiftAnnotation: "true", + }, }, DockerImageReference: fmt.Sprintf("%s/%s/%s@%s", r.registryAddr, r.namespace, r.name, dgst.String()), DockerImageManifest: string(payload), diff --git a/pkg/image/api/types.go b/pkg/image/api/types.go index 816d57c0e3cb..48818acb6f0b 100644 --- a/pkg/image/api/types.go +++ b/pkg/image/api/types.go @@ -13,6 +13,8 @@ type ImageList struct { Items []Image `json:"items"` } +const ManagedByOpenShiftAnnotation = "openshift.io/image.managed" + // Image is an immutable representation of a Docker image and metadata at a point in time. type Image struct { kapi.TypeMeta `json:",inline"` diff --git a/pkg/image/prune/imagepruner.go b/pkg/image/prune/imagepruner.go index a7575edb2a33..9202b44603a7 100644 --- a/pkg/image/prune/imagepruner.go +++ b/pkg/image/prune/imagepruner.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "strings" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" @@ -28,23 +27,10 @@ import ( // pruneAlgorithm contains the various settings to use when evaluating images // and layers for pruning. type pruneAlgorithm struct { - registryURLs []string minimumAgeInMinutesToPrune int tagRevisionsToKeep int } -// externalImage returns true if the image belongs to an external Docker -// registry; i.e., a registry not controlled by OpenShift. -func (pa pruneAlgorithm) externalImage(image string) bool { - for _, url := range pa.registryURLs { - if strings.HasPrefix(image, url) { - return false - } - } - - return true -} - // ImagePruneFunc is a function that is invoked for each image that is // prunable, along with the list of image streams that reference it. type ImagePruneFunc func(image *imageapi.Image, streams []*imageapi.ImageStream) []error @@ -71,9 +57,6 @@ var _ ImagePruner = &imagePruner{} /* NewImagePruner creates a new ImagePruner. -registryURLs is a list of OpenShift registries. Only images with URLs -belonging to this list are candidates for pruning. - minimumAgeInMinutesToPrune is the minimum age, in minutes, that a resource must be in order for the image it references (or an image itself) to be a candidate for pruning. For example, if minimumAgeInMinutesToPrune is 60, and @@ -87,9 +70,9 @@ than tagRevisionsToKeep is eligible for pruning. images, streams, pods, rcs, bcs, builds, and dcs are client interfaces for retrieving each respective resource type. -The ImagePruner performs the following logic: remove any image belonging to the -specified registry URL(s) that was created at least *n* minutes ago and is -*not* currently referenced by: +The ImagePruner performs the following logic: remove any image contaning the +annotation openshift.io/image.managed=true that was created at least *n* +minutes ago and is *not* currently referenced by: - any pod created less than *n* minutes ago - any image stream created less than *n* minutes ago @@ -107,7 +90,7 @@ ImageStreams having a reference to the image in `status.tags`. Also automatically remove any image layer that is no longer referenced by any images. */ -func NewImagePruner(registryURLs []string, minimumAgeInMinutesToPrune int, tagRevisionsToKeep int, images client.ImagesInterfacer, streams client.ImageStreamsNamespacer, pods kclient.PodsNamespacer, rcs kclient.ReplicationControllersNamespacer, bcs client.BuildConfigsNamespacer, builds client.BuildsNamespacer, dcs client.DeploymentConfigsNamespacer) (ImagePruner, error) { +func NewImagePruner(minimumAgeInMinutesToPrune int, tagRevisionsToKeep int, images client.ImagesInterfacer, streams client.ImageStreamsNamespacer, pods kclient.PodsNamespacer, rcs kclient.ReplicationControllersNamespacer, bcs client.BuildConfigsNamespacer, builds client.BuildsNamespacer, dcs client.DeploymentConfigsNamespacer) (ImagePruner, error) { allImages, err := images.Images().List(labels.Everything(), fields.Everything()) if err != nil { return nil, fmt.Errorf("Error listing images: %v", err) @@ -143,17 +126,16 @@ func NewImagePruner(registryURLs []string, minimumAgeInMinutesToPrune int, tagRe return nil, fmt.Errorf("Error listing deployment configs: %v", err) } - return newImagePruner(registryURLs, minimumAgeInMinutesToPrune, tagRevisionsToKeep, allImages, allStreams, allPods, allRCs, allBCs, allBuilds, allDCs), nil + return newImagePruner(minimumAgeInMinutesToPrune, tagRevisionsToKeep, allImages, allStreams, allPods, allRCs, allBCs, allBuilds, allDCs), nil } // newImagePruner creates a new ImagePruner. -func newImagePruner(registryURLs []string, minimumAgeInMinutesToPrune int, tagRevisionsToKeep int, images *imageapi.ImageList, streams *imageapi.ImageStreamList, pods *kapi.PodList, rcs *kapi.ReplicationControllerList, bcs *buildapi.BuildConfigList, builds *buildapi.BuildList, dcs *deployapi.DeploymentConfigList) ImagePruner { +func newImagePruner(minimumAgeInMinutesToPrune int, tagRevisionsToKeep int, images *imageapi.ImageList, streams *imageapi.ImageStreamList, pods *kapi.PodList, rcs *kapi.ReplicationControllerList, bcs *buildapi.BuildConfigList, builds *buildapi.BuildList, dcs *deployapi.DeploymentConfigList) ImagePruner { g := graph.New() - glog.V(1).Infof("Creating image pruner with registryURLs=%v, minimumAgeInMinutesToPrune=%d, tagRevisionsToKeep=%d", registryURLs, minimumAgeInMinutesToPrune, tagRevisionsToKeep) + glog.V(1).Infof("Creating image pruner with minimumAgeInMinutesToPrune=%d, tagRevisionsToKeep=%d", minimumAgeInMinutesToPrune, tagRevisionsToKeep) algorithm := pruneAlgorithm{ - registryURLs: registryURLs, minimumAgeInMinutesToPrune: minimumAgeInMinutesToPrune, tagRevisionsToKeep: tagRevisionsToKeep, } @@ -161,10 +143,10 @@ func newImagePruner(registryURLs []string, minimumAgeInMinutesToPrune int, tagRe addImagesToGraph(g, images, algorithm) addImageStreamsToGraph(g, streams, algorithm) addPodsToGraph(g, pods, algorithm) - addReplicationControllersToGraph(g, rcs, algorithm) - addBuildConfigsToGraph(g, bcs, algorithm) - addBuildsToGraph(g, builds, algorithm) - addDeploymentConfigsToGraph(g, dcs, algorithm) + addReplicationControllersToGraph(g, rcs) + addBuildConfigsToGraph(g, bcs) + addBuildsToGraph(g, builds) + addDeploymentConfigsToGraph(g, dcs) return &imagePruner{ g: g, @@ -180,8 +162,14 @@ func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm prune for i := range images.Items { image := &images.Items[i] - if algorithm.externalImage(image.DockerImageReference) { - glog.V(4).Infof("Image %q belongs to an external registry - skipping", image.DockerImageReference) + glog.V(4).Infof("Examining image %q", image.Name) + + if image.Annotations == nil { + glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference) + continue + } + if value, ok := image.Annotations[imageapi.ManagedByOpenShiftAnnotation]; !ok || value != "true" { + glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference) continue } @@ -221,6 +209,8 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, al for i := range streams.Items { stream := &streams.Items[i] + glog.V(4).Infof("Examining image stream %s/%s", stream.Namespace, stream.Name) + // use a weak reference for old image revisions by default oldImageRevisionReferenceKind := graph.WeakReferencedImageGraphEdgeKind @@ -237,14 +227,9 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, al for tag, history := range stream.Status.Tags { for i := range history.Items { - if algorithm.externalImage(history.Items[i].DockerImageReference) { - glog.V(4).Infof("Tag %q revision %d points to %s which is part of an external registry; skipping", tag, i, history.Items[i].DockerImageReference) - continue - } - n := graph.FindImage(g, history.Items[i].Image) if n == nil { - glog.V(1).Infof("Unable to find image %q in graph", history.Items[i].Image) + glog.V(1).Infof("Unable to find image %q in graph (from tag %q, revision %d, dockerImageReference %s)", history.Items[i].Image, tag, i, history.Items[i].DockerImageReference) continue } imageNode := n.(*graph.ImageNode) @@ -280,12 +265,13 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, al // defined by algorithm. // // Edges are added to the graph from each pod to the images specified by that -// pod's list of containers, as long as the image belongs to one of the -// registries specified in algorithm. +// pod's list of containers, as long as the image is managed by OpenShift. func addPodsToGraph(g graph.Graph, pods *kapi.PodList, algorithm pruneAlgorithm) { for i := range pods.Items { pod := &pods.Items[i] + glog.V(4).Infof("Examining pod %s/%s", pod.Namespace, pod.Name) + if pod.Status.Phase != kapi.PodRunning && pod.Status.Phase != kapi.PodPending { age := util.Now().Sub(pod.CreationTimestamp.Time) if int(age.Minutes()) >= algorithm.minimumAgeInMinutesToPrune { @@ -298,22 +284,18 @@ func addPodsToGraph(g graph.Graph, pods *kapi.PodList, algorithm pruneAlgorithm) glog.V(4).Infof("Adding pod %s/%s to graph", pod.Namespace, pod.Name) podNode := graph.Pod(g, pod) - addPodSpecToGraph(g, &pod.Spec, podNode, algorithm) + addPodSpecToGraph(g, &pod.Spec, podNode) } } // Edges are added to the graph from each predecessor (pod or replication // controller) to the images specified by the pod spec's list of containers, as -// long as the image belongs to one of the registries specified in algorithm. -func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node, algorithm pruneAlgorithm) { +// long as the image is managed by OpenShift. +func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node) { for j := range spec.Containers { container := spec.Containers[j] glog.V(4).Infof("Examining container image %q", container.Image) - if algorithm.externalImage(container.Image) { - glog.V(4).Infof("Image belongs to an external registry - skipping") - continue - } ref, err := imageapi.ParseDockerImageReference(container.Image) if err != nil { @@ -322,13 +304,13 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node } if len(ref.ID) == 0 { - glog.Errorf("Missing image ID") + glog.V(4).Infof("%q has no image ID", container.Image) continue } imageNode := graph.FindImage(g, ref.ID) if imageNode == nil { - glog.Errorf("Expected to find image %q in the graph, but it was missing", ref.ID) + glog.Infof("Unable to find image %q in the graph", ref.ID) continue } @@ -340,75 +322,83 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node // addReplicationControllersToGraph adds replication controllers to the graph. // // Edges are added to the graph from each replication controller to the images -// specified by its pod spec's list of containers, as long as the image belongs -// to one of the registries specified in algorithm. -func addReplicationControllersToGraph(g graph.Graph, rcs *kapi.ReplicationControllerList, algorithm pruneAlgorithm) { +// specified by its pod spec's list of containers, as long as the image is +// managed by OpenShift. +func addReplicationControllersToGraph(g graph.Graph, rcs *kapi.ReplicationControllerList) { for i := range rcs.Items { rc := &rcs.Items[i] + glog.V(4).Infof("Examining replication controller %s/%s", rc.Namespace, rc.Name) rcNode := graph.ReplicationController(g, rc) - addPodSpecToGraph(g, &rc.Spec.Template.Spec, rcNode, algorithm) + addPodSpecToGraph(g, &rc.Spec.Template.Spec, rcNode) } } // addDeploymentConfigsToGraph adds deployment configs to the graph. // // Edges are added to the graph from each deployment config to the images -// specified by its pod spec's list of containers, as long as the image belongs -// to one of the registries specified in algorithm. -func addDeploymentConfigsToGraph(g graph.Graph, dcs *deployapi.DeploymentConfigList, algorithm pruneAlgorithm) { +// specified by its pod spec's list of containers, as long as the image is +// managed by OpenShift. +func addDeploymentConfigsToGraph(g graph.Graph, dcs *deployapi.DeploymentConfigList) { for i := range dcs.Items { dc := &dcs.Items[i] + glog.V(4).Infof("Examining deployment config %s/%s", dc.Namespace, dc.Name) dcNode := graph.DeploymentConfig(g, dc) - addPodSpecToGraph(g, &dc.Template.ControllerTemplate.Template.Spec, dcNode, algorithm) + addPodSpecToGraph(g, &dc.Template.ControllerTemplate.Template.Spec, dcNode) } } // addBuildConfigsToGraph adds build configs to the graph. // // Edges are added to the graph from each build config to the image specified by its strategy.from. -func addBuildConfigsToGraph(g graph.Graph, bcs *buildapi.BuildConfigList, algorithm pruneAlgorithm) { +func addBuildConfigsToGraph(g graph.Graph, bcs *buildapi.BuildConfigList) { for i := range bcs.Items { bc := &bcs.Items[i] + glog.V(4).Infof("Examining build config %s/%s", bc.Namespace, bc.Name) bcNode := graph.BuildConfig(g, bc) - addBuildStrategyImageReferencesToGraph(g, bc.Parameters.Strategy, bcNode, algorithm) + addBuildStrategyImageReferencesToGraph(g, bc.Parameters.Strategy, bcNode) } } // addBuildsToGraph adds builds to the graph. // // Edges are added to the graph from each build to the image specified by its strategy.from. -func addBuildsToGraph(g graph.Graph, builds *buildapi.BuildList, algorithm pruneAlgorithm) { +func addBuildsToGraph(g graph.Graph, builds *buildapi.BuildList) { for i := range builds.Items { build := &builds.Items[i] + glog.V(4).Infof("Examining build %s/%s", build.Namespace, build.Name) buildNode := graph.Build(g, build) - addBuildStrategyImageReferencesToGraph(g, build.Parameters.Strategy, buildNode, algorithm) + addBuildStrategyImageReferencesToGraph(g, build.Parameters.Strategy, buildNode) } } // Edges are added to the graph from each predecessor (build or build config) -// to the image specified by strategy.from, as long as the image belongs to one -// of the registries specified in algorithm. -func addBuildStrategyImageReferencesToGraph(g graph.Graph, strategy buildapi.BuildStrategy, predecessor gonum.Node, algorithm pruneAlgorithm) { +// to the image specified by strategy.from, as long as the image is managed by +// OpenShift. +func addBuildStrategyImageReferencesToGraph(g graph.Graph, strategy buildapi.BuildStrategy, predecessor gonum.Node) { + glog.V(4).Infof("Examining build strategy with type %q", strategy.Type) + from := buildutil.GetImageStreamForStrategy(strategy) if from == nil { + glog.V(4).Infof("Unable to determine 'from' reference - skipping") return } + glog.V(4).Infof("Examining build strategy with from: %#v", from) + var imageID string switch from.Kind { case "ImageStreamImage": _, id, err := imagestreamimage.ParseNameAndID(from.Name) if err != nil { + glog.V(4).Infof("Error parsing ImageStreamImage name %q: %v - skipping", from.Name, err) return } imageID = id case "DockerImage": - if algorithm.externalImage(from.Name) { - return - } ref, err := imageapi.ParseDockerImageReference(from.Name) if err != nil { + glog.V(4).Infof("Error parsing DockerImage name %q: %v - skipping", from.Name, err) return } imageID = ref.ID @@ -416,11 +406,14 @@ func addBuildStrategyImageReferencesToGraph(g graph.Graph, strategy buildapi.Bui return } + glog.V(4).Infof("Looking for image %q in graph", imageID) imageNode := graph.FindImage(g, imageID) if imageNode == nil { + glog.V(4).Infof("Unable to find image %q in graph - skipping", imageID) return } + glog.V(4).Infof("Adding edge from %v to %v", predecessor, imageNode) g.AddEdge(predecessor, imageNode, graph.ReferencedImageGraphEdgeKind) } @@ -568,7 +561,7 @@ func pruneLayers(g graph.Graph, layerNodes []*graph.ImageLayerNode, layerPruneFu ref, err := imageapi.DockerImageReferenceForStream(stream) if err != nil { //TODO - glog.Errorf("Error constructing DockerImageReference for %q", streamName) + glog.Errorf("Error constructing DockerImageReference for %q: %v", streamName, err) continue } diff --git a/pkg/image/prune/imagepruner_test.go b/pkg/image/prune/imagepruner_test.go index 21337538eebf..c434f88a73c4 100644 --- a/pkg/image/prune/imagepruner_test.go +++ b/pkg/image/prune/imagepruner_test.go @@ -1,6 +1,7 @@ package prune import ( + "encoding/json" "flag" "fmt" "reflect" @@ -23,31 +24,13 @@ func imageList(images ...imageapi.Image) imageapi.ImageList { } func agedImage(id, ref string, ageInMinutes int64) imageapi.Image { - image := imageapi.Image{ - ObjectMeta: kapi.ObjectMeta{ - Name: id, - }, - DockerImageReference: ref, - DockerImageManifest: `{ - "fsLayers": [ - { - "blobSum": "tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - }, - { - "blobSum": "tarsum.dev+sha256:b194de3772ebbcdc8f244f663669799ac1cb141834b7cb8b69100285d357a2b0" - }, - { - "blobSum": "tarsum.dev+sha256:c937c4bb1c1a21cc6d94340812262c6472092028972ae69b551b1a70d4276171" - }, - { - "blobSum": "tarsum.dev+sha256:2aaacc362ac6be2b9e9ae8c6029f6f616bb50aec63746521858e47841b90fabd" - }, - { - "blobSum": "tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - } - ] - }`, - } + image := imageWithLayers(id, ref, + "tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tarsum.dev+sha256:b194de3772ebbcdc8f244f663669799ac1cb141834b7cb8b69100285d357a2b0", + "tarsum.dev+sha256:c937c4bb1c1a21cc6d94340812262c6472092028972ae69b551b1a70d4276171", + "tarsum.dev+sha256:2aaacc362ac6be2b9e9ae8c6029f6f616bb50aec63746521858e47841b90fabd", + "tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) if ageInMinutes >= 0 { image.CreationTimestamp = util.NewTime(util.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute)) @@ -60,6 +43,35 @@ func image(id, ref string) imageapi.Image { return agedImage(id, ref, -1) } +func imageWithLayers(id, ref string, layers ...string) imageapi.Image { + image := imageapi.Image{ + ObjectMeta: kapi.ObjectMeta{ + Name: id, + Annotations: map[string]string{ + imageapi.ManagedByOpenShiftAnnotation: "true", + }, + }, + DockerImageReference: ref, + } + + manifest := imageapi.DockerImageManifest{ + FSLayers: []imageapi.DockerFSLayer{}, + } + + for _, layer := range layers { + manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer}) + } + + manifestBytes, err := json.Marshal(&manifest) + if err != nil { + panic(err) + } + + image.DockerImageManifest = string(manifestBytes) + + return image +} + func podList(pods ...kapi.Pod) kapi.PodList { return kapi.PodList{ Items: pods, @@ -108,17 +120,18 @@ func streamList(streams ...imageapi.ImageStream) imageapi.ImageStreamList { } } -func stream(namespace, name string, tags map[string]imageapi.TagEventList) imageapi.ImageStream { - return agedStream(namespace, name, -1, tags) +func stream(registry, namespace, name string, tags map[string]imageapi.TagEventList) imageapi.ImageStream { + return agedStream(registry, namespace, name, -1, tags) } -func agedStream(namespace, name string, ageInMinutes int64, tags map[string]imageapi.TagEventList) imageapi.ImageStream { +func agedStream(registry, namespace, name string, ageInMinutes int64, tags map[string]imageapi.TagEventList) imageapi.ImageStream { stream := imageapi.ImageStream{ ObjectMeta: kapi.ObjectMeta{ Namespace: namespace, Name: name, }, Status: imageapi.ImageStreamStatus{ + DockerImageRepository: fmt.Sprintf("%s/%s/%s", registry, namespace, name), Tags: tags, }, } @@ -130,8 +143,8 @@ func agedStream(namespace, name string, ageInMinutes int64, tags map[string]imag return stream } -func streamPtr(namespace, name string, tags map[string]imageapi.TagEventList) *imageapi.ImageStream { - s := stream(namespace, name, tags) +func streamPtr(registry, namespace, name string, tags map[string]imageapi.TagEventList) *imageapi.ImageStream { + s := stream(registry, namespace, name, tags) return &s } @@ -276,7 +289,7 @@ func buildParameters(strategyType buildapi.BuildStrategyType, fromKind, fromName var logLevel = flag.Int("loglevel", 0, "") -func TestRun(t *testing.T) { +func TestImagePruning(t *testing.T) { flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel)) registryURL := "registry" @@ -431,7 +444,7 @@ func TestRun(t *testing.T) { image("id4", registryURL+"/foo/bar@id4"), ), streams: streamList( - stream("foo", "bar", tags( + stream(registryURL, "foo", "bar", tags( tag("latest", tagEvent("id", registryURL+"/foo/bar@id"), tagEvent("id2", registryURL+"/foo/bar@id2"), @@ -449,7 +462,7 @@ func TestRun(t *testing.T) { image("id2", registryURL+"/foo/bar@id2"), ), streams: streamList( - agedStream("foo", "bar", 5, tags( + agedStream(registryURL, "foo", "bar", 5, tags( tag("latest", tagEvent("id", registryURL+"/foo/bar@id"), tagEvent("id2", registryURL+"/foo/bar@id2"), @@ -465,7 +478,7 @@ func TestRun(t *testing.T) { image("id2", registryURL+"/foo/bar@id2"), ), streams: streamList( - stream("foo", "bar", tags( + stream(registryURL, "foo", "bar", tags( tag("latest", tagEvent("id", registryURL+"/foo/bar@id"), tagEvent("id2", registryURL+"/foo/bar@id2"), @@ -483,11 +496,7 @@ func TestRun(t *testing.T) { } for name, test := range tests { - registryURLs := []string{registryURL} - if len(test.registryURLs) > 0 { - registryURLs = test.registryURLs - } - p := newImagePruner(registryURLs, 60, 3, &test.images, &test.streams, &test.pods, &test.rcs, &test.bcs, &test.builds, &test.dcs) + p := newImagePruner(60, 3, &test.images, &test.streams, &test.pods, &test.rcs, &test.bcs, &test.builds, &test.dcs) actualDeletions := util.NewStringSet() actualUpdatedStreams := util.NewStringSet() @@ -518,6 +527,8 @@ func TestRun(t *testing.T) { } func TestDefaultImagePruneFunc(t *testing.T) { + registryURL := "registry" + tests := map[string]struct { referencedStreams []*imageapi.ImageStream expectedUpdates []*imageapi.ImageStream @@ -528,7 +539,7 @@ func TestDefaultImagePruneFunc(t *testing.T) { }, "1 tag, 1 image revision": { referencedStreams: []*imageapi.ImageStream{ - streamPtr("foo", "bar", tags( + streamPtr(registryURL, "foo", "bar", tags( tag("latest", tagEvent("id", "registry/foo/bar@id"), ), @@ -538,7 +549,7 @@ func TestDefaultImagePruneFunc(t *testing.T) { }, "1 tag, multiple image revisions": { referencedStreams: []*imageapi.ImageStream{ - streamPtr("foo", "bar", tags( + streamPtr(registryURL, "foo", "bar", tags( tag("latest", tagEvent("id", "registry/foo/bar@id"), tagEvent("id2", "registry/foo/bar@id2"), @@ -546,7 +557,7 @@ func TestDefaultImagePruneFunc(t *testing.T) { )), }, expectedUpdates: []*imageapi.ImageStream{ - streamPtr("foo", "bar", tags( + streamPtr(registryURL, "foo", "bar", tags( tag("latest", tagEvent("id", "registry/foo/bar@id"), ), @@ -580,3 +591,79 @@ func TestDefaultImagePruneFunc(t *testing.T) { } } } + +func TestLayerPruning(t *testing.T) { + flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel)) + registryURL := "registry1" + + tests := map[string]struct { + images imageapi.ImageList + streams imageapi.ImageStreamList + expectedDeletions map[string]util.StringSet + expectedStreamUpdates map[string]util.StringSet + }{ + "layers unique to id1 pruned": { + images: imageList( + imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), + imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"), + ), + streams: streamList( + stream(registryURL, "foo", "bar", tags( + tag("latest", + tagEvent("id2", registryURL+"/foo/bar@id2"), + tagEvent("id1", registryURL+"/foo/bar@id1"), + ), + )), + ), + expectedDeletions: map[string]util.StringSet{ + "registry1": util.NewStringSet("layer1", "layer2"), + }, + }, + } + + for name, test := range tests { + actualDeletions := map[string]util.StringSet{} + actualUpdatedStreams := map[string]util.StringSet{} + + imagePruneFunc := func(image *imageapi.Image, streams []*imageapi.ImageStream) []error { + return []error{} + } + + layerPruneFunc := func(registryURL string, req dockerregistry.DeleteLayersRequest) []error { + registryDeletions, ok := actualDeletions[registryURL] + if !ok { + registryDeletions = util.NewStringSet() + } + streamUpdates, ok := actualUpdatedStreams[registryURL] + if !ok { + streamUpdates = util.NewStringSet() + } + + for layer, streams := range req { + registryDeletions.Insert(layer) + streamUpdates.Insert(streams...) + } + + actualDeletions[registryURL] = registryDeletions + actualUpdatedStreams[registryURL] = streamUpdates + + return []error{} + } + + p := newImagePruner(60, 1, &test.images, &test.streams, &kapi.PodList{}, &kapi.ReplicationControllerList{}, &buildapi.BuildConfigList{}, &buildapi.BuildList{}, &deployapi.DeploymentConfigList{}) + + p.Run(imagePruneFunc, layerPruneFunc) + + if !reflect.DeepEqual(test.expectedDeletions, actualDeletions) { + t.Errorf("%s: expected layer deletions %#v, got %#v", name, test.expectedDeletions, actualDeletions) + } + + /* + expectedUpdatedStreams := util.NewStringSet(test.expectedUpdatedStreams...) + if !reflect.DeepEqual(expectedUpdatedStreams, actualUpdatedStreams) { + t.Errorf("%s: expected stream updates %q, got %q", name, expectedUpdatedStreams.List(), actualUpdatedStreams.List()) + } + */ + + } +}