Merge branch 'master' into dont-assume-docker-PATH-in-image_util

README.md

@@ -318,6 +318,7 @@ To address this, we publish variants of the `distroless` runtime images tagged
 to make debugging easier.
 
 For example (in this repo):
+
 ```shell
 $ bazel run -c dbg testdata:go_image
 ...
@@ -379,6 +380,7 @@ _cc_image_repos()
 
 Then in your `BUILD` file, simply rewrite `cc_binary` to `cc_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//cc:image.bzl", "cc_image")
 
@@ -394,6 +396,7 @@ cc_image(
 To use `cc_image` (or `go_image`, `d_image`, `rust_image`) with an external
 `cc_binary` (or the like) target, then your `BUILD` file should instead look
 like:
+
 ```python
 load("@io_bazel_rules_docker//cc:image.bzl", "cc_image")
 
@@ -436,6 +439,7 @@ _py_image_repos()
 
 Then in your `BUILD` file, simply rewrite `py_binary` to `py_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//python:image.bzl", "py_image")
 
@@ -461,6 +465,7 @@ in a location different to the default base, please see
 For Python and Java's `lang_image` rules, you can factor
 dependencies that don't change into their own layers by overriding the
 `layers=[]` attribute.  Consider this sample from the `rules_k8s` repository:
+
 ```python
 py_image(
     name = "server",
@@ -480,6 +485,7 @@ py_image(
 
 You can also implement more complex fine layering strategies by using the
 `py_layer` rule and its `filter` attribute.  For example:
+
 ```python
 # Suppose that we are synthesizing an image that depends on a complex set
 # of libraries that we want to break into layers.
@@ -590,6 +596,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `nodejs_binary` to `nodejs_image` with
 the following import:
+
 ```python
 load("@io_bazel_rules_docker//nodejs:image.bzl", "nodejs_image")
 
@@ -636,6 +643,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `go_binary` to `go_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 
@@ -659,6 +667,7 @@ see example below.
 To use a custom base image, with any of the `lang_image`
 rules, you can override the default `base="..."` attribute.  Consider this
 modified sample from the `distroless` repository:
+
 ```python
 load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 
@@ -733,6 +742,7 @@ _java_image_repos()
 
 Then in your `BUILD` file, simply rewrite `java_binary` to `java_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//java:image.bzl", "java_image")
 
@@ -776,6 +786,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `java_war` to `war_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//java:image.bzl", "war_image")
 
@@ -841,6 +852,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `scala_binary` to `scala_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//scala:image.bzl", "scala_image")
 
@@ -898,6 +910,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `groovy_binary` to `groovy_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//groovy:image.bzl", "groovy_image")
 
@@ -955,6 +968,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `rust_binary` to `rust_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//rust:image.bzl", "rust_image")
 
@@ -1011,6 +1025,7 @@ lines to your `WORKSPACE`.
 
 Then in your `BUILD` file, simply rewrite `d_binary` to `d_image` with the
 following import:
+
 ```python
 load("@io_bazel_rules_docker//d:image.bzl", "d_image")
 
@@ -1088,6 +1103,7 @@ to use container_push with custom docker authentication credentials.
 ### container_push (Custom client configuration)
 If you wish to use container_push using custom docker authentication credentials,
 in `WORKSPACE`:
+
 ```python
 # Download the rules_docker repository
 http_archive(
@@ -1112,6 +1128,7 @@ docker_toolchain_configure(
 )
 ```
 In `BUILD` file:
+
 ```python
 load("@io_bazel_rules_docker//container:container.bzl", "container_push")
 
@@ -1263,6 +1280,7 @@ The digest references to the `distroless` base images must be updated over time
 to pick up bug fixes and security patches.  To facilitate this, the files
 containing the digest references are generated by `tools/update_deps.py`.  To
 update all of the dependencies, please run (from the root of the repository):
+
 ```shell
 ./update_deps.sh
 ```

WORKSPACE

@@ -323,9 +323,9 @@ _go_image_repos()
 # For our rust_image test
 http_archive(
     name = "io_bazel_rules_rust",
-    sha256 = "2f33532e19e08cb4f5808685bb8c3360e5d56c3e486a23a5cab5de4724b84632",
-    strip_prefix = "rules_rust-29acd8fe69dd20d9a306b3f12cd1e7393682e239",
-    urls = ["https://github.com/bazelbuild/rules_rust/archive/29acd8fe69dd20d9a306b3f12cd1e7393682e239.tar.gz"],
+    sha256 = "cb3630589dd8041ce11edee299182c960de78f7dfb8627842aa53c23ecd2f5f7",
+    strip_prefix = "rules_rust-b94d3a40787343c0a08fdc15f63927b298dca561",
+    urls = ["https://github.com/bazelbuild/rules_rust/archive/b94d3a40787343c0a08fdc15f63927b298dca561.tar.gz"],
 )
 
 load("@io_bazel_rules_rust//rust:repositories.bzl", "rust_repositories")

docker/util/BUILD

@@ -40,3 +40,20 @@ bzl_library(
     name = "util",
     srcs = ["run.bzl"],
 )
+
+# Re-package config_stripper as a library to be used in unit tests.
+py_library(
+    name = "config_stripper_lib",
+    srcs = [
+        "config_stripper.py",
+    ],
+)
+
+py_test(
+    name = "config_stripper_test",
+    size = "medium",
+    srcs = ["config_stripper_test.py"],
+    data = ["//docker/util/testdata:image_with_symlinked_layer.tar"],
+    python_version = "PY3",
+    deps = [":config_stripper_lib"],
+)

docker/util/config_stripper.py

@@ -59,11 +59,22 @@ def strip_tar(input, output):
         # Scrape each layer for any timestamps
         new_layers = []
         new_diff_ids = []
-        for layer in image['Layers']:
+        # Process the layers in reverse to fix https://github.com/bazelbuild/rules_docker/issues/1104.
+        # Image tarballs generated by "docker save" can include layer tarballs
+        # that are just symlinks to the previous layer. In the Docker manifest,
+        # the layers are ordered from top to bottom. Thus, as the stripper goes
+        # through these layers in top to bottom order, it may encounter a layer
+        # that symlinks to a lower layer that hasn't been extracted yet. Just
+        # reversing the iteration order avoids this problem.
+        for layer in reversed(image['Layers']):
           (new_layer_name, new_diff_id) = strip_layer(os.path.join(tempdir, layer))
 
           new_layers.append(new_layer_name)
           new_diff_ids.append(new_diff_id)
+        # Reverse the layer order to go back to the topmost layer to bottom as
+        # required by the Docker manifest format.
+        new_layers = [l for l in reversed(new_layers)]
+        new_diff_ids = [d for d in reversed(new_diff_ids)]
 
         # Change the manifest to reflect the new layer name
         image['Layers'] = new_layers

docker/util/config_stripper_test.py

@@ -0,0 +1,40 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Lint as: python3
+"""Tests for config_stripper."""
+
+import os
+import unittest
+
+from docker.util.config_stripper import strip_tar
+
+class ConfigStripperTest(unittest.TestCase):
+    def test_image_with_symlinked_layers(self):
+        # Ensure the config stripper can strip a tar with symlinked layers.
+        # https://github.com/bazelbuild/rules_docker/issues/1104.
+        img_tar = os.path.join(
+            os.environ['TEST_SRCDIR'],
+            'io_bazel_rules_docker',
+            'docker/util/testdata/image_with_symlinked_layer.tar')
+        out_tar = os.path.join(
+            os.environ["TEST_TMPDIR"],
+            "test_image_with_symlinked_layers_out.tar")
+        strip_tar(img_tar, out_tar)
+        if not os.path.exists(out_tar):
+            self.fail(
+                "Config stripper did not produce stripped tarball {}".format(
+                    out_tar))
+
+if __name__ == '__main__':
+  unittest.main()

docker/util/testdata/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files([
+    "image_with_symlinked_layer.tar",
+])

repositories/repositories.bzl

@@ -105,18 +105,18 @@ def repositories():
     if "io_bazel_rules_go" not in excludes:
         http_archive(
             name = "io_bazel_rules_go",
-            sha256 = "842ec0e6b4fbfdd3de6150b61af92901eeb73681fd4d185746644c338f51d4c0",
+            sha256 = "b9aa86ec08a292b97ec4591cf578e020b35f98e12173bbd4a921f84f583aebd9",
             urls = [
-                "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.1/rules_go-v0.20.1.tar.gz",
-                "https://github.com/bazelbuild/rules_go/releases/download/0.20.1/rules_go-0.20.1.tar.gz",
+                "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.2/rules_go-v0.20.2.tar.gz",
+                "https://github.com/bazelbuild/rules_go/releases/download/v0.20.2/rules_go-v0.20.2.tar.gz",
             ],
         )
     if "rules_python" not in excludes:
         http_archive(
             name = "rules_python",
-            sha256 = "4ef098d1c89e25f2e14bde2714a1f2f1899009d4b6ccab864f347252ff1ef746",
-            strip_prefix = "rules_python-f46e953f6e0315a3f884154f9395a32ec9999eab",
-            urls = ["https://github.com/bazelbuild/rules_python/archive/f46e953f6e0315a3f884154f9395a32ec9999eab.tar.gz"],
+            sha256 = "43fb8fd1dc950f1def50b04b220eda8a27e0608a5492aa4f164dc5b34803ff5e",
+            strip_prefix = "rules_python-a610acec1653001b736338e4bc2dbe9609ed86b2",
+            urls = ["https://github.com/bazelbuild/rules_python/archive/a610acec1653001b736338e4bc2dbe9609ed86b2.tar.gz"],
         )
 
     if "httplib2" not in excludes: