tests/e2e: build the pre-install-payload image

Currently changes on install/pre-install-payload directory aren't tested
because the scripts aren't re-building the pre-install-payload image.
With this change the image will always be built and used.

It was added more two dependencies:
- kustomize: used to edit the kustomization file so to update the pre-install-payload
   image
- qemu-user-static: used by docker buildx to build the pre-install-payload image for
  multiple architectures. It also needs to pass the `--insecure` to
`docker manifest` commands because the image is pushed/pulled to a local
insecure registry, otherwise `docker manifest` fails

Fixes #177
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>

tests/e2e/ansible/group_vars/all

@@ -5,6 +5,7 @@ build_pkgs:
   ubuntu:
     - make
     - gcc
+    - qemu-user-static
   centos:
     - make
     - gcc

tests/e2e/ansible/install_build_deps.yml

@@ -32,6 +32,19 @@
         dest: /usr/local/bin/operator-sdk
         mode: '+x'
     - import_tasks: "install_docker.yml"
+      # Docker buildx relies on qemu-user-static to multi-arch builds, but
+      # qemu-user-static is not packaged for CentOS. Let's get it installed via
+      # https://github.com/multiarch/qemu-user-static
+    - name: Handle qemu-user-static installation on CentOS.
+      block:
+        - name: Check qemu-user-static is installed
+          shell: docker run --rm -t s390x/ubuntu uname -m
+          register: qemu_user_static_exist
+          ignore_errors: yes
+        - name: Install qemu-user-static
+          shell: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+          when: qemu_user_static_exist.rc != 0
+      when: ansible_distribution == "CentOS"
     # Undo the installation.
     #
     - name: Uninstall build dependencies

tests/e2e/ansible/install_test_deps.yml

@@ -29,6 +29,17 @@
                 path: bats-core
                 state: absent
           when: bats_exist.rc != 0
+    - name: Check kustomize is installed
+      shell: command -v kustomize >/dev/null 2>&1
+      register: kustomize_exist
+      ignore_errors: yes
+    - name: Install kustomize
+      shell: |
+        curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash
+        cp -f ./kustomize /usr/local/bin
+      args:
+        creates: /usr/local/bin/kustomize
+      when: kustomize_exist.rc != 0
     - block:
       - name: Download and extract Go tarball
         unarchive:
@@ -56,3 +67,4 @@
             - /usr/local/bin/go
             - /usr/local/go
             - /usr/local/bin/bats
+            - /usr/local/bin/kustomize

tests/e2e/operator.sh

@@ -18,6 +18,7 @@ source "${script_dir}/lib.sh"
 readonly op_ns="confidential-containers-system"
 # There should be a registry running locally on port 5000.
 export IMG=localhost:5000/cc-operator
+export PRE_INSTALL_IMG=localhost:5000/container-engine-for-cc-payload
 
 # Build the operator and push images to a local registry.
 #
@@ -42,6 +43,17 @@ build_operator () {
 	popd >/dev/null
 }
 
+# Build the container-engine-for-cc-payload and push images to a local registry.
+#
+build_pre_install_img() {
+	start_local_registry
+
+	pushd "${project_dir}/install/pre-install-payload" >/dev/null
+	make containerd registry="${PRE_INSTALL_IMG}" \
+		extra_docker_manifest_flags="--insecure"
+	popd >/dev/null
+}
+
 # Install the operator.
 #
 install_operator() {
@@ -79,8 +91,15 @@ install_operator() {
 #
 install_ccruntime() {
 	local runtimeclass="${RUNTIMECLASS:-kata-qemu}"
-	pushd "$project_dir" >/dev/null
-	kubectl create -k config/samples/ccruntime/${ccruntime_overlay}
+	local overlay_dir="${project_dir}/config/samples/ccruntime/${ccruntime_overlay}"
+
+	# Use the built pre-install image
+	kustomization_set_image  "$overlay_dir" \
+		"quay.io/confidential-containers/container-engine-for-cc-payload" \
+		"${PRE_INSTALL_IMG}"
+
+	pushd "$overlay_dir" >/dev/null
+	kubectl create -k .
 	popd >/dev/null
 
 	local pod=""
@@ -110,6 +129,31 @@ install_ccruntime() {
 	start_local_registry
 }
 
+# Set image on a kustomize's kustomization.yaml.
+#
+# Parameters:
+#	$1 - path to the overlay directory
+#	$2 - name of the old image
+#	$3 - name of the new image
+#
+kustomization_set_image() {
+	local overlay_dir="$1"
+	local old="$2"
+	local new="$3"
+
+	pushd "$overlay_dir" >/dev/null
+	# The kustomize tool will silently add a new image name if the old one does not exist,
+	# and this can introduce false-positive on the tests. So let's check the old image really
+	# exist.
+	if ! grep -q "name: ${old}$" ./kustomization.yaml; then
+		echo "ERROR: expected image ${old} in ${overlay_dir}/kustomization.yaml"
+		return 1
+	fi
+
+	kustomize edit set image "${old}=${new}"
+	popd >/dev/null
+}
+
 # Start a local registry where images can be stored.
 # The ansible playbooks should start it however it can get stopped when,
 # for example, the operator is unistalled.
@@ -159,11 +203,15 @@ main() {
 	if [ $# -eq 0 ]; then
 		build_operator
 		install_operator
+		build_pre_install_img
 		install_ccruntime
 	else
 		case $1 in
 			-h|--help) usage && exit 0;;
-			build) build_operator;;
+			build)
+				build_operator
+				build_pre_install_img
+				;;
 			install)
 				install_operator
 				install_ccruntime