Add bootstrapInPlace to insall config

Added new asset for SingleNodeBootstrapInPlace Updated bootkube to set BOOTSTRAP_INPLACE variable and evaluate it at run time. This change allow someone looking at the script to follow the shell logic for single-node vs. multi-node deployment while debugging. When creating single-node-ignition-config we now validate that the install-config contain configuration for bootstrapInPlace. Added install-to-disk serivce that will complete the installation by writing the OS to the desired instnallation disk and reboot the node. Signed-off-by: Eran Cohen <eranco@redhat.com>
openshift · Feb 11, 2021 · 3088a01 · 3088a01
1 parent 756212b
commit 3088a01
Show file tree

Hide file tree

Showing 14 changed files with 283 additions and 119 deletions.
diff --git a/...n-place/bootstrap-in-place-post-reboot.sh → ...n-place/bootstrap-in-place-post-reboot.sh b/...n-place/bootstrap-in-place-post-reboot.sh → ...n-place/bootstrap-in-place-post-reboot.sh
@@ -1,19 +1,20 @@
 #!/usr/bin/env bash
 set -euoE pipefail ## -E option will cause functions to inherit trap
 
+echo "Running bootkube bootstrap-in-place post reboot"
 export KUBECONFIG=/etc/kubernetes/bootstrap-secrets/kubeconfig
 
 function wait_for_api {
   until oc get csr &> /dev/null
   do
-      echo "Waiting for api ..."
-      sleep 5
+    echo "Waiting for api ..."
+    sleep 5
   done
 }
 
 function restart_kubelet {
   echo "Restarting kubelet"
-  until [ "$(oc get pod -n openshift-kube-apiserver-operator --selector='app=kube-apiserver-operator'  -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' | grep -c "True")" -eq 1 ];
+  until [ "$(oc get pod -n openshift-kube-apiserver-operator --selector='app=kube-apiserver-operator' -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' | grep -c "True")" -eq 1 ];
   do
     echo "Waiting for kube-apiserver-operator ready condition to be True"
     sleep 10
@@ -22,7 +23,7 @@ function restart_kubelet {
   systemctl daemon-reload
   systemctl restart kubelet
 
-  while grep  bootstrap-kube-apiserver /etc/kubernetes/manifests/kube-apiserver-pod.yaml;
+  while grep bootstrap-kube-apiserver /etc/kubernetes/manifests/kube-apiserver-pod.yaml;
   do
     echo "Waiting for kube-apiserver to apply the new static pod configuration"
     sleep 10
@@ -34,9 +35,9 @@ function approve_csr {
   echo "Approving csrs ..."
   until [ "$(oc get nodes --selector='node-role.kubernetes.io/master' -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' | grep -c "True")" -eq 1 ];
   do
-     echo "Approving csrs ..."
-     oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve &> /dev/null || true
-     sleep 30
+    echo "Approving csrs ..."
+    oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve &> /dev/null || true
+    sleep 30
   done
 }
 
@@ -58,7 +59,7 @@ function clean {
   rm -rf /usr/local/bin/installer-masters-gather.sh
   rm -rf /var/log/log-bundle-bootstrap.tar.gz
 
-  systemctl disable bootstrap-in-place-post-reboot.service
+  systemctl disable bootkube.service
 }
 
 wait_for_api

diff --git a/...hift/bootstrap-in-place/master-update.fcc → ...hift/bootstrap-in-place/master-update.fcc b/...hift/bootstrap-in-place/master-update.fcc → ...hift/bootstrap-in-place/master-update.fcc
@@ -50,11 +50,11 @@ storage:
       mode: 0555
 systemd:
   units:
-    - name: bootstrap-in-place-post-reboot.service
+    - name: bootkube.service
       enabled: true
       contents: |
         [Unit]
-        Description=bootstrap-in-place-post-reboot
+        Description=Bootkube - bootstrap in place post reboot
         Wants=kubelet.service
         After=kubelet.service
         ConditionPathExists=/etc/kubernetes/bootstrap-secrets/kubeconfig

diff --git a/...place/usr/local/bin/bootstrap-in-place.sh → ...files/usr/local/bin/bootstrap-in-place.sh b/...place/usr/local/bin/bootstrap-in-place.sh → ...files/usr/local/bin/bootstrap-in-place.sh
@@ -13,7 +13,7 @@ bootkube_podman_run() {
 
 if [ ! -f stop-etcd.done ]; then
   echo "Stop etcd static pod by moving the manifest"
-  mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes
+  mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes || echo "already moved etcd-member-pod.yaml"
 
   until ! crictl ps | grep etcd; do
     echo "Waiting for etcd to go down"
@@ -26,14 +26,14 @@ fi
 if [ ! -f master-ignition.done ]; then
   echo "Creating master ignition and writing it to disk"
   # Get the master ignition from MCS
-  curl --header "Accept:'application/vnd.coreos.ignition+json;version=3.1.0 ;q=0.1'" \
+  curl --header 'Accept:application/vnd.coreos.ignition+json;version=3.1.0' \
     http://localhost:22624/config/master -o /opt/openshift/original-master.ign
 
   GATHER_ID="bootstrap"
   GATHER_TAR_FILE=log-bundle-${GATHER_ID}.tar.gz
 
   echo "Gathering installer bootstrap logs"
-  TAR_FILE=${GATHER_TAR_FILE} /usr/local/bin/installer-gather.sh --id $GATHER_ID
+  TAR_FILE=${GATHER_TAR_FILE} /usr/local/bin/installer-gather.sh --id ${GATHER_ID}
 
   echo "Adding bootstrap control plane and bootstrap installer-gather bundle to master ignition"
   bootkube_podman_run \
@@ -51,6 +51,3 @@ if [ ! -f master-ignition.done ]; then
 
   touch master-ignition.done
 fi
-
-echo "Done"
-echo "To complete the installation execute: 'coreos-installer install -i /opt/openshift/master.ign <device>' and reboot the node"
diff --git a/data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/install-to-disk.sh.template b/data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/install-to-disk.sh.template
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euoE pipefail ## -E option will cause functions to inherit trap
+
+# This script is executed by install-to-disk service when installing single node with bootstrap in place
+
+echo "Waiting for /opt/openshift/.bootkube.done"
+until [ -f /opt/openshift/.bootkube.done ]; do
+  sleep 5
+done
+
+if [ ! -f coreos-installer.done ]; then
+  # Write image + ignition to disk
+  echo "Executing coreos-installer with the following options: install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}"
+  coreos-installer install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}
++
+  touch coreos-installer.done
+fi
+
+echo "Going to reboot"
+shutdown -r +1 "Bootstrap completed, server is going to reboot."
+touch /opt/openshift/.install-to-disk.done
+echo "Done"
diff --git a/data/data/bootstrap/bootstrap-in-place/systemd/units/install-to-disk.service b/data/data/bootstrap/bootstrap-in-place/systemd/units/install-to-disk.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Install to disk
+Requires=bootkube.service
+Wants=bootkube.service
+After=bootkube.service
+ConditionPathExists=!/opt/openshift/.install-to-disk.done
+
+[Service]
+WorkingDirectory=/opt/openshift
+ExecStart=/usr/local/bin/install-to-disk.sh
+
+Restart=on-failure
+RestartSec=5s
diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
@@ -5,6 +5,12 @@ set -euoE pipefail ## -E option will cause functions to inherit trap
 
 mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
 
+{{- if .BootstrapInPlace }}
+BOOTSTRAP_INPLACE=true
+{{ else }}
+BOOTSTRAP_INPLACE=false
+{{ end -}}
+
 ETCD_ENDPOINTS=
 
 bootkube_podman_run() {
@@ -14,23 +20,23 @@ bootkube_podman_run() {
 }
 
 wait_for_etcd_cluster() {
-  until bootkube_podman_run \
-      --rm \
-      --name etcdctl \
-      --env ETCDCTL_API=3 \
-      --volume /opt/openshift/tls:/opt/openshift/tls:ro,z \
-      --entrypoint etcdctl \
-      "${MACHINE_CONFIG_ETCD_IMAGE}" \
-      --dial-timeout=10m \
-      --cacert=/opt/openshift/tls/etcd-ca-bundle.crt \
-      --cert=/opt/openshift/tls/etcd-client.crt \
-      --key=/opt/openshift/tls/etcd-client.key \
-      --endpoints="${ETCD_ENDPOINTS}" \
-      endpoint health
-  do
-    echo "etcdctl failed. Retrying in 5 seconds..."
-    sleep 5
-  done
+    until bootkube_podman_run \
+        --rm \
+        --name etcdctl \
+        --env ETCDCTL_API=3 \
+        --volume /opt/openshift/tls:/opt/openshift/tls:ro,z \
+        --entrypoint etcdctl \
+        "${MACHINE_CONFIG_ETCD_IMAGE}" \
+        --dial-timeout=10m \
+        --cacert=/opt/openshift/tls/etcd-ca-bundle.crt \
+        --cert=/opt/openshift/tls/etcd-client.crt \
+        --key=/opt/openshift/tls/etcd-client.key \
+        --endpoints="${ETCD_ENDPOINTS}" \
+        endpoint health
+    do
+        echo "etcdctl failed. Retrying in 5 seconds..."
+        sleep 5
+    done
 }
 
 MACHINE_CONFIG_OPERATOR_IMAGE=$(image_for machine-config-operator)
@@ -338,16 +344,17 @@ then
 fi
 
 # in case of single node, if we removed etcd, there is no point to wait for it on restart
-{{- if .SingleNodeBootstrapInPlace}}
 if [ ! -f stop-etcd.done ]
 then
-  # Wait for the etcd cluster to come up.
-  wait_for_etcd_cluster
+	# Wait for the etcd cluster to come up.
+	wait_for_etcd_cluster
 fi
-{{- else}}
-wait_for_etcd_cluster
-{{- end}}
 
+REQUIRED_PODS="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
+if [ "$BOOTSTRAP_INPLACE" = true ]
+    REQUIRED_PODS=""
+then
+fi
 
 echo "Starting cluster-bootstrap..."
 
@@ -358,31 +365,28 @@ then
         --volume "$PWD:/assets:z" \
         --volume /etc/kubernetes:/etc/kubernetes:z \
         "${CLUSTER_BOOTSTRAP_IMAGE}" \
-{{- if .SingleNodeBootstrapInPlace}}
-        start --tear-down-early=false --asset-dir=/assets --required-pods=""  --assets-create-timeout=5m
-{{- else}}
-        start --tear-down-early=false --asset-dir=/assets --required-pods="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
-{{- end}}
+        start --tear-down-early=false --asset-dir=/assets --required-pods="${REQUIRED_PODS}"
+
     touch cb-bootstrap.done
 fi
 
-{{- if .SingleNodeBootstrapInPlace}}
-. /usr/local/bin/bootstrap-in-place.sh "${CLUSTER_BOOTSTRAP_IMAGE}"
-{{- else}}
-rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml
-
-if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ]
+if [ "$BOOTSTRAP_INPLACE" = true ]
 then
-	echo "Waiting for CEO to finish..."
-	bootkube_podman_run \
-		--volume "$PWD:/assets:z" \
-		"${CLUSTER_ETCD_OPERATOR_IMAGE}" \
-		/usr/bin/cluster-etcd-operator \
-			wait-for-ceo \
-			--kubeconfig /assets/auth/kubeconfig
+    . /usr/local/bin/bootstrap-in-place.sh "${CLUSTER_BOOTSTRAP_IMAGE}"
+else
+    rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml
+
+    if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ]
+    then
+        echo "Waiting for CEO to finish..."
+        bootkube_podman_run \
+            --volume "$PWD:/assets:z" \
+            "${CLUSTER_ETCD_OPERATOR_IMAGE}" \
+            /usr/bin/cluster-etcd-operator \
+            wait-for-ceo \
+            --kubeconfig /assets/auth/kubeconfig
+    fi
 fi
-{{- end}}
-
 # Workaround for https://github.com/opencontainers/runc/pull/1807
 touch /opt/openshift/.bootkube.done
 echo "bootkube.service complete"
diff --git a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh
@@ -148,7 +148,7 @@ if [[ -f ${LOG_BUNDLE_BOOTSTRAP_ARCHIVE_NAME} ]]; then
     MASTER_ARTIFACTS="/tmp/artifacts-${MASTER_GATHER_ID}"
     mkdir -p "${ARTIFACTS}/control-plane/master"
     sudo /usr/local/bin/installer-masters-gather.sh --id "${MASTER_GATHER_ID}" </dev/null
-    cp -r "$MASTER_ARTIFACTS"/* "${ARTIFACTS}/control-plane/master/"
+    cp -r "${MASTER_ARTIFACTS}"/* "${ARTIFACTS}/control-plane/master/"
 elif [ "$#" -ne 0 ]; then
     MASTERS=( "$@" )
 elif test -s "${ARTIFACTS}/resources/masters.list"; then

diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml
@@ -34,6 +34,16 @@ spec:
             description: BaseDomain is the base domain to which the cluster should
               belong.
             type: string
+          bootstrapInPlace:
+            description: BootstrapInPlace is the configuration for installing a single
+              node with bootstrap in place installation.
+            properties:
+              installationDisk:
+                description: InstallationDisk is the target disk drive for coreos-installer
+                type: string
+            required:
+            - installationDisk
+            type: object
           compute:
             description: Compute is the configuration for the machines that comprise
               the compute nodes.

diff --git a/pkg/asset/ignition/bootstrap/bootstrap-in-place.go b/pkg/asset/ignition/bootstrap/bootstrap-in-place.go