Single Node deployment with bootstrap-in-place

Updated bootkube.sh.template
Override cluster-bootstrap to wait for API and accept assets-creation-timeout
Update master ignition (currently using CB, should move to MCO)
Added after_reboot.sh for compleating single node installation
Added machineconfig manifest with the after_reboot script and service

data/data/bootstrap/files/usr/local/bin/bootkube.sh.template

@@ -319,7 +319,7 @@ fi
 
 # Wait for the etcd cluster to come up.
 until bootkube_podman_run \
-		--rm \
+        --rm \
 		--name etcdctl \
 		--env ETCDCTL_API=3 \
 		--volume /opt/openshift/tls:/opt/openshift/tls:ro,z \
@@ -336,8 +336,69 @@ do
 	sleep 5
 done
 
-echo "Starting cluster-bootstrap..."
+
+{{if .SingleNode}}
 
+if [ ! -f cb-bootstrap.done ]
+then
+    bootkube_podman_run \
+        --rm \
+        --volume "$PWD:/assets:z" \
+        --volume /etc/kubernetes:/etc/kubernetes:z \
+        quay.io/eranco74/cluster-bootstrap:bootstrap-in-place \
+        start --tear-down-early=false --asset-dir=/assets --required-pods=""  --assets-create-timeout=5m
+        touch cb-bootstrap.done
+fi
+
+if [ ! -f create-static-pods-data.done ]
+then
+    echo "Move etcd static pod"
+    mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes
+
+    until ! crictl ps | grep etcd
+    do
+        echo "Waiting for etcd to go down"
+        sleep 10
+    done
+
+    echo "Creating master ignition and writing it to disk"
+    # Get the master ignition for MCS
+    curl -k  -H "Accept:'application/vnd.coreos.ignition+json;version=3.1.0, */*;q=0.1'" \
+    https://localhost:22623/config/master -o /opt/openshift/master.ign
+
+    echo "Creating master ignition"
+    bootkube_podman_run \
+        --rm \
+        --privileged \
+        --volume "/var/lib/etcd:/var/lib/etcd" \
+        --volume "$PWD:/assets:z" \
+        --volume "/etc/kubernetes:/etc/kubernetes" \
+        quay.io/eranco74/cluster-bootstrap:bip \
+        bootstrap-in-place --asset-dir=/assets --ignition-path=/assets/master.ign
+
+    touch create-static-pods-data.done
+fi
+
+if [ ! -f write-to-disk.done ]
+then
+    # Write image + ignition to disk
+    echo "Getting installation disk"
+    INSTALL_DISK=$(lsblk | grep disk | awk 'NR==1{print $1}')
+    echo "Installation disk: $INSTALL_DISK"
+    coreos-installer install --insecure -i /opt/openshift/master.ign  /dev/$INSTALL_DISK
+
+    touch write-to-disk.done
+fi
+
+if [ ! -f reboot.done ]
+then
+
+    echo "Going to reboot"
+    shutdown -r +1 "Bootstrap completed, server is going to reboot."
+    touch reboot.done
+fi
+
+{{else}}
 if [ ! -f cb-bootstrap.done ]
 then
     bootkube_podman_run \
@@ -365,3 +426,5 @@ fi
 # Workaround for https://github.com/opencontainers/runc/pull/1807
 touch /opt/openshift/.bootkube.done
 echo "bootkube.service complete"
+
+{{end}}

hack/after_reboot.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -x
+export KUBECONFIG=/etc/kubernetes/bootstrap-secrets/kubeconfig
+
+function wait_for_api {
+  until oc get csr &> /dev/null
+    do
+        echo "Waiting for api ..."
+        sleep 30
+    done
+}
+function restart_kubelet {
+  echo "Restarting kubelet"
+  while cat /etc/kubernetes/manifests/kube-apiserver-pod.yaml  | grep bootstrap-kube-apiserver; do
+    echo "Waiting for kube-apiserver to apply the new static pod configuration"
+    sleep 10
+  done
+  systemctl daemon-reload
+  systemctl restart kubelet
+}
+function approve_csr {
+  echo "Approving csrs ..."
+  needed_to_approve=false
+  until [ $(oc get nodes | grep master | grep -v NotReady | grep Ready | wc -l) -eq 1 ]; do
+      needed_to_approve=true
+      echo "Approving csrs ..."
+     oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve &> /dev/null || true
+     sleep 30
+    done
+  # Restart kubelet only if node was added
+  if $needed_to_approve ; then
+    sleep 60
+    restart_kubelet
+  fi
+}
+function wait_for_cvo {
+  echo "Waiting for cvo"
+  until [ "$(oc get clusterversion -o jsonpath='{.items[0].status.conditions[?(@.type=="Available")].status}')" == "True" ]; do
+      echo "Still waiting for cvo ..."
+     sleep 30
+    done
+}
+function clean {
+  if [ -d "/etc/kubernetes/bootstrap-secrets" ]; then
+     rm -rf /etc/kubernetes/bootstrap-*
+  fi
+}
+
+wait_for_api
+approve_csr
+wait_for_cvo
+clean

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -59,6 +59,7 @@ type bootstrapTemplateData struct {
 	Registries            []sysregistriesv2.Registry
 	BootImage             string
 	PlatformData          platformTemplateData
+	SingleNode            bool
 }
 
 // platformTemplateData is the data to use to replace values in bootstrap
@@ -278,6 +279,7 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, releaseI
 		BootImage:             string(*rhcosImage),
 		PlatformData:          platformData,
 		ClusterProfile:        clusterProfile,
+		SingleNode:            *installConfig.ControlPlane.Replicas == 1,
 	}, nil
 }
 

sno_manifest.yaml

@@ -0,0 +1,22 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: master
+  name: after-reboot
+spec:
+  config:
+    ignition:
+      version: 3.1.0
+    storage:
+      files:
+        - contents:
+            source: data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2ggLXgKZXhwb3J0IEtVQkVDT05GSUc9L2V0Yy9rdWJlcm5ldGVzL2Jvb3RzdHJhcC1zZWNyZXRzL2t1YmVjb25maWcKCmZ1bmN0aW9uIHdhaXRfZm9yX2FwaSB7CiAgdW50aWwgb2MgZ2V0IGNzciAmPiAvZGV2L251bGwKICAgIGRvCiAgICAgICAgZWNobyAiV2FpdGluZyBmb3IgYXBpIC4uLiIKICAgICAgICBzbGVlcCAzMAogICAgZG9uZQp9CmZ1bmN0aW9uIHJlc3RhcnRfa3ViZWxldCB7CiAgZWNobyAiUmVzdGFydGluZyBrdWJlbGV0IgogIHdoaWxlIGNhdCAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzL2t1YmUtYXBpc2VydmVyLXBvZC55YW1sICB8IGdyZXAgYm9vdHN0cmFwLWt1YmUtYXBpc2VydmVyOyBkbwogICAgZWNobyAiV2FpdGluZyBmb3Iga3ViZS1hcGlzZXJ2ZXIgdG8gYXBwbHkgdGhlIG5ldyBzdGF0aWMgcG9kIGNvbmZpZ3VyYXRpb24iCiAgICBzbGVlcCAxMAogIGRvbmUKICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZAogIHN5c3RlbWN0bCByZXN0YXJ0IGt1YmVsZXQKfQpmdW5jdGlvbiBhcHByb3ZlX2NzciB7CiAgZWNobyAiQXBwcm92aW5nIGNzcnMgLi4uIgogIG5lZWRlZF90b19hcHByb3ZlPWZhbHNlCiAgdW50aWwgWyAkKG9jIGdldCBub2RlcyB8IGdyZXAgbWFzdGVyIHwgZ3JlcCAtdiBOb3RSZWFkeSB8IGdyZXAgUmVhZHkgfCB3YyAtbCkgLWVxIDEgXTsgZG8KICAgICAgbmVlZGVkX3RvX2FwcHJvdmU9dHJ1ZQogICAgICBlY2hvICJBcHByb3ZpbmcgY3NycyAuLi4iCiAgICAgb2MgZ2V0IGNzciAtbyBnby10ZW1wbGF0ZT0ne3tyYW5nZSAuaXRlbXN9fXt7aWYgbm90IC5zdGF0dXN9fXt7Lm1ldGFkYXRhLm5hbWV9fXt7IlxuIn19e3tlbmR9fXt7ZW5kfX0nIHwgeGFyZ3Mgb2MgYWRtIGNlcnRpZmljYXRlIGFwcHJvdmUgJj4gL2Rldi9udWxsIHx8IHRydWUKICAgICBzbGVlcCAzMAogICAgZG9uZQogICMgUmVzdGFydCBrdWJlbGV0IG9ubHkgaWYgbm9kZSB3YXMgYWRkZWQKICBpZiAkbmVlZGVkX3RvX2FwcHJvdmUgOyB0aGVuCiAgICBzbGVlcCA2MAogICAgcmVzdGFydF9rdWJlbGV0CiAgZmkKfQpmdW5jdGlvbiB3YWl0X2Zvcl9jdm8gewogIGVjaG8gIldhaXRpbmcgZm9yIGN2byIKICB1bnRpbCBbICIkKG9jIGdldCBjbHVzdGVydmVyc2lvbiAtbyBqc29ucGF0aD0ney5pdGVtc1swXS5zdGF0dXMuY29uZGl0aW9uc1s/KEAudHlwZT09IkF2YWlsYWJsZSIpXS5zdGF0dXN9JykiID09ICJUcnVlIiBdOyBkbwogICAgICBlY2hvICJTdGlsbCB3YWl0aW5nIGZvciBjdm8gLi4uIgogICAgIHNsZWVwIDMwCiAgICBkb25lCn0KZnVuY3Rpb24gY2xlYW4gewogIGlmIFsgLWQgIi9ldGMva3ViZXJuZXRlcy9ib290c3RyYXAtc2VjcmV0cyIgXTsgdGhlbgogICAgIHJtIC1yZiAvZXRjL2t1YmVybmV0ZXMvYm9vdHN0cmFwLSoKICBmaQp9Cgp3YWl0X2Zvcl9hcGkKYXBwcm92ZV9jc3IKd2FpdF9mb3JfY3ZvCmNsZWFu
+          mode: 365
+          overwrite: true
+          path: /usr/local/bin/after_reboot.sh
+    systemd:
+      units:
+        - name:  after_reboot.service
+          contents: "[Unit]\nDescription=Master Install\nWants=kubelet.service\nAfter=kubelet.service\n[Service]\nType=oneshot\nExecStart=/usr/local/bin/after_reboot.sh\n\nRestartSec=5s\n\n[Install]\nWantedBy=multi-user.target\n"
+          enabled: true