diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index f72194a54af..431a7338809 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -319,7 +319,7 @@ fi # Wait for the etcd cluster to come up. until bootkube_podman_run \ - --rm \ + --rm \ --name etcdctl \ --env ETCDCTL_API=3 \ --volume /opt/openshift/tls:/opt/openshift/tls:ro,z \ @@ -336,8 +336,69 @@ do sleep 5 done -echo "Starting cluster-bootstrap..." + +{{if .SingleNode}} +if [ ! -f cb-bootstrap.done ] +then + bootkube_podman_run \ + --rm \ + --volume "$PWD:/assets:z" \ + --volume /etc/kubernetes:/etc/kubernetes:z \ + quay.io/eranco74/cluster-bootstrap:bootstrap-in-place \ + start --tear-down-early=false --asset-dir=/assets --required-pods="" --assets-create-timeout=5m + touch cb-bootstrap.done +fi + +if [ ! -f create-static-pods-data.done ] +then + echo "Move etcd static pod" + mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes + + until ! crictl ps | grep etcd + do + echo "Waiting for etcd to go down" + sleep 10 + done + + echo "Creating master ignition and writing it to disk" + # Get the master ignition for MCS + curl -k -H "Accept:'application/vnd.coreos.ignition+json;version=3.1.0, */*;q=0.1'" \ + https://localhost:22623/config/master -o /opt/openshift/master.ign + + echo "Creating master ignition" + bootkube_podman_run \ + --rm \ + --privileged \ + --volume "/var/lib/etcd:/var/lib/etcd" \ + --volume "$PWD:/assets:z" \ + --volume "/etc/kubernetes:/etc/kubernetes" \ + quay.io/eranco74/cluster-bootstrap:bip \ + bootstrap-in-place --asset-dir=/assets --ignition-path=/assets/master.ign + + touch create-static-pods-data.done +fi + +if [ ! -f write-to-disk.done ] +then + # Write image + ignition to disk + echo "Getting installation disk" + INSTALL_DISK=$(lsblk | grep disk | awk 'NR==1{print $1}') + echo "Installation disk: $INSTALL_DISK" + coreos-installer install --insecure -i /opt/openshift/master.ign /dev/$INSTALL_DISK + + touch write-to-disk.done +fi + +if [ ! -f reboot.done ] +then + + echo "Going to reboot" + shutdown -r +1 "Bootstrap completed, server is going to reboot." + touch reboot.done +fi + +{{else}} if [ ! -f cb-bootstrap.done ] then bootkube_podman_run \ @@ -365,3 +426,5 @@ fi # Workaround for https://github.com/opencontainers/runc/pull/1807 touch /opt/openshift/.bootkube.done echo "bootkube.service complete" + +{{end}} diff --git a/hack/after_reboot.sh b/hack/after_reboot.sh new file mode 100644 index 00000000000..327a1a7838c --- /dev/null +++ b/hack/after_reboot.sh @@ -0,0 +1,51 @@ +#!/bin/bash -x +export KUBECONFIG=/etc/kubernetes/bootstrap-secrets/kubeconfig + +function wait_for_api { + until oc get csr &> /dev/null + do + echo "Waiting for api ..." + sleep 30 + done +} +function restart_kubelet { + echo "Restarting kubelet" + while cat /etc/kubernetes/manifests/kube-apiserver-pod.yaml | grep bootstrap-kube-apiserver; do + echo "Waiting for kube-apiserver to apply the new static pod configuration" + sleep 10 + done + systemctl daemon-reload + systemctl restart kubelet +} +function approve_csr { + echo "Approving csrs ..." + needed_to_approve=false + until [ $(oc get nodes | grep master | grep -v NotReady | grep Ready | wc -l) -eq 1 ]; do + needed_to_approve=true + echo "Approving csrs ..." + oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve &> /dev/null || true + sleep 30 + done + # Restart kubelet only if node was added + if $needed_to_approve ; then + sleep 60 + restart_kubelet + fi +} +function wait_for_cvo { + echo "Waiting for cvo" + until [ "$(oc get clusterversion -o jsonpath='{.items[0].status.conditions[?(@.type=="Available")].status}')" == "True" ]; do + echo "Still waiting for cvo ..." + sleep 30 + done +} +function clean { + if [ -d "/etc/kubernetes/bootstrap-secrets" ]; then + rm -rf /etc/kubernetes/bootstrap-* + fi +} + +wait_for_api +approve_csr +wait_for_cvo +clean \ No newline at end of file diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 0c40e84a1ff..68580b60d05 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -59,6 +59,7 @@ type bootstrapTemplateData struct { Registries []sysregistriesv2.Registry BootImage string PlatformData platformTemplateData + SingleNode bool } // platformTemplateData is the data to use to replace values in bootstrap @@ -278,6 +279,7 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, releaseI BootImage: string(*rhcosImage), PlatformData: platformData, ClusterProfile: clusterProfile, + SingleNode: *installConfig.ControlPlane.Replicas == 1, }, nil } diff --git a/sno_manifest.yaml b/sno_manifest.yaml new file mode 100644 index 00000000000..c9e43370726 --- /dev/null +++ b/sno_manifest.yaml @@ -0,0 +1,22 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: after-reboot +spec: + config: + ignition: + version: 3.1.0 + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2ggLXgKZXhwb3J0IEtVQkVDT05GSUc9L2V0Yy9rdWJlcm5ldGVzL2Jvb3RzdHJhcC1zZWNyZXRzL2t1YmVjb25maWcKCmZ1bmN0aW9uIHdhaXRfZm9yX2FwaSB7CiAgdW50aWwgb2MgZ2V0IGNzciAmPiAvZGV2L251bGwKICAgIGRvCiAgICAgICAgZWNobyAiV2FpdGluZyBmb3IgYXBpIC4uLiIKICAgICAgICBzbGVlcCAzMAogICAgZG9uZQp9CmZ1bmN0aW9uIHJlc3RhcnRfa3ViZWxldCB7CiAgZWNobyAiUmVzdGFydGluZyBrdWJlbGV0IgogIHdoaWxlIGNhdCAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzL2t1YmUtYXBpc2VydmVyLXBvZC55YW1sICB8IGdyZXAgYm9vdHN0cmFwLWt1YmUtYXBpc2VydmVyOyBkbwogICAgZWNobyAiV2FpdGluZyBmb3Iga3ViZS1hcGlzZXJ2ZXIgdG8gYXBwbHkgdGhlIG5ldyBzdGF0aWMgcG9kIGNvbmZpZ3VyYXRpb24iCiAgICBzbGVlcCAxMAogIGRvbmUKICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZAogIHN5c3RlbWN0bCByZXN0YXJ0IGt1YmVsZXQKfQpmdW5jdGlvbiBhcHByb3ZlX2NzciB7CiAgZWNobyAiQXBwcm92aW5nIGNzcnMgLi4uIgogIG5lZWRlZF90b19hcHByb3ZlPWZhbHNlCiAgdW50aWwgWyAkKG9jIGdldCBub2RlcyB8IGdyZXAgbWFzdGVyIHwgZ3JlcCAtdiBOb3RSZWFkeSB8IGdyZXAgUmVhZHkgfCB3YyAtbCkgLWVxIDEgXTsgZG8KICAgICAgbmVlZGVkX3RvX2FwcHJvdmU9dHJ1ZQogICAgICBlY2hvICJBcHByb3ZpbmcgY3NycyAuLi4iCiAgICAgb2MgZ2V0IGNzciAtbyBnby10ZW1wbGF0ZT0ne3tyYW5nZSAuaXRlbXN9fXt7aWYgbm90IC5zdGF0dXN9fXt7Lm1ldGFkYXRhLm5hbWV9fXt7IlxuIn19e3tlbmR9fXt7ZW5kfX0nIHwgeGFyZ3Mgb2MgYWRtIGNlcnRpZmljYXRlIGFwcHJvdmUgJj4gL2Rldi9udWxsIHx8IHRydWUKICAgICBzbGVlcCAzMAogICAgZG9uZQogICMgUmVzdGFydCBrdWJlbGV0IG9ubHkgaWYgbm9kZSB3YXMgYWRkZWQKICBpZiAkbmVlZGVkX3RvX2FwcHJvdmUgOyB0aGVuCiAgICBzbGVlcCA2MAogICAgcmVzdGFydF9rdWJlbGV0CiAgZmkKfQpmdW5jdGlvbiB3YWl0X2Zvcl9jdm8gewogIGVjaG8gIldhaXRpbmcgZm9yIGN2byIKICB1bnRpbCBbICIkKG9jIGdldCBjbHVzdGVydmVyc2lvbiAtbyBqc29ucGF0aD0ney5pdGVtc1swXS5zdGF0dXMuY29uZGl0aW9uc1s/KEAudHlwZT09IkF2YWlsYWJsZSIpXS5zdGF0dXN9JykiID09ICJUcnVlIiBdOyBkbwogICAgICBlY2hvICJTdGlsbCB3YWl0aW5nIGZvciBjdm8gLi4uIgogICAgIHNsZWVwIDMwCiAgICBkb25lCn0KZnVuY3Rpb24gY2xlYW4gewogIGlmIFsgLWQgIi9ldGMva3ViZXJuZXRlcy9ib290c3RyYXAtc2VjcmV0cyIgXTsgdGhlbgogICAgIHJtIC1yZiAvZXRjL2t1YmVybmV0ZXMvYm9vdHN0cmFwLSoKICBmaQp9Cgp3YWl0X2Zvcl9hcGkKYXBwcm92ZV9jc3IKd2FpdF9mb3JfY3ZvCmNsZWFu + mode: 365 + overwrite: true + path: /usr/local/bin/after_reboot.sh + systemd: + units: + - name: after_reboot.service + contents: "[Unit]\nDescription=Master Install\nWants=kubelet.service\nAfter=kubelet.service\n[Service]\nType=oneshot\nExecStart=/usr/local/bin/after_reboot.sh\n\nRestartSec=5s\n\n[Install]\nWantedBy=multi-user.target\n" + enabled: true