-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #4482 from eranco74/bootstrap-in-place
Single Node deployment with bootstrap-in-place
- Loading branch information
Showing
14 changed files
with
530 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
...otstrap-in-place/files/opt/openshift/bootstrap-in-place/bootstrap-in-place-post-reboot.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| #!/usr/bin/env bash | ||
| set -euoE pipefail ## -E option will cause functions to inherit trap | ||
|
|
||
| echo "Running bootkube bootstrap-in-place post reboot" | ||
| export KUBECONFIG=/etc/kubernetes/bootstrap-secrets/kubeconfig | ||
|
|
||
| function wait_for_api { | ||
| until oc get csr &> /dev/null | ||
| do | ||
| echo "Waiting for api ..." | ||
| sleep 5 | ||
| done | ||
| } | ||
|
|
||
| function restart_kubelet { | ||
| echo "Restarting kubelet" | ||
| until [ "$(oc get pod -n openshift-kube-apiserver-operator --selector='app=kube-apiserver-operator' -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' | grep -c "True")" -eq 1 ]; | ||
| do | ||
| echo "Waiting for kube-apiserver-operator ready condition to be True" | ||
| sleep 10 | ||
| done | ||
| # daemon-reload is required because /etc/systemd/system/kubelet.service.d/20-nodenet.conf is added after kubelet started | ||
| systemctl daemon-reload | ||
| systemctl restart kubelet | ||
|
|
||
| while grep bootstrap-kube-apiserver /etc/kubernetes/manifests/kube-apiserver-pod.yaml; | ||
| do | ||
| echo "Waiting for kube-apiserver to apply the new static pod configuration" | ||
| sleep 10 | ||
| done | ||
| systemctl restart kubelet | ||
| } | ||
|
|
||
| function approve_csr { | ||
| echo "Approving csrs ..." | ||
| until [ "$(oc get nodes --selector='node-role.kubernetes.io/master' -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' | grep -c "True")" -eq 1 ]; | ||
| do | ||
| echo "Approving csrs ..." | ||
| oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve &> /dev/null || true | ||
| sleep 30 | ||
| done | ||
| } | ||
|
|
||
| function wait_for_cvo { | ||
| echo "Waiting for cvo" | ||
| until [ "$(oc get clusterversion -o jsonpath='{.items[0].status.conditions[?(@.type=="Available")].status}')" == "True" ]; | ||
| do | ||
| echo "Still waiting for cvo ..." | ||
| sleep 30 | ||
| done | ||
| } | ||
|
|
||
| function clean { | ||
| if [ -d "/etc/kubernetes/bootstrap-secrets" ]; then | ||
| rm -rf /etc/kubernetes/bootstrap-* | ||
| fi | ||
|
|
||
| rm -rf /usr/local/bin/installer-gather.sh | ||
| rm -rf /usr/local/bin/installer-masters-gather.sh | ||
| rm -rf /var/log/log-bundle-bootstrap.tar.gz | ||
|
|
||
| systemctl disable bootkube.service | ||
| } | ||
|
|
||
| wait_for_api | ||
| approve_csr | ||
| restart_kubelet | ||
| wait_for_cvo | ||
| clean |
66 changes: 66 additions & 0 deletions
66
...ata/bootstrap/bootstrap-in-place/files/opt/openshift/bootstrap-in-place/master-update.fcc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| variant: fcos | ||
| version: 1.1.0 | ||
| ignition: | ||
| config: | ||
| merge: | ||
| - local: original-master.ign | ||
| storage: | ||
| trees: | ||
| - local: kubernetes/bootstrap-configs | ||
| path: /etc/kubernetes/bootstrap-configs | ||
| - local: tls/ | ||
| path: /etc/kubernetes/bootstrap-secrets | ||
| - local: etcd-bootstrap/bootstrap-manifests/secrets/ | ||
| path: /etc/kubernetes/static-pod-resources/etcd-member | ||
| - local: etcd-data | ||
| path: /var/lib/etcd | ||
| files: | ||
| - path: /etc/kubernetes/bootstrap-secrets/kubeconfig | ||
| contents: | ||
| local: auth/kubeconfig-loopback | ||
| - path: /etc/kubernetes/static-pod-resources/etcd-member/ca.crt | ||
| contents: | ||
| local: tls/etcd-ca-bundle.crt | ||
| - path: /etc/kubernetes/manifests/etcd-pod.yaml | ||
| contents: | ||
| local: etcd-bootstrap/bootstrap-manifests/etcd-member-pod.yaml | ||
| - path: /etc/kubernetes/manifests/kube-apiserver-pod.yaml | ||
| contents: | ||
| local: bootstrap-manifests/kube-apiserver-pod.yaml | ||
| - path: /etc/kubernetes/manifests/kube-controller-manager-pod.yaml | ||
| contents: | ||
| local: bootstrap-manifests/kube-controller-manager-pod.yaml | ||
| - path: /etc/kubernetes/manifests/kube-scheduler-pod.yaml | ||
| contents: | ||
| local: bootstrap-manifests/kube-scheduler-pod.yaml | ||
| - path: /usr/local/bin/bootstrap-in-place-post-reboot.sh | ||
| contents: | ||
| local: bootstrap-in-place/bootstrap-in-place-post-reboot.sh | ||
| mode: 0555 | ||
| - path: /var/log/log-bundle-bootstrap.tar.gz | ||
| contents: | ||
| local: log-bundle-bootstrap.tar.gz | ||
| - path: /usr/local/bin/installer-masters-gather.sh | ||
| contents: | ||
| local: bin/installer-masters-gather.sh | ||
| mode: 0555 | ||
| - path: /usr/local/bin/installer-gather.sh | ||
| contents: | ||
| local: bin/installer-gather.sh | ||
| mode: 0555 | ||
| systemd: | ||
| units: | ||
| - name: bootkube.service | ||
| enabled: true | ||
| contents: | | ||
| [Unit] | ||
| Description=Bootkube - bootstrap in place post reboot | ||
| Wants=kubelet.service | ||
| After=kubelet.service | ||
| ConditionPathExists=/etc/kubernetes/bootstrap-secrets/kubeconfig | ||
| [Service] | ||
| Type=oneshot | ||
| ExecStart=/usr/local/bin/bootstrap-in-place-post-reboot.sh | ||
| RestartSec=5s | ||
| [Install] | ||
| WantedBy=multi-user.target |
53 changes: 53 additions & 0 deletions
53
data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/bootstrap-in-place.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| #!/usr/bin/env bash | ||
| set -euoE pipefail ## -E option will cause functions to inherit trap | ||
|
|
||
| # This script is executed by bootkube.sh when installing single node with bootstrap in place | ||
| CLUSTER_BOOTSTRAP_IMAGE=$1 | ||
|
|
||
|
|
||
| bootkube_podman_run() { | ||
| # we run all commands in the host-network to prevent IP conflicts with | ||
| # end-user infrastructure. | ||
| podman run --quiet --net=host "${@}" | ||
| } | ||
|
|
||
| if [ ! -f stop-etcd.done ]; then | ||
| echo "Stop etcd static pod by moving the manifest" | ||
| mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes || echo "already moved etcd-member-pod.yaml" | ||
|
|
||
| until ! crictl ps | grep etcd; do | ||
| echo "Waiting for etcd to go down" | ||
| sleep 10 | ||
| done | ||
|
|
||
| touch stop-etcd.done | ||
| fi | ||
|
|
||
| if [ ! -f master-ignition.done ]; then | ||
| echo "Creating master ignition and writing it to disk" | ||
| # Get the master ignition from MCS | ||
| curl --header 'Accept:application/vnd.coreos.ignition+json;version=3.1.0' \ | ||
| http://localhost:22624/config/master -o /opt/openshift/original-master.ign | ||
|
|
||
| GATHER_ID="bootstrap" | ||
| GATHER_TAR_FILE=log-bundle-${GATHER_ID}.tar.gz | ||
|
|
||
| echo "Gathering installer bootstrap logs" | ||
| TAR_FILE=${GATHER_TAR_FILE} /usr/local/bin/installer-gather.sh --id ${GATHER_ID} | ||
|
|
||
| echo "Adding bootstrap control plane and bootstrap installer-gather bundle to master ignition" | ||
| bootkube_podman_run \ | ||
| --rm \ | ||
| --privileged \ | ||
| --volume "$PWD:/assets:z" \ | ||
| --volume "/usr/local/bin/:/assets/bin" \ | ||
| --volume "/var/lib/etcd/:/assets/etcd-data" \ | ||
| --volume "/etc/kubernetes:/assets/kubernetes" \ | ||
| "${CLUSTER_BOOTSTRAP_IMAGE}" \ | ||
| bootstrap-in-place \ | ||
| --asset-dir /assets \ | ||
| --input /assets/bootstrap-in-place/master-update.fcc \ | ||
| --output /assets/master.ign | ||
|
|
||
| touch master-ignition.done | ||
| fi |
22 changes: 22 additions & 0 deletions
22
data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/install-to-disk.sh.template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| #!/usr/bin/env bash | ||
| set -euoE pipefail ## -E option will cause functions to inherit trap | ||
|
|
||
| # This script is executed by install-to-disk service when installing single node with bootstrap in place | ||
|
|
||
| echo "Waiting for /opt/openshift/.bootkube.done" | ||
| until [ -f /opt/openshift/.bootkube.done ]; do | ||
| sleep 5 | ||
| done | ||
|
|
||
| if [ ! -f coreos-installer.done ]; then | ||
| # Write image + ignition to disk | ||
| echo "Executing coreos-installer with the following options: install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}" | ||
| coreos-installer install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}} | ||
| + | ||
| touch coreos-installer.done | ||
| fi | ||
|
|
||
| echo "Going to reboot" | ||
| shutdown -r +1 "Bootstrap completed, server is going to reboot." | ||
| touch /opt/openshift/.install-to-disk.done | ||
| echo "Done" |
13 changes: 13 additions & 0 deletions
13
data/data/bootstrap/bootstrap-in-place/systemd/units/install-to-disk.service
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| [Unit] | ||
| Description=Install to disk | ||
| Requires=bootkube.service | ||
| Wants=bootkube.service | ||
| After=bootkube.service | ||
| ConditionPathExists=!/opt/openshift/.install-to-disk.done | ||
|
|
||
| [Service] | ||
| WorkingDirectory=/opt/openshift | ||
| ExecStart=/usr/local/bin/install-to-disk.sh | ||
|
|
||
| Restart=on-failure | ||
| RestartSec=5s |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.