test(e2e): new k3s versions break our dev/e2e test setup (#627)

Hardcode versions of k3s to the last versions before the broken
behaviour was introduced. We plan to unpin the versions after we figure
out where the bug is coming from and fix it upstream.

.github/workflows/test_e2e.yml

@@ -11,11 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        k3s: [ v1.26, v1.27, v1.28, v1.29 ]
+        # All k3s after January 2024 break our e2e tests, we hardcode
+        # the versions for now until we can fix the source of this.
+        k3s: [ v1.26.12+k3s1, v1.27.9+k3s1, v1.28.5+k3s1, v1.29.0+k3s1 ]
       fail-fast: false
 
     env:
-      K3S_CHANNEL: ${{ matrix.k3s }}
+      K3S_VERSION: ${{ matrix.k3s }}
       SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }}
 
       # Domain must be available in the account running the tests. This domain is available in the account
@@ -119,7 +121,7 @@ jobs:
     environment: e2e-robot
 
     env:
-      K3S_CHANNEL: v1.29
+      K3S_VERSION: v1.29.0+k3s1
       SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-robot
 
       # Disable routes in dev-env, not supported for Robot.

hack/dev-up.sh

@@ -19,13 +19,16 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
 
   trap error ERR
 
-  image_name=${IMAGE_NAME:-ubuntu-20.04}
+  image_name=${IMAGE_NAME:-ubuntu-22.04}
   instance_count=${INSTANCES:-1}
   instance_type=${INSTANCE_TYPE:-cpx11}
   location=${LOCATION:-fsn1}
   network_zone=${NETWORK_ZONE:-eu-central}
   ssh_keys=${SSH_KEYS:-}
-  channel=${K3S_CHANNEL:-stable}
+  # All k3s after January 2024 break our e2e tests, we hardcode
+  # the versions for now until we can fix the source of this.
+  # channel=${K3S_CHANNEL:-stable}
+  k3s_version=${K3S_VERSION:-v1.28.5+k3s1}
   network_cidr=${NETWORK_CIDR:-10.0.0.0/8}
   subnet_cidr=${SUBNET_CIDR:-10.0.0.0/24}
   cluster_cidr=${CLUSTER_CIDR:-10.244.0.0/16}
@@ -105,7 +108,7 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
 
       if [[ "$num" == "1" ]]; then
         # First node is control plane.
-        k3sup install --print-config=false --ip $ip --k3s-channel $channel --k3s-extra-args "${k3s_server_opts} ${k3s_opts} ${k3s_node_ip_opts}" --local-path $KUBECONFIG --ssh-key $ssh_private_key
+        k3sup install --print-config=false --ip $ip --k3s-version "${k3s_version}" --k3s-extra-args "${k3s_server_opts} ${k3s_opts} ${k3s_node_ip_opts}" --local-path $KUBECONFIG --ssh-key $ssh_private_key
       else
         # All subsequent nodes are initialized as workers.