Update CNI Manifests

Add makefile target to update manifests automatically.
nutanix-cloud-native · Jul 15, 2024 · a03dd89 · a03dd89
1 parent 6601c72
commit a03dd89
Show file tree

Hide file tree

Showing 9 changed files with 767 additions and 251 deletions.
diff --git a/Makefile b/Makefile
@@ -47,9 +47,10 @@ RELEASE_DIR ?= $(REPO_ROOT)/out
 
 # CNI paths for e2e tests
 CNI_PATH_CALICO ?= "${E2E_DIR}/data/cni/calico/calico.yaml"
-CNI_PATH_FLANNEL ?= "${E2E_DIR}/data/cni/flannel/flannel.yaml" # From https://github.com/flannel-io/flannel/blob/master/Documentation/kube-flannel.yml
-CNI_PATH_CILIUM ?= "${E2E_DIR}/data/cni/cilium/cilium.yaml" # helm template cilium cilium/cilium --version 1.13.0 -n kube-system --set hubble.enabled=false --set cni.chainingMode=portmap  --set sessionAffinity=true | sed 's/${BIN_PATH}/$BIN_PATH/g'
-CNI_PATH_CILIUM_NO_KUBEPROXY ?= "${E2E_DIR}/data/cni/cilium/cilium-no-kubeproxy.yaml" # helm template cilium cilium/cilium --version 1.13.0 -n kube-system --set hubble.enabled=false --set cni.chainingMode=portmap  --set sessionAffinity=true --set kubeProxyReplacement=strict | sed 's/${BIN_PATH}/$BIN_PATH/g'
+CNI_PATH_CILIUM ?= "${E2E_DIR}/data/cni/cilium/cilium.yaml"
+CNI_PATH_CILIUM_NO_KUBEPROXY ?= "${E2E_DIR}/data/cni/cilium/cilium-no-kubeproxy.yaml"
+CNI_PATH_FLANNEL ?= "${E2E_DIR}/data/cni/flannel/flannel.yaml"
+CNI_PATH_KINDNET ?= "${E2E_DIR}/data/cni/kindnet/kindnet.yaml"
 
 # CRD_OPTIONS define options to add to the CONTROLLER_GEN
 CRD_OPTIONS ?= "crd:crdVersions=v1"
@@ -78,8 +79,8 @@ ifneq ($(LABEL_FILTERS),)
 		LABEL_FILTER_ARGS := "$(LABEL_FILTER_ARGS) && $(LABEL_FILTERS)"
 endif
 JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
-GINKGO_SKIP ?= ""
-GINKGO_FOCUS ?= ""
+GINKGO_SKIP ?=
+GINKGO_FOCUS ?=
 GINKGO_NODES  ?= 1
 E2E_CONF_FILE  ?= ${E2E_DIR}/config/nutanix.yaml
 E2E_CONF_FILE_TMP = ${E2E_CONF_FILE}.tmp
@@ -89,15 +90,10 @@ USE_EXISTING_CLUSTER ?= false
 GINKGO_NOCOLOR ?= false
 FLAVOR ?= e2e
 
-# set ginkgo focus flags, if any
-ifneq ($(strip $(GINKGO_FOCUS)),)
-_FOCUS_ARGS := $(foreach arg,$(strip $(GINKGO_FOCUS)),--focus="$(arg)")
-endif
+define ginkgo_option
+--$(1)="$(shell echo '$(2)' | sed -E 's/^[[:space:]]+//' | sed -E 's/"[[:space:]]+"/" --$(1)="/g')"
+endef
 
-# to set multiple ginkgo skip flags, if any
-ifneq ($(strip $(GINKGO_SKIP)),)
-_SKIP_ARGS := $(foreach arg,$(strip $(GINKGO_SKIP)),--skip="$(arg)")
-endif
 .PHONY: all
 all: build
 
@@ -159,6 +155,31 @@ kind-delete: ## Delete the kind cluster
 nutanix-cp-endpoint-ip: ## Gets a random free IP from the control plane endpoint range set in the environment.
 	@shuf --head-count=1 < <(fping -g -u "$(CONTROL_PLANE_ENDPOINT_RANGE_START)" "$(CONTROL_PLANE_ENDPOINT_RANGE_END)")
 
+update-calico-cni: ## Updates the calico CNI manifests
+	@echo "Updating calico CNI manifest"
+	@curl -sL https://docs.projectcalico.org/manifests/calico.yaml -o $(CNI_PATH_CALICO)
+
+update-cilium-cni: ## Updates the cilium CNI manifests
+	@echo "Updating cilium CNI manifest"
+	@helm repo add cilium https://helm.cilium.io/
+	@helm repo update
+	# TODO(use the latest version of cilium instead of hardcoding v1.15.4 once fix for 1.15.5+ failure is identified)
+	# as 1.15.5+ fails due to mount-cgroup init container failing with nsenter: cannot open /hostproc/1/ns/cgroup: Permission denied
+	# Also, add k8s-service-proxy-name: "cilium" to cilium-config ConfigMap to ensure multi-protocol sig-network conformance tests pass
+	@helm template cilium cilium/cilium --version 1.15.4 -n kube-system --set cni.chainingMode=portmap --set sessionAffinity=true --set k8s.serviceProxyName=cilium | awk '{gsub(/\$\{BIN_PATH\}/,"$ BIN_PATH"); print}' > $(CNI_PATH_CILIUM)
+	@helm template cilium cilium/cilium --version 1.15.4 -n kube-system --set cni.chainingMode=portmap --set sessionAffinity=true --set kubeProxyReplacement=strict --set k8s.serviceProxyName=cilium | awk '{gsub(/\$\{BIN_PATH\}/,"$ BIN_PATH"); print}' > $(CNI_PATH_CILIUM_NO_KUBEPROXY)
+
+update-flannel-cni: ## Updates the flannel CNI manifests
+	@echo "Updating flannel CNI manifest"
+	@curl -sL https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml -o $(CNI_PATH_FLANNEL)
+
+update-kindnet-cni: ## Updates the kindnet CNI manifests
+	@echo "Updating kindnet CNI manifest"
+	@curl -sL https://github.com/kubernetes-sigs/cluster-api/raw/main/test/e2e/data/cni/kindnet/kindnet.yaml -o $(CNI_PATH_KINDNET)
+
+.PHONY: update-cni-manifests ## Updates all the CNI manifests to latest variants from upstream
+update-cni-manifests: update-calico-cni update-cilium-cni update-flannel-cni update-kindnet-cni  ## Updates all the CNI manifests to latest variants from upstream
+
 ##@ Build
 
 .PHONY: build
@@ -320,18 +341,17 @@ test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
-		--progress \
 		--tags=e2e \
 		--label-filter=$(LABEL_FILTER_ARGS) \
-		$(_SKIP_ARGS) \
-		$(_FOCUS_ARGS) \
+		$(call ginkgo_option,skip,$(GINKGO_SKIP)) \
+		$(call ginkgo_option,focus,$(GINKGO_FOCUS)) \
 		--nodes=$(GINKGO_NODES) \
 		--no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" \
 		--junit-report=${JUNIT_REPORT_FILE} \
 		--timeout="24h" \
-		--always-emit-ginkgo-writer \
-		$(GINKGO_ARGS) ./test/e2e -- \
+		$(GINKGO_ARGS) \
+		./test/e2e -- \
 		-e2e.artifacts-folder="$(ARTIFACTS)" \
 		-e2e.config="$(E2E_CONF_FILE_TMP)" \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
@@ -346,17 +366,17 @@ test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy clust
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
-		--progress \
 		--tags=e2e \
 		--label-filter=$(LABEL_FILTER_ARGS) \
-		$(_SKIP_ARGS) \
+		$(call ginkgo_option,skip,$(GINKGO_SKIP)) \
+		$(call ginkgo_option,focus,$(GINKGO_FOCUS)) \
 		--nodes=$(GINKGO_NODES) \
 		--no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" \
 		--junit-report=${JUNIT_REPORT_FILE} \
 		--timeout="24h" \
-		--always-emit-ginkgo-writer \
-		$(GINKGO_ARGS) ./test/e2e -- \
+		$(GINKGO_ARGS) \
+		./test/e2e -- \
 		-e2e.artifacts-folder="$(ARTIFACTS)" \
 		-e2e.config="$(E2E_CONF_FILE)" \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
@@ -365,12 +385,22 @@ test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy clust
 .PHONY: list-e2e
 list-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
 	mkdir -p $(ARTIFACTS)
-	ginkgo -v --trace --dry-run --tags=e2e --label-filter="$(LABEL_FILTERS)" $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) \
-	    --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" \
-	    $(GINKGO_ARGS) ./test/e2e -- \
+	ginkgo -v \
+	    --trace \
+	    --dry-run \
+	    --tags=e2e \
+	    --label-filter="$(LABEL_FILTERS)" \
+		$(call ginkgo_option,skip,$(GINKGO_SKIP)) \
+		$(call ginkgo_option,focus,$(GINKGO_FOCUS)) \
+	    --nodes=$(GINKGO_NODES) \
+	    --no-color=$(GINKGO_NOCOLOR) \
+	    --output-dir="$(ARTIFACTS)" \
+	    $(GINKGO_ARGS) \
+	    ./test/e2e -- \
 	    -e2e.artifacts-folder="$(ARTIFACTS)" \
 	    -e2e.config="$(E2E_CONF_FILE)" \
-	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
+	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
+	    -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
 
 .PHONY: test-e2e-calico
 test-e2e-calico:
@@ -382,7 +412,7 @@ test-e2e-flannel:
 
 .PHONY: test-e2e-cilium
 test-e2e-cilium:
-	CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
+	CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" GINKGO_SKIP=$(GINKGO_SKIP) $(MAKE) test-e2e
 
 .PHONY: test-e2e-cilium-no-kubeproxy
 test-e2e-cilium-no-kubeproxy:

diff --git a/devbox.json b/devbox.json
@@ -9,6 +9,7 @@
     "ginkgo@2.17.0",
     "go@1.22.1",
     "gotestsum@1.6.4",
+    "kubernetes-helm@latest",
     "kind@0.22.0",
     "ko@0.15.2",
     "kubectl@latest",

diff --git a/devbox.lock b/devbox.lock
@@ -447,6 +447,54 @@
         }
       }
     },
+    "kubernetes-helm@latest": {
+      "last_modified": "2024-07-07T16:08:25Z",
+      "resolved": "github:NixOS/nixpkgs/ab82a9612aa45284d4adf69ee81871a389669a9e#kubernetes-helm",
+      "source": "devbox-search",
+      "version": "3.15.2",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/svgmhkl47asx8r999aiv8db9gc79mn09-kubernetes-helm-3.15.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/svgmhkl47asx8r999aiv8db9gc79mn09-kubernetes-helm-3.15.2"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/zg9a8pk65mp7gy645j408lck3psc1m45-kubernetes-helm-3.15.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/zg9a8pk65mp7gy645j408lck3psc1m45-kubernetes-helm-3.15.2"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/dnpl9y8vwxx130hg8i5s7zigcpq6a67i-kubernetes-helm-3.15.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/dnpl9y8vwxx130hg8i5s7zigcpq6a67i-kubernetes-helm-3.15.2"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/p0hahjjjvnixs864zsfd3cn9k586wca3-kubernetes-helm-3.15.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/p0hahjjjvnixs864zsfd3cn9k586wca3-kubernetes-helm-3.15.2"
+        }
+      }
+    },
     "kustomize@5.3.0": {
       "last_modified": "2024-02-24T23:06:34Z",
       "resolved": "github:NixOS/nixpkgs/9a9dae8f6319600fa9aebde37f340975cab4b8c0#kustomize",

diff --git a/test/e2e/data/cni/calico/calico.yaml b/test/e2e/data/cni/calico/calico.yaml
@@ -1,8 +1,4 @@
 ---
-# From: https://projectcalico.docs.tigera.io/v3.25/manifests/calico.yaml
-# Source: calico/templates/calico-config.yaml
-# Modified original file in order to use quay.io instead of docker.io
----
 # Source: calico/templates/calico-kube-controllers.yaml
 # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
 
@@ -977,10 +973,10 @@ spec:
                 - type: string
                 description: 'BPFPSNATPorts sets the range from which we randomly
                   pick a port if there is a source port collision. This should be
-                  within the ephemeral range as defined by RFC 6056 (1024â€“65535) and
+                  within the ephemeral range as defined by RFC 6056 (1024–65535) and
                   preferably outside the  ephemeral ranges used by common operating
-                  systems. Linux uses 32768â€“60999, while others mostly use the IANA
-                  defined range 49152â€“65535. It is not necessarily a problem if this
+                  systems. Linux uses 32768–60999, while others mostly use the IANA
+                  defined range 49152–65535. It is not necessarily a problem if this
                   range overlaps with the operating systems. Both ends of the range
                   are inclusive. [Default: 20000:29999]'
                 pattern: ^.*
@@ -4233,7 +4229,7 @@ rules:
     resources:
       - endpointslices
     verbs:
-      - watch
+      - watch 
       - list
   - apiGroups: [""]
     resources:
@@ -4314,7 +4310,7 @@ rules:
       - create
       - update
   # Calico must update some CRDs.
-  - apiGroups: ["crd.projectcalico.org"]
+  - apiGroups: [ "crd.projectcalico.org" ]
     resources:
       - caliconodestatuses
     verbs:
@@ -4444,7 +4440,7 @@ spec:
         # It can be deleted if this is a fresh installation, or if you have already
         # upgraded to use calico-ipam.
         - name: upgrade-ipam
-          image: quay.io/calico/cni:v3.25.0
+          image: docker.io/calico/cni:v3.25.0
           imagePullPolicy: IfNotPresent
           command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
           envFrom:
@@ -4472,7 +4468,7 @@ spec:
         # This container installs the CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: quay.io/calico/cni:v3.25.0
+          image: docker.io/calico/cni:v3.25.0
           imagePullPolicy: IfNotPresent
           command: ["/opt/cni/bin/install"]
           envFrom:
@@ -4515,7 +4511,7 @@ spec:
         # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed
         # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode.
         - name: "mount-bpffs"
-          image: quay.io/calico/node:v3.25.0
+          image: docker.io/calico/node:v3.25.0
           imagePullPolicy: IfNotPresent
           command: ["calico-node", "-init", "-best-effort"]
           volumeMounts:
@@ -4541,7 +4537,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: quay.io/calico/node:v3.25.0
+          image: docker.io/calico/node:v3.25.0
           imagePullPolicy: IfNotPresent
           envFrom:
           - configMapRef:
@@ -4758,7 +4754,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: calico-kube-controllers
-          image: quay.io/calico/kube-controllers:v3.25.0
+          image: docker.io/calico/kube-controllers:v3.25.0
           imagePullPolicy: IfNotPresent
           env:
             # Choose which controllers to run.