enable interconnect in helm chart

add `-ic` option to `kind-helm.sh` to enable interconnect - if npz is not specified, create single-node zone interconnect - if npz is greater than 1, create multi-node zone interconnect Fixes: #4371 Signed-off-by: Xiaobin Qu <xqu@nvidia.com>
ovn-org · Aug 19, 2024 · 9d03215 · 9d03215
1 parent 8219205
commit 9d03215
Show file tree

Hide file tree

Showing 17 changed files with 561 additions and 64 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -407,14 +407,14 @@ jobs:
         # forwarding : ["", "disable-forwarding"]
         # dns-name-resolver : ["", "enable-dns-name-resolver"]
         include:
-          - {"target": "shard-conformance", "ha": "HA",   "gateway-mode": "local",  "ipfamily": "ipv6",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "shard-conformance", "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled",          "dns-name-resolver": "enable-dns-name-resolver"}
           - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
-          - {"target": "control-plane-helm", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled", "dns-name-resolver": "enable-dns-name-resolver"}
+          - {"target": "control-plane-helm","ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled", "dns-name-resolver": "enable-dns-name-resolver"}
+          - {"target": "control-plane-helm","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones", "dns-name-resolver": "enable-dns-name-resolver"}
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "dns-name-resolver": "enable-dns-name-resolver"}
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}

diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -exo pipefail
+set -eo pipefail
 
 # Returns the full directory name of the script
 export DIR="$( cd -- "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -57,6 +57,24 @@ set_default_params() {
     KIND_NUM_MASTER=3
   fi
 
+  OVN_ENABLE_INTERCONNECT=${OVN_ENABLE_INTERCONNECT:-false}
+  if [ "$OVN_COMPACT_MODE" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != false ]; then
+     echo "Compact mode cannot be used together with Interconnect"
+     exit 1
+  fi
+
+
+  if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then
+    KIND_NUM_NODES_PER_ZONE=${KIND_NUM_NODES_PER_ZONE:-1}
+    TOTAL_NODES=$((KIND_NUM_WORKER + KIND_NUM_MASTER))
+    if [[ ${KIND_NUM_NODES_PER_ZONE} -gt 1 ]] && [[ $((TOTAL_NODES % KIND_NUM_NODES_PER_ZONE)) -ne 0 ]]; then
+      echo "(Total k8s nodes / number of nodes per zone) should be zero"
+      exit 1
+    fi
+  else
+    KIND_NUM_NODES_PER_ZONE=0
+  fi
+
   # Hard code ipv4 support until IPv6 is implemented
   export KIND_IPV4_SUPPORT=true
 
@@ -77,6 +95,8 @@ usage() {
     echo "       [ -ikv | --install-kubevirt ]"
     echo "       [ -mne | --multi-network-enable ]"
     echo "       [ -wk  | --num-workers <num> ]"
+    echo "       [ -ic  | --enable-interconnect]"
+    echo "       [ -npz | --node-per-zone ]"
     echo "       [ -cn  | --cluster-name ]"
     echo "       [ -h ]"
     echo ""
@@ -97,6 +117,8 @@ usage() {
     echo "-wk  | --num-workers                Number of worker nodes. DEFAULT: 2 workers"
     echo "-cn  | --cluster-name               Configure the kind cluster's name"
     echo "-dns | --enable-dnsnameresolver     Enable DNSNameResolver for resolving the DNS names used in the DNS rules of EgressFirewall."
+    echo "-ic  | --enable-interconnect        Enable interconnect with each node as a zone (only valid if OVN_HA is false)"
+    echo "-npz | --nodes-per-zone             Specify number of nodes per zone (Default 0, which means global zone; >0 means interconnect zone, where 1 for single-node zone, >1 for multi-node zone). If this value > 1, then (total k8s nodes (workers + 1) / num of nodes per zone) should be zero."
     echo ""
 
 }
@@ -151,6 +173,16 @@ parse_args() {
                                                 ;;
             -dns | --enable-dnsnameresolver )   OVN_ENABLE_DNSNAMERESOLVER=true
                                                 ;;
+            -ic | --enable-interconnect )       OVN_ENABLE_INTERCONNECT=true
+                                                ;;
+            -npz | --nodes-per-zone )           shift
+                                                if ! [[ "$1" =~ ^[0-9]+$ ]]; then
+                                                    echo "Invalid num-nodes-per-zone: $1"
+                                                    usage
+                                                    exit 1
+                                                fi
+                                                KIND_NUM_NODES_PER_ZONE=$1
+                                                ;;
             * )                                 usage
                                                 exit 1
         esac
@@ -178,6 +210,14 @@ print_params() {
      echo "KIND_NUM_MASTER = $KIND_NUM_MASTER"
      echo "KIND_NUM_WORKER = $KIND_NUM_WORKER"
      echo "OVN_ENABLE_DNSNAMERESOLVER= $OVN_ENABLE_DNSNAMERESOLVER"
+     echo "OVN_ENABLE_INTERCONNECT = $OVN_ENABLE_INTERCONNECT"
+     if [[ $OVN_ENABLE_INTERCONNECT == true ]]; then
+       echo "KIND_NUM_NODES_PER_ZONE = $KIND_NUM_NODES_PER_ZONE"
+       if [ "${KIND_NUM_NODES_PER_ZONE}" -gt 1 ] && [ "${OVN_ENABLE_OVNKUBE_IDENTITY}" = "true" ]; then
+         echo "multi_node_zone is not compatible with ovnkube_identity, disabling ovnkube_identity"
+         OVN_ENABLE_OVNKUBE_IDENTITY="false"
+       fi
+     fi
      echo ""
 }
 
@@ -191,7 +231,6 @@ check_dependencies() {
 
     # check for currently unsupported features
     [ "${KIND_IPV6_SUPPORT}" == "true" ] && { &>1 echo "Fatal: KIND_IPV6_SUPPORT support not implemented yet"; exit 1; } ||:
-    [ "${OVN_ENABLE_INTERCONNECT}" == "true" ] && { &>1 echo "Fatal: OVN_ENABLE_INTERCONNECT support not implemented yet"; exit 1; } ||:
 }
 
 helm_prereqs() {
@@ -302,28 +341,66 @@ EOT
     fi
 }
 
+label_ovn_single_node_zones() {
+  KIND_NODES=$(kind_get_nodes)
+  for n in $KIND_NODES; do
+    kubectl label node "${n}" k8s.ovn.org/zone-name=${n} --overwrite
+  done
+}
+
+label_ovn_multiple_nodes_zones() {
+  KIND_NODES=$(kind_get_nodes | sort)
+  zone_idx=1
+  n=1
+  for node in $KIND_NODES; do
+    zone="zone-${zone_idx}"
+    kubectl label node "${node}" k8s.ovn.org/zone-name=${zone} --overwrite
+    if [ "${n}" == "1" ]; then
+      # Mark 1st node of each zone as zone control plane
+      kubectl label node "${node}" node-role.kubernetes.io/zone-controller="" --overwrite
+    fi
+
+    if [ "${n}" == "${KIND_NUM_NODES_PER_ZONE}" ]; then
+      n=1
+      zone_idx=$((zone_idx+1))
+    else
+      n=$((n+1))
+    fi
+  done
+}
+
 create_ovn_kubernetes() {
     cd ${DIR}/../helm/ovn-kubernetes
-
     MASTER_REPLICAS=$(kubectl get node -l node-role.kubernetes.io/control-plane --no-headers | wc -l)
-    helm install ovn-kubernetes . -f values.yaml \
-        --set k8sAPIServer=${API_URL} \
-        --set podNetwork="${NET_CIDR_IPV4}/24" \
-        --set serviceNetwork=${SVC_CIDR_IPV4} \
-        --set ovnkube-identity.replicas=${MASTER_REPLICAS} \
-        --set ovnkube-master.replicas=${MASTER_REPLICAS} \
-        --set global.image.repository=$(get_image) \
-        --set global.image.tag=$(get_tag) \
-        --set global.enableAdminNetworkPolicy=true \
-        --set global.enableMulticast=$(if [ "${OVN_MULTICAST_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set global.enableMultiNetwork=$(if [ "${ENABLE_MULTI_NET}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set global.enableHybridOverlay=$(if [ "${OVN_HYBRID_OVERLAY_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set global.enableDNSNameResolver=$(if [ "${OVN_ENABLE_DNSNAMERESOLVER}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set tags.ovnkube-db-raft=$(if [ "${OVN_HA}" == "true" ]; then echo "true"; else echo "false"; fi) \
-        --set tags.ovnkube-db=$(if [ "${OVN_HA}" == "false" ]; then echo "true"; else echo "false"; fi) \
-        --set global.v4MasqueradeSubnet=${MASQUERADE_SUBNET_IPV4} \
-        --set global.v6MasqueradeSubnet=${MASQUERADE_SUBNET_IPV6}
+    if [[ $KIND_NUM_NODES_PER_ZONE == 1 ]]; then
+      label_ovn_single_node_zones
+      value_file="values-single-node-zone.yaml"
+      ovnkube_db_options=""
+    elif [[ $KIND_NUM_NODES_PER_ZONE > 1 ]]; then
+      label_ovn_multiple_nodes_zones
+      value_file="values-multi-node-zone.yaml"
+      ovnkube_db_options=""
+    else
+      value_file="values-no-ic.yaml"
+      ovnkube_db_options="--set tags.ovnkube-db-raft=$(if [ "${OVN_HA}" == "true" ]; then echo "true"; else echo "false"; fi) \
+                          --set tags.ovnkube-db=$(if [ "${OVN_HA}" == "false" ]; then echo "true"; else echo "false"; fi)"
+    fi
+    echo "value_file=${value_file}"
+    helm install ovn-kubernetes . -f ${value_file} \
+          --set k8sAPIServer=${API_URL} \
+          --set podNetwork="${NET_CIDR_IPV4}/24" \
+          --set serviceNetwork=${SVC_CIDR_IPV4} \
+          --set ovnkube-identity.replicas=${MASTER_REPLICAS} \
+          --set ovnkube-master.replicas=${MASTER_REPLICAS} \
+          --set global.image.repository=$(get_image) \
+          --set global.image.tag=$(get_tag) \
+          --set global.enableAdminNetworkPolicy=true \
+          --set global.enableMulticast=$(if [ "${OVN_MULTICAST_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          --set global.enableMultiNetwork=$(if [ "${ENABLE_MULTI_NET}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          --set global.enableHybridOverlay=$(if [ "${OVN_HYBRID_OVERLAY_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          --set global.enableDNSNameResolver=$(if [ "${OVN_ENABLE_DNSNAMERESOLVER}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          ${ovnkube_db_options}
 }
 
 delete() {
@@ -341,8 +418,8 @@ install_online_ovn_kubernetes_crds() {
 }
 
 check_dependencies
-set_default_params
 parse_args "$@"
+set_default_params
 print_params
 helm_prereqs
 build_ovn_image
@@ -369,7 +446,15 @@ if [ "$ENABLE_MULTI_NET" == true ]; then
   enable_multi_net
 fi
 
+# if ! kubectl wait -n ovn-kubernetes --for=condition=ready pods --all --timeout=300s ; then
+#  echo "some pods in the system are not running"
+#  kubectl get pods -A -o wide || true
+#  kubectl describe po -A
+#  exit 1
+# fi
+
 kubectl_wait_pods
+
 if [ "$OVN_ENABLE_DNSNAMERESOLVER" == true ]; then
     kubectl_wait_dnsnameresolver_pods
 fi

diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh
@@ -2011,7 +2011,7 @@ ovnkube-controller-with-node() {
     ${ovn_enable_dnsnameresolver_flag} \
     --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \
     --export-ovs-metrics \
-    --gateway-mode=${ovn_gateway_mode} \
+    --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \
     --gateway-router-subnet=${ovn_gateway_router_subnet} \
     --host-network-namespace ${ovn_host_network_namespace} \
     --inactivity-probe=${ovn_remote_probe_interval} \

diff --git a/docs/installation/launching-ovn-kubernetes-with-helm.md b/docs/installation/launching-ovn-kubernetes-with-helm.md
@@ -233,7 +233,7 @@ false
 			<td>Whether or not to enable hybrid overlay functionality</td>
 		</tr>
 		<tr>
-			<td>global.enableInterConnect</td>
+			<td>global.enableInterconnect</td>
 			<td>bool</td>
 			<td><pre lang="json">
 false

diff --git a/helm/ovn-kubernetes/README.md b/helm/ovn-kubernetes/README.md
@@ -56,7 +56,16 @@ some of these subcharts are installed to provide the aforementioned OVN K8s
 CNI features, this can be done by editing `tags` section in values.yaml file.
 
 ## Quickstart:
-Run script `helm/basic-deploy.sh` to set up a basic OVN/Kubernetes cluster.
+- Install Kind, see https://kind.sigs.k8s.io
+- Run script `contrib/kind-helm.sh` to set up a basic OVN/Kubernetes cluster.
+- Run following command to set up a OVN/Kubernetes cluster with single-node-zone interconnect enabled.
+  ```
+  contrib/kind-helm.sh -ic
+  ```
+- Add `-npz` (node-per-zone) to set up cluster with multi-node-zone interconnect
+  ```
+  contrib/kind-helm.sh -ic -wk 3 -npz 2
+  ```
 
 ## Manual steps:
 - Disable IPv6 of `kind` docker network, otherwise ovnkube-node will fail to start
@@ -211,6 +220,15 @@ false
 </td>
 			<td>Enables monitoring OVN-Kubernetes master and OVN configuration duration</td>
 		</tr>
+		<tr>
+			<td>global.enableDNSNameResolver</td>
+			<td>bool</td>
+			<td><pre lang="json">
+false
+</pre>
+</td>
+			<td>Configure to use DNSNameResolver feature with ovn-kubernetes</td>
+		</tr>
 		<tr>
 			<td>global.enableEgressFirewall</td>
 			<td>bool</td>
@@ -257,7 +275,7 @@ true
 			<td>Whether or not to enable hybrid overlay functionality</td>
 		</tr>
 		<tr>
-			<td>global.enableInterConnect</td>
+			<td>global.enableInterconnect</td>
 			<td>bool</td>
 			<td><pre lang="json">
 false
@@ -602,7 +620,7 @@ false
 			<td>global.v4MasqueradeSubnet</td>
 			<td>string</td>
 			<td><pre lang="json">
-""
+"169.254.0.0/17"
 </pre>
 </td>
 			<td>The v4 masquerade subnet used for assigning masquerade IPv4 addresses</td>
@@ -620,7 +638,7 @@ false
 			<td>global.v6MasqueradeSubnet</td>
 			<td>string</td>
 			<td><pre lang="json">
-""
+"fd69::/112"
 </pre>
 </td>
 			<td>The v6 masquerade subnet used for assigning masquerade IPv6 addresses</td>
@@ -676,6 +694,15 @@ false
 </td>
 			<td>number of ovnube-identity pods, co-located with kube-apiserver process, so need to be the same number of control plane nodes</td>
 		</tr>
+		<tr>
+			<td>ovnkube-master.replicas</td>
+			<td>int</td>
+			<td><pre lang="json">
+1
+</pre>
+</td>
+			<td>number of ovnkube-master pods</td>
+		</tr>
 		<tr>
 			<td>podNetwork</td>
 			<td>string</td>

diff --git a/helm/ovn-kubernetes/README.md.gotmpl b/helm/ovn-kubernetes/README.md.gotmpl
@@ -54,7 +54,16 @@ some of these subcharts are installed to provide the aforementioned OVN K8s
 CNI features, this can be done by editing `tags` section in values.yaml file.
 
 ## Quickstart:
-Run script `helm/basic-deploy.sh` to set up a basic OVN/Kubernetes cluster.
+- Install Kind, see https://kind.sigs.k8s.io
+- Run script `contrib/kind-helm.sh` to set up a basic OVN/Kubernetes cluster.
+- Run following command to set up a OVN/Kubernetes cluster with single-node-zone interconnect enabled.
+  ```
+  contrib/kind-helm.sh -ic
+  ```
+- Add `-npz` (node-per-zone) to set up cluster with multi-node-zone interconnect 
+  ```
+  contrib/kind-helm.sh -ic -wk 3 -npz 2
+  ```
 
 ## Manual steps:
 - Disable IPv6 of `kind` docker network, otherwise ovnkube-node will fail to start

diff --git a/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml b/helm/ovn-kubernetes/charts/ovnkube-control-plane/templates/ovnkube-control-plane.yaml
@@ -127,7 +127,7 @@ spec:
         - name: OVN_HYBRID_OVERLAY_NET_CIDR
           value: {{ default "" .Values.global.hybridOverlayNetCidr | quote }}
         - name: OVN_DISABLE_SNAT_MULTIPLE_GWS
-          value: {{ default "" .Values.disableSnatMultipleGws | quote }}
+          value: {{ default "" .Values.global.disableSnatMultipleGws | quote }}
         - name: OVN_EMPTY_LB_EVENTS
           value: {{ default "" .Values.global.emptyLbEvents | quote }}
         - name: OVN_V4_JOIN_SUBNET
@@ -151,6 +151,12 @@ spec:
           value: {{ hasKey .Values.global "enableInterconnect" | ternary .Values.global.enableInterconnect false | quote }}
         - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
           value: {{ hasKey .Values.global "enableMultiExternalGateway" | ternary .Values.global.enableMultiExternalGateway false | quote }}
+        - name: OVN_V4_TRANSIT_SWITCH_SUBNET
+          value: {{ default "" .Values.global.v4TransitSwitchSubnet | quote }}
+        - name: OVN_V6_TRANSIT_SWITCH_SUBNET
+          value: {{ default "" .Values.global.v6TransitSwitchSubnet | quote }}
+        - name: OVN_ENABLE_PERSISTENT_IPS
+          value: {{ hasKey .Values.global "enablePersistentIPs" | ternary .Values.global.enablePersistentIPs false | quote }}
         - name: OVN_ENABLE_DNSNAMERESOLVER
           value: {{ hasKey .Values.global "enableDNSNameResolver" | ternary .Values.global.enableDNSNameResolver false | quote }}
       # end of container