operator: API to configure kubelet directory path

Not all clusters use /var/lib/kubelet as state directory for kubelet
(kubernetes-csi/csi-driver-host-path#71). There is no way for the operator
to auto-detect this path. Hence, added a new field to deployment spec that
could be used to provide the kubelet path to driver deployment.

FIXES: intel#668

docs/design.md

@@ -33,7 +33,7 @@ There is a more detailed explanation in the following paragraphs.
 <sup>1 </sup> **Free space fragmentation** is a problem when there appears to
 be enough free capacity for a new namespace, but there isn't a contiguous
 region big enough to allocate it. The PMEM-CSI driver is only capable of
-allocating continguous memory to a namespace and cannot de-fragment or combine
+allocating continuous memory to a namespace and cannot de-fragment or combine
 smaller blocks. For example, this could happen when you create a 63 GB
 namespace, followed by a 1 GB namespace, and then delete the 63 GB namespace.
 Eventhough there is 127 GB available, the driver cannot create a namespace

docs/install.md

@@ -842,8 +842,9 @@ The current API for PMEM-CSI `Deployment` resources is:
 | nodeControllerKey | string | Encoded RSA private key used for signing by `nodeControllerCert` | generated by the operator |
 | caCert | string | Certificate of the CA by which the `registryCert` and `controllerCert` are signed | self-signed certificate generated by the operator |
 | nodeSelector | string map | [Labels to use for selecting Nodes](../docs/install.md#run-pmem-csi-on-kubernetes) on which PMEM-CSI driver should run. | `{ "storage": "pmem" }`|
-| pmemPercentage | integer | Percentage of PMEM space to be used by the driver on each node. This is only valid for a driver deployed in `lvm` mode. This field can be modified, but by that time the old value may have been used already. Reducing the percentage is not supported. | 100 |
-| labels | string map | Additional labels for all objects created by the operator. Can be modified after the initial creation, but removed labels will not be removed from existing objects because the operator cannot know which labels it needs to remove and which it has to leave in place. |
+| pmemPercentage | integer | Percentage of PMEM space to be used by the driver on each node. This is only valid for a driver deployed in `lvm` mode. This field can be modified, but by that time the old value may have been used already. Reducing the percentage is not supported | 100 |
+| labels | string map | Additional labels for all objects created by the operator. Can be modified after the initial creation, but removed labels will not be removed from existing objects because the operator cannot know which labels it needs to remove and which it has to leave in place |
+| kubeletDir | string | Kubelet's root directory path | /var/lib/kubelet |
 
 <sup>1</sup> To use the same container image as default driver image
 the operator pod must set with below environment variables with

pkg/apis/pmemcsi/v1alpha1/deployment_types.go

@@ -91,6 +91,8 @@ type DeploymentSpec struct {
 	PMEMPercentage uint16 `json:"pmemPercentage,omitempty"`
 	// Labels contains additional labels for all objects created by the operator.
 	Labels map[string]string `json:"labels,omitempty"`
+	// KubeletDir kubelet's root directory path
+	KubeletDir string `json:"kubeletDir,omitempty"`
 }
 
 // DeploymentStatus defines the observed state of Deployment
@@ -163,6 +165,8 @@ const (
 	DefaultDeviceMode = DeviceModeLVM
 	// DefaultPMEMPercentage PMEM space to reserve for the driver
 	DefaultPMEMPercentage = 100
+	// DefaultKubeletDir default kubelet's path
+	DefaultKubeletDir = "/var/lib/kubelet"
 )
 
 var (
@@ -204,6 +208,7 @@ const (
 	RegistryKey
 	NodeControllerCertificate
 	NodeControllerKey
+	KubeletDir
 )
 
 func (c DeploymentChange) String() string {
@@ -224,6 +229,7 @@ func (c DeploymentChange) String() string {
 		RegistryKey:               "registryKey",
 		NodeControllerCertificate: "nodeControllerCert",
 		NodeControllerKey:         "nodeControllerKey",
+		KubeletDir:                "kubeletDir",
 	}[c]
 }
 
@@ -293,6 +299,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error {
 		d.Spec.PMEMPercentage = DefaultPMEMPercentage
 	}
 
+	if d.Spec.KubeletDir == "" {
+		d.Spec.KubeletDir = DefaultKubeletDir
+	}
+
 	return nil
 }
 
@@ -357,6 +367,9 @@ func (d *Deployment) Compare(other *Deployment) map[DeploymentChange]struct{} {
 	if bytes.Compare(d.Spec.NodeControllerPrivateKey, other.Spec.NodeControllerPrivateKey) != 0 {
 		changes[NodeControllerKey] = struct{}{}
 	}
+	if d.Spec.KubeletDir != other.Spec.KubeletDir {
+		changes[KubeletDir] = struct{}{}
+	}
 
 	return changes
 }
@@ -460,7 +473,12 @@ func GetDeploymentCRDSchema() *apiextensions.JSONSchemaProps {
 								Type: "string",
 							},
 						},
-					}},
+					},
+					"kubeletDir": apiextensions.JSONSchemaProps{
+						Type:        "string",
+						Description: "Kubelet root directory path",
+					},
+				},
 			},
 			"status": apiextensions.JSONSchemaProps{
 				Type:        "object",

pkg/deployments/load.go

@@ -47,6 +47,11 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo
 		*yaml = bytes.ReplaceAll(*yaml, []byte("path: /var/lib/pmem-csi.intel.com"), []byte("path: /var/lib/"+deployment.Name))
 		*yaml = bytes.ReplaceAll(*yaml, []byte("mountPath: /var/lib/pmem-csi.intel.com"), []byte("mountPath: /var/lib/"+deployment.Name))
 
+		// Update kubelet path
+		if deployment.Spec.KubeletDir != api.DefaultKubeletDir {
+			*yaml = bytes.ReplaceAll(*yaml, []byte("/var/lib/kubelet"), []byte(deployment.Spec.KubeletDir))
+		}
+
 		// This assumes that all namespaced objects actually have "namespace: default".
 		*yaml = bytes.ReplaceAll(*yaml, []byte("namespace: default"), []byte("namespace: "+namespace))
 
@@ -198,7 +203,7 @@ func loadObjects(kubernetes version.Version, deviceMode api.DeviceMode,
 	// item. Only works for .yaml.
 	//
 	// We need to split ourselves because we need access to each
-	// original chunk of data for decoding.  kubectl has its own
+	// original chunk of data for decoding. kubectl has its own
 	// infrastructure for this, but that is a lot of code with
 	// many dependencies.
 	items := bytes.Split(yaml, []byte("\n---"))

pkg/pmem-csi-operator/controller/deployment/controller_driver.go

@@ -105,6 +105,8 @@ func (d *PmemCSIDriver) reconcileDeploymentChanges(r *ReconcileDeployment, chang
 			updateAll = true
 		case api.CACertificate, api.RegistryCertificate, api.NodeControllerCertificate:
 			updateSecrets = true
+		case api.KubeletDir:
+			updateNodeDriver = true
 		}
 
 		if err != nil {
@@ -717,7 +719,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "registration-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/plugins_registry/",
+									Path: d.Spec.KubeletDir + "/plugins_registry/",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -726,7 +728,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "mountpoint-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/plugins/kubernetes.io/csi",
+									Path: d.Spec.KubeletDir + "/plugins/kubernetes.io/csi",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -735,7 +737,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "pods-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/pods",
+									Path: d.Spec.KubeletDir + "/pods",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -908,12 +910,12 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container {
 		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:             "mountpoint-dir",
-				MountPath:        "/var/lib/kubelet/plugins/kubernetes.io/csi",
+				MountPath:        d.Spec.KubeletDir + "/plugins/kubernetes.io/csi",
 				MountPropagation: &bidirectional,
 			},
 			{
 				Name:             "pods-dir",
-				MountPath:        "/var/lib/kubelet/pods",
+				MountPath:        d.Spec.KubeletDir + "/pods",
 				MountPropagation: &bidirectional,
 			},
 			{

pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go

@@ -45,6 +45,7 @@ type pmemDeployment struct {
 	controllerCPU, controllerMemory                     string
 	nodeCPU, nodeMemory                                 string
 	caCert, regCert, regKey, ncCert, ncKey              []byte
+	kubeletDir                                          string
 }
 
 func getDeployment(d *pmemDeployment) *api.Deployment {
@@ -91,6 +92,9 @@ func getDeployment(d *pmemDeployment) *api.Deployment {
 	spec.RegistryPrivateKey = d.regKey
 	spec.NodeControllerCert = d.ncCert
 	spec.NodeControllerPrivateKey = d.ncKey
+	if d.kubeletDir != "" {
+		spec.KubeletDir = d.kubeletDir
+	}
 
 	return dep
 }
@@ -236,6 +240,7 @@ func TestDeploymentController(t *testing.T) {
 				controllerMemory: "300Mi",
 				nodeCPU:          "1000m",
 				nodeMemory:       "500Mi",
+				kubeletDir:       "/some/directory",
 			}
 
 			dep := getDeployment(d)
@@ -308,7 +313,7 @@ func TestDeploymentController(t *testing.T) {
 		t.Run("provided private keys and certificates", func(t *testing.T) {
 			tc := setup(t)
 			ca, err := pmemtls.NewCA(nil, nil)
-			require.NoError(t, err, "faield to instantiate CA")
+			require.NoError(t, err, "failed to instantiate CA")
 
 			regKey, err := pmemtls.NewPrivateKey()
 			require.NoError(t, err, "failed to generate a private key: %v", err)

pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go

@@ -77,6 +77,9 @@ func UpdateTests() []UpdateTest {
 			}
 			d.Spec.Labels["foo"] = "bar"
 		},
+		"kubeletDir": func(d *api.Deployment) {
+			d.Spec.KubeletDir = "/foo/bar"
+		},
 	}
 
 	full := api.Deployment{

test/e2e/operator/validate/validate.go

@@ -281,7 +281,7 @@ StatefulSet:` + defaultsApps + `
 // In addition, it fills in default values in the expected spec if they are missing
 // before comparing against the actual value. This makes it possible to
 // compare the on-disk YAML files which are typically not complete against
-// objects frome the apiserver which have all defaults filled in.
+// objects from the apiserver which have all defaults filled in.
 func compareSpec(kind string, expected, actual interface{}) []string {
 	defaults := defaultSpecValues[kind]
 	path := "spec"