Add --kubelet-dir and --image-registry switches to installer

Closes: #311
Closes: #314

CHANGELOG.md

@@ -12,6 +12,8 @@
 - **Kubernetes:** Added support for Prometheus metrics.
 - Switched from glide to go modules for dependency management.
 - ONTAP drivers now support `Virtual Pools`.
+- **Kubernetes:** Added --image-registry switch to installer. (Issue [#311](https://github.com/NetApp/trident/issues/311))
+- **Kubernetes:** Added --kubelet-dir switch to installer to simplify installation on some Kubernetes distributions. (Issue [#314](https://github.com/NetApp/trident/issues/314))
 
 ## v19.10.0
 

cli/cmd/install.go

@@ -66,6 +66,8 @@ var (
 	pvcName         string
 	tridentImage    string
 	etcdImage       string
+	kubeletDir      string
+	imageRegistry   string
 	logFormat       string
 	k8sTimeout      time.Duration
 	migratorTimeout time.Duration
@@ -119,6 +121,8 @@ func init() {
 	installCmd.Flags().StringVar(&tridentImage, "trident-image", "", "The Trident image to install.")
 	installCmd.Flags().StringVar(&etcdImage, "etcd-image", "", "The etcd image to install.")
 	installCmd.Flags().StringVar(&logFormat, "log-format", "text", "The Trident logging format (text, json).")
+	installCmd.Flags().StringVar(&kubeletDir, "kubelet-dir", "/var/lib/kubelet", "The host location of kubelet's internal state.")
+	installCmd.Flags().StringVar(&imageRegistry, "image-registry", "", "The address/port of an internal image registry.")
 
 	installCmd.Flags().DurationVar(&k8sTimeout, "k8s-timeout", 180*time.Second, "The timeout for all Kubernetes operations.")
 	installCmd.Flags().DurationVar(&migratorTimeout, "migrator-timeout", 300*time.Minute, "The timeout for etcd-to-CRD migration.")
@@ -210,11 +214,22 @@ func discoverInstallationEnvironment() error {
 	// Default deployment image to what Trident was built with
 	if tridentImage == "" {
 		tridentImage = tridentconfig.BuildImage
+
+		// Override registry only if using the default Trident image name and an alternate registry was supplied
+		if imageRegistry != "" {
+			tridentImage = utils.ReplaceImageRegistry(tridentImage, imageRegistry)
+		}
 	}
+	log.Debugf("Trident image: %s", tridentImage)
 
 	// Default deployment image to what etcd was built with
 	if etcdImage == "" {
 		etcdImage = tridentconfig.BuildEtcdImage
+
+		// Override registry only if using the default etcd image name and an alternate registry was supplied
+		if imageRegistry != "" {
+			etcdImage = utils.ReplaceImageRegistry(etcdImage, imageRegistry)
+		}
 	} else if !strings.Contains(etcdImage, tridentconfig.BuildEtcdVersion) {
 		log.Warningf("Trident was qualified with etcd %s. You appear to be using a different version.", tridentconfig.BuildEtcdVersion)
 	}
@@ -456,13 +471,13 @@ func prepareCSIYAMLFiles() error {
 	}
 
 	deploymentYAML := k8sclient.GetCSIDeploymentYAML(
-		tridentImage, appLabelValue, logFormat, Debug, useIPv6, client.ServerVersion())
+		tridentImage, imageRegistry, appLabelValue, logFormat, Debug, useIPv6, client.ServerVersion())
 	if err = writeFile(deploymentPath, deploymentYAML); err != nil {
 		return fmt.Errorf("could not write deployment YAML file; %v", err)
 	}
 
 	daemonSetYAML := k8sclient.GetCSIDaemonSetYAML(
-		tridentImage, TridentNodeLabelValue, logFormat, Debug, client.ServerVersion())
+		tridentImage, imageRegistry, kubeletDir, TridentNodeLabelValue, logFormat, Debug, client.ServerVersion())
 	if err = writeFile(csiDaemonSetPath, daemonSetYAML); err != nil {
 		return fmt.Errorf("could not write daemonset YAML file; %v", err)
 	}
@@ -741,7 +756,8 @@ func installTrident() (returnError error) {
 			logFields = log.Fields{"path": deploymentPath}
 		} else {
 			returnError = client.CreateObjectByYAML(
-				k8sclient.GetCSIDeploymentYAML(tridentImage, appLabelValue, logFormat, Debug, useIPv6, client.ServerVersion()))
+				k8sclient.GetCSIDeploymentYAML(
+					tridentImage, imageRegistry, appLabelValue, logFormat, Debug, useIPv6, client.ServerVersion()))
 			logFields = log.Fields{}
 		}
 		if returnError != nil {
@@ -762,7 +778,9 @@ func installTrident() (returnError error) {
 		} else {
 			returnError = client.CreateObjectByYAML(
 				k8sclient.GetCSIDaemonSetYAML(
-					tridentImage, TridentNodeLabelValue, logFormat, Debug, client.ServerVersion()))
+					tridentImage, imageRegistry, kubeletDir,
+					TridentNodeLabelValue, logFormat, Debug,
+					client.ServerVersion()))
 			logFields = log.Fields{}
 		}
 		if returnError != nil {
@@ -1616,6 +1634,14 @@ func installTridentInCluster() (returnError error) {
 		commandArgs = append(commandArgs, "--log-format")
 		commandArgs = append(commandArgs, logFormat)
 	}
+	if kubeletDir != "" {
+		commandArgs = append(commandArgs, "--kubelet-dir")
+		commandArgs = append(commandArgs, kubeletDir)
+	}
+	if imageRegistry != "" {
+		commandArgs = append(commandArgs, "--image-registry")
+		commandArgs = append(commandArgs, imageRegistry)
+	}
 	commandArgs = append(commandArgs, "--in-cluster=false")
 
 	// Create the install pod

cli/k8s_client/yaml_factory.go

@@ -287,7 +287,9 @@ spec:
     targetPort: 8001
 `
 
-func GetCSIDeploymentYAML(tridentImage, label, logFormat string, debug, useIPv6 bool, version *utils.Version) string {
+func GetCSIDeploymentYAML(
+	tridentImage, imageRegistry, label, logFormat string, debug, useIPv6 bool, version *utils.Version,
+) string {
 
 	var debugLine string
 	var logLevel string
@@ -306,6 +308,11 @@ func GetCSIDeploymentYAML(tridentImage, label, logFormat string, debug, useIPv6
 		ipLocalhost = "127.0.0.1"
 	}
 
+	csiSidecarRegistry := "quay.io"
+	if imageRegistry != "" {
+		csiSidecarRegistry = imageRegistry
+	}
+
 	var deploymentYAML string
 	switch version.MinorVersion() {
 	case 13:
@@ -321,6 +328,7 @@ func GetCSIDeploymentYAML(tridentImage, label, logFormat string, debug, useIPv6
 	}
 
 	deploymentYAML = strings.Replace(deploymentYAML, "{TRIDENT_IMAGE}", tridentImage, 1)
+	deploymentYAML = strings.Replace(deploymentYAML, "{CSI_SIDECAR_REGISTRY}", csiSidecarRegistry, -1)
 	deploymentYAML = strings.Replace(deploymentYAML, "{DEBUG}", debugLine, 1)
 	deploymentYAML = strings.Replace(deploymentYAML, "{LABEL}", label, -1)
 	deploymentYAML = strings.Replace(deploymentYAML, "{LOG_LEVEL}", logLevel, -1)
@@ -397,7 +405,7 @@ spec:
           mountPath: /certs
           readOnly: true
       - name: csi-provisioner
-        image: quay.io/k8scsi/csi-provisioner:v1.0.2
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-provisioner:v1.0.2
         args:
         - "--v={LOG_LEVEL}"
         - "--connection-timeout=24h"
@@ -409,7 +417,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-attacher
-        image: quay.io/k8scsi/csi-attacher:v1.0.1
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-attacher:v1.0.1
         args:
         - "--v={LOG_LEVEL}"
         - "--connection-timeout=24h"
@@ -422,7 +430,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-snapshotter
-        image: quay.io/k8scsi/csi-snapshotter:v1.0.2
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-snapshotter:v1.0.2
         args:
         - "--v={LOG_LEVEL}"
         - "--connection-timeout=24h"
@@ -434,7 +442,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-cluster-driver-registrar
-        image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-cluster-driver-registrar:v1.0.1
         args:
         - "--v={LOG_LEVEL}"
         - "--connection-timeout=24h"
@@ -524,7 +532,7 @@ spec:
           mountPath: /certs
           readOnly: true
       - name: csi-provisioner
-        image: quay.io/k8scsi/csi-provisioner:v1.3.1
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-provisioner:v1.3.1
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=600s"
@@ -536,7 +544,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-attacher
-        image: quay.io/k8scsi/csi-attacher:v2.0.0
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-attacher:v2.0.0
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=60s"
@@ -549,7 +557,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-snapshotter
-        image: quay.io/k8scsi/csi-snapshotter:v1.2.2
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-snapshotter:v1.2.2
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=300s"
@@ -639,7 +647,7 @@ spec:
           mountPath: /certs
           readOnly: true
       - name: csi-provisioner
-        image: quay.io/k8scsi/csi-provisioner:v1.3.1
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-provisioner:v1.3.1
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=600s"
@@ -651,7 +659,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-attacher
-        image: quay.io/k8scsi/csi-attacher:v2.0.0
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-attacher:v2.0.0
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=60s"
@@ -664,7 +672,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-resizer
-        image: quay.io/k8scsi/csi-resizer:v0.3.0
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-resizer:v0.3.0
         args:
         - "--v=9"
         - "--csiTimeout=300s"
@@ -676,7 +684,7 @@ spec:
         - name: socket-dir
           mountPath: /var/lib/csi/sockets/pluginproxy/
       - name: csi-snapshotter
-        image: quay.io/k8scsi/csi-snapshotter:v1.2.2
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-snapshotter:v1.2.2
         args:
         - "--v={LOG_LEVEL}"
         - "--timeout=300s"
@@ -698,7 +706,9 @@ spec:
           secretName: trident-csi
 `
 
-func GetCSIDaemonSetYAML(tridentImage, label, logFormat string, debug bool, version *utils.Version) string {
+func GetCSIDaemonSetYAML(
+	tridentImage, imageRegistry, kubeletDir, label, logFormat string, debug bool, version *utils.Version,
+) string {
 
 	var debugLine string
 	var logLevel string
@@ -711,14 +721,23 @@ func GetCSIDaemonSetYAML(tridentImage, label, logFormat string, debug bool, vers
 		logLevel = "2"
 	}
 
+	csiSidecarRegistry := "quay.io"
+	if imageRegistry != "" {
+		csiSidecarRegistry = imageRegistry
+	}
+
 	var daemonSetYAML string
 	if version.MajorVersion() == 1 && version.MinorVersion() == 13 {
 		daemonSetYAML = daemonSet113YAMLTemplate
 	} else {
 		daemonSetYAML = daemonSet114YAMLTemplate
 	}
 
+	kubeletDir = strings.TrimRight(kubeletDir, "/")
+
 	daemonSetYAML = strings.Replace(daemonSetYAML, "{TRIDENT_IMAGE}", tridentImage, 1)
+	daemonSetYAML = strings.Replace(daemonSetYAML, "{CSI_SIDECAR_REGISTRY}", csiSidecarRegistry, -1)
+	daemonSetYAML = strings.Replace(daemonSetYAML, "{KUBELET_DIR}", kubeletDir, -1)
 	daemonSetYAML = strings.Replace(daemonSetYAML, "{LABEL}", label, -1)
 	daemonSetYAML = strings.Replace(daemonSetYAML, "{DEBUG}", debugLine, 1)
 	daemonSetYAML = strings.Replace(daemonSetYAML, "{LOG_LEVEL}", logLevel, -1)
@@ -778,9 +797,9 @@ spec:
         - name: plugin-dir
           mountPath: /plugin
         - name: plugins-mount-dir
-          mountPath: /var/lib/kubelet/plugins
+          mountPath: {KUBELET_DIR}/plugins
         - name: pods-mount-dir
-          mountPath: /var/lib/kubelet/pods
+          mountPath: {KUBELET_DIR}/pods
           mountPropagation: "Bidirectional"
         - name: dev-dir
           mountPath: /dev
@@ -795,7 +814,7 @@ spec:
           mountPath: /certs
           readOnly: true
       - name: driver-registrar
-        image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-node-driver-registrar:v1.0.2
         args:
         - "--v={LOG_LEVEL}"
         - "--connection-timeout=24h"
@@ -805,7 +824,7 @@ spec:
         - name: ADDRESS
           value: /plugin/csi.sock
         - name: REGISTRATION_PATH
-          value: "/var/lib/kubelet/plugins/csi.trident.netapp.io/csi.sock"
+          value: "{KUBELET_DIR}/plugins/csi.trident.netapp.io/csi.sock"
         - name: KUBE_NODE_NAME
           valueFrom:
             fieldRef:
@@ -826,19 +845,19 @@ spec:
       volumes:
       - name: plugin-dir
         hostPath:
-          path: /var/lib/kubelet/plugins/csi.trident.netapp.io/
+          path: {KUBELET_DIR}/plugins/csi.trident.netapp.io/
           type: DirectoryOrCreate
       - name: registration-dir
         hostPath:
-          path: /var/lib/kubelet/plugins_registry/
+          path: {KUBELET_DIR}/plugins_registry/
           type: Directory
       - name: plugins-mount-dir
         hostPath:
-          path: /var/lib/kubelet/plugins
+          path: {KUBELET_DIR}/plugins
           type: DirectoryOrCreate
       - name: pods-mount-dir
         hostPath:
-          path: /var/lib/kubelet/pods
+          path: {KUBELET_DIR}/pods
           type: DirectoryOrCreate
       - name: dev-dir
         hostPath:
@@ -913,9 +932,9 @@ spec:
         - name: plugin-dir
           mountPath: /plugin
         - name: plugins-mount-dir
-          mountPath: /var/lib/kubelet/plugins
+          mountPath: {KUBELET_DIR}/plugins
         - name: pods-mount-dir
-          mountPath: /var/lib/kubelet/pods
+          mountPath: {KUBELET_DIR}/pods
           mountPropagation: "Bidirectional"
         - name: dev-dir
           mountPath: /dev
@@ -930,7 +949,7 @@ spec:
           mountPath: /certs
           readOnly: true
       - name: driver-registrar
-        image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
+        image: {CSI_SIDECAR_REGISTRY}/k8scsi/csi-node-driver-registrar:v1.2.0
         args:
         - "--v={LOG_LEVEL}"
         - "--csi-address=$(ADDRESS)"
@@ -939,7 +958,7 @@ spec:
         - name: ADDRESS
           value: /plugin/csi.sock
         - name: REGISTRATION_PATH
-          value: "/var/lib/kubelet/plugins/csi.trident.netapp.io/csi.sock"
+          value: "{KUBELET_DIR}/plugins/csi.trident.netapp.io/csi.sock"
         - name: KUBE_NODE_NAME
           valueFrom:
             fieldRef:
@@ -960,19 +979,19 @@ spec:
       volumes:
       - name: plugin-dir
         hostPath:
-          path: /var/lib/kubelet/plugins/csi.trident.netapp.io/
+          path: {KUBELET_DIR}/plugins/csi.trident.netapp.io/
           type: DirectoryOrCreate
       - name: registration-dir
         hostPath:
-          path: /var/lib/kubelet/plugins_registry/
+          path: {KUBELET_DIR}/plugins_registry/
           type: Directory
       - name: plugins-mount-dir
         hostPath:
-          path: /var/lib/kubelet/plugins
+          path: {KUBELET_DIR}/plugins
           type: DirectoryOrCreate
       - name: pods-mount-dir
         hostPath:
-          path: /var/lib/kubelet/pods
+          path: {KUBELET_DIR}/pods
           type: DirectoryOrCreate
       - name: dev-dir
         hostPath:

docs/kubernetes/deploying.rst

@@ -195,10 +195,19 @@ Customized Installation
 
 Trident's installer allows you to customize attributes. For example, if you have
 copied the Trident image to a private repository, you can specify the image name by using
-``--trident-image``.
-
-Users can also customize Trident's deployment files. Using the ``--generate-custom-yaml``
-parameter will create the following YAML files in the installer's ``setup`` directory:
+``--trident-image``.  If you have copied the Trident image as well as the needed CSI
+sidecar images to a private repository, it may be preferable to specify the location
+of that repository by using the ``--image-registry`` switch, which takes the form
+``<registry FQDN>[:port]``.
+
+If you are using a distribution of Kubernetes where kubelet keeps its data on a path
+other than the usual ``/var/lib/kubelet``, you can specify the alternate path by using
+``--kubelet-dir``.
+
+As a last resort, if you need to customize Trident's installation beyond what the
+installer's arguments allow, you can also customize Trident's deployment files. Using
+the ``--generate-custom-yaml`` parameter will create the following YAML files in the
+installer's ``setup`` directory:
 
 - trident-clusterrolebinding.yaml
 - trident-deployment.yaml