Support Graceful Shutdown

If the kubelet where the the sriov pod is running has gracefulShutdown
configured, we'll delay in preStop for a while if and while /tmp/sriov-delay-shutdown
exists, up to a maximum wait that's less than 15 minutes (which is now specified in
the daemonset pod's terminationGracePeriodSeconds).

bindata/manifests/daemon/daemonset.yaml

@@ -95,6 +95,7 @@ spec:
           preStop:
             exec:
               command: ["/bindata/scripts/clean-k8s-services.sh"]
+      terminationGracePeriodSeconds: 900
       volumes:
       - name: host
         hostPath:

bindata/scripts/clean-k8s-services.sh

@@ -6,6 +6,13 @@ if [ "$CLUSTER_TYPE" == "openshift" ]; then
 fi
 
 chroot_path="/host"
+delay_shutdown_path="$chroot_path/tmp/sriov-delay-shutdown"
+kubelet_config_path="$chroot_path/etc/kubernetes/kubelet.conf"
+
+# 10 minutes - this should be shorter than the time that is specifed for the
+# terminationGracePeriodSeconds in the daemonset's pod spec, so that everything
+# else in the preStop hook has time to run and the Pod can be terminated properly.
+wait_time=600
 
 function clean_services() {
   # Remove switchdev service files
@@ -33,6 +40,19 @@ function clean_services() {
   fi
 }
 
+# If the kubelet is configured to shutdown gracefully (>0s shutdownGracePeriod), we need to wait for
+# things to settle before shutting down the node.
+if grep "$kubelet_config_path" -e shutdownGracePeriod | grep -qv \"0s\"; then
+  start=$(date +%s)
+  while [ $(( $(date +%s) - $start )) -lt $wait_time ]; do
+    if [ ! -f "$delay_shutdown_path" ]; then  # don't have to wait anymore
+      break
+    fi
+    sleep 1
+  done
+  rm -f "$delay_shutdown_path"
+fi
+
 clean_services
 # Reload host services
 chroot $chroot_path /bin/bash -c systemctl daemon-reload >/dev/null 2>&1 || true

pkg/daemon/daemon.go

@@ -52,6 +52,9 @@ const (
 	// maxUpdateBackoff is the maximum time to react to a change as we back off
 	// in the face of errors.
 	maxUpdateBackoff = 60 * time.Second
+
+	// the presence of this file indicates that the sriov shutdown should be delayed
+	delayShutdownPath = "/host/tmp/sriov-delay-shutdown"
 )
 
 type Message struct {
@@ -604,6 +607,17 @@ func (dn *Daemon) completeDrain() error {
 		glog.Errorf("completeDrain(): failed to annotate node: %v", err)
 		return err
 	}
+
+	if _, err := os.Stat(delayShutdownPath); err == nil {
+		if err := os.Remove(delayShutdownPath); err != nil {
+			glog.Errorf("completeDrain(): failed to remove file %v: %v", delayShutdownPath, err)
+			return err
+		}
+	} else if !os.IsNotExist(err) { // error is not "not exist"
+		glog.Errorf("completeDrain(): error checking file status %v: %v", delayShutdownPath, err)
+		return err
+	}
+
 	return nil
 }
 
@@ -925,6 +939,14 @@ func (dn *Daemon) drainNode() error {
 		return err
 	}
 	glog.Info("drainNode(): drain complete")
+
+	file, err := os.Create(delayShutdownPath)
+	if err != nil {
+		glog.Errorf("drainNode(): failed to create file %v %v", delayShutdownPath, err)
+		return err
+	}
+	defer file.Close()
+
 	return nil
 }