[ML] Reinstate ML daily maintenance actions (#47103)

A refactoring in 6.6 meant that the ML daily maintenance actions have not been run at all since then. This change installs the local master listener that schedules the ML daily maintenance, and also defends against some subtle race conditions that could occur in the future if a node flipped very quickly between master and non-master. Fixes #47003
elastic · Sep 30, 2019 · 0807d40 · 0807d40
1 parent 4675968
commit 0807d40
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java
@@ -79,12 +79,12 @@ private static TimeValue delayToNextTime(ClusterName clusterName) {
         return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli());
     }
 
-    public void start() {
+    public synchronized void start() {
         LOGGER.debug("Starting ML daily maintenance service");
         scheduleNext();
     }
 
-    public void stop() {
+    public synchronized void stop() {
         LOGGER.debug("Stopping ML daily maintenance service");
         if (cancellable != null && cancellable.isCancelled() == false) {
             cancellable.cancel();
@@ -100,7 +100,7 @@ public void close() {
         stop();
     }
 
-    private void scheduleNext() {
+    private synchronized void scheduleNext() {
         try {
             cancellable = threadPool.schedule(this::triggerTasks, schedulerProvider.get(), ThreadPool.Names.GENERIC);
         } catch (EsRejectedExecutionException e) {

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
@@ -39,6 +39,7 @@ class MlInitializationService implements LocalNodeMasterListener, ClusterStateLi
         this.clusterService = clusterService;
         this.client = client;
         clusterService.addListener(this);
+        clusterService.addLocalNodeMasterListener(this);
     }
 
     @Override
@@ -80,7 +81,7 @@ public String executorName() {
         return ThreadPool.Names.GENERIC;
     }
 
-    private void installDailyMaintenanceService() {
+    private synchronized void installDailyMaintenanceService() {
         if (mlDailyMaintenanceService == null) {
             mlDailyMaintenanceService = new MlDailyMaintenanceService(clusterService.getClusterName(), threadPool, client);
             mlDailyMaintenanceService.start();
@@ -93,7 +94,7 @@ public void beforeStop() {
         }
     }
 
-    private void uninstallDailyMaintenanceService() {
+    private synchronized void uninstallDailyMaintenanceService() {
         if (mlDailyMaintenanceService != null) {
             mlDailyMaintenanceService.stop();
             mlDailyMaintenanceService = null;
@@ -106,7 +107,7 @@ MlDailyMaintenanceService getDailyMaintenanceService() {
     }
 
     /** For testing */
-    void setDailyMaintenanceService(MlDailyMaintenanceService service) {
+    synchronized void setDailyMaintenanceService(MlDailyMaintenanceService service) {
         mlDailyMaintenanceService = service;
     }
 }