Do not close bad indices on startup

server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java

@@ -31,8 +31,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.IndicesService;
 
 import java.util.Map;
 
@@ -92,26 +90,6 @@ static ClusterState recoverClusterBlocks(final ClusterState state) {
         return ClusterState.builder(state).blocks(blocks).build();
     }
 
-    static ClusterState closeBadIndices(final ClusterState clusterState, final IndicesService indicesService) {
-        final MetaData.Builder builder = MetaData.builder(clusterState.metaData()).removeAllIndices();
-
-        for (IndexMetaData metaData : clusterState.metaData()) {
-            try {
-                if (metaData.getState() == IndexMetaData.State.OPEN) {
-                    // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
-                    indicesService.verifyIndexMetadata(metaData, metaData);
-                }
-            } catch (final Exception e) {
-                final Index electedIndex = metaData.getIndex();
-                logger.warn(() -> new ParameterizedMessage("recovering index {} failed - recovering as closed", electedIndex), e);
-                metaData = IndexMetaData.builder(metaData).state(IndexMetaData.State.CLOSE).build();
-            }
-            builder.put(metaData, false);
-        }
-
-        return ClusterState.builder(clusterState).metaData(builder).build();
-    }
-
     static ClusterState updateRoutingTable(final ClusterState state) {
         // initialize all index routing tables as empty
         final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable());

server/src/main/java/org/elasticsearch/gateway/Gateway.java

@@ -126,7 +126,6 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
         }
         ClusterState recoveredState = Function.<ClusterState>identity()
             .andThen(state -> ClusterStateUpdaters.upgradeAndArchiveUnknownOrInvalidSettings(state, clusterService.getClusterSettings()))
-            .andThen(state -> ClusterStateUpdaters.closeBadIndices(state, indicesService))
             .apply(ClusterState.builder(clusterService.getClusterName()).metaData(metaDataBuilder).build());
 
         listener.onSuccess(recoveredState);

server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -137,7 +137,6 @@ public void applyClusterStateUpdaters() {
             .andThen(ClusterStateUpdaters::addStateNotRecoveredBlock)
             .andThen(state -> ClusterStateUpdaters.setLocalNode(state, transportService.getLocalNode()))
             .andThen(state -> ClusterStateUpdaters.upgradeAndArchiveUnknownOrInvalidSettings(state, clusterService.getClusterSettings()))
-            .andThen(state -> ClusterStateUpdaters.closeBadIndices(state, indicesService))
             .andThen(ClusterStateUpdaters::recoverClusterBlocks)
             .apply(previousClusterState);
     }

server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java

@@ -47,7 +47,6 @@
 
 import static org.elasticsearch.cluster.metadata.MetaData.CLUSTER_READ_ONLY_BLOCK;
 import static org.elasticsearch.gateway.ClusterStateUpdaters.addStateNotRecoveredBlock;
-import static org.elasticsearch.gateway.ClusterStateUpdaters.closeBadIndices;
 import static org.elasticsearch.gateway.ClusterStateUpdaters.hideStateIfNotRecovered;
 import static org.elasticsearch.gateway.ClusterStateUpdaters.mixCurrentStateAndRecoveredState;
 import static org.elasticsearch.gateway.ClusterStateUpdaters.recoverClusterBlocks;
@@ -201,32 +200,6 @@ public void testAddStateNotRecoveredBlock() {
         assertTrue(newState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK));
     }
 
-    public void testCloseBadIndices() throws IOException {
-        final IndicesService indicesService = mock(IndicesService.class);
-        final IndexMetaData good = createIndexMetaData("good", Settings.EMPTY);
-        final IndexMetaData bad = createIndexMetaData("bad", Settings.EMPTY);
-        final IndexMetaData ugly = IndexMetaData.builder(createIndexMetaData("ugly", Settings.EMPTY))
-                .state(IndexMetaData.State.CLOSE)
-                .build();
-
-        final ClusterState initialState = ClusterState
-                .builder(ClusterState.EMPTY_STATE)
-                .metaData(MetaData.builder()
-                        .put(good, false)
-                        .put(bad, false)
-                        .put(ugly, false)
-                        .build())
-                .build();
-
-        doThrow(new RuntimeException("test")).when(indicesService).verifyIndexMetadata(bad, bad);
-        doThrow(new AssertionError("ugly index is already closed")).when(indicesService).verifyIndexMetadata(ugly, ugly);
-
-        final ClusterState newState = closeBadIndices(initialState, indicesService);
-        assertThat(newState.metaData().index(good.getIndex()).getState(), equalTo(IndexMetaData.State.OPEN));
-        assertThat(newState.metaData().index(bad.getIndex()).getState(), equalTo(IndexMetaData.State.CLOSE));
-        assertThat(newState.metaData().index(ugly.getIndex()).getState(), equalTo(IndexMetaData.State.CLOSE));
-    }
-
     public void testUpdateRoutingTable() {
         final int numOfShards = randomIntBetween(1, 10);
 

server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java

@@ -34,7 +34,11 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -51,12 +55,14 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.nullValue;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
@@ -369,9 +375,20 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             }
         });
 
-        // ensureGreen(closedIndex) waits for the index to show up in the metadata
-        // this is crucial otherwise the state call below might not contain the index yet
-        ensureGreen(metaData.getIndex().getName());
+        // check that the cluster does not keep reallocating shards
+        assertBusy(() -> {
+            final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
+            final IndexRoutingTable indexRoutingTable = routingTable.index("test");
+            assertNotNull(indexRoutingTable);
+            for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
+                assertTrue(shardRoutingTable.primaryShard().unassigned());
+                assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO,
+                    shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
+                assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
+            }
+        }, 60, TimeUnit.SECONDS);
+        client().admin().indices().prepareClose("test").get();
+
         state = client().admin().cluster().prepareState().get().getState();
         assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
         assertEquals("classic", state.getMetaData().index(metaData.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
@@ -432,11 +449,19 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             }
         });
 
-        // ensureGreen(closedIndex) waits for the index to show up in the metadata
-        // this is crucial otherwise the state call below might not contain the index yet
-        ensureGreen(metaData.getIndex().getName());
-        state = client().admin().cluster().prepareState().get().getState();
-        assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
+        // check that the cluster does not keep reallocating shards
+        assertBusy(() -> {
+            final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
+            final IndexRoutingTable indexRoutingTable = routingTable.index("test");
+            assertNotNull(indexRoutingTable);
+            for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
+                assertTrue(shardRoutingTable.primaryShard().unassigned());
+                assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO,
+                    shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
+                assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
+            }
+        }, 60, TimeUnit.SECONDS);
+        client().admin().indices().prepareClose("test").get();
 
         // try to open it with the broken setting - fail again!
         ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());