allow deletion of index ranges that are no longer manged (#17841)

* allow deletion of index ranges that are no longer managed by an index set

* add changelog

* clean up

changelog/unreleased/issue-17815.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Allow the index range clean up periodcal to delete index ranges that are no longer managed by an index set"
+
+issues = ["17815"]
+pulls = ["17841"]

full-backend-tests/src/test/java/org/graylog2/periodical/IndexRangesCleanUpIT.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.periodical;
+
+import com.github.rholder.retry.RetryException;
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
+import org.graylog.testing.completebackend.Lifecycle;
+import org.graylog.testing.completebackend.apis.GraylogApis;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS)
+public class IndexRangesCleanUpIT {
+    public static final String RANGE_CLEANUP_PREFIX = "range-cleanup";
+    public static final String INDEX_TWO = RANGE_CLEANUP_PREFIX + "_1";
+    public static final String INDEX_ONE = RANGE_CLEANUP_PREFIX + "_0";
+    private final GraylogApis api;
+
+    public IndexRangesCleanUpIT(GraylogApis api) {
+        this.api = api;
+    }
+
+    @ContainerMatrixTest
+    void testCleanUp() throws ExecutionException, RetryException {
+        String indexSetId = api.indices().createIndexSet("Range clean up", "test index range clean up", RANGE_CLEANUP_PREFIX);
+
+        //Rotate to create indices 0 & 1
+        api.indices().rotateIndexSet(indexSetId);
+        api.indices().rotateIndexSet(indexSetId);
+
+        assertThat(getIndexRangesList()).isNotEmpty().contains(INDEX_ONE, INDEX_TWO);
+
+        //Deleting index should automatically remove the range
+        api.indices().deleteIndex(INDEX_ONE);
+
+        assertThat(getIndexRangesList()).isNotEmpty().doesNotContain(INDEX_ONE);
+
+        //Deleting index set without deleting underlying indices
+        api.indices().deleteIndexSet(indexSetId, false);
+        assertThat(getIndexRangesList()).isNotEmpty().contains(INDEX_TWO);
+
+        //Trigger clean up periodical over api
+        api.indices().rebuildIndexRanges();
+        assertThat(getIndexRangesList()).isNotEmpty().doesNotContain(INDEX_TWO);
+    }
+
+    private List<String> getIndexRangesList() throws ExecutionException, RetryException {
+        return RetryerBuilder.<List<String>>newBuilder()
+                .withWaitStrategy(WaitStrategies.fixedWait(1, TimeUnit.SECONDS))
+                .withStopStrategy(StopStrategies.stopAfterAttempt(3))
+                .retryIfResult(List::isEmpty)
+                .build()
+                .call(() -> api.indices().listIndexRanges().properJSONPath().read("ranges.*.index_name"));
+    }
+}

graylog2-server/src/main/java/org/graylog2/indexer/ranges/MongoIndexRangeService.java

@@ -173,10 +173,6 @@ public boolean remove(String index) {
     @AllowConcurrentEvents
     public void handleIndexDeletion(IndicesDeletedEvent event) {
         for (String index : event.indices()) {
-            if (!indexSetRegistry.isManagedIndex(index)) {
-                LOG.debug("Not handling deleted index <{}> because it's not managed by any index set.", index);
-                continue;
-            }
             LOG.debug("Index \"{}\" has been deleted. Removing index range.", index);
             if (remove(index)) {
                 auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RANGE_DELETE, ImmutableMap.of("index_name", index));

graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/Indices.java

@@ -104,4 +104,69 @@ public GraylogApiResponse listOpenIndices(String indexSetId) {
                 .statusCode(200);
         return new GraylogApiResponse(response);
     }
+
+    public void rotateIndexSet(String indexSetId) {
+        given()
+                .spec(api.requestSpecification())
+                .log().ifValidationFails()
+                .when()
+                .post("/system/deflector/" + indexSetId + "/cycle")
+                .then()
+                .log().ifError()
+                .log()
+                .ifValidationFails()
+                .statusCode(204);
+    }
+
+    public void deleteIndexSet(String indexSetId, boolean deleteIndices) {
+        given()
+                .spec(api.requestSpecification())
+                .log().ifValidationFails()
+                .when()
+                .param("delete_indices", deleteIndices)
+                .delete("/system/indices/index_sets/" + indexSetId)
+                .then()
+                .log().ifError()
+                .log().ifValidationFails()
+                .statusCode(204);
+    }
+
+    public void deleteIndex(String index) {
+        given()
+                .spec(api.requestSpecification())
+                .log().ifValidationFails()
+                .when()
+                .delete("/system/indexer/indices/" + index)
+                .then()
+                .log().ifError()
+                .log().ifValidationFails()
+                .statusCode(204);
+    }
+
+    public GraylogApiResponse listIndexRanges() {
+        final ValidatableResponse response = given()
+                .spec(api.requestSpecification())
+                .log().ifValidationFails()
+                .when()
+                .get("/system/indices/ranges")
+                .then()
+                .log().ifError()
+                .log()
+                .ifValidationFails()
+                .statusCode(200);
+        return new GraylogApiResponse(response);
+    }
+
+    public void rebuildIndexRanges() {
+        given()
+                .spec(api.requestSpecification())
+                .log().ifValidationFails()
+                .when()
+                .post("/system/indices/ranges/rebuild")
+                .then()
+                .log().ifError()
+                .log()
+                .ifValidationFails()
+                .statusCode(202);
+    }
 }

graylog2-server/src/test/java/org/graylog2/indexer/ranges/MongoIndexRangeServiceTest.java

@@ -256,8 +256,6 @@ public void remove() throws Exception {
     @Test
     @MongoDBFixtures("MongoIndexRangeServiceTest.json")
     public void testHandleIndexDeletion() throws Exception {
-        when(indexSetRegistry.isManagedIndex("graylog_1")).thenReturn(true);
-
         assertThat(indexRangeService.findAll()).hasSize(2);
 
         localEventBus.post(IndicesDeletedEvent.create(Collections.singleton("graylog_1")));