diff --git a/changelog/unreleased/issue-17815.toml b/changelog/unreleased/issue-17815.toml new file mode 100644 index 000000000000..f84df47264a2 --- /dev/null +++ b/changelog/unreleased/issue-17815.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Allow the index range clean up periodcal to delete index ranges that are no longer managed by an index set" + +issues = ["17815"] +pulls = ["17841"] diff --git a/full-backend-tests/src/test/java/org/graylog2/periodical/IndexRangesCleanUpIT.java b/full-backend-tests/src/test/java/org/graylog2/periodical/IndexRangesCleanUpIT.java new file mode 100644 index 000000000000..5ffea5e78252 --- /dev/null +++ b/full-backend-tests/src/test/java/org/graylog2/periodical/IndexRangesCleanUpIT.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.periodical; + +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; +import com.github.rholder.retry.WaitStrategies; +import org.graylog.testing.completebackend.Lifecycle; +import org.graylog.testing.completebackend.apis.GraylogApis; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS) +public class IndexRangesCleanUpIT { + public static final String RANGE_CLEANUP_PREFIX = "range-cleanup"; + public static final String INDEX_TWO = RANGE_CLEANUP_PREFIX + "_1"; + public static final String INDEX_ONE = RANGE_CLEANUP_PREFIX + "_0"; + private final GraylogApis api; + + public IndexRangesCleanUpIT(GraylogApis api) { + this.api = api; + } + + @ContainerMatrixTest + void testCleanUp() throws ExecutionException, RetryException { + String indexSetId = api.indices().createIndexSet("Range clean up", "test index range clean up", RANGE_CLEANUP_PREFIX); + + //Rotate to create indices 0 & 1 + api.indices().rotateIndexSet(indexSetId); + api.indices().rotateIndexSet(indexSetId); + + assertThat(getIndexRangesList()).isNotEmpty().contains(INDEX_ONE, INDEX_TWO); + + //Deleting index should automatically remove the range + api.indices().deleteIndex(INDEX_ONE); + + assertThat(getIndexRangesList()).isNotEmpty().doesNotContain(INDEX_ONE); + + //Deleting index set without deleting underlying indices + api.indices().deleteIndexSet(indexSetId, false); + assertThat(getIndexRangesList()).isNotEmpty().contains(INDEX_TWO); + + //Trigger clean up periodical over api + api.indices().rebuildIndexRanges(); + assertThat(getIndexRangesList()).isNotEmpty().doesNotContain(INDEX_TWO); + } + + private List getIndexRangesList() throws ExecutionException, RetryException { + return RetryerBuilder.>newBuilder() + .withWaitStrategy(WaitStrategies.fixedWait(1, TimeUnit.SECONDS)) + .withStopStrategy(StopStrategies.stopAfterAttempt(3)) + .retryIfResult(List::isEmpty) + .build() + .call(() -> api.indices().listIndexRanges().properJSONPath().read("ranges.*.index_name")); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/MongoIndexRangeService.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/MongoIndexRangeService.java index 71a87e50c91b..16ed1d0cab68 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/MongoIndexRangeService.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/MongoIndexRangeService.java @@ -173,10 +173,6 @@ public boolean remove(String index) { @AllowConcurrentEvents public void handleIndexDeletion(IndicesDeletedEvent event) { for (String index : event.indices()) { - if (!indexSetRegistry.isManagedIndex(index)) { - LOG.debug("Not handling deleted index <{}> because it's not managed by any index set.", index); - continue; - } LOG.debug("Index \"{}\" has been deleted. Removing index range.", index); if (remove(index)) { auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_RANGE_DELETE, ImmutableMap.of("index_name", index)); diff --git a/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/Indices.java b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/Indices.java index 04187e726059..fe47cdfa13e4 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/Indices.java +++ b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/Indices.java @@ -104,4 +104,69 @@ public GraylogApiResponse listOpenIndices(String indexSetId) { .statusCode(200); return new GraylogApiResponse(response); } + + public void rotateIndexSet(String indexSetId) { + given() + .spec(api.requestSpecification()) + .log().ifValidationFails() + .when() + .post("/system/deflector/" + indexSetId + "/cycle") + .then() + .log().ifError() + .log() + .ifValidationFails() + .statusCode(204); + } + + public void deleteIndexSet(String indexSetId, boolean deleteIndices) { + given() + .spec(api.requestSpecification()) + .log().ifValidationFails() + .when() + .param("delete_indices", deleteIndices) + .delete("/system/indices/index_sets/" + indexSetId) + .then() + .log().ifError() + .log().ifValidationFails() + .statusCode(204); + } + + public void deleteIndex(String index) { + given() + .spec(api.requestSpecification()) + .log().ifValidationFails() + .when() + .delete("/system/indexer/indices/" + index) + .then() + .log().ifError() + .log().ifValidationFails() + .statusCode(204); + } + + public GraylogApiResponse listIndexRanges() { + final ValidatableResponse response = given() + .spec(api.requestSpecification()) + .log().ifValidationFails() + .when() + .get("/system/indices/ranges") + .then() + .log().ifError() + .log() + .ifValidationFails() + .statusCode(200); + return new GraylogApiResponse(response); + } + + public void rebuildIndexRanges() { + given() + .spec(api.requestSpecification()) + .log().ifValidationFails() + .when() + .post("/system/indices/ranges/rebuild") + .then() + .log().ifError() + .log() + .ifValidationFails() + .statusCode(202); + } } diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/MongoIndexRangeServiceTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/MongoIndexRangeServiceTest.java index 6ce2d77ebef9..616a33023248 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/MongoIndexRangeServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/MongoIndexRangeServiceTest.java @@ -256,8 +256,6 @@ public void remove() throws Exception { @Test @MongoDBFixtures("MongoIndexRangeServiceTest.json") public void testHandleIndexDeletion() throws Exception { - when(indexSetRegistry.isManagedIndex("graylog_1")).thenReturn(true); - assertThat(indexRangeService.findAll()).hasSize(2); localEventBus.post(IndicesDeletedEvent.create(Collections.singleton("graylog_1")));