From aeba58af67f63f74e7396c18735deb618ee444df Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Mon, 1 Oct 2018 16:21:14 +0200 Subject: [PATCH] drop elasticsearch-translog for 7.0 (#33373) #32281 adds elasticsearch-shard to provide bwc version of elasticsearch-translog for 6.x; have to remove elasticsearch-translog for 7.0 Relates to #31389 --- distribution/src/bin/elasticsearch-translog | 5 -- .../src/bin/elasticsearch-translog.bat | 12 ---- .../reference/index-modules/translog.asciidoc | 62 ------------------- .../migration/migrate_7_0/indices.asciidoc | 6 +- .../packaging/util/Archives.java | 3 +- .../packaging/util/Installation.java | 1 - .../packaging/util/Packages.java | 3 +- .../resources/packaging/utils/packages.bash | 1 - .../test/resources/packaging/utils/tar.bash | 1 - .../RemoveCorruptedShardDataCommand.java | 50 +++++---------- .../index/translog/TranslogToolCli.java | 42 ------------- .../RemoveCorruptedShardDataCommandIT.java | 8 +-- 12 files changed, 25 insertions(+), 169 deletions(-) delete mode 100755 distribution/src/bin/elasticsearch-translog delete mode 100644 distribution/src/bin/elasticsearch-translog.bat delete mode 100644 server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java diff --git a/distribution/src/bin/elasticsearch-translog b/distribution/src/bin/elasticsearch-translog deleted file mode 100755 index aa5bfb32df199..0000000000000 --- a/distribution/src/bin/elasticsearch-translog +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -ES_MAIN_CLASS=org.elasticsearch.index.translog.TranslogToolCli \ - "`dirname "$0"`"/elasticsearch-cli \ - "$@" diff --git a/distribution/src/bin/elasticsearch-translog.bat b/distribution/src/bin/elasticsearch-translog.bat deleted file mode 100644 index 6a2e3046205fe..0000000000000 --- a/distribution/src/bin/elasticsearch-translog.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -setlocal enabledelayedexpansion -setlocal enableextensions - -set ES_MAIN_CLASS=org.elasticsearch.index.translog.TranslogToolCli -call "%~dp0elasticsearch-cli.bat" ^ - %%* ^ - || exit /b 1 - -endlocal -endlocal diff --git a/docs/reference/index-modules/translog.asciidoc b/docs/reference/index-modules/translog.asciidoc index 713a352210054..705fb81b09c8c 100644 --- a/docs/reference/index-modules/translog.asciidoc +++ b/docs/reference/index-modules/translog.asciidoc @@ -86,65 +86,3 @@ file based sync. Defaults to `512mb` `index.translog.retention.age`:: The maximum duration for which translog files will be kept. Defaults to `12h`. - - -[float] -[[corrupt-translog-truncation]] -=== What to do if the translog becomes corrupted? - -[WARNING] -This tool is deprecated and will be completely removed in 7.0. -Use the <> instead of this one. - -In some cases (a bad drive, user error) the translog on a shard copy can become -corrupted. When this corruption is detected by Elasticsearch due to mismatching -checksums, Elasticsearch will fail that shard copy and refuse to use that copy -of the data. If there are other copies of the shard available then -Elasticsearch will automatically recover from one of them using the normal -shard allocation and recovery mechanism. In particular, if the corrupt shard -copy was the primary when the corruption was detected then one of its replicas -will be promoted in its place. - -If there is no copy of the data from which Elasticsearch can recover -successfully, a user may want to recover the data that is part of the shard at -the cost of losing the data that is currently contained in the translog. We -provide a command-line tool for this, `elasticsearch-translog`. - -[WARNING] -The `elasticsearch-translog` tool should *not* be run while Elasticsearch is -running. If you attempt to run this tool while Elasticsearch is running, you -will permanently lose the documents that were contained only in the translog! - -In order to run the `elasticsearch-translog` tool, specify the `truncate` -subcommand as well as the directory for the corrupted translog with the `-d` -option: - -[source,txt] --------------------------------------------------- -$ bin/elasticsearch-translog truncate -d /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/ -Checking existing translog files -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! WARNING: Elasticsearch MUST be stopped before running this tool ! -! ! -! WARNING: Documents inside of translog files will be lost ! -! ! -! WARNING: The following files will be DELETED! ! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-41.ckp ---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-6.ckp ---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-37.ckp ---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-24.ckp ---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-11.ckp - -Continue and DELETE files? [y/N] y -Reading translog UUID information from Lucene commit from shard at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index] -Translog Generation: 3 -Translog UUID : AxqC4rocTC6e0fwsljAh-Q -Removing existing translog files -Creating new empty checkpoint at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog.ckp] -Creating new empty translog at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-3.tlog] -Done. --------------------------------------------------- - -You can also use the `-h` option to get a list of all options and parameters -that the `elasticsearch-translog` tool supports. diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc index d040343d32eb4..cc17a37a96633 100644 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ b/docs/reference/migration/migrate_7_0/indices.asciidoc @@ -90,4 +90,8 @@ The previous behavior can be restored by setting `indices.breaker.total.use_real [float] ==== `fix` value for `index.shard.check_on_startup` is removed -Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported. \ No newline at end of file +Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported. + +==== `elasticsearch-translog` is removed + +Use the `elasticsearch-shard` tool to remove corrupted translog data. \ No newline at end of file diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index 45629f286fcc4..cbee11ea41c5d 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -186,8 +186,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu "elasticsearch-env", "elasticsearch-keystore", "elasticsearch-plugin", - "elasticsearch-shard", - "elasticsearch-translog" + "elasticsearch-shard" ).forEach(executable -> { assertThat(es.bin(executable), file(File, owner, owner, p755)); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java index 620ccd5e442d7..8dea694492bc2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java @@ -102,7 +102,6 @@ public class Executables { public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore"); public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil"); public final Path elasticsearchShard = platformExecutable("elasticsearch-shard"); - public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog"); private Path platformExecutable(String name) { final String platformExecutableName = Platforms.WINDOWS diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java index 56de822316634..bdad7d56791cf 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java @@ -187,8 +187,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu "elasticsearch", "elasticsearch-plugin", "elasticsearch-keystore", - "elasticsearch-shard", - "elasticsearch-translog" + "elasticsearch-shard" ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755))); Stream.of( diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/src/test/resources/packaging/utils/packages.bash index f6ba68d84d483..0e218ce21f36c 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/packages.bash @@ -96,7 +96,6 @@ verify_package_installation() { assert_file "$ESHOME/bin/elasticsearch" f root root 755 assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755 assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755 - assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755 assert_file "$ESHOME/lib" d root root 755 assert_file "$ESCONFIG" d root elasticsearch 2750 assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660 diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/src/test/resources/packaging/utils/tar.bash index 23901cbae99b7..95e293df05423 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/tar.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/tar.bash @@ -95,7 +95,6 @@ verify_archive_installation() { assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755 assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755 assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755 - assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755 assert_file "$ESCONFIG" d elasticsearch elasticsearch 755 assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660 assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660 diff --git a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java index de22903efb334..54c1dd7c1db69 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java +++ b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java @@ -85,10 +85,6 @@ public class RemoveCorruptedShardDataCommand extends EnvironmentAwareCommand { private final NamedXContentRegistry namedXContentRegistry; public RemoveCorruptedShardDataCommand() { - this(false); - } - - public RemoveCorruptedShardDataCommand(boolean translogOnly) { super("Removes corrupted shard files"); folderOption = parser.acceptsAll(Arrays.asList("d", "dir"), @@ -104,18 +100,13 @@ public RemoveCorruptedShardDataCommand(boolean translogOnly) { namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); - removeCorruptedLuceneSegmentsAction = translogOnly ? null : new RemoveCorruptedLuceneSegmentsAction(); + removeCorruptedLuceneSegmentsAction = new RemoveCorruptedLuceneSegmentsAction(); truncateTranslogAction = new TruncateTranslogAction(namedXContentRegistry); } @Override protected void printAdditionalHelp(Terminal terminal) { - if (removeCorruptedLuceneSegmentsAction == null) { - // that's only for 6.x branch for bwc with elasticsearch-translog - terminal.println("This tool truncates the translog and translog checkpoint files to create a new translog"); - } else { - terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard."); - } + terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard."); } // Visible for testing @@ -277,12 +268,6 @@ private void warnAboutESShouldBeStopped(Terminal terminal) { terminal.println(""); terminal.println(" WARNING: Elasticsearch MUST be stopped before running this tool."); terminal.println(""); - // that's only for 6.x branch for bwc with elasticsearch-translog - if (removeCorruptedLuceneSegmentsAction == null) { - terminal.println(" This tool is deprecated and will be completely removed in 7.0."); - terminal.println(" It is replaced by the elasticsearch-shard tool. "); - terminal.println(""); - } terminal.println(" Please make a complete backup of your index before using this tool."); terminal.println(""); terminal.println("-----------------------------------------------------------------------"); @@ -318,26 +303,21 @@ public void write(int b) { // keep the index lock to block any runs of older versions of this tool try (Lock writeIndexLock = indexDir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { ////////// Index - // that's only for 6.x branch for bwc with elasticsearch-translog - if (removeCorruptedLuceneSegmentsAction != null) { - terminal.println(""); - terminal.println("Opening Lucene index at " + indexPath); - terminal.println(""); - try { - indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir, - writeIndexLock, printStream, verbose); - } catch (Exception e) { - terminal.println(e.getMessage()); - throw e; - } - - terminal.println(""); - terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath); - terminal.println(""); - } else { - indexCleanStatus = Tuple.tuple(CleanStatus.CLEAN, null); + terminal.println(""); + terminal.println("Opening Lucene index at " + indexPath); + terminal.println(""); + try { + indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir, + writeIndexLock, printStream, verbose); + } catch (Exception e) { + terminal.println(e.getMessage()); + throw e; } + terminal.println(""); + terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath); + terminal.println(""); + ////////// Translog // as translog relies on data stored in an index commit - we have to have non unrecoverable index to truncate translog if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) { diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java deleted file mode 100644 index a8a8d735f9a0c..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.translog; - -import org.elasticsearch.cli.LoggingAwareMultiCommand; -import org.elasticsearch.cli.Terminal; -import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand; - -/** - * Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool - */ -@Deprecated -public class TranslogToolCli extends LoggingAwareMultiCommand { - - private TranslogToolCli() { - // that's only for 6.x branch for bwc with elasticsearch-translog - super("A CLI tool for various Elasticsearch translog actions"); - subcommands.put("truncate", new RemoveCorruptedShardDataCommand(true)); - } - - public static void main(String[] args) throws Exception { - exit(new TranslogToolCli().main(args, Terminal.DEFAULT)); - } - -} diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index dc3be31734d5c..3c52497539ed0 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -293,11 +293,9 @@ public void testCorruptTranslogTruncation() throws Exception { indexRandom(false, false, false, Arrays.asList(builders)); Set translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME); - // that's only for 6.x branch for bwc with elasticsearch-translog - final boolean translogOnly = randomBoolean(); - final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(translogOnly); - final MockTerminal terminal = new MockTerminal(); - final OptionParser parser = command.getParser(); + RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(); + MockTerminal terminal = new MockTerminal(); + OptionParser parser = command.getParser(); if (randomBoolean() && numDocsToTruncate > 0) { // flush the replica, so it will have more docs than what the primary will have