drop elasticsearch-translog for 7.0 (#33373)

#32281 adds elasticsearch-shard to provide bwc version of elasticsearch-translog for 6.x; have to remove elasticsearch-translog for 7.0

Relates to #31389

Author: vladimirdolzhenko

distribution/src/bin/elasticsearch-translog

@@ -86,65 +86,3 @@ file based sync. Defaults to `512mb`
 `index.translog.retention.age`::
 
 The maximum duration for which translog files will be kept. Defaults to `12h`.
-
-
-[float]
-[[corrupt-translog-truncation]]
-=== What to do if the translog becomes corrupted?
-
-[WARNING]
-This tool is deprecated and will be completely removed in 7.0.
-Use the <<shard-tool,elasticsearch-shard tool>> instead of this one.
-
-In some cases (a bad drive, user error) the translog on a shard copy can become
-corrupted. When this corruption is detected by Elasticsearch due to mismatching
-checksums, Elasticsearch will fail that shard copy and refuse to use that copy
-of the data.  If there are other copies of the shard available then
-Elasticsearch will automatically recover from one of them using the normal
-shard allocation and recovery mechanism.  In particular, if the corrupt shard
-copy was the primary when the corruption was detected then one of its replicas
-will be promoted in its place.
-
-If there is no copy of the data from which Elasticsearch can recover
-successfully, a user may want to recover the data that is part of the shard at
-the cost of losing the data that is currently contained in the translog. We
-provide a command-line tool for this, `elasticsearch-translog`.
-
-[WARNING]
-The `elasticsearch-translog` tool should *not* be run while Elasticsearch is
-running. If you attempt to run this tool while Elasticsearch is running, you 
-will permanently lose the documents that were contained only in the translog!
-
-In order to run the `elasticsearch-translog` tool, specify the `truncate`
-subcommand as well as the directory for the corrupted translog with the `-d`
-option:
-
-[source,txt]
---------------------------------------------------
-$ bin/elasticsearch-translog truncate -d /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
-Checking existing translog files
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-!   WARNING: Elasticsearch MUST be stopped before running this tool   !
-!                                                                     !
-!   WARNING:    Documents inside of translog files will be lost       !
-!                                                                     !
-!   WARNING:          The following files will be DELETED!            !
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-41.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-6.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-37.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-24.ckp
---> data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-11.ckp
-
-Continue and DELETE files? [y/N] y
-Reading translog UUID information from Lucene commit from shard at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index]
-Translog Generation: 3
-Translog UUID      : AxqC4rocTC6e0fwsljAh-Q
-Removing existing translog files
-Creating new empty checkpoint at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog.ckp]
-Creating new empty translog at [data/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/translog-3.tlog]
-Done.
---------------------------------------------------
-
-You can also use the `-h` option to get a list of all options and parameters
-that the `elasticsearch-translog` tool supports.

distribution/src/bin/elasticsearch-translog.bat

@@ -90,4 +90,8 @@ The previous behavior can be restored by setting `indices.breaker.total.use_real
 [float]
 ==== `fix` value for `index.shard.check_on_startup` is removed
 
-Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported.
+Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported.
+
+==== `elasticsearch-translog` is removed
+
+Use the `elasticsearch-shard` tool to remove corrupted translog data.

docs/reference/index-modules/translog.asciidoc

@@ -186,8 +186,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch-env",
             "elasticsearch-keystore",
             "elasticsearch-plugin",
-            "elasticsearch-shard",
-            "elasticsearch-translog"
+            "elasticsearch-shard"
         ).forEach(executable -> {
 
             assertThat(es.bin(executable), file(File, owner, owner, p755));

docs/reference/migration/migrate_7_0/indices.asciidoc

@@ -102,7 +102,6 @@ public class Executables {
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
         public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
         public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
-        public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java

@@ -187,8 +187,7 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch",
             "elasticsearch-plugin",
             "elasticsearch-keystore",
-            "elasticsearch-shard",
-            "elasticsearch-translog"
+            "elasticsearch-shard"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 
         Stream.of(

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -96,7 +96,6 @@ verify_package_installation() {
     assert_file "$ESHOME/bin/elasticsearch" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
-    assert_file "$ESHOME/bin/elasticsearch-translog" f root root 755
     assert_file "$ESHOME/lib" d root root 755
     assert_file "$ESCONFIG" d root elasticsearch 2750
     assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -95,7 +95,6 @@ verify_archive_installation() {
     assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
-    assert_file "$ESHOME/bin/elasticsearch-translog" f elasticsearch elasticsearch 755
     assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
     assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
     assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -85,10 +85,6 @@ public class RemoveCorruptedShardDataCommand extends EnvironmentAwareCommand {
     private final NamedXContentRegistry namedXContentRegistry;
 
     public RemoveCorruptedShardDataCommand() {
-        this(false);
-    }
-
-    public RemoveCorruptedShardDataCommand(boolean translogOnly) {
         super("Removes corrupted shard files");
 
         folderOption = parser.acceptsAll(Arrays.asList("d", "dir"),
@@ -104,18 +100,13 @@ public RemoveCorruptedShardDataCommand(boolean translogOnly) {
 
         namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
 
-        removeCorruptedLuceneSegmentsAction = translogOnly ? null : new RemoveCorruptedLuceneSegmentsAction();
+        removeCorruptedLuceneSegmentsAction = new RemoveCorruptedLuceneSegmentsAction();
         truncateTranslogAction = new TruncateTranslogAction(namedXContentRegistry);
     }
 
     @Override
     protected void printAdditionalHelp(Terminal terminal) {
-        if (removeCorruptedLuceneSegmentsAction == null) {
-            // that's only for 6.x branch for bwc with elasticsearch-translog
-            terminal.println("This tool truncates the translog and translog checkpoint files to create a new translog");
-        } else {
-            terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard.");
-        }
+        terminal.println("This tool attempts to detect and remove unrecoverable corrupted data in a shard.");
     }
 
     // Visible for testing
@@ -277,12 +268,6 @@ private void warnAboutESShouldBeStopped(Terminal terminal) {
         terminal.println("");
         terminal.println("    WARNING: Elasticsearch MUST be stopped before running this tool.");
         terminal.println("");
-        // that's only for 6.x branch for bwc with elasticsearch-translog
-        if (removeCorruptedLuceneSegmentsAction == null) {
-            terminal.println("  This tool is deprecated and will be completely removed in 7.0.");
-            terminal.println("  It is replaced by the elasticsearch-shard tool. ");
-            terminal.println("");
-        }
         terminal.println("  Please make a complete backup of your index before using this tool.");
         terminal.println("");
         terminal.println("-----------------------------------------------------------------------");
@@ -318,26 +303,21 @@ public void write(int b) {
                 // keep the index lock to block any runs of older versions of this tool
                 try (Lock writeIndexLock = indexDir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
                     ////////// Index
-                    // that's only for 6.x branch for bwc with elasticsearch-translog
-                    if (removeCorruptedLuceneSegmentsAction != null) {
-                        terminal.println("");
-                        terminal.println("Opening Lucene index at " + indexPath);
-                        terminal.println("");
-                        try {
-                            indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir,
-                                writeIndexLock, printStream, verbose);
-                        } catch (Exception e) {
-                            terminal.println(e.getMessage());
-                            throw e;
-                        }
-
-                        terminal.println("");
-                        terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath);
-                        terminal.println("");
-                    } else {
-                        indexCleanStatus = Tuple.tuple(CleanStatus.CLEAN, null);
+                    terminal.println("");
+                    terminal.println("Opening Lucene index at " + indexPath);
+                    terminal.println("");
+                    try {
+                        indexCleanStatus = removeCorruptedLuceneSegmentsAction.getCleanStatus(shardPath, indexDir,
+                            writeIndexLock, printStream, verbose);
+                    } catch (Exception e) {
+                        terminal.println(e.getMessage());
+                        throw e;
                     }
 
+                    terminal.println("");
+                    terminal.println(" >> Lucene index is " + indexCleanStatus.v1().getMessage() + " at " + indexPath);
+                    terminal.println("");
+
                     ////////// Translog
                     // as translog relies on data stored in an index commit - we have to have non unrecoverable index to truncate translog
                     if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) {

qa/vagrant/src/test/resources/packaging/utils/tar.bash

@@ -293,11 +293,9 @@ public void testCorruptTranslogTruncation() throws Exception {
         indexRandom(false, false, false, Arrays.asList(builders));
         Set<Path> translogDirs = getDirs(indexName, ShardPath.TRANSLOG_FOLDER_NAME);
 
-        // that's only for 6.x branch for bwc with elasticsearch-translog
-        final boolean translogOnly = randomBoolean();
-        final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(translogOnly);
-        final MockTerminal terminal = new MockTerminal();
-        final OptionParser parser = command.getParser();
+        RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
+        MockTerminal terminal = new MockTerminal();
+        OptionParser parser = command.getParser();
 
         if (randomBoolean() && numDocsToTruncate > 0) {
             // flush the replica, so it will have more docs than what the primary will have