Merge remote-tracking branch 'elastic/master' into sync-retention-leases

* elastic/master:
  Docs be explicit on how to turn off deprecated auditing (elastic#37316)
  Fix line length for monitor and remove suppressions (elastic#37456)
  Fix IndexShardTestCase.recoverReplica(IndexShard, IndexShard, boolean) (elastic#37414)
  Update the Flush API documentation (elastic#33551)
  [TEST] Muted testDifferentRolesMaintainPathOnRestart
  Remove dead code from ShardSearchStats (elastic#37421)
  Simplify testSendSnapshotSendsOps (elastic#37445)
  SQL: Fix issue with field names containing "." (elastic#37364)
  Restore lost @Inject annotation (elastic#37452)

Author: jasontedor

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -48,11 +48,8 @@
 
   <!-- Hopefully temporary suppression of LineLength on files that don't pass it. We should remove these when we the
     files start to pass. -->
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]aliases[/\\]IndexAliasesIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="buildSrc[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

docs/reference/indices/flush.asciidoc

@@ -2,11 +2,12 @@
 == Flush
 
 The flush API allows to flush one or more indices through an API. The
-flush process of an index basically frees memory from the index by
-flushing data to the index storage and clearing the internal
-<<index-modules-translog,transaction log>>. By
-default, Elasticsearch uses memory heuristics in order to automatically
-trigger flush operations as required in order to clear memory.
+flush process of an index makes sure that any data that is currently only
+persisted in the <<index-modules-translog,transaction log>> is also permanently
+persisted in Lucene. This reduces recovery times as that data doesn't need to be
+reindexed from the transaction logs after the Lucene indexed is opened. By
+default, Elasticsearch uses heuristics in order to automatically
+trigger flushes as required. It is rare for users to need to call the API directly.
 
 [source,js]
 --------------------------------------------------

server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java

@@ -122,27 +122,11 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         });
     }
 
-    public void clear() {
-        totalStats.clear();
-        synchronized (this) {
-            if (!groupsStats.isEmpty()) {
-                MapBuilder<String, StatsHolder> typesStatsBuilder = MapBuilder.newMapBuilder();
-                for (Map.Entry<String, StatsHolder> typeStats : groupsStats.entrySet()) {
-                    if (typeStats.getValue().totalCurrent() > 0) {
-                        typeStats.getValue().clear();
-                        typesStatsBuilder.put(typeStats.getKey(), typeStats.getValue());
-                    }
-                }
-                groupsStats = typesStatsBuilder.immutableMap();
-            }
-        }
-    }
-
     private void computeStats(SearchContext searchContext, Consumer<StatsHolder> consumer) {
         consumer.accept(totalStats);
         if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                consumer.accept(groupStats(searchContext.groupStats().get(i)));
+            for (String group : searchContext.groupStats()) {
+                consumer.accept(groupStats(group));
             }
         }
     }
@@ -184,40 +168,29 @@ public void onFreeScrollContext(SearchContext context) {
     }
 
     static final class StatsHolder {
-        public final MeanMetric queryMetric = new MeanMetric();
-        public final MeanMetric fetchMetric = new MeanMetric();
+        final MeanMetric queryMetric = new MeanMetric();
+        final MeanMetric fetchMetric = new MeanMetric();
         /* We store scroll statistics in microseconds because with nanoseconds we run the risk of overflowing the total stats if there are
          * many scrolls. For example, on a system with 2^24 scrolls that have been executed, each executing for 2^10 seconds, then using
          * nanoseconds would require a numeric representation that can represent at least 2^24 * 2^10 * 10^9 > 2^24 * 2^10 * 2^29 = 2^63
          * which exceeds the largest value that can be represented by a long. By using microseconds, we enable capturing one-thousand
          * times as many scrolls (i.e., billions of scrolls which at one per second would take 32 years to occur), or scrolls that execute
          * for one-thousand times as long (i.e., scrolls that execute for almost twelve days on average).
          */
-        public final MeanMetric scrollMetric = new MeanMetric();
-        public final MeanMetric suggestMetric = new MeanMetric();
-        public final CounterMetric queryCurrent = new CounterMetric();
-        public final CounterMetric fetchCurrent = new CounterMetric();
-        public final CounterMetric scrollCurrent = new CounterMetric();
-        public final CounterMetric suggestCurrent = new CounterMetric();
-
-        public SearchStats.Stats stats() {
+        final MeanMetric scrollMetric = new MeanMetric();
+        final MeanMetric suggestMetric = new MeanMetric();
+        final CounterMetric queryCurrent = new CounterMetric();
+        final CounterMetric fetchCurrent = new CounterMetric();
+        final CounterMetric scrollCurrent = new CounterMetric();
+        final CounterMetric suggestCurrent = new CounterMetric();
+
+        SearchStats.Stats stats() {
             return new SearchStats.Stats(
                     queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(),
                     fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(),
                     scrollMetric.count(), TimeUnit.MICROSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(),
                     suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), suggestCurrent.count()
             );
         }
-
-        public long totalCurrent() {
-            return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count() + suggestCurrent.count();
-        }
-
-        public void clear() {
-            queryMetric.clear();
-            fetchMetric.clear();
-            scrollMetric.clear();
-            suggestMetric.clear();
-        }
     }
 }

server/src/main/java/org/elasticsearch/monitor/jvm/GcNames.java

@@ -29,13 +29,16 @@ public class GcNames {
      * Resolves the GC type by its memory pool name ({@link java.lang.management.MemoryPoolMXBean#getName()}.
      */
     public static String getByMemoryPoolName(String poolName, String defaultName) {
-        if ("Eden Space".equals(poolName) || "PS Eden Space".equals(poolName) || "Par Eden Space".equals(poolName) || "G1 Eden Space".equals(poolName)) {
+        if ("Eden Space".equals(poolName) || "PS Eden Space".equals(poolName)
+            || "Par Eden Space".equals(poolName) || "G1 Eden Space".equals(poolName)) {
             return YOUNG;
         }
-        if ("Survivor Space".equals(poolName) || "PS Survivor Space".equals(poolName) || "Par Survivor Space".equals(poolName) || "G1 Survivor Space".equals(poolName)) {
+        if ("Survivor Space".equals(poolName) || "PS Survivor Space".equals(poolName)
+            || "Par Survivor Space".equals(poolName) || "G1 Survivor Space".equals(poolName)) {
             return SURVIVOR;
         }
-        if ("Tenured Gen".equals(poolName) || "PS Old Gen".equals(poolName) || "CMS Old Gen".equals(poolName) || "G1 Old Gen".equals(poolName)) {
+        if ("Tenured Gen".equals(poolName) || "PS Old Gen".equals(poolName)
+            || "CMS Old Gen".equals(poolName) || "G1 Old Gen".equals(poolName)) {
             return OLD;
         }
         return defaultName;
@@ -45,7 +48,8 @@ public static String getByGcName(String gcName, String defaultName) {
         if ("Copy".equals(gcName) || "PS Scavenge".equals(gcName) || "ParNew".equals(gcName) || "G1 Young Generation".equals(gcName)) {
             return YOUNG;
         }
-        if ("MarkSweepCompact".equals(gcName) || "PS MarkSweep".equals(gcName) || "ConcurrentMarkSweep".equals(gcName) || "G1 Old Generation".equals(gcName)) {
+        if ("MarkSweepCompact".equals(gcName) || "PS MarkSweep".equals(gcName)
+            || "ConcurrentMarkSweep".equals(gcName) || "G1 Old Generation".equals(gcName)) {
             return OLD;
         }
         return defaultName;

server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java

@@ -234,7 +234,8 @@ private String innerDetect() throws Exception {
                 continue; // thread is not alive yet or died before the first snapshot - ignore it!
             }
             double percent = (((double) time) / interval.nanos()) * 100;
-            sb.append(String.format(Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n", percent, TimeValue.timeValueNanos(time), interval, type, threadName));
+            sb.append(String.format(Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n",
+                percent, TimeValue.timeValueNanos(time), interval, type, threadName));
             // for each snapshot (2nd array index) find later snapshot for same thread with max number of
             // identical StackTraceElements (starting from end of each)
             boolean[] done = new boolean[threadElementsSnapshotCount];
@@ -267,7 +268,8 @@ private String innerDetect() throws Exception {
                             sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
                         }
                     } else {
-                        sb.append(String.format(Locale.ROOT, "  %d/%d snapshots sharing following %d elements%n", count, threadElementsSnapshotCount, maxSim));
+                        sb.append(String.format(Locale.ROOT, "  %d/%d snapshots sharing following %d elements%n",
+                            count, threadElementsSnapshotCount, maxSim));
                         for (int l = show.length - maxSim; l < show.length; l++) {
                             sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
                         }

server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java

@@ -76,7 +76,6 @@
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
-import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -108,7 +107,6 @@
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class RecoverySourceHandlerTests extends ESTestCase {
@@ -205,9 +203,6 @@ public void testSendSnapshotSendsOps() throws IOException {
         final StartRecoveryRequest request = getStartRecoveryRequest();
         final IndexShard shard = mock(IndexShard.class);
         when(shard.state()).thenReturn(IndexShardState.STARTED);
-        final RecoveryTargetHandler recoveryTarget = mock(RecoveryTargetHandler.class);
-        final RecoverySourceHandler handler =
-            new RecoverySourceHandler(shard, recoveryTarget, request, fileChunkSizeInBytes, between(1, 10));
         final List<Translog.Operation> operations = new ArrayList<>();
         final int initialNumberOfDocs = randomIntBetween(16, 64);
         for (int i = 0; i < initialNumberOfDocs; i++) {
@@ -219,38 +214,23 @@ public void testSendSnapshotSendsOps() throws IOException {
             final Engine.Index index = getIndex(Integer.toString(i));
             operations.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i - initialNumberOfDocs, true)));
         }
-        operations.add(null);
         final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1);
         final long requiredStartingSeqNo = randomIntBetween((int) startingSeqNo, numberOfDocsWithValidSequenceNumbers - 1);
         final long endingSeqNo = randomIntBetween((int) requiredStartingSeqNo - 1, numberOfDocsWithValidSequenceNumbers - 1);
-        RecoverySourceHandler.SendSnapshotResult result = handler.phase2(startingSeqNo, requiredStartingSeqNo,
-            endingSeqNo, new Translog.Snapshot() {
-                @Override
-                public void close() {
-
-                }
-
-                private int counter = 0;
 
-                @Override
-                public int totalOperations() {
-                    return operations.size() - 1;
-                }
-
-                @Override
-                public Translog.Operation next() throws IOException {
-                    return operations.get(counter++);
-                }
-            }, randomNonNegativeLong(), randomNonNegativeLong());
+        final List<Translog.Operation> shippedOps = new ArrayList<>();
+        RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {
+            @Override
+            public long indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp, long msu) {
+                shippedOps.addAll(operations);
+                return SequenceNumbers.NO_OPS_PERFORMED;
+            }
+        };
+        RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, request, fileChunkSizeInBytes, between(1, 10));
+        RecoverySourceHandler.SendSnapshotResult result = handler.phase2(startingSeqNo, requiredStartingSeqNo,
+            endingSeqNo, newTranslogSnapshot(operations, Collections.emptyList()), randomNonNegativeLong(), randomNonNegativeLong());
         final int expectedOps = (int) (endingSeqNo - startingSeqNo + 1);
         assertThat(result.totalOperations, equalTo(expectedOps));
-        final ArgumentCaptor<List> shippedOpsCaptor = ArgumentCaptor.forClass(List.class);
-        verify(recoveryTarget).indexTranslogOperations(shippedOpsCaptor.capture(), ArgumentCaptor.forClass(Integer.class).capture(),
-            ArgumentCaptor.forClass(Long.class).capture(), ArgumentCaptor.forClass(Long.class).capture());
-        List<Translog.Operation> shippedOps = new ArrayList<>();
-        for (List list: shippedOpsCaptor.getAllValues()) {
-            shippedOps.addAll(list);
-        }
         shippedOps.sort(Comparator.comparing(Translog.Operation::seqNo));
         assertThat(shippedOps.size(), equalTo(expectedOps));
         for (int i = 0; i < shippedOps.size(); i++) {
@@ -261,30 +241,8 @@ public Translog.Operation next() throws IOException {
             List<Translog.Operation> requiredOps = operations.subList(0, operations.size() - 1).stream() // remove last null marker
                 .filter(o -> o.seqNo() >= requiredStartingSeqNo && o.seqNo() <= endingSeqNo).collect(Collectors.toList());
             List<Translog.Operation> opsToSkip = randomSubsetOf(randomIntBetween(1, requiredOps.size()), requiredOps);
-            expectThrows(IllegalStateException.class, () ->
-                handler.phase2(startingSeqNo, requiredStartingSeqNo,
-                    endingSeqNo, new Translog.Snapshot() {
-                        @Override
-                        public void close() {
-
-                        }
-
-                        private int counter = 0;
-
-                        @Override
-                        public int totalOperations() {
-                            return operations.size() - 1 - opsToSkip.size();
-                        }
-
-                        @Override
-                        public Translog.Operation next() throws IOException {
-                            Translog.Operation op;
-                            do {
-                                op = operations.get(counter++);
-                            } while (op != null && opsToSkip.contains(op));
-                            return op;
-                        }
-                    }, randomNonNegativeLong(), randomNonNegativeLong()));
+            expectThrows(IllegalStateException.class, () -> handler.phase2(startingSeqNo, requiredStartingSeqNo,
+                endingSeqNo, newTranslogSnapshot(operations, opsToSkip), randomNonNegativeLong(), randomNonNegativeLong()));
         }
     }
 
@@ -716,4 +674,39 @@ public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesR
                                    int totalTranslogOps, ActionListener<Void> listener) {
         }
     }
+
+    private Translog.Snapshot newTranslogSnapshot(List<Translog.Operation> operations, List<Translog.Operation> operationsToSkip) {
+        return new Translog.Snapshot() {
+            int index = 0;
+            int skippedCount = 0;
+
+            @Override
+            public int totalOperations() {
+                return operations.size();
+            }
+
+            @Override
+            public int skippedOperations() {
+                return skippedCount;
+            }
+
+            @Override
+            public Translog.Operation next() {
+                while (index < operations.size()) {
+                    Translog.Operation op = operations.get(index++);
+                    if (operationsToSkip.contains(op)) {
+                        skippedCount++;
+                    } else {
+                        return op;
+                    }
+                }
+                return null;
+            }
+
+            @Override
+            public void close() {
+
+            }
+        };
+    }
 }