Merge branch 'master' into byte-size-value-setting-limit-error-message

jasontedor · jasontedor · commit 1415e2b025d9 · 2018-04-03T23:05:47.000-04:00
* master: Remove silent batch mode from install plugin (elastic#29359) Align cat thread pool info to thread pool config (elastic#29195) Track Lucene operations in engine explicitly (elastic#29357)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -494,7 +494,7 @@ class ClusterFormationTasks {
          * the short name requiring the path to already exist.
          */
         final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
-        final Object[] args = [esPluginUtil, 'install', file]
+        final Object[] args = [esPluginUtil, 'install', '--batch', file]
         return configureExecTask(name, project, setup, node, args)
     }
 
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -208,7 +208,7 @@ protected void printAdditionalHelp(Terminal terminal) {
     @Override
     protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         String pluginId = arguments.value(options);
-        boolean isBatch = options.has(batchOption) || System.console() == null;
+        final boolean isBatch = options.has(batchOption);
         execute(terminal, pluginId, isBatch, env);
     }
 
diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc
@@ -113,14 +113,15 @@ in the table below.
 |Field Name |Alias |Description
 |`type` |`t` |The current (*) type of thread pool (`fixed` or `scaling`)
 |`active` |`a` |The number of active threads in the current thread pool
-|`size` |`s` |The number of threads in the current thread pool
+|`pool_size` |`psz` |The number of threads in the current thread pool
 |`queue` |`q` |The number of tasks in the queue for the current thread pool
 |`queue_size` |`qs` |The maximum number of tasks permitted in the queue for the current thread pool
 |`rejected` |`r` |The number of tasks rejected by the thread pool executor
 |`largest` |`l` |The highest number of active threads in the current thread pool
 |`completed` |`c` |The number of tasks completed by the thread pool executor
-|`min` |`mi` |The configured minimum number of active threads allowed in the current thread pool
-|`max` |`ma` |The configured maximum number of active threads allowed in the current thread pool
+|`core` |`cr` |The configured core number of active threads allowed in the current thread pool
+|`max` |`mx` |The configured maximum number of active threads allowed in the current thread pool
+|`size` |`sz` |The configured fixed number of active threads allowed in the current thread pool
 |`keep_alive` |`k` |The configured keep alive time for threads
 |=======================================================================
 
diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash
@@ -416,7 +416,7 @@ fi
 
 @test "[$GROUP] install a sample plugin with different logging modes and check output" {
     local relativePath=${1:-$(readlink -m custom-settings-*.zip)}
-    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
+    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install --batch "file://$relativePath" > /tmp/plugin-cli-output
     # exclude progress line
     local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
     [ "$loglines" -eq "2" ] || {
@@ -427,7 +427,7 @@ fi
     remove_plugin_example
 
     local relativePath=${1:-$(readlink -m custom-settings-*.zip)}
-    sudo -E -u $ESPLUGIN_COMMAND_USER ES_JAVA_OPTS="-Des.logger.level=DEBUG" "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
+    sudo -E -u $ESPLUGIN_COMMAND_USER ES_JAVA_OPTS="-Des.logger.level=DEBUG" "$ESHOME/bin/elasticsearch-plugin" install --batch "file://$relativePath" > /tmp/plugin-cli-output
     local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
     [ "$loglines" -gt "2" ] || {
         echo "Expected more than 2 lines excluding progress bar but the output had $loglines lines and was:"
diff --git a/qa/vagrant/src/test/resources/packaging/utils/plugins.bash b/qa/vagrant/src/test/resources/packaging/utils/plugins.bash
@@ -47,9 +47,9 @@ install_plugin() {
     fi
 
     if [ -z "$umask" ]; then
-      sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install -batch "file://$path"
+      sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install --batch "file://$path"
     else
-      sudo -E -u $ESPLUGIN_COMMAND_USER bash -c "umask $umask && \"$ESHOME/bin/elasticsearch-plugin\" install -batch \"file://$path\""
+      sudo -E -u $ESPLUGIN_COMMAND_USER bash -c "umask $umask && \"$ESHOME/bin/elasticsearch-plugin\" install --batch \"file://$path\""
     fi
 
     #check we did not accidentially create a log file as root as /usr/share/elasticsearch
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml
@@ -1,6 +1,10 @@
 ---
 "Test cat thread_pool output":
 
+  - skip:
+        version: " - 6.99.99"
+        reason: this API was changed in a backwards-incompatible fashion in 7.0.0 so we need to skip in a mixed cluster
+
   - do:
       cat.thread_pool: {}
 
@@ -46,25 +50,25 @@
   - do:
       cat.thread_pool:
           thread_pool_patterns: bulk
-          h: id,name,type,active,size,queue,queue_size,rejected,largest,completed,min,max,keep_alive
+          h: id,name,type,active,pool_size,queue,queue_size,rejected,largest,completed,core,max,size,keep_alive
           v: true
 
   - match:
       $body: |
-               /^  id  \s+ name \s+ type  \s+ active \s+ size \s+ queue \s+ queue_size \s+ rejected \s+ largest \s+ completed \s+ min \s+ max \s+ keep_alive \n
-                  (\S+ \s+ bulk \s+ fixed \s+ \d+    \s+ \d+  \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d* \s+ \d* \s+ \S*        \n)+  $/
+               /^  id  \s+ name \s+ type  \s+ active \s+ pool_size \s+ queue \s+ queue_size \s+ rejected \s+ largest \s+ completed \s+ core \s+ max \s+ size \s+ keep_alive \n
+                  (\S+ \s+ bulk \s+ fixed \s+ \d+    \s+ \d+       \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d*  \s+ \d* \s+ \d*  \s+ \S*        \n)+  $/
 
   - do:
       cat.thread_pool:
           thread_pool_patterns: fetch*
-          h: id,name,type,active,size,queue,queue_size,rejected,largest,completed,min,max,keep_alive
+          h: id,name,type,active,pool_size,queue,queue_size,rejected,largest,completed,core,max,size,keep_alive
           v: true
 
   - match:
       $body: |
-               /^  id  \s+ name                \s+ type    \s+ active \s+ size \s+ queue \s+ queue_size \s+ rejected \s+ largest \s+ completed \s+ min \s+ max \s+ keep_alive \n
-                  (\S+ \s+ fetch_shard_started \s+ scaling \s+ \d+    \s+ \d+  \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d* \s+ \d* \s+ \S*        \n
-                   \S+ \s+ fetch_shard_store   \s+ scaling \s+ \d+    \s+ \d+  \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d* \s+ \d* \s+ \S*        \n)+  $/
+               /^  id  \s+ name                \s+ type    \s+ active \s+ pool_size \s+ queue \s+ queue_size \s+ rejected \s+ largest \s+ completed \s+ core \s+ max \s+ size \s+ keep_alive \n
+                  (\S+ \s+ fetch_shard_started \s+ scaling \s+ \d+    \s+ \d+       \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d*  \s+ \d* \s+ \d*  \s+ \S*        \n
+                   \S+ \s+ fetch_shard_store   \s+ scaling \s+ \d+    \s+ \d+       \s+ \d+   \s+ (-1|\d+)   \s+ \d+      \s+ \d+     \s+ \d+       \s+ \d*  \s+ \d* \s+ \d*  \s+ \S*        \n)+  $/
 
   - do:
       cat.thread_pool:
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -136,6 +136,11 @@ public class InternalEngine extends Engine {
     private final AtomicLong maxSeqNoOfNonAppendOnlyOperations = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
     private final CounterMetric numVersionLookups = new CounterMetric();
     private final CounterMetric numIndexVersionsLookups = new CounterMetric();
+    // Lucene operations since this engine was opened - not include operations from existing segments.
+    private final CounterMetric numDocDeletes = new CounterMetric();
+    private final CounterMetric numDocAppends = new CounterMetric();
+    private final CounterMetric numDocUpdates = new CounterMetric();
+
     /**
      * How many bytes we are currently moving to disk, via either IndexWriter.flush or refresh.  IndexingMemoryController polls this
      * across all shards to decide if throttling is necessary because moving bytes to disk is falling behind vs incoming documents
@@ -907,11 +912,11 @@ private IndexResult indexIntoLucene(Index index, IndexingStrategy plan)
         index.parsedDoc().version().setLongValue(plan.versionForIndexing);
         try {
             if (plan.useLuceneUpdateDocument) {
-                update(index.uid(), index.docs(), indexWriter);
+                updateDocs(index.uid(), index.docs(), indexWriter);
             } else {
                 // document does not exists, we can optimize for create, but double check if assertions are running
                 assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false);
-                index(index.docs(), indexWriter);
+                addDocs(index.docs(), indexWriter);
             }
             return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
         } catch (Exception ex) {
@@ -968,12 +973,13 @@ long getMaxSeqNoOfNonAppendOnlyOperations() {
         return maxSeqNoOfNonAppendOnlyOperations.get();
     }
 
-    private static void index(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
+    private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
         if (docs.size() > 1) {
             indexWriter.addDocuments(docs);
         } else {
             indexWriter.addDocument(docs.get(0));
         }
+        numDocAppends.inc(docs.size());
     }
 
     private static final class IndexingStrategy {
@@ -1054,12 +1060,13 @@ private boolean assertDocDoesNotExist(final Index index, final boolean allowDele
         return true;
     }
 
-    private static void update(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
+    private void updateDocs(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
         if (docs.size() > 1) {
             indexWriter.updateDocuments(uid, docs);
         } else {
             indexWriter.updateDocument(uid, docs.get(0));
         }
+        numDocUpdates.inc(docs.size());
     }
 
     @Override
@@ -1188,6 +1195,7 @@ private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan)
                 // any exception that comes from this is a either an ACE or a fatal exception there
                 // can't be any document failures  coming from this
                 indexWriter.deleteDocuments(delete.uid());
+                numDocDeletes.inc();
             }
             versionMap.putUnderLock(delete.uid().bytes(),
                 new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(),
@@ -2205,13 +2213,28 @@ boolean isSafeAccessRequired() {
         return versionMap.isSafeAccessRequired();
     }
 
+    /**
+     * Returns the number of documents have been deleted since this engine was opened.
+     * This count does not include the deletions from the existing segments before opening engine.
+     */
+    long getNumDocDeletes() {
+        return numDocDeletes.count();
+    }
+
+    /**
+     * Returns the number of documents have been appended since this engine was opened.
+     * This count does not include the appends from the existing segments before opening engine.
+     */
+    long getNumDocAppends() {
+        return numDocAppends.count();
+    }
 
     /**
-     * Returns <code>true</code> iff the index writer has any deletions either buffered in memory or
-     * in the index.
+     * Returns the number of documents have been updated since this engine was opened.
+     * This count does not include the updates from the existing segments before opening engine.
      */
-    boolean indexWriterHasDeletions() {
-        return indexWriter.hasDeletions();
+    long getNumDocUpdates() {
+        return numDocUpdates.count();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -124,14 +124,15 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("name", "default:true;alias:n;desc:thread pool name");
         table.addCell("type", "alias:t;default:false;desc:thread pool type");
         table.addCell("active", "alias:a;default:true;text-align:right;desc:number of active threads");
-        table.addCell("size", "alias:s;default:false;text-align:right;desc:number of threads");
+        table.addCell("pool_size", "alias:psz;default:false;text-align:right;desc:number of threads");
         table.addCell("queue", "alias:q;default:true;text-align:right;desc:number of tasks currently in queue");
         table.addCell("queue_size", "alias:qs;default:false;text-align:right;desc:maximum number of tasks permitted in queue");
         table.addCell("rejected", "alias:r;default:true;text-align:right;desc:number of rejected tasks");
         table.addCell("largest", "alias:l;default:false;text-align:right;desc:highest number of seen active threads");
         table.addCell("completed", "alias:c;default:false;text-align:right;desc:number of completed tasks");
-        table.addCell("min", "alias:mi;default:false;text-align:right;desc:minimum number of threads");
-        table.addCell("max", "alias:ma;default:false;text-align:right;desc:maximum number of threads");
+        table.addCell("core", "alias:cr;default:false;text-align:right;desc:core number of threads in a scaling thread pool");
+        table.addCell("max", "alias:mx;default:false;text-align:right;desc:maximum number of threads in a scaling thread pool");
+        table.addCell("size", "alias:sz;default:false;text-align:right;desc:number of threads in a fixed thread pool");
         table.addCell("keep_alive", "alias:ka;default:false;text-align:right;desc:thread keep alive time");
         table.endHeaders();
         return table;
@@ -201,8 +202,9 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
 
                 Long maxQueueSize = null;
                 String keepAlive = null;
-                Integer minThreads = null;
-                Integer maxThreads = null;
+                Integer core = null;
+                Integer max = null;
+                Integer size = null;
 
                 if (poolInfo != null) {
                     if (poolInfo.getQueueSize() != null) {
@@ -211,11 +213,15 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
                     if (poolInfo.getKeepAlive() != null) {
                         keepAlive = poolInfo.getKeepAlive().toString();
                     }
-                    if (poolInfo.getMin() >= 0) {
-                        minThreads = poolInfo.getMin();
-                    }
-                    if (poolInfo.getMax() >= 0) {
-                        maxThreads = poolInfo.getMax();
+
+                    if (poolInfo.getThreadPoolType() == ThreadPool.ThreadPoolType.SCALING) {
+                        assert poolInfo.getMin() >= 0;
+                        core = poolInfo.getMin();
+                        assert poolInfo.getMax() > 0;
+                        max = poolInfo.getMax();
+                    } else {
+                        assert poolInfo.getMin() == poolInfo.getMax() && poolInfo.getMax() > 0;
+                        size = poolInfo.getMax();
                     }
                 }
 
@@ -228,8 +234,9 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
                 table.addCell(poolStats == null ? null : poolStats.getRejected());
                 table.addCell(poolStats == null ? null : poolStats.getLargest());
                 table.addCell(poolStats == null ? null : poolStats.getCompleted());
-                table.addCell(minThreads);
-                table.addCell(maxThreads);
+                table.addCell(core);
+                table.addCell(max);
+                table.addCell(size);
                 table.addCell(keepAlive);
 
                 table.endRow();
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java

Original file line number	Diff line number	Diff line change
`@@ -494,7 +494,7 @@ class ClusterFormationTasks {`
`494`	`494`	`* the short name requiring the path to already exist.`
`495`	`495`	`*/`
`496`	`496`	`final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"`
`497`		`- final Object[] args = [esPluginUtil, 'install', file]`
	`497`	`+ final Object[] args = [esPluginUtil, 'install', '--batch', file]`
`498`	`498`	`return configureExecTask(name, project, setup, node, args)`
`499`	`499`	`}`
`500`	`500`
Original file line number	Diff line number	Diff line change
`@@ -208,7 +208,7 @@ protected void printAdditionalHelp(Terminal terminal) {`
`208`	`208`	`@Override`
`209`	`209`	`protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {`
`210`	`210`	`String pluginId = arguments.value(options);`
`211`		`- boolean isBatch = options.has(batchOption) \|\| System.console() == null;`
	`211`	`+ final boolean isBatch = options.has(batchOption);`
`212`	`212`	`execute(terminal, pluginId, isBatch, env);`
`213`	`213`	`}`
`214`	`214`