Core: throttle delete-by-query when merges are falling behind

Delete-by-query is incredibly costly because it forces a refresh each
time, so if you are also indexing this can cause massive segment
explosion.

This change throttles delete-by-query when merges can't keep up.  It's
likely not enough (elastic#7052 is the long-term solution) but can only
help.

Closes elastic#9986

Author: mikemccand

src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java

@@ -618,6 +618,7 @@ public void delete(Delete delete) throws EngineException {
             if (writer == null) {
                 throw new EngineClosedException(shardId, failedEngine);
             }
+            // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
             innerDelete(delete, writer);
             dirty = true;
             possibleMergeNeeded = true;
@@ -690,7 +691,19 @@ public void delete(DeleteByQuery delete) throws EngineException {
             if (writer == null) {
                 throw new EngineClosedException(shardId);
             }
+            if (delete.origin() == Operation.Origin.RECOVERY) {
+                // Don't throttle recovery operations
+                innerDelete(delete, writer);
+            } else {
+                try (Releasable r = throttle.acquireThrottle()) {
+                    innerDelete(delete, writer);
+                }
+            }
+        }
+    }
 
+    private void innerDelete(DeleteByQuery delete, IndexWriter writer) throws EngineException {
+        try {
             Query query;
             if (delete.nested() && delete.aliasFilter() != null) {
                 query = new IncludeNestedDocsQuery(new XFilteredQuery(delete.query(), delete.aliasFilter()), delete.parentFilter());