Core: throttle delete-by-query when merges are falling behind

Delete-by-query is incredibly costly because it forces a refresh each
time, so if you are also indexing this can cause massive segment
explosion.

This change throttles delete-by-query when merges can't keep up.  It's
likely not enough (#7052 is the long-term solution) but can only
help.

Closes #9986

Author: mikemccand

src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -452,6 +452,7 @@ private void innerIndex(Index index) throws IOException {
     public void delete(Delete delete) throws EngineException {
         try (ReleasableLock _ = readLock.acquire()) {
             ensureOpen();
+            // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
             innerDelete(delete);
             flushNeeded = true;
         } catch (OutOfMemoryError | IllegalStateException | IOException t) {
@@ -519,6 +520,19 @@ private void innerDelete(Delete delete) throws IOException {
     public void delete(DeleteByQuery delete) throws EngineException {
         try (ReleasableLock _ = readLock.acquire()) {
             ensureOpen();
+            if (delete.origin() == Operation.Origin.RECOVERY) {
+                // Don't throttle recovery operations
+                innerDelete(delete);
+            } else {
+                try (Releasable r = throttle.acquireThrottle()) {
+                    innerDelete(delete);
+                }
+            }
+        }
+    }
+
+    private void innerDelete(DeleteByQuery delete) throws EngineException {
+        try {
             Query query;
             if (delete.nested() && delete.aliasFilter() != null) {
                 query = new IncludeNestedDocsQuery(new XFilteredQuery(delete.query(), delete.aliasFilter()), delete.parentFilter());