Merge branch 'ccr' into follower-primary-term

* ccr:
  [DOCS] Creates rest-api folder in docs
  [Rollup] Disallow index patterns that match the rollup index (elastic#30491)
  Replace exact numDocs by soft-del count in SegmentInfo (elastic#31086)
  Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (elastic#31073)
  Add cors support to NioHttpServerTransport (elastic#30827)
  [DOCS] Fixes security example (elastic#31082)
  Allow terms query in _rollup_search (elastic#30973)
  Removing erroneous repeat
  Adapt bwc versions after backporting elastic#30983 to 6.4
  [Tests] Muting RatedRequestsTests#testXContentParsingIsNotLenient
  TEST:  Retry synced-flush if ongoing ops on primary (elastic#30978)
  Fix docs build.
  Only auto-update license signature if all nodes ready (elastic#30859)
  Add BlobContainer.writeBlobAtomic() (elastic#30902)
  Add a doc value format to binary fields. (elastic#30860)

Author: jasontedor

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -505,8 +505,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]settings[/\\]ClusterSettingsIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]structure[/\\]RoutingIteratorTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreContainerTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]breaker[/\\]MemoryCircuitBreakerTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]ShapeBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]hash[/\\]MessageDigestsTests.java" checks="LineLength" />

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-1cbadda4d3
+lucene            = 7.4.0-snapshot-0a7c3f462f
 
 # optional dependencies
 spatial4j         = 0.7

docs/reference/index.asciidoc

@@ -63,7 +63,7 @@ include::{xes-repo-dir}/monitoring/index.asciidoc[]
 
 include::{xes-repo-dir}/rollup/index.asciidoc[]
 
-include::{xes-repo-dir}/rest-api/index.asciidoc[]
+include::rest-api/index.asciidoc[]
 
 include::{xes-repo-dir}/commands/index.asciidoc[]
 

docs/reference/query-dsl/query-string-syntax.asciidoc

@@ -254,7 +254,6 @@ would look like this:
         }
     }
 
-****
 
 ===== Grouping
 

docs/reference/rest-api/index.asciidoc

@@ -0,0 +1,29 @@
+[role="xpack"]
+[[xpack-api]]
+= {xpack} APIs
+
+[partintro]
+--
+{xpack} exposes REST APIs that are used by the UI components and can be called
+directly to configure and access {xpack} features.
+
+* <<info-api,Info API>>
+* <<graph-explore-api,Graph Explore API>>
+* <<licensing-apis,Licensing APIs>>
+* <<ml-apis,Machine Learning APIs>>
+* <<security-api,Security APIs>>
+* <<watcher-api,Watcher APIs>>
+* <<rollup-apis,Rollup APIs>>
+* <<migration-api,Migration APIs>>
+--
+
+
+include::{xes-repo-dir}/rest-api/info.asciidoc[]
+include::{xes-repo-dir}/rest-api/graph/explore.asciidoc[]
+include::{xes-repo-dir}/rest-api/licensing.asciidoc[]
+include::{xes-repo-dir}/rest-api/migration.asciidoc[]
+include::{xes-repo-dir}/rest-api/ml-api.asciidoc[]
+include::{xes-repo-dir}/rest-api/rollup-api.asciidoc[]
+include::{xes-repo-dir}/rest-api/security.asciidoc[]
+include::{xes-repo-dir}/rest-api/watcher.asciidoc[]
+include::{xes-repo-dir}/rest-api/defs.asciidoc[]

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+bf2cfa0551ebdf08a2cf3079f3c74643bd9dbb76

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-1cbadda4d3.jar.sha1

@@ -131,6 +131,7 @@ public void testXContentRoundtrip() throws IOException {
         }
     }
 
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/31104")
     public void testXContentParsingIsNotLenient() throws IOException {
         RatedRequest testItem = createTestItem(randomBoolean());
         XContentType xContentType = randomFrom(XContentType.values());

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java

@@ -76,7 +76,8 @@ public boolean isCorsSupportEnabled() {
     }
 
     /**
-     * Determines whether a wildcard origin, '*', is supported.
+     * Determines whether a wildcard origin, '*', is supported. This also means that null origins are
+     * supported.
      *
      * @return {@code boolean} true if any origin is allowed.
      */
@@ -121,21 +122,21 @@ public boolean isNullOriginAllowed() {
     }
 
     /**
-     * Determines if cookies are supported for CORS requests.
+     * Determines if credentials are supported for CORS requests.
      *
-     * By default cookies are not included in CORS requests but if isCredentialsAllowed returns
-     * true cookies will be added to CORS requests. Setting this value to true will set the
+     * By default credentials are not included in CORS requests but if isCredentialsAllowed returns
+     * true credentials will be added to CORS requests. Setting this value to true will set the
      * CORS 'Access-Control-Allow-Credentials' response header to true.
      *
-     * Please note that cookie support needs to be enabled on the client side as well.
-     * The client needs to opt-in to send cookies by calling:
+     * Please note that credentials support needs to be enabled on the client side as well.
+     * The client needs to opt-in to send credentials by calling:
      * <pre>
      * xhr.withCredentials = true;
      * </pre>
-     * The default value for 'withCredentials' is false in which case no cookies are sent.
-     * Setting this to true will included cookies in cross origin requests.
+     * The default value for 'withCredentials' is false in which case no credentials are sent.
+     * Setting this to true will included credentials in cross origin requests.
      *
-     * @return {@code true} if cookies are supported.
+     * @return {@code true} if credentials are supported.
      */
     public boolean isCredentialsAllowed() {
         return allowCredentials;

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java

@@ -0,0 +1 @@
+82d83fcac1d9c8948aa0247fc9c87f177ddbd59b

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+73fd4364f2931e7c8303b5927e140a7d21116c36

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-1cbadda4d3.jar.sha1

@@ -0,0 +1 @@
+0a2c4417fa9a8be078864f590a5a66b98d551cf5

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+6fa179924f139a30fc0e5399256e1a44562ed32b

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-1cbadda4d3.jar.sha1

@@ -0,0 +1 @@
+5ed135d34d7868b71a725257a46dc8d8735a15d4

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+875911b36b99c2103719f94559878a0ecb862fb6

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-1cbadda4d3.jar.sha1

@@ -0,0 +1 @@
+e7191628df8cb72382a20da79224aef677117849

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -36,6 +36,8 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpPipelinedRequest;
+import org.elasticsearch.http.nio.cors.NioCorsConfig;
+import org.elasticsearch.http.nio.cors.NioCorsHandler;
 import org.elasticsearch.nio.FlushOperation;
 import org.elasticsearch.nio.InboundChannelBuffer;
 import org.elasticsearch.nio.NioSocketChannel;
@@ -50,21 +52,25 @@
 import java.util.List;
 import java.util.function.BiConsumer;
 
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+
 public class HttpReadWriteHandler implements ReadWriteHandler {
 
     private final NettyAdaptor adaptor;
     private final NioSocketChannel nioChannel;
     private final NioHttpServerTransport transport;
     private final HttpHandlingSettings settings;
     private final NamedXContentRegistry xContentRegistry;
+    private final NioCorsConfig corsConfig;
     private final ThreadContext threadContext;
 
     HttpReadWriteHandler(NioSocketChannel nioChannel, NioHttpServerTransport transport, HttpHandlingSettings settings,
-                         NamedXContentRegistry xContentRegistry, ThreadContext threadContext) {
+                         NamedXContentRegistry xContentRegistry, NioCorsConfig corsConfig, ThreadContext threadContext) {
         this.nioChannel = nioChannel;
         this.transport = transport;
         this.settings = settings;
         this.xContentRegistry = xContentRegistry;
+        this.corsConfig = corsConfig;
         this.threadContext = threadContext;
 
         List<ChannelHandler> handlers = new ArrayList<>(5);
@@ -78,6 +84,9 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
         if (settings.isCompression()) {
             handlers.add(new HttpContentCompressor(settings.getCompressionLevel()));
         }
+        if (settings.isCorsEnabled()) {
+            handlers.add(new NioCorsHandler(corsConfig));
+        }
         handlers.add(new NioHttpPipeliningHandler(transport.getLogger(), settings.getPipeliningMaxEvents()));
 
         adaptor = new NettyAdaptor(handlers.toArray(new ChannelHandler[0]));
@@ -178,7 +187,7 @@ private void handleRequest(Object msg) {
                 int sequence = pipelinedRequest.getSequence();
                 BigArrays bigArrays = transport.getBigArrays();
                 try {
-                    innerChannel = new NioHttpChannel(nioChannel, bigArrays, httpRequest, sequence, settings, threadContext);
+                    innerChannel = new NioHttpChannel(nioChannel, bigArrays, httpRequest, sequence, settings, corsConfig, threadContext);
                 } catch (final IllegalArgumentException e) {
                     if (badRequestCause == null) {
                         badRequestCause = e;
@@ -191,7 +200,7 @@ private void handleRequest(Object msg) {
                             Collections.emptyMap(), // we are going to dispatch the request as a bad request, drop all parameters
                             copiedRequest.uri(),
                             copiedRequest);
-                    innerChannel = new NioHttpChannel(nioChannel, bigArrays, innerRequest, sequence, settings, threadContext);
+                    innerChannel = new NioHttpChannel(nioChannel, bigArrays, innerRequest, sequence, settings, corsConfig, threadContext);
                 }
                 channel = innerChannel;
             }

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-1cbadda4d3.jar.sha1

@@ -41,6 +41,8 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpHandlingSettings;
+import org.elasticsearch.http.nio.cors.NioCorsConfig;
+import org.elasticsearch.http.nio.cors.NioCorsHandler;
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestResponse;
@@ -58,17 +60,19 @@ public class NioHttpChannel extends AbstractRestChannel {
 
     private final BigArrays bigArrays;
     private final int sequence;
+    private final NioCorsConfig corsConfig;
     private final ThreadContext threadContext;
     private final FullHttpRequest nettyRequest;
     private final NioSocketChannel nioChannel;
     private final boolean resetCookies;
 
     NioHttpChannel(NioSocketChannel nioChannel, BigArrays bigArrays, NioHttpRequest request, int sequence,
-                   HttpHandlingSettings settings, ThreadContext threadContext) {
+                   HttpHandlingSettings settings, NioCorsConfig corsConfig, ThreadContext threadContext) {
         super(request, settings.getDetailedErrorsEnabled());
         this.nioChannel = nioChannel;
         this.bigArrays = bigArrays;
         this.sequence = sequence;
+        this.corsConfig = corsConfig;
         this.threadContext = threadContext;
         this.nettyRequest = request.getRequest();
         this.resetCookies = settings.isResetCookies();
@@ -87,6 +91,8 @@ public void sendResponse(RestResponse response) {
         }
         resp.setStatus(getStatus(response.status()));
 
+        NioCorsHandler.setCorsResponseHeaders(nettyRequest, resp, corsConfig);
+
         String opaque = nettyRequest.headers().get("X-Opaque-Id");
         if (opaque != null) {
             setHeaderField(resp, "X-Opaque-Id", opaque);

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.http.nio;
 
+import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.timeout.ReadTimeoutException;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -28,6 +29,7 @@
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -38,11 +40,13 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.AbstractHttpServerTransport;
 import org.elasticsearch.http.BindHttpException;
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
-import org.elasticsearch.http.AbstractHttpServerTransport;
+import org.elasticsearch.http.nio.cors.NioCorsConfig;
+import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
 import org.elasticsearch.nio.AcceptingSelector;
 import org.elasticsearch.nio.AcceptorEventHandler;
 import org.elasticsearch.nio.BytesChannelContext;
@@ -56,6 +60,7 @@
 import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.nio.SocketEventHandler;
 import org.elasticsearch.nio.SocketSelector;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -64,15 +69,23 @@
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.common.settings.Setting.intSetting;
 import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
@@ -86,6 +99,7 @@
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
+import static org.elasticsearch.http.nio.cors.NioCorsHandler.ANY_ORIGIN;
 
 public class NioHttpServerTransport extends AbstractHttpServerTransport {
 
@@ -115,6 +129,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
     private final Set<NioSocketChannel> socketChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
     private NioGroup nioGroup;
     private HttpChannelFactory channelFactory;
+    private final NioCorsConfig corsConfig;
 
     public NioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
                                   NamedXContentRegistry xContentRegistry, HttpServerTransport.Dispatcher dispatcher) {
@@ -136,6 +151,7 @@ public NioHttpServerTransport(Settings settings, NetworkService networkService,
             SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
             SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
             pipeliningMaxEvents);
+        this.corsConfig = buildCorsConfig(settings);
 
         this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings);
         this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings);
@@ -279,6 +295,38 @@ protected void nonChannelExceptionCaught(Exception ex) {
         logger.warn(new ParameterizedMessage("exception caught on transport layer [thread={}]", Thread.currentThread().getName()), ex);
     }
 
+    static NioCorsConfig buildCorsConfig(Settings settings) {
+        if (SETTING_CORS_ENABLED.get(settings) == false) {
+            return NioCorsConfigBuilder.forOrigins().disable().build();
+        }
+        String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
+        final NioCorsConfigBuilder builder;
+        if (Strings.isNullOrEmpty(origin)) {
+            builder = NioCorsConfigBuilder.forOrigins();
+        } else if (origin.equals(ANY_ORIGIN)) {
+            builder = NioCorsConfigBuilder.forAnyOrigin();
+        } else {
+            Pattern p = RestUtils.checkCorsSettingForRegex(origin);
+            if (p == null) {
+                builder = NioCorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin));
+            } else {
+                builder = NioCorsConfigBuilder.forPattern(p);
+            }
+        }
+        if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
+            builder.allowCredentials();
+        }
+        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
+        HttpMethod[] methods = Arrays.stream(strMethods)
+            .map(HttpMethod::valueOf)
+            .toArray(HttpMethod[]::new);
+        return builder.allowedRequestMethods(methods)
+            .maxAge(SETTING_CORS_MAX_AGE.get(settings))
+            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
+            .shortCircuit()
+            .build();
+    }
+
     private void closeChannels(List<NioChannel> channels) {
         List<ActionFuture<Void>> futures = new ArrayList<>(channels.size());
 
@@ -315,7 +363,7 @@ private HttpChannelFactory() {
         public NioSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException {
             NioSocketChannel nioChannel = new NioSocketChannel(channel);
             HttpReadWriteHandler httpReadWritePipeline = new HttpReadWriteHandler(nioChannel,NioHttpServerTransport.this,
-                httpHandlingSettings, xContentRegistry, threadPool.getThreadContext());
+                httpHandlingSettings, xContentRegistry, corsConfig, threadPool.getThreadContext());
             Consumer<Exception> exceptionHandler = (e) -> exceptionCaught(nioChannel, e);
             SocketChannelContext context = new BytesChannelContext(nioChannel, selector, exceptionHandler, httpReadWritePipeline,
                 InboundChannelBuffer.allocatingInstance());