ODFE Release 1.8.0 (#124)

This PR made the following changes:
1. Update to JDK 14
2. Hack to overstep potential ES 7.7 bug when installing plugins for test cluster
3. Upgrade gradle, nebula and jacoco versions
4. Change how we register rest endpoint handler as required by ES 7.7
5. Impelement SystemIndexPlugin and register the descriptor according to elastic/elasticsearch#49959
6. added new version of job scheduler zip

Testing done:
1. gralde build passes
2. Manually tested all public APIs

Author: hamersu9t

.github/workflows/test-workflow.yml

@@ -8,7 +8,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [13]
+        java: [14]
     # Job name
     name: Build Anomaly detection with JDK ${{ matrix.java }}
     # This job runs on Linux

README.md

@@ -22,7 +22,7 @@ Please see our [documentation](https://opendistro.github.io/for-elasticsearch-do
 
 1. Checkout source code of this package from Github repo.
 1. Launch Intellij IDEA, Choose Import Project and select the settings.gradle file in the root of this package.
-1. To build from command line set JAVA_HOME to point to a JDK 13 before running ./gradlew
+1. To build from command line set JAVA_HOME to point to a JDK 14 before running ./gradlew
 
   * Unix System
     * export JAVA_HOME=jdk-install-dir: Replace jdk-install-dir by the JAVA_HOME directory of your system.

build.gradle

@@ -1,5 +1,5 @@
 /*
- *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *   Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *   
  *   Licensed under the Apache License, Version 2.0 (the "License").
  *   You may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 buildscript {
     ext {
         es_group = "org.elasticsearch"
-        es_version = '7.6.1'
+        es_version = '7.7.0'
     }
 
     repositories {
@@ -31,9 +31,10 @@ buildscript {
 }
 
 plugins {
-    id 'nebula.ospackage' version "8.2.0" apply false
+    id 'nebula.ospackage' version "8.3.0" apply false
     id "com.diffplug.gradle.spotless" version "3.26.1"
     id 'java-library'
+    id 'checkstyle'
 }
 
 repositories {
@@ -43,7 +44,7 @@ repositories {
 }
 
 ext {
-    opendistroVersion = '1.7.0'
+    opendistroVersion = '1.8.0'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
 }
 
@@ -165,6 +166,30 @@ testClusters.integTest {
         }
     }
     plugin(fileTree("src/test/resources/job-scheduler").getSingleFile())
+
+    // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before
+    // the opendistro-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug.
+    //
+    // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opendistro-job-scheduler], dependency of [opendistro-anomaly-detection]
+    //       at org.elasticsearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452)
+    //
+    // One explanation is that ES build script sort plugins according to the natural ordering of their names.
+    // opendistro-anomaly-detection comes before opendistro-job-scheduler.
+    //
+    // The following is a comparison of different plugin installation order:
+    // Before 7.7:
+    // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opendistro-job-scheduler.zip
+    //
+    // After 7.7:
+    // ./bin/elasticsearch-plugin install --batch file:opendistro-job-scheduler.zip file:opendistro-anomaly-detection.zip 
+    //
+    // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
+    nodes.each { node ->
+        def plugins = node.plugins
+        def firstPlugin = plugins.get(0)
+        plugins.remove(0)
+        plugins.add(firstPlugin)
+    }
 }
 
 run {
@@ -242,6 +267,10 @@ jacocoTestCoverageVerification {
 check.dependsOn jacocoTestCoverageVerification
 jacocoTestCoverageVerification.dependsOn jacocoTestReport
 
+checkstyle {
+    toolVersion = '8.20'
+}
+
 dependencies {
     compile "org.elasticsearch:elasticsearch:${es_version}"
     compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
@@ -254,10 +283,10 @@ dependencies {
     compile group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
     compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
 
-    compile "org.jacoco:org.jacoco.agent:0.8.3"
-    compile "org.jacoco:org.jacoco.ant:0.8.3"
-    compile "org.jacoco:org.jacoco.core:0.8.3"
-    compile "org.jacoco:org.jacoco.report:0.8.3"
+    compile "org.jacoco:org.jacoco.agent:0.8.5"
+    compile "org.jacoco:org.jacoco.ant:0.8.5"
+    compile "org.jacoco:org.jacoco.core:0.8.5"
+    compile "org.jacoco:org.jacoco.report:0.8.5"
 
     testCompile group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.0.0'

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,20 @@
-#Wed Mar 13 15:07:47 IST 2019
+/*
+ *   Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *   
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *   
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *   
+ *   or in the "license" file accompanying this file. This file is distributed 
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
+ *   express or implied. See the License for the specific language governing 
+ *   permissions and limitations under the License.
+ */
+ 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip

src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java

@@ -104,9 +104,11 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.monitor.jvm.JvmService;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
@@ -130,7 +132,7 @@
 /**
  * Entry point of AD plugin.
  */
-public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension {
+public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension, SystemIndexPlugin {
 
     public static final String AD_BASE_URI = "/_opendistro/_anomaly_detection";
     public static final String AD_BASE_DETECTORS_URI = AD_BASE_URI + "/detectors";
@@ -241,7 +243,8 @@ public Collection<Object> createComponents(
         NamedXContentRegistry xContentRegistry,
         Environment environment,
         NodeEnvironment nodeEnvironment,
-        NamedWriteableRegistry namedWriteableRegistry
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         this.client = client;
         this.threadPool = threadPool;
@@ -452,4 +455,17 @@ public ScheduledJobParser getJobParser() {
         };
     }
 
+    @Override
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
+        return Collections
+            .unmodifiableList(
+                Arrays
+                    .asList(
+                        new SystemIndexDescriptor(AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN, "anomaly result"),
+                        new SystemIndexDescriptor(AnomalyDetector.ANOMALY_DETECTORS_INDEX, "detector definition"),
+                        new SystemIndexDescriptor(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, "detector job"),
+                        new SystemIndexDescriptor(CommonName.CHECKPOINT_INDEX_NAME, "model checkpoint")
+                    )
+            );
+    }
 }

src/main/java/com/amazon/opendistroforelasticsearch/ad/ml/ModelManager.java

@@ -141,6 +141,7 @@ public String getName() {
      * @param minPreviewSize minimum number of data points for preview
      * @param modelTtl time to live for hosted models
      * @param checkpointInterval interval between checkpoints
+     * @param shingleSize required shingle size before RCF emitting anomaly scores
      */
     public ModelManager(
         ClusterService clusterService,

src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/AbstractSearchAction.java

@@ -16,6 +16,8 @@
 package com.amazon.opendistroforelasticsearch.ad.rest;
 
 import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector;
+import com.google.common.collect.ImmutableList;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.SearchRequest;
@@ -39,6 +41,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.util.List;
 
 import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.getSourceContext;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
@@ -52,14 +55,14 @@ public abstract class AbstractSearchAction<T extends ToXContentObject> extends B
 
     private final String index;
     private final Class<T> clazz;
+    private final String urlPath;
 
     private final Logger logger = LogManager.getLogger(AbstractSearchAction.class);
 
     public AbstractSearchAction(RestController controller, String urlPath, String index, Class<T> clazz) {
         this.index = index;
         this.clazz = clazz;
-        controller.registerHandler(RestRequest.Method.POST, urlPath, this);
-        controller.registerHandler(RestRequest.Method.GET, urlPath, this);
+        this.urlPath = urlPath;
     }
 
     @Override
@@ -102,4 +105,9 @@ public RestResponse buildResponse(SearchResponse response) throws Exception {
             }
         };
     }
+
+    @Override
+    public List<Route> routes() {
+        return ImmutableList.of(new Route(RestRequest.Method.POST, urlPath), new Route(RestRequest.Method.GET, urlPath));
+    }
 }

src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestAnomalyDetectorJobAction.java

@@ -18,6 +18,8 @@
 import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin;
 import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices;
 import com.amazon.opendistroforelasticsearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler;
+import com.google.common.collect.ImmutableList;
+
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -29,6 +31,7 @@
 import org.elasticsearch.rest.RestRequest;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Locale;
 
 import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT;
@@ -61,22 +64,6 @@ public RestAnomalyDetectorJobAction(
         this.requestTimeout = REQUEST_TIMEOUT.get(settings);
         this.clusterService = clusterService;
         clusterService.getClusterSettings().addSettingsUpdateConsumer(REQUEST_TIMEOUT, it -> requestTimeout = it);
-
-        // start AD job
-        controller
-            .registerHandler(
-                RestRequest.Method.POST,
-                String.format(Locale.ROOT, "%s/{%s}/%s", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID, START_JOB),
-                this
-            );
-
-        // stop AD job
-        controller
-            .registerHandler(
-                RestRequest.Method.POST,
-                String.format(Locale.ROOT, "%s/{%s}/%s", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID, STOP_JOB),
-                this
-            );
     }
 
     @Override
@@ -116,4 +103,21 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             }
         };
     }
+
+    @Override
+    public List<Route> routes() {
+        return ImmutableList
+            .of(
+                // start AD job
+                new Route(
+                    RestRequest.Method.POST,
+                    String.format(Locale.ROOT, "%s/{%s}/%s", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID, START_JOB)
+                ),
+                // stop AD job
+                new Route(
+                    RestRequest.Method.POST,
+                    String.format(Locale.ROOT, "%s/{%s}/%s", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID, STOP_JOB)
+                )
+            );
+    }
 }

src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestDeleteAnomalyDetectorAction.java

@@ -18,6 +18,7 @@
 import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector;
 import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob;
 import com.amazon.opendistroforelasticsearch.ad.rest.handler.AnomalyDetectorActionHandler;
+import com.google.common.collect.ImmutableList;
 import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -36,6 +37,7 @@
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Locale;
 
 import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.DETECTOR_ID;
@@ -53,13 +55,6 @@ public class RestDeleteAnomalyDetectorAction extends BaseRestHandler {
 
     public RestDeleteAnomalyDetectorAction(RestController controller, ClusterService clusterService) {
         this.clusterService = clusterService;
-        // delete anomaly detector document
-        controller
-            .registerHandler(
-                RestRequest.Method.DELETE,
-                String.format(Locale.ROOT, "%s/{%s}", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID),
-                this
-            );
     }
 
     @Override
@@ -115,4 +110,15 @@ private void deleteAnomalyDetectorDoc(NodeClient client, String detectorId, Rest
         client.delete(deleteRequest, new RestStatusToXContentListener<>(channel));
     }
 
+    @Override
+    public List<Route> routes() {
+        return ImmutableList
+            .of(
+                // delete anomaly detector document
+                new Route(
+                    RestRequest.Method.DELETE,
+                    String.format(Locale.ROOT, "%s/{%s}", AnomalyDetectorPlugin.AD_BASE_DETECTORS_URI, DETECTOR_ID)
+                )
+            );
+    }
 }