[ML] Adds support for a global calendar via _all (elastic#50372)

This adds `_all` to Calendar searches. This enables users to supply the `_all` string in the `job_ids` array when creating a Calendar. That calendar will now be applied to all jobs (existing and newly created).

Closes elastic#45013

Author: benwtrent

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -335,7 +336,54 @@ public void testAddOpenedJobToGroupWithCalendar() throws Exception {
         assertEquals(0, buckets.get(8).getScheduledEvents().size());
     }
 
-    private Job.Builder createJob(String jobId, TimeValue bucketSpan) {
+    /**
+     * An open job that later gets added to a calendar, should take the scheduled events into account
+     */
+    public void testNewJobWithGlobalCalendar() throws Exception {
+        String calendarId = "test-global-calendar";
+
+        // Create a new calendar referencing groupName
+        putCalendar(calendarId, Collections.singletonList(MetaData.ALL), "testNewJobWithGlobalCalendar calendar");
+
+        long startTime = 1514764800000L;
+        final int bucketCount = 3;
+        TimeValue bucketSpan = TimeValue.timeValueMinutes(30);
+
+        // Put events in the calendar
+        List<ScheduledEvent> events = new ArrayList<>();
+        long eventStartTime = startTime;
+        long eventEndTime = eventStartTime + (long) (1.5 * bucketSpan.millis());
+        events.add(new ScheduledEvent.Builder().description("Some Event")
+            .startTime((Instant.ofEpochMilli(eventStartTime)))
+            .endTime((Instant.ofEpochMilli(eventEndTime)))
+            .calendarId(calendarId).build());
+
+        postScheduledEvents(calendarId, events);
+
+        Job.Builder job = createJob("scheduled-events-add-to-new-job--with-global-calendar", bucketSpan);
+
+        // Open the job
+        openJob(job.getId());
+
+        // write some buckets of data
+        postData(job.getId(), generateData(startTime, bucketSpan, bucketCount + 1, bucketIndex -> randomIntBetween(100, 200))
+            .stream().collect(Collectors.joining()));
+
+        // and close
+        closeJob(job.getId());
+
+        GetBucketsAction.Request getBucketsRequest = new GetBucketsAction.Request(job.getId());
+        List<Bucket> buckets = getBuckets(getBucketsRequest);
+
+        // 1st and 2nd buckets have the event but the last one does not
+        assertEquals(1, buckets.get(0).getScheduledEvents().size());
+        assertEquals("Some Event", buckets.get(0).getScheduledEvents().get(0));
+        assertEquals(1, buckets.get(1).getScheduledEvents().size());
+        assertEquals("Some Event", buckets.get(1).getScheduledEvents().get(0));
+        assertEquals(0, buckets.get(2).getScheduledEvents().size());
+    }
+
+        private Job.Builder createJob(String jobId, TimeValue bucketSpan) {
         Detector.Builder detector = new Detector.Builder("count", null);
         AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
         analysisConfig.setBucketSpan(bucketSpan);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
@@ -66,6 +67,7 @@ public SearchSourceBuilder build() {
         }
 
         if (jobIdAndGroups.isEmpty() == false) {
+            jobIdAndGroups.add(MetaData.ALL);
             qb = new BoolQueryBuilder()
                     .filter(new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE))
                     .filter(new TermsQueryBuilder(Calendar.JOB_IDS.getPreferredName(), jobIdAndGroups));