Merge remote-tracking branch 'origin/snapshot-lifecycle-management' into slm-retention

Author: dakrone

.gitignore

@@ -41,3 +41,6 @@ html_docs
 # random old stuff that we should look at the necessity of...
 /tmp/
 eclipse-build
+
+# projects using testfixtures 
+testfixtures_shared/

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -825,7 +825,7 @@ class BuildPlugin implements Plugin<Project> {
 
                 test.executable = "${ext.get('runtimeJavaHome')}/bin/java"
                 test.workingDir = project.file("${project.buildDir}/testrun/${test.name}")
-                test.maxParallelForks = project.rootProject.extensions.getByType(ExtraPropertiesExtension).get('defaultParallel') as Integer
+                test.maxParallelForks = System.getProperty('tests.jvms', project.rootProject.extensions.extraProperties.get('defaultParallel').toString()) as Integer
 
                 test.exclude '**/*$*.class'
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -353,7 +353,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
 
         private void testSetup(Snippet snippet) {
             if (lastDocsPath == snippet.path) {
-                throw new InvalidUserDataException("$snippet: wasn't first")
+                throw new InvalidUserDataException("$snippet: wasn't first. TESTSETUP can only be used in the first snippet of a document.")
             }
             setupCurrent(snippet)
             current.println('---')

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -27,7 +27,9 @@ import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.plugins.JavaBasePlugin
+import org.gradle.api.plugins.JavaPluginConvention
 import org.gradle.api.plugins.quality.Checkstyle
 
 /**
@@ -45,13 +47,23 @@ class PrecommitTasks {
             forbiddenApisCliJar('de.thetaphi:forbiddenapis:2.6')
         }
 
+        Configuration jarHellConfig = project.configurations.create("jarHell")
+        if (ClasspathUtils.isElasticsearchProject() && project.path.equals(":libs:elasticsearch-core") == false) {
+            // External plugins will depend on this already via transitive dependencies.
+            // Internal projects are not all plugins, so make sure the check is available
+            // we are not doing this for this project itself to avoid jar hell with itself
+            project.dependencies {
+                jarHell project.project(":libs:elasticsearch-core")
+            }
+        }
+
         List<Task> precommitTasks = [
                 configureCheckstyle(project),
                 configureForbiddenApisCli(project),
                 project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
                 project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
                 project.tasks.create('filepermissions', FilePermissionsTask.class),
-                configureJarHell(project),
+                configureJarHell(project, jarHellConfig),
                 configureThirdPartyAudit(project),
                 configureTestingConventions(project)
         ]
@@ -108,12 +120,13 @@ class PrecommitTasks {
         return task
     }
 
-    private static Task configureJarHell(Project project) {
+    private static Task configureJarHell(Project project, Configuration jarHelConfig) {
         return project.tasks.create('jarHell', JarHellTask) { task ->
-            task.classpath = project.sourceSets.test.runtimeClasspath
+            task.classpath = project.sourceSets.test.runtimeClasspath + jarHelConfig;
             if (project.plugins.hasPlugin(ShadowPlugin)) {
                 task.classpath += project.configurations.bundle
             }
+            task.dependsOn(jarHelConfig);
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -58,6 +58,9 @@ public class AntFixture extends AntTask implements Fixture {
     @Input
     boolean useShell = false
 
+    @Input
+    int maxWaitInSeconds = 30
+
     /**
      * A flag to indicate whether the fixture should be run in the foreground, or spawned.
      * It is protected so subclasses can override (eg RunTask).
@@ -128,7 +131,7 @@ public class AntFixture extends AntTask implements Fixture {
 
         String failedProp = "failed${name}"
         // first wait for resources, or the failure marker from the wrapper script
-        ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
+        ant.waitfor(maxwait: maxWaitInSeconds, maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
             or {
                 resourceexists {
                     file(file: failureMarker.toString())

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -59,7 +59,7 @@ class RestIntegTestTask extends DefaultTask {
     Boolean includePackaged = false
 
     RestIntegTestTask() {
-        runner = project.tasks.create("${name}Runner", Test.class)
+        runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
         clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
         runner.dependsOn(clusterInit)
@@ -77,10 +77,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.useCluster project.testClusters."$name"
         }
 
-        // disable the build cache for rest test tasks
-        // there are a number of inputs we aren't properly tracking here so we'll just not cache these for now
-        runner.getOutputs().doNotCacheIf("Caching is disabled for REST integration tests", Specs.SATISFIES_ALL)
-
         // override/add more for rest tests
         runner.maxParallelForks = 1
         runner.include('**/*IT.class')

buildSrc/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java

@@ -0,0 +1,27 @@
+package org.elasticsearch.gradle;
+
+import java.util.List;
+
+public abstract class AbstractLazyPropertyCollection {
+
+    final String name;
+    final Object owner;
+
+    public AbstractLazyPropertyCollection(String name) {
+        this(name, null);
+    }
+
+    public AbstractLazyPropertyCollection(String name, Object owner) {
+        this.name = name;
+        this.owner = owner;
+    }
+
+    abstract List<? extends Object> getNormalizedCollection();
+
+    void assertNotNull(Object value, String description) {
+        if (value == null) {
+            throw new NullPointerException(name + " " + description + " was null" + (owner != null ? " when configuring " + owner : ""));
+        }
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java

@@ -0,0 +1,205 @@
+package org.elasticsearch.gradle;
+
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Nested;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public class LazyPropertyList<T> extends AbstractLazyPropertyCollection implements List<T> {
+
+    private final List<PropertyListEntry<T>> delegate = new ArrayList<>();
+
+    public LazyPropertyList(String name) {
+        super(name);
+    }
+
+    public LazyPropertyList(String name, Object owner) {
+        super(name, owner);
+    }
+
+    @Override
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        return delegate.stream().anyMatch(entry -> entry.getValue().equals(o));
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).iterator();
+    }
+
+    @Override
+    public Object[] toArray() {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).toArray();
+    }
+
+    @Override
+    public <T1> T1[] toArray(T1[] a) {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).collect(Collectors.toList()).toArray(a);
+    }
+
+    @Override
+    public boolean add(T t) {
+        return delegate.add(new PropertyListEntry<>(() -> t, PropertyNormalization.DEFAULT));
+    }
+
+    public boolean add(Supplier<T> supplier) {
+        return delegate.add(new PropertyListEntry<>(supplier, PropertyNormalization.DEFAULT));
+    }
+
+    public boolean add(Supplier<T> supplier, PropertyNormalization normalization) {
+        return delegate.add(new PropertyListEntry<>(supplier, normalization));
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support remove()");
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+        return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toList()).containsAll(c);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends T> c) {
+        c.forEach(this::add);
+        return true;
+    }
+
+    @Override
+    public boolean addAll(int index, Collection<? extends T> c) {
+        int i = index;
+        for (T item : c) {
+            this.add(i++, item);
+        }
+        return true;
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support removeAll()");
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support retainAll()");
+    }
+
+    @Override
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public T get(int index) {
+        PropertyListEntry<T> entry = delegate.get(index);
+        validate(entry);
+        return entry.getValue();
+    }
+
+    @Override
+    public T set(int index, T element) {
+        return delegate.set(index, new PropertyListEntry<>(() -> element, PropertyNormalization.DEFAULT)).getValue();
+    }
+
+    @Override
+    public void add(int index, T element) {
+        delegate.add(index, new PropertyListEntry<>(() -> element, PropertyNormalization.DEFAULT));
+    }
+
+    @Override
+    public T remove(int index) {
+        return delegate.remove(index).getValue();
+    }
+
+    @Override
+    public int indexOf(Object o) {
+        for (int i = 0; i < delegate.size(); i++) {
+            if (delegate.get(i).getValue().equals(o)) {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object o) {
+        int lastIndex = -1;
+        for (int i = 0; i < delegate.size(); i++) {
+            if (delegate.get(i).getValue().equals(o)) {
+                lastIndex = i;
+            }
+        }
+
+        return lastIndex;
+    }
+
+    @Override
+    public ListIterator<T> listIterator() {
+        return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toList()).listIterator();
+    }
+
+    @Override
+    public ListIterator<T> listIterator(int index) {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).collect(Collectors.toList()).listIterator(index);
+    }
+
+    @Override
+    public List<T> subList(int fromIndex, int toIndex) {
+        return delegate.stream()
+            .peek(this::validate)
+            .map(PropertyListEntry::getValue)
+            .collect(Collectors.toList())
+            .subList(fromIndex, toIndex);
+    }
+
+    @Override
+    @Nested
+    List<? extends Object> getNormalizedCollection() {
+        return delegate.stream()
+            .peek(this::validate)
+            .filter(entry -> entry.getNormalization() != PropertyNormalization.IGNORE_VALUE)
+            .collect(Collectors.toList());
+    }
+
+    private void validate(PropertyListEntry<T> entry) {
+        assertNotNull(entry.getValue(), "entry");
+    }
+
+    private class PropertyListEntry<T> {
+        private final Supplier<T> supplier;
+        private final PropertyNormalization normalization;
+
+        PropertyListEntry(Supplier<T> supplier, PropertyNormalization normalization) {
+            this.supplier = supplier;
+            this.normalization = normalization;
+        }
+
+        public PropertyNormalization getNormalization() {
+            return normalization;
+        }
+
+        @Input
+        public T getValue() {
+            assertNotNull(supplier, "supplier");
+            return supplier.get();
+        }
+    }
+}