Merge pull request #193 from elastic/main

🤖 ESQL: Merge upstream

Author: bpintea

BUILDING.md

@@ -63,6 +63,38 @@ E.g. [configuration-cache support](https://github.com/elastic/elasticsearch/issu
 There are a few guidelines to follow that should make your life easier to make changes to the elasticsearch build.
 Please add a member of the `es-delivery` team as a reviewer if you're making non-trivial changes to the build.
 
+#### Adding or updating a dependency
+
+We rely on [Gradle dependency verification](https://docs.gradle.org/current/userguide/dependency_verification.html) to mitigate the security risks and avoid integrating compromised dependencies.
+
+This requires to have third party dependencies and their checksums listed in `gradle/verification-metadata.xml`.
+
+For updated or newly added dependencies you need to add an entry to this verification file or update the existing one:
+```
+      <component group="asm" name="asm" version="3.1">
+         <artifact name="asm-3.1.jar">
+            <sha256 value="333ff5369043975b7e031b8b27206937441854738e038c1f47f98d072a20437a" origin="official site"/>
+         </artifact>
+      </component>
+```
+
+You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
+```
+>./gradlew --write-verification-metadata sha256 precommit
+```
+
+The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations, 
+but we use detached configurations for a certain set of plugins and tasks. Therefore, please ensure you run this option with a task that
+uses the changed dependencies. In most cases, `precommit` or `check` are good candidates.
+
+We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry 
+will have the `origin` attribute been set to `Generated by Gradle`. 
+
+>A manual confirmation of the Gradle generated checksums is currently not mandatory.
+>If you want to add a level of verification you can manually confirm the checksum (e.g by looking it up on the website of the library)
+>Please replace the content of the `origin` attribute by `official site` in that case.
+>
+
 #### Custom Plugin and Task implementations
 
 Build logic that is used across multiple subprojects should considered to be moved into a Gradle plugin with according Gradle task implmentation.

CHANGELOG.md

@@ -1,3 +1,3 @@
-# Elasticsearch Changlog
+# Elasticsearch Changelog
 
 Please see the [release notes](https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html) in the reference manual.

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -6,10 +6,11 @@
  * Side Public License, v 1.
  */
 
+
 import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.internal.info.BuildParams
-import org.elasticsearch.gradle.testclusters.TestDistribution
 import org.elasticsearch.gradle.testclusters.TestClustersAware
+import org.elasticsearch.gradle.testclusters.TestDistribution
 
 // Common config when running with a FIPS-140 runtime JVM
 if (BuildParams.inFipsJvm) {
@@ -31,7 +32,13 @@ if (BuildParams.inFipsJvm) {
         copy 'fips_java.policy'
         copy 'cacerts.bcfks'
       }
-      def extraFipsJarsConfiguration = configurations.detachedConfiguration(bcFips, bcTlsFips)
+
+      def extraFipsJarsConfiguration = configurations.create("fipsImplementation") {
+        withDependencies {
+          add(bcFips)
+          add(bcTlsFips)
+        }
+      }
 
       project.afterEvaluate {
         // ensure that bouncycastle is on classpath for the all of test types, must happen in evaluateAfter since the rest tests explicitly

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java

@@ -34,9 +34,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
                 runtimeClasspath.fileCollection(dependency -> dependency instanceof ProjectDependency == false).minus(compileOnly)
             );
         });
-
-        // we also create the updateShas helper task that is associated with dependencyLicenses
-        project.getTasks().register("updateShas", UpdateShasTask.class, t -> t.setParentTask(dependencyLicenses));
         return dependencyLicenses;
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -7,7 +7,6 @@
  */
 package org.elasticsearch.gradle.internal.precommit;
 
-import org.apache.commons.codec.binary.Hex;
 import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer.LicenseInfo;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
@@ -23,30 +22,21 @@
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
-import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputDirectory;
 import org.gradle.api.tasks.TaskAction;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
@@ -193,7 +183,7 @@ public void ignoreFile(String file) {
     }
 
     @TaskAction
-    public void checkDependencies() throws IOException, NoSuchAlgorithmException {
+    public void checkDependencies() {
         if (dependencies == null) {
             throw new GradleException("No dependencies variable defined.");
         }
@@ -214,12 +204,9 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
         Map<String, Boolean> licenses = new HashMap<>();
         Map<String, Boolean> notices = new HashMap<>();
         Map<String, Boolean> sources = new HashMap<>();
-        Set<File> shaFiles = new HashSet<>();
         for (File file : licensesDirAsFile.listFiles()) {
             String name = file.getName();
-            if (name.endsWith(SHA_EXTENSION)) {
-                shaFiles.add(file);
-            } else if (name.endsWith("-LICENSE") || name.endsWith("-LICENSE.txt")) {
+            if (name.endsWith("-LICENSE") || name.endsWith("-LICENSE.txt")) {
                 // TODO: why do we support suffix of LICENSE *and* LICENSE.txt??
                 licenses.put(name, false);
             } else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) {
@@ -233,18 +220,13 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
         notices.keySet().removeAll(ignoreFiles);
         sources.keySet().removeAll(ignoreFiles);
 
-        checkDependencies(licenses, notices, sources, shaFiles);
+        checkDependencies(licenses, notices, sources);
 
         licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));
 
         notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice"));
 
         sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources"));
-
-        if (shaFiles.isEmpty() == false) {
-            throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles));
-        }
-
     }
 
     // This is just a marker output folder to allow this task being up-to-date.
@@ -261,18 +243,10 @@ private void failIfAnyMissing(String item, Boolean exists, String type) {
         }
     }
 
-    private void checkDependencies(
-        Map<String, Boolean> licenses,
-        Map<String, Boolean> notices,
-        Map<String, Boolean> sources,
-        Set<File> shaFiles
-    ) throws NoSuchAlgorithmException, IOException {
+    private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolean> notices, Map<String, Boolean> sources) {
         for (File dependency : dependencies) {
             String jarName = dependency.getName();
             String depName = regex.matcher(jarName).replaceFirst("");
-
-            validateSha(shaFiles, dependency, jarName, depName);
-
             String dependencyName = getDependencyName(mappings, depName);
             logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
             checkFile(dependencyName, jarName, licenses, "LICENSE");
@@ -286,24 +260,6 @@ private void checkDependencies(
         }
     }
 
-    private void validateSha(Set<File> shaFiles, File dependency, String jarName, String depName) throws NoSuchAlgorithmException,
-        IOException {
-        if (ignoreShas.contains(depName)) {
-            // local deps should not have sha files!
-            if (getShaFile(jarName).exists()) {
-                throw new GradleException("SHA file " + getShaFile(jarName) + " exists for ignored dependency " + depName);
-            }
-        } else {
-            logger.info("Checking sha for {}", jarName);
-            checkSha(dependency, jarName, shaFiles);
-        }
-    }
-
-    private String joinFilenames(Set<File> shaFiles) {
-        List<String> names = shaFiles.stream().map(File::getName).collect(Collectors.toList());
-        return String.join("\n", names);
-    }
-
     public static String getDependencyName(Map<String, String> mappings, String dependencyName) {
         // order is the same for keys and values iteration since we use a linked hashmap
         List<String> mapped = new ArrayList<>(mappings.values());
@@ -319,30 +275,6 @@ public static String getDependencyName(Map<String, String> mappings, String depe
         return dependencyName;
     }
 
-    private void checkSha(File jar, String jarName, Set<File> shaFiles) throws NoSuchAlgorithmException, IOException {
-        File shaFile = getShaFile(jarName);
-        if (shaFile.exists() == false) {
-            throw new GradleException("Missing SHA for " + jarName + ". Run \"gradle updateSHAs\" to create them");
-        }
-
-        // TODO: shouldn't have to trim, sha files should not have trailing newline
-        byte[] fileBytes = Files.readAllBytes(shaFile.toPath());
-        String expectedSha = new String(fileBytes, StandardCharsets.UTF_8).trim();
-
-        String sha = getSha1(jar);
-
-        if (expectedSha.equals(sha) == false) {
-            final String exceptionMessage = String.format(Locale.ROOT, """
-                SHA has changed! Expected %s for %s but got %s.
-                This usually indicates a corrupt dependency cache or artifacts changed upstream.
-                Either wipe your cache, fix the upstream artifact, or delete %s and run updateShas
-                """, expectedSha, jarName, sha, shaFile);
-
-            throw new GradleException(exceptionMessage);
-        }
-        shaFiles.remove(shaFile);
-    }
-
     private void checkFile(String name, String jarName, Map<String, Boolean> counters, String type) {
         String fileName = getFileName(name, counters, type);
 
@@ -375,27 +307,4 @@ public LinkedHashMap<String, String> getMappings() {
         return new LinkedHashMap<>(mappings);
     }
 
-    File getShaFile(String jarName) {
-        return new File(licensesDir.get().getAsFile(), jarName + SHA_EXTENSION);
-    }
-
-    @Internal
-    Set<File> getShaFiles() {
-        File licenseDirAsFile = licensesDir.get().getAsFile();
-        File[] array = licenseDirAsFile.listFiles();
-        if (array == null) {
-            throw new GradleException("\"" + licenseDirAsFile.getPath() + "\" isn't a valid directory");
-        }
-
-        return Arrays.stream(array).filter(file -> file.getName().endsWith(SHA_EXTENSION)).collect(Collectors.toSet());
-    }
-
-    String getSha1(File file) throws IOException, NoSuchAlgorithmException {
-        byte[] bytes = Files.readAllBytes(file.toPath());
-
-        MessageDigest digest = MessageDigest.getInstance("SHA-1");
-        char[] encoded = Hex.encodeHex(digest.digest(bytes));
-        return String.copyValueOf(encoded);
-    }
-
 }