diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy deleted file mode 100644 index 928298db7bfc2..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle - -import org.gradle.api.DefaultTask -import org.gradle.api.Project -import org.gradle.api.artifacts.Configuration -import org.gradle.api.tasks.InputFile -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.TaskAction - -/** - * A task to create a notice file which includes dependencies' notices. - */ -public class NoticeTask extends DefaultTask { - - @InputFile - File inputFile = project.rootProject.file('NOTICE.txt') - - @OutputFile - File outputFile = new File(project.buildDir, "notices/${name}/NOTICE.txt") - - /** Directories to include notices from */ - private List licensesDirs = new ArrayList<>() - - public NoticeTask() { - description = 'Create a notice file from dependencies' - // Default licenses directory is ${projectDir}/licenses (if it exists) - File licensesDir = new File(project.projectDir, 'licenses') - if (licensesDir.exists()) { - licensesDirs.add(licensesDir) - } - } - - /** Add notices from the specified directory. */ - public void licensesDir(File licensesDir) { - licensesDirs.add(licensesDir) - } - - @TaskAction - public void generateNotice() { - StringBuilder output = new StringBuilder() - output.append(inputFile.getText('UTF-8')) - output.append('\n\n') - // This is a map rather than a set so that the sort order is the 3rd - // party component names, unaffected by the full path to the various files - Map seen = new TreeMap<>() - for (File licensesDir : licensesDirs) { - licensesDir.eachFileMatch({ it ==~ /.*-NOTICE\.txt/ }) { File file -> - String name = file.name.substring(0, file.name.length() - '-NOTICE.txt'.length()) - if (seen.containsKey(name)) { - File prevFile = seen.get(name) - if (prevFile.text != file.text) { - throw new RuntimeException("Two different notices exist for dependency '" + - name + "': " + prevFile + " and " + file) - } - } else { - seen.put(name, file) - } - } - } - for (Map.Entry entry : seen.entrySet()) { - String name = entry.getKey() - File file = entry.getValue() - appendFile(file, name, 'NOTICE', output) - appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output) - } - outputFile.setText(output.toString(), 'UTF-8') - } - - static void appendFile(File file, String name, String type, StringBuilder output) { - String text = file.getText('UTF-8') - if (text.trim().isEmpty()) { - return - } - output.append('================================================================================\n') - output.append("${name} ${type}\n") - output.append('================================================================================\n') - output.append(text) - output.append('\n\n') - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java new file mode 100644 index 0000000000000..a0f3a9bdd3a11 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -0,0 +1,182 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle; + +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.nio.charset.Charset; +import java.nio.charset.MalformedInputException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.charset.StandardCharsets; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.stream.Stream; + +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; +import java.util.Collections; +import java.util.stream.Collectors; + +/** + * A task to create a notice file which includes dependencies' notices. + */ +public class NoticeTask extends DefaultTask { + private File inputFile = getProject().getRootProject().file("NOTICE.txt"); + private File outputFile = new File(getProject().getBuildDir(), "notices/" + getName() + "/NOTICE.txt"); + /** + * Directories to include notices from + */ + private List licensesDirs = new ArrayList<>(); + + @InputFile + public File getInputFile() { + return inputFile; + } + + public void setInputFile(File inputFile) { + this.inputFile = inputFile; + } + + @OutputFile + public File getOutputFile() { + return outputFile; + } + + public void setOutputFile(File outputFile) { + this.outputFile = outputFile; + } + + public void licensesDir(File licensesDir) { + licensesDirs.add(licensesDir); + } + + @InputFiles + public List getLicensesDirs() { + return Collections.unmodifiableList(this.licensesDirs.stream() + .map(file -> new File(file.toString())).collect(Collectors.toList())); + } + + /** + * Add notices from the specified directory. + */ + public NoticeTask() { + setDescription("Create a notice file from dependencies"); + // Default licenses directory is ${projectDir}/licenses (if it exists) + File licensesDir = new File(getProject().getProjectDir(), "licenses"); + + if (licensesDir.exists()) { + licensesDirs.add(licensesDir); + } + } + + @TaskAction + public void generateNotice() throws IOException { + final StringBuilder output = new StringBuilder(); + + output.append(readFileToString(this.inputFile,StandardCharsets.UTF_8)); + output.append("\n\n"); + + // This is a map rather than a set so that the sort order is the 3rd + // party component names, unaffected by the full path to the various files + final Map seen = this.getFilesToAppend(licensesDirs); + + for (Map.Entry entry : seen.entrySet()) { + final String name = entry.getKey(); + final File file = entry.getValue(); + final File licenseFile = new File(file.getParentFile(), name + "-LICENSE.txt"); + + appendFileToOutput(file, name, "NOTICE", output); + appendFileToOutput(licenseFile, name, "LICENSE", output); + } + + write(outputFile,output.toString(),StandardCharsets.UTF_8); + } + + private static void appendFileToOutput(File file, final String name, final String type, + StringBuilder output) throws IOException { + String text = readFileToString(file,StandardCharsets.UTF_8); + if (text.trim().isEmpty() == false) { + output.append("================================================================================\n"); + output.append(name + " " + type + "\n"); + output.append("================================================================================\n"); + output.append(text); + output.append("\n\n"); + } + } + + private Map getFilesToAppend(List licensesDirectories) throws IOException{ + final Map licensesSeen = new TreeMap<>(); + + for (File directory: licensesDirectories) { + try(DirectoryStream stream = Files.newDirectoryStream(directory.toPath())){ + for (Path path : stream){ + if (Files.isRegularFile(path) && path.toString().endsWith("-NOTICE.txt")){ + File licenseFile = path.toFile(); + + final String name = + licenseFile.getName().substring(0, licenseFile.getName().length() - "-NOTICE.txt".length()); + + if (licensesSeen.containsKey(name)) { + File prevLicenseFile = licensesSeen.get(name); + + if (readFileToString(prevLicenseFile,StandardCharsets.UTF_8) + .equals(readFileToString(licenseFile,StandardCharsets.UTF_8)) == false) { + throw new RuntimeException("Two different notices exist for dependency '" + + name + "': " + prevLicenseFile + " and " + licenseFile); + } + } else { + licensesSeen.put(name, licenseFile); + } + } + } + } + } + return licensesSeen; + } + private static String readFileToString(File file, Charset charset) throws IOException{ + final StringBuilder builder = new StringBuilder(); + + try(Stream stream = Files.lines(file.toPath(),charset)){ + stream.forEach(builder::append); + + }catch (MalformedInputException | UncheckedIOException e){ + // Files.lines() wraps MalformedInputException inside of an UncheckedIOException + throw new RuntimeException("Unable to process file " + file.toString(),e); + + } + return builder.toString(); + } + + private static void write(File outputFile, String output,Charset charset) throws IOException{ + List lines = Arrays.asList(output.split("\\r?\\n")); + Files.write(outputFile.toPath(),lines,charset, StandardOpenOption.CREATE); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/NoticeTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/NoticeTaskTests.java new file mode 100644 index 0000000000000..8fa8b925d3d68 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/NoticeTaskTests.java @@ -0,0 +1,162 @@ +package org.elasticsearch.gradle; + +import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.gradle.api.Project; +import org.junit.Rule; +import org.junit.Test; +import org.gradle.testfixtures.ProjectBuilder; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Arrays; +import java.util.stream.Collectors; + + +public class NoticeTaskTests extends GradleUnitTestCase { + + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + private NoticeTask noticeTask; + private Project project; + private final String outputHeader = "This is the header for the output file\nIt should contain:\n3 lines & 2 spaces"; + + public List getListWithoutCopies() throws IOException { + File directory1 = new File(noticeTask.getTemporaryDir(), "directoryA"); + File directory2 = new File(noticeTask.getTemporaryDir(), "directoryB"); + + Files.createDirectories(directory1.toPath()); + Files.createDirectories(directory2.toPath()); + + File d1Notice = new File(directory1, "test-NOTICE.txt"); + File d1License = new File(directory1, "test-LICENSE.txt"); + File d2Notice = new File(directory2, "test2-NOTICE.txt"); + File d2License = new File(directory2, "test2-LICENSE.txt"); + + write(d1Notice, "d1 Notice text file",StandardCharsets.UTF_8); + write(d2Notice, "d2 Notice text file",StandardCharsets.UTF_8); + write(d1License, "d1 License text file",StandardCharsets.UTF_8); + write(d2License, "d2 License text file",StandardCharsets.UTF_8); + + List files = new ArrayList<>(); + + files.add(d1License.getParentFile()); + files.add(d1Notice.getParentFile()); + files.add(d2License.getParentFile()); + files.add(d2Notice.getParentFile()); + + return files; + } + + public List getListWithCopies() throws IOException { + List files = this.getListWithoutCopies(); + + File directory2 = new File(noticeTask.getTemporaryDir(), "directoryC"); + Files.createDirectories(directory2.toPath()); + + File d1NoticeCopy = new File(directory2, "test-NOTICE.txt"); + File d1LicenseCopy = new File(directory2, "test-LICENSE.txt"); + + write(d1NoticeCopy, "d1 Copy Notice text file",StandardCharsets.UTF_8); + write(d1LicenseCopy, "d1 License text file",StandardCharsets.UTF_8); + + files.add(d1LicenseCopy.getParentFile()); + files.add(d1NoticeCopy.getParentFile()); + + return files; + } + + @Test + public void verifyGenerateNoticeWithException() throws IOException { + // Setup everything so we can test out the task + project = createProject(); + noticeTask = createTask(project); + File inputFile = new File(project.getProjectDir(), "NOTICE.txt"); + File outputFile = new File(project.getProjectDir(), "OUTPUT.txt"); + Files.write(inputFile.toPath(), this.outputHeader.getBytes()); + + // Set the input and output files on the NoticeTask so we can compare them later + noticeTask.setInputFile(inputFile); + noticeTask.setOutputFile(outputFile); + this.getListWithCopies().forEach(noticeTask::licensesDir); + + // Should see an exception because there will be different notices for the same item + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Two different notices exist for dependency"); + noticeTask.generateNotice(); + } + + @Test + public void verifyGenerateNotice() throws IOException { + // Setup everything so we can test out the task + project = createProject(); + noticeTask = createTask(project); + final File inputFile = new File(project.getProjectDir(), "NOTICE.txt"); + final File outputFile = new File(project.getProjectDir(), "OUTPUT.txt"); + + // Set the input and output files on the NoticeTask so we can compare them later + noticeTask.setInputFile(inputFile); + noticeTask.setOutputFile(outputFile); + + // Give us some dummy data to work with + Files.write(inputFile.toPath(), this.outputHeader.getBytes()); + + // Add each element to the list of license directories to check in the NoticeTask + this.getListWithoutCopies().forEach(noticeTask::licensesDir); + + // Generate the notice output + noticeTask.generateNotice(); + + // Get the output String from the output file so we can compare it + final String outputText = readFileToString(outputFile, StandardCharsets.UTF_8); + + final String lineDivider = + "================================================================================"; + + // We shouldn't have any of the 'copy' notices + assertFalse(outputText.contains("d1 Copy Notice text file")); + assertFalse(outputText.contains("d1 copy License text file")); + + // We should have the non-copy notice and licenses text: + assertTrue(outputText.contains("d1 Notice text file")); + assertTrue(outputText.contains("d2 Notice text file")); + assertTrue(outputText.contains("d1 License text file")); + assertTrue(outputText.contains("d2 License text file")); + + assertTrue(outputText.contains(lineDivider)); + } + + @Test + public void EnsureDirectoriesAreLoaded(){ + project = createProject(); + noticeTask = createTask(project); + noticeTask.licensesDir(new File(project.getProjectDir(), "NOTICE.txt")); + + var list = noticeTask.getLicensesDirs(); + assertTrue(list.get(0).toString().endsWith("NOTICE.txt")); + } + + private Project createProject() { + return ProjectBuilder.builder().build(); + } + + private NoticeTask createTask(Project project) { + return project.getTasks().create("NoticeTask", NoticeTask.class); + } + + private static String readFileToString(File file, Charset charset) throws IOException{ + return Files.readAllLines(file.toPath(),charset).stream() + .collect(Collectors.joining("\n")); + } + + private static void write(File outputFile, String output,Charset charset) throws IOException{ + List lines = Arrays.asList(output.split("\\r?\\n")); + Files.write(outputFile.toPath(),lines,charset, StandardOpenOption.CREATE); + } +}