Skip to content

Commit 007ec1f

Browse files
mark-vieiramark-vieira
authored
Support parallel building of Docker images (#52920)
1 parent f2b2241 commit 007ec1f

File tree

4 files changed

+158
-43
lines changed

4 files changed

+158
-43
lines changed

‎buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
package org.elasticsearch.gradle.docker;
2+
3+
import org.elasticsearch.gradle.LoggedExec;
4+
import org.gradle.api.DefaultTask;
5+
import org.gradle.api.file.DirectoryProperty;
6+
import org.gradle.api.file.RegularFileProperty;
7+
import org.gradle.api.provider.ListProperty;
8+
import org.gradle.api.provider.Property;
9+
import org.gradle.api.tasks.Input;
10+
import org.gradle.api.tasks.InputDirectory;
11+
import org.gradle.api.tasks.OutputFile;
12+
import org.gradle.api.tasks.PathSensitive;
13+
import org.gradle.api.tasks.PathSensitivity;
14+
import org.gradle.api.tasks.TaskAction;
15+
import org.gradle.process.ExecOperations;
16+
import org.gradle.workers.WorkAction;
17+
import org.gradle.workers.WorkParameters;
18+
import org.gradle.workers.WorkerExecutor;
19+
20+
import javax.inject.Inject;
21+
import java.io.IOException;
22+
import java.util.Arrays;
23+
24+
public class DockerBuildTask extends DefaultTask {
25+
private final WorkerExecutor workerExecutor;
26+
private final RegularFileProperty markerFile = getProject().getObjects().fileProperty();
27+
private final DirectoryProperty dockerContext = getProject().getObjects().directoryProperty();
28+
29+
private String[] tags;
30+
private boolean pull = true;
31+
private boolean noCache = true;
32+
33+
@Inject
34+
public DockerBuildTask(WorkerExecutor workerExecutor) {
35+
this.workerExecutor = workerExecutor;
36+
this.markerFile.set(getProject().getLayout().getBuildDirectory().file("markers/" + this.getName() + ".marker"));
37+
}
38+
39+
@TaskAction
40+
public void build() {
41+
workerExecutor.noIsolation().submit(DockerBuildAction.class, params -> {
42+
params.getDockerContext().set(dockerContext);
43+
params.getMarkerFile().set(markerFile);
44+
params.getTags().set(Arrays.asList(tags));
45+
params.getPull().set(pull);
46+
params.getNoCache().set(noCache);
47+
});
48+
}
49+
50+
@InputDirectory
51+
@PathSensitive(PathSensitivity.RELATIVE)
52+
public DirectoryProperty getDockerContext() {
53+
return dockerContext;
54+
}
55+
56+
@Input
57+
public String[] getTags() {
58+
return tags;
59+
}
60+
61+
public void setTags(String[] tags) {
62+
this.tags = tags;
63+
}
64+
65+
@Input
66+
public boolean isPull() {
67+
return pull;
68+
}
69+
70+
public void setPull(boolean pull) {
71+
this.pull = pull;
72+
}
73+
74+
@Input
75+
public boolean isNoCache() {
76+
return noCache;
77+
}
78+
79+
public void setNoCache(boolean noCache) {
80+
this.noCache = noCache;
81+
}
82+
83+
@OutputFile
84+
public RegularFileProperty getMarkerFile() {
85+
return markerFile;
86+
}
87+
88+
public abstract static class DockerBuildAction implements WorkAction<Parameters> {
89+
private final ExecOperations execOperations;
90+
91+
@Inject
92+
public DockerBuildAction(ExecOperations execOperations) {
93+
this.execOperations = execOperations;
94+
}
95+
96+
@Override
97+
public void execute() {
98+
LoggedExec.exec(execOperations, spec -> {
99+
spec.executable("docker");
100+
101+
spec.args("build", getParameters().getDockerContext().get().getAsFile().getAbsolutePath());
102+
103+
if (getParameters().getPull().get()) {
104+
spec.args("--pull");
105+
}
106+
107+
if (getParameters().getNoCache().get()) {
108+
spec.args("--no-cache");
109+
}
110+
111+
getParameters().getTags().get().forEach(tag -> spec.args("--tag", tag));
112+
});
113+
114+
try {
115+
getParameters().getMarkerFile().getAsFile().get().createNewFile();
116+
} catch (IOException e) {
117+
throw new RuntimeException("Failed to create marker file", e);
118+
}
119+
}
120+
}
121+
122+
interface Parameters extends WorkParameters {
123+
DirectoryProperty getDockerContext();
124+
125+
RegularFileProperty getMarkerFile();
126+
127+
ListProperty<String> getTags();
128+
129+
Property<Boolean> getPull();
130+
131+
Property<Boolean> getNoCache();
132+
}
133+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import org.gradle.api.Plugin;
44
import org.gradle.api.Project;
55
import org.gradle.api.Task;
6-
import org.gradle.api.plugins.ExtraPropertiesExtension;
76
import org.gradle.api.provider.Provider;
87

98
import java.io.File;
@@ -13,21 +12,10 @@
1312
/**
1413
* Plugin providing {@link DockerSupportService} for detecting Docker installations and determining requirements for Docker-based
1514
* Elasticsearch build tasks.
16-
* <p>
17-
* Additionally registers a task graph listener used to assert a compatible Docker installation exists when task requiring Docker are
18-
* scheduled for execution. Tasks may declare a Docker requirement via an extra property. If a compatible Docker installation is not
19-
* available on the build system an exception will be thrown prior to task execution.
20-
*
21-
* <pre>
22-
* task myDockerTask {
23-
* ext.requiresDocker = true
24-
* }
25-
* </pre>
2615
*/
2716
public class DockerSupportPlugin implements Plugin<Project> {
2817
public static final String DOCKER_SUPPORT_SERVICE_NAME = "dockerSupportService";
2918
public static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions";
30-
public static final String REQUIRES_DOCKER_ATTRIBUTE = "requiresDocker";
3119

3220
@Override
3321
public void apply(Project project) {
@@ -45,12 +33,13 @@ public void apply(Project project) {
4533
)
4634
);
4735

48-
// Ensure that if any tasks declare they require docker, we assert an available Docker installation exists
36+
// Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists
4937
project.getGradle().getTaskGraph().whenReady(graph -> {
50-
List<String> dockerTasks = graph.getAllTasks().stream().filter(task -> {
51-
ExtraPropertiesExtension ext = task.getExtensions().getExtraProperties();
52-
return ext.has(REQUIRES_DOCKER_ATTRIBUTE) && (boolean) ext.get(REQUIRES_DOCKER_ATTRIBUTE);
53-
}).map(Task::getPath).collect(Collectors.toList());
38+
List<String> dockerTasks = graph.getAllTasks()
39+
.stream()
40+
.filter(task -> task instanceof DockerBuildTask)
41+
.map(Task::getPath)
42+
.collect(Collectors.toList());
5443

5544
if (dockerTasks.isEmpty() == false) {
5645
dockerSupportServiceProvider.get().failIfDockerUnavailable(dockerTasks);

‎buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
import org.gradle.api.Project;
66
import org.gradle.api.Task;
77
import org.gradle.api.logging.Logger;
8+
import org.gradle.api.logging.Logging;
89
import org.gradle.api.tasks.Exec;
910
import org.gradle.api.tasks.Internal;
1011
import org.gradle.process.BaseExecSpec;
12+
import org.gradle.process.ExecOperations;
1113
import org.gradle.process.ExecResult;
1214
import org.gradle.process.ExecSpec;
1315
import org.gradle.process.JavaExecSpec;
@@ -30,6 +32,7 @@
3032
@SuppressWarnings("unchecked")
3133
public class LoggedExec extends Exec {
3234

35+
private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
3336
private Consumer<Logger> outputLogger;
3437

3538
public LoggedExec() {
@@ -94,21 +97,21 @@ public void setSpoolOutput(boolean spoolOutput) {
9497
}
9598

9699
public static ExecResult exec(Project project, Action<ExecSpec> action) {
97-
return genericExec(project, project::exec, action);
100+
return genericExec(project::exec, action);
101+
}
102+
103+
public static ExecResult exec(ExecOperations execOperations, Action<ExecSpec> action) {
104+
return genericExec(execOperations::exec, action);
98105
}
99106

100107
public static ExecResult javaexec(Project project, Action<JavaExecSpec> action) {
101-
return genericExec(project, project::javaexec, action);
108+
return genericExec(project::javaexec, action);
102109
}
103110

104111
private static final Pattern NEWLINE = Pattern.compile(System.lineSeparator());
105112

106-
private static <T extends BaseExecSpec> ExecResult genericExec(
107-
Project project,
108-
Function<Action<T>, ExecResult> function,
109-
Action<T> action
110-
) {
111-
if (project.getLogger().isInfoEnabled()) {
113+
private static <T extends BaseExecSpec> ExecResult genericExec(Function<Action<T>, ExecResult> function, Action<T> action) {
114+
if (LOGGER.isInfoEnabled()) {
112115
return function.apply(action);
113116
}
114117
ByteArrayOutputStream output = new ByteArrayOutputStream();
@@ -126,8 +129,8 @@ private static <T extends BaseExecSpec> ExecResult genericExec(
126129
} catch (Exception e) {
127130
try {
128131
if (output.size() != 0) {
129-
project.getLogger().error("Exec output and error:");
130-
NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> project.getLogger().error("| " + s));
132+
LOGGER.error("Exec output and error:");
133+
NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> LOGGER.error("| " + s));
131134
}
132135
} catch (UnsupportedEncodingException ue) {
133136
throw new GradleException("Failed to read exec output", ue);

‎distribution/docker/build.gradle

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor
22
import org.elasticsearch.gradle.LoggedExec
33
import org.elasticsearch.gradle.VersionProperties
4+
import org.elasticsearch.gradle.docker.DockerBuildTask
45
import org.elasticsearch.gradle.info.BuildParams
56
import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
67

@@ -149,10 +150,11 @@ task integTest(type: Test) {
149150
check.dependsOn integTest
150151

151152
void addBuildDockerImage(final boolean oss) {
152-
final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
153-
ext.requiresDocker = true // mark this task as requiring docker to execute
154-
inputs.files(tasks.named(taskName("copy", oss, "DockerContext")))
155-
List<String> tags
153+
final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: DockerBuildTask) {
154+
TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", oss, "DockerContext"))
155+
dependsOn(copyContextTask)
156+
dockerContext.fileProvider(copyContextTask.map { it.destinationDir })
157+
156158
if (oss) {
157159
tags = [
158160
"docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}",
@@ -166,18 +168,6 @@ void addBuildDockerImage(final boolean oss) {
166168
"elasticsearch:test",
167169
]
168170
}
169-
executable 'docker'
170-
final List<String> dockerArgs = ['build', buildPath(oss), '--pull', '--no-cache']
171-
for (final String tag : tags) {
172-
dockerArgs.add('--tag')
173-
dockerArgs.add(tag)
174-
}
175-
args dockerArgs.toArray()
176-
File markerFile = file("build/markers/${it.name}.marker")
177-
outputs.file(markerFile)
178-
doLast {
179-
markerFile.setText('', 'UTF-8')
180-
}
181171
}
182172
assemble.dependsOn(buildDockerImageTask)
183173
}

0 commit comments

Comments
 (0)