diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000000..afe60c8936815 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,63 @@ +# Elasticsearch Microbenchmark Suite + +This directory contains the microbenchmark suite of Elasticsearch. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/). + +## Purpose + +We do not want to microbenchmark everything but the kitchen sink and should typically rely on our +[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with +[Rally](http://github.com/elastic/rally). Microbenchmarks are intended for performance-critical components to spot performance +regressions. The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR. + +## Getting Started + +Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result. + +## Running Microbenchmarks + +Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`. +``` + +Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks). + +If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have any other special requirements +generate the uberjar with `gradle :benchmarks:jmhJar` and run the it directly with: + +``` +java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar +``` + +JMH supports lots of command line parameters. Add `-h` to the command above for more information about the available command line options. + +## Adding Microbenchmarks + +Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the +[JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). + +In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and +end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. + +## Tips and Best Practices + +To get realistic results, you should exercise care when running your benchmarks. Here are a few tips: + +### Do + +* Ensure that the system executing your microbenchmarks has as little load as possible and shutdown every process that can cause unnecessary + runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance. +* Ensure to run enough warmup iterations to get into a stable state. If you are unsure, don't change the defaults. +* Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`. +* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the + `performance` CPU governor. +* Vary problem input size with `@Param`. +* Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses: +** Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews + your results. If so, try to force a GC between runs (`-gc true`). +** Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots. +* Have your benchmarks peer-reviewed. + +### Don't + +* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.e. with `-prof perfasm`. +* Run run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks). +* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable. \ No newline at end of file diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle new file mode 100644 index 0000000000000..26fb5105af3e1 --- /dev/null +++ b/benchmarks/build.gradle @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +buildscript { + repositories { + maven { + url 'https://plugins.gradle.org/m2/' + } + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' + } +} + +apply plugin: 'elasticsearch.build' +// build an uberjar with all benchmarks +apply plugin: 'com.github.johnrengelman.shadow' +// have the shadow plugin provide the runShadow task +apply plugin: 'application' + +archivesBaseName = 'elasticsearch-benchmarks' +mainClassName = 'org.openjdk.jmh.Main' + +// never try to invoke tests on the benchmark project - there aren't any +check.dependsOn.remove(test) +// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip +task test(type: Test, overwrite: true) + +dependencies { + compile("org.elasticsearch:elasticsearch:${version}") { + // JMH ships with the conflicting version 4.6 (JMH will not update this dependency as it is Java 6 compatible and joptsimple is one + // of the most recent compatible version). This prevents us from using jopt-simple in benchmarks (which should be ok) but allows us + // to invoke the JMH uberjar as usual. + exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' + } + compile "org.openjdk.jmh:jmh-core:$versions.jmh" + compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" + //TODO: Transitive dependencies of JMH. Add them here explicitly for the time being and find out why they are not included implicitly. + runtime 'net.sf.jopt-simple:jopt-simple:4.6' + runtime 'org.apache.commons:commons-math3:3.2' +} + +compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" + +forbiddenApis { + // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes + ignoreFailures = true +} + +// No licenses for our benchmark deps (we don't ship benchmarks) +dependencyLicenses.enabled = false + +thirdPartyAudit.excludes = [ + // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees) + 'org.openjdk.jmh.profile.AbstractHotspotProfiler', + 'org.openjdk.jmh.profile.HotspotThreadProfiler', + 'org.openjdk.jmh.profile.HotspotClassloadingProfiler', + 'org.openjdk.jmh.profile.HotspotCompilationProfiler', + 'org.openjdk.jmh.profile.HotspotMemoryProfiler', + 'org.openjdk.jmh.profile.HotspotRuntimeProfiler', + 'org.openjdk.jmh.util.Utils' +] + +shadowJar { + classifier = 'benchmarks' +} + +// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface +task jmhJar( + dependsOn: shadowJar, + description: 'Generates an uberjar with the microbenchmarks and all dependencies', + group: 'Benchmark' +) + +task jmh( + dependsOn: runShadow, + description: 'Runs all microbenchmarks', + group: 'Benchmark' +) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java new file mode 100644 index 0000000000000..87f4f0ccbdf48 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark; + +import org.joda.time.DateTimeZone; +import org.joda.time.MutableDateTime; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +@SuppressWarnings("unused") //invoked by benchmarking framework +@State(Scope.Benchmark) +public class DateBenchmark { + private long instant; + + private MutableDateTime jodaDate = new MutableDateTime(0, DateTimeZone.UTC); + + private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); + + private ZonedDateTime javaDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneOffset.UTC); + + private TemporalAmount diff = Duration.ofMillis(1L); + + @Benchmark + public int mutableDateTimeSetMillisGetDayOfMonth() { + jodaDate.setMillis(instant++); + return jodaDate.getDayOfMonth(); + } + + @Benchmark + public int calendarSetMillisGetDayOfMonth() { + calendar.setTimeInMillis(instant++); + return calendar.get(Calendar.DAY_OF_MONTH); + } + + @Benchmark + public int javaDateTimeSetMillisGetDayOfMonth() { + // all classes in java.time are immutable, we have to use a new instance + javaDateTime = javaDateTime.plus(diff); + return javaDateTime.getDayOfMonth(); + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java new file mode 100644 index 0000000000000..0af853c18f77c --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark; + +import org.openjdk.jmh.annotations.Benchmark; + +@SuppressWarnings("unused") //invoked by benchmarking framework +public class HelloBenchmark { + @Benchmark + public void benchmarkRuntimeOverhead() { + //intentionally left blank + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java new file mode 100644 index 0000000000000..d5bb0916eca78 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.settings.Settings; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +@Fork(3) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@SuppressWarnings("unused") //invoked by benchmarking framework +public class AllocationBenchmark { + // Do NOT make any field final (even if it is not annotated with @Param)! See also + // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java + + // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no + // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would + // need its own main method and we cannot execute more than one class with a main method per JAR. + @Param({ + // indices, shards, replicas, nodes + " 10, 1, 0, 1", + " 10, 3, 0, 1", + " 10, 10, 0, 1", + " 100, 1, 0, 1", + " 100, 3, 0, 1", + " 100, 10, 0, 1", + + " 10, 1, 0, 10", + " 10, 3, 0, 10", + " 10, 10, 0, 10", + " 100, 1, 0, 10", + " 100, 3, 0, 10", + " 100, 10, 0, 10", + + " 10, 1, 1, 10", + " 10, 3, 1, 10", + " 10, 10, 1, 10", + " 100, 1, 1, 10", + " 100, 3, 1, 10", + " 100, 10, 1, 10", + + " 10, 1, 2, 10", + " 10, 3, 2, 10", + " 10, 10, 2, 10", + " 100, 1, 2, 10", + " 100, 3, 2, 10", + " 100, 10, 2, 10", + + " 10, 1, 0, 50", + " 10, 3, 0, 50", + " 10, 10, 0, 50", + " 100, 1, 0, 50", + " 100, 3, 0, 50", + " 100, 10, 0, 50", + + " 10, 1, 1, 50", + " 10, 3, 1, 50", + " 10, 10, 1, 50", + " 100, 1, 1, 50", + " 100, 3, 1, 50", + " 100, 10, 1, 50", + + " 10, 1, 2, 50", + " 10, 3, 2, 50", + " 10, 10, 2, 50", + " 100, 1, 2, 50", + " 100, 3, 2, 50", + " 100, 10, 2, 50" + }) + public String indicesShardsReplicasNodes = "10,1,0,1"; + + public int numTags = 2; + + private AllocationService strategy; + private ClusterState initialClusterState; + + @Setup + public void setUp() throws Exception { + final String[] params = indicesShardsReplicasNodes.split(","); + + int numIndices = toInt(params[0]); + int numShards = toInt(params[1]); + int numReplicas = toInt(params[2]); + int numNodes = toInt(params[3]); + + strategy = Allocators.createAllocationService(Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "tag") + .build()); + + MetaData.Builder mb = MetaData.builder(); + for (int i = 1; i <= numIndices; i++) { + mb.put(IndexMetaData.builder("test_" + i) + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(numShards) + .numberOfReplicas(numReplicas) + ); + } + MetaData metaData = mb.build(); + RoutingTable.Builder rb = RoutingTable.builder(); + for (int i = 1; i <= numIndices; i++) { + rb.addAsNew(metaData.index("test_" + i)); + } + RoutingTable routingTable = rb.build(); + DiscoveryNodes.Builder nb = DiscoveryNodes.builder(); + for (int i = 1; i <= numNodes; i++) { + nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags)))); + } + initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes + (nb).build(); + } + + private int toInt(String v) { + return Integer.valueOf(v.trim()); + } + + @Benchmark + public ClusterState measureAllocation() { + ClusterState clusterState = initialClusterState; + while (clusterState.getRoutingNodes().hasUnassignedShards()) { + RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes() + .shardsWithState(ShardRoutingState.INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + result = strategy.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + } + return clusterState; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java new file mode 100644 index 0000000000000..aba7fda1021d7 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.gateway.GatewayAllocator; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class Allocators { + private static class NoopGatewayAllocator extends GatewayAllocator { + public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator(); + + protected NoopGatewayAllocator() { + super(Settings.EMPTY, null, null); + } + + @Override + public void applyStartedShards(StartedRerouteAllocation allocation) { + // noop + } + + @Override + public void applyFailedShards(FailedRerouteAllocation allocation) { + // noop + } + + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + return false; + } + } + + private Allocators() { + throw new AssertionError("Do not instantiate"); + } + + + public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException, + IllegalAccessException, InvocationTargetException { + return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings + .BUILT_IN_CLUSTER_SETTINGS)); + } + + public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws + InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + return new AllocationService(settings, + defaultAllocationDeciders(settings, clusterSettings), + NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE); + } + + public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws + IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException { + List list = new ArrayList<>(); + // Keep a deterministic order of allocation deciders for the benchmark + for (Class deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) { + try { + Constructor constructor = deciderClass.getConstructor(Settings.class, ClusterSettings + .class); + list.add(constructor.newInstance(settings, clusterSettings)); + } catch (NoSuchMethodException e) { + Constructor constructor = deciderClass.getConstructor(Settings.class); + list.add(constructor.newInstance(settings)); + } + } + return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0])); + + } + + public static DiscoveryNode newNode(String nodeId, Map attributes) { + return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER, + DiscoveryNode.Role.DATA), Version.CURRENT); + } +} diff --git a/benchmarks/src/main/resources/log4j.properties b/benchmarks/src/main/resources/log4j.properties new file mode 100644 index 0000000000000..8ca1bc8729567 --- /dev/null +++ b/benchmarks/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Do not log at all if it is not really critical - we're in a benchmark +benchmarks.es.logger.level=ERROR +log4j.rootLogger=${benchmarks.es.logger.level}, out + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n + diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 4ee6febdfa531..f8d8b5848c943 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -17,3 +17,6 @@ httpclient = 4.5.2 httpcore = 4.4.4 commonslogging = 1.1.3 commonscodec = 1.10 + +# benchmark dependencies +jmh = 1.12 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 5aa675eac78cc..82cfa58af6268 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,6 +5,7 @@ List projects = [ 'rest-api-spec', 'core', 'docs', + 'benchmarks', 'distribution:integ-test-zip', 'distribution:zip', 'distribution:tar',