Skip to content

Commit 64815f1

Browse files
BukhtawarBukhtawar
committed
Merge remote-tracking branch 'upstream/master'
2 parents 569e8cc + 4e003fe commit 64815f1

File tree

11,019 files changed

+516457
-244276
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

11,019 files changed

+516457
-244276
lines changed

‎.ci/README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
CI configuration for Elasticsearch
2+
==================================
3+
4+
Layout and Syntax
5+
-----------------
6+
7+
CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/).
8+
Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax
9+
simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/).
10+
Macros are not allowed, and each job must be defined in its own file.
11+
Merging of the default configuration is customized so unlike in standard JJB,
12+
it recurses into YML objects.
13+
Further (internal) documentation on the setup
14+
[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md)
15+
.
16+

‎.ci/build-cache.gradle

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.ci/build.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
# drop page cache and kernel slab objects on linux
4+
[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
5+
6+
rm -Rfv ~/.gradle/init.d/init.gradle
7+
mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
8+
9+
if [ -f /proc/cpuinfo ] ; then
10+
MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2`
11+
else
12+
if [[ "$OSTYPE" == "darwin"* ]]; then
13+
# Parallel is disabled at this time (eventually set to 1) due to errors on the Mac workers
14+
# We'll have to do more testing to see if this can be re-enabled or what the proper value is.
15+
# MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
16+
MAX_WORKERS=2
17+
else
18+
echo "Unsupported OS Type: $OSTYPE"
19+
exit 1
20+
fi
21+
fi
22+
23+
if pwd | grep -v -q ^/dev/shm ; then
24+
echo "Not running on a ramdisk, reducing number of workers"
25+
MAX_WORKERS=$(($MAX_WORKERS*2/3))
26+
fi
27+
28+
set -e
29+
./gradlew --parallel --scan \
30+
-Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \
31+
--parallel --max-workers=$MAX_WORKERS \
32+
"$@"

‎.ci/bwcVersions

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
BWC_VERSION:
2+
- "7.0.0"
3+
- "7.0.1"
4+
- "7.1.0"
5+
- "7.1.1"
6+
- "7.2.0"
7+
- "7.2.1"
8+
- "7.3.0"
9+
- "7.3.1"
10+
- "7.3.2"
11+
- "7.4.0"
12+
- "7.4.1"
13+
- "7.4.2"
14+
- "7.5.0"
15+
- "7.5.1"
16+
- "7.5.2"
17+
- "7.6.0"
18+
- "7.6.1"
19+
- "7.7.0"
20+
- "8.0.0"

‎.ci/dockerOnLinuxExclusions

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# This file specifies the Linux OS versions on which we can't build and
2+
# test Docker images for some reason. These values correspond to ID and
3+
# VERSION_ID from /etc/os-release, and a matching value will cause the
4+
# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist
5+
# (as is the case on centos-6, for example) then that OS will again be
6+
# excluded.
7+
debian-8
8+
opensuse-15-1
9+
ol-7.7
10+
sles-12
11+
12+
# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
13+
# for PR checks
14+
centos-6
15+
ol-6.10

‎.ci/init.gradle

Lines changed: 109 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,114 @@
1-
if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
2-
throw new GradleException("Using init script without configuration")
3-
} else {
4-
logger.info("Using elastic artifactory repos")
5-
settingsEvaluated { settings ->
6-
settings.pluginManagement {
7-
repositories {
8-
maven {
9-
name "artifactory-gradle-plugins"
10-
url "https://artifactory.elstc.co/artifactory/gradle-plugins"
11-
credentials {
12-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
13-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
14-
}
15-
}
16-
gradlePluginPortal()
17-
}
1+
import com.bettercloud.vault.VaultConfig
2+
import com.bettercloud.vault.Vault
3+
4+
initscript {
5+
repositories {
6+
mavenCentral()
7+
}
8+
dependencies {
9+
classpath 'com.bettercloud:vault-java-driver:4.1.0'
10+
}
11+
}
12+
13+
boolean USE_ARTIFACTORY = false
14+
15+
if (System.getenv('VAULT_ADDR') == null) {
16+
throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
17+
}
18+
19+
if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
20+
throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
21+
"or the VAULT_TOKEN environment variable to use this init script.")
22+
}
23+
24+
final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
25+
new VaultConfig()
26+
.address(System.env.VAULT_ADDR)
27+
.engineVersion(1)
28+
.build()
29+
)
30+
.withRetries(5, 1000)
31+
.auth()
32+
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
33+
.getAuthClientToken()
34+
35+
final Vault vault = new Vault(
36+
new VaultConfig()
37+
.address(System.env.VAULT_ADDR)
38+
.engineVersion(1)
39+
.token(vaultToken)
40+
.build()
41+
)
42+
.withRetries(5, 1000)
43+
44+
45+
if (USE_ARTIFACTORY) {
46+
final Map<String, String> artifactoryCredentials = vault.logical()
47+
.read("secret/elasticsearch-ci/artifactory.elstc.co")
48+
.getData()
49+
logger.info("Using elastic artifactory repos")
50+
Closure configCache = {
51+
return {
52+
name "artifactory-gradle-release"
53+
url "https://artifactory.elstc.co/artifactory/gradle-release"
54+
credentials {
55+
username artifactoryCredentials.get("username")
56+
password artifactoryCredentials.get("token")
57+
}
58+
}
59+
}
60+
settingsEvaluated { settings ->
61+
settings.pluginManagement {
62+
repositories {
63+
maven configCache()
64+
}
65+
}
66+
}
67+
projectsLoaded {
68+
allprojects {
69+
buildscript {
70+
repositories {
71+
maven configCache()
1872
}
73+
}
74+
repositories {
75+
maven configCache()
76+
}
77+
}
78+
}
79+
}
80+
81+
projectsLoaded {
82+
rootProject {
83+
project.pluginManager.withPlugin('com.gradle.build-scan') {
84+
buildScan.server = 'https://gradle-enterprise.elastic.co'
1985
}
20-
projectsLoaded {
21-
allprojects {
22-
buildscript {
23-
repositories {
24-
maven {
25-
name "artifactory-gradle-release"
26-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
27-
credentials {
28-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
29-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
30-
}
31-
}
32-
}
33-
}
34-
repositories {
35-
maven {
36-
name "artifactory-gradle-release"
37-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
38-
credentials {
39-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
40-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
41-
}
42-
}
43-
}
86+
}
87+
}
88+
89+
90+
final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
91+
final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
92+
93+
if (buildCacheUrl) {
94+
final Map<String, String> buildCacheCredentials = vault.logical()
95+
.read("secret/elasticsearch-ci/gradle-build-cache")
96+
.getData()
97+
gradle.settingsEvaluated { settings ->
98+
settings.buildCache {
99+
local {
100+
// Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
101+
enabled = false
102+
}
103+
remote(HttpBuildCache) {
104+
url = buildCacheUrl
105+
push = buildCachePush
106+
credentials {
107+
username = buildCacheCredentials.get("username")
108+
password = buildCacheCredentials.get("password")
44109
}
110+
}
45111
}
112+
}
46113
}
114+

‎.ci/java-versions.properties

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
# build and test Elasticsearch for this branch. Valid Java versions
55
# are 'java' or 'openjdk' followed by the major release number.
66

7-
ES_BUILD_JAVA=openjdk12
8-
ES_RUNTIME_JAVA=java11
7+
ES_BUILD_JAVA=openjdk13
8+
ES_RUNTIME_JAVA=openjdk11
99
GRADLE_TASK=build
10-
11-

‎.ci/jobs.t/defaults.yml

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
---
2+
3+
##### GLOBAL METADATA
4+
5+
- meta:
6+
cluster: elasticsearch-ci
7+
8+
##### JOB DEFAULTS
9+
10+
- job:
11+
vault:
12+
url: https://secrets.elastic.co:8200
13+
role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db
14+
node: "general-purpose"
15+
concurrent: true
16+
logrotate:
17+
daysToKeep: 30
18+
numToKeep: 90
19+
artifactDaysToKeep: 7
20+
scm:
21+
- git:
22+
name: origin
23+
# master node jenkins user ~/.ssh
24+
credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
25+
reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
26+
branches:
27+
- "%BRANCH%"
28+
url: "https://github.com/elastic/elasticsearch.git"
29+
basedir: ""
30+
wipe-workspace: "yes"
31+
wrappers:
32+
- timeout:
33+
type: absolute
34+
timeout: "120"
35+
fail: true
36+
- ansicolor
37+
- timestamps
38+
- gradle-build-scan
39+
- inject:
40+
properties-file: '.ci/java-versions.properties'
41+
properties-content: |
42+
WORKSPACE=/dev/shm/$JOB_NAME
43+
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
44+
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
45+
JAVA7_HOME=$HOME/.java/java7
46+
JAVA8_HOME=$HOME/.java/java8
47+
JAVA9_HOME=$HOME/.java/java9
48+
JAVA10_HOME=$HOME/.java/java10
49+
JAVA11_HOME=$HOME/.java/java11
50+
JAVA12_HOME=$HOME/.java/openjdk12
51+
JAVA13_HOME=$HOME/.java/openjdk13
52+
GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m
53+
properties:
54+
- github:
55+
url: https://github.com/elastic/elasticsearch/
56+
- inject:
57+
properties-content: |
58+
HOME=$JENKINS_HOME
59+
builders:
60+
- shell: |
61+
#!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
62+
.ci/build.sh -Dbwc.checkout.align=true check
63+
publishers:
64+
- email:
65+
recipients: [email protected]
66+
# Upload additional logs
67+
- google-cloud-storage:
68+
credentials-id: 'elasticsearch-ci-gcs-plugin'
69+
uploads:
70+
- classic:
71+
file-pattern: 'build/*.tar.bz2'
72+
storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME'
73+
share-publicly: false
74+
upload-for-failed-jobs: true
75+
show-inline: true
76+
# Notify homer
77+
- postbuildscript:
78+
builders:
79+
- role: SLAVE
80+
build-on:
81+
- SUCCESS
82+
- FAILURE
83+
- UNSTABLE
84+
build-steps:
85+
- shell: |
86+
curl -sS -X POST \
87+
-H "Content-Type: text/plain" \
88+
--data "$BUILD_URL" \
89+
"https://homer.app.elstc.co/webhook/jenkins/build-finished" || true
90+
91+
92+
93+

‎.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
- job:
2+
name: elastic+elasticsearch+%BRANCH%+periodic-next
3+
workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic
4+
display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)"
5+
description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n"
6+
triggers: []

‎.ci/make-branch-config.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
if [ -z "$BRANCH" ] ; then
4+
echo "BRANCH is unset"
5+
exit 1
6+
fi
7+
8+
rm -Rf .ci/jobs
9+
cp -r .ci/jobs.t .ci/jobs
10+
11+
sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml

‎.ci/matrix-build-javas.yml

Lines changed: 0 additions & 9 deletions
This file was deleted.

0 commit comments

Comments
 (0)