From 3fe2504da06bd5c8b64af9314ebcf2e2c2ee1d1a Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 28 Oct 2019 10:53:05 +0200 Subject: [PATCH 1/7] Move periodic job to ES repo This change kickstarts the process of moving CI job definitions to this repo. --- .ci/build.sh | 32 +++++++ .ci/jobs.t/defaults.yml | 92 +++++++++++++++++++ ...c+elasticsearch+{branch}+periodic-next.yml | 11 +++ .ci/make-branch-config.sh | 11 +++ .gitignore | 3 + 5 files changed, 149 insertions(+) create mode 100755 .ci/build.sh create mode 100644 .ci/jobs.t/defaults.yml create mode 100644 .ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml create mode 100755 .ci/make-branch-config.sh diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 0000000000000..0a04728d872c9 --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# drop page cache and kernel slab objects on linux +[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + +rm -Rfv ~/.gradle/init.d/init.gradle +mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d + +if [ -f /proc/cpuinfo ] ; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` +else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + # Looks like it's too much for our workers so reduce it further + MAX_WORKERS=$(($MAX_WORKERS/2)) + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi +fi + +if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) +fi + +export GRADLE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m" +set -e +./gradlew --parallel --scan \ + -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \ + --parallel --max-workers=$MAX_WORKERS \ + "$@" \ No newline at end of file diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml new file mode 100644 index 0000000000000..26b9c615c728d --- /dev/null +++ b/.ci/jobs.t/defaults.yml @@ -0,0 +1,92 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: elasticsearch-ci + +##### JOB DEFAULTS + +- job: + vault: + url: https://secrets.elastic.co:8200 + role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db + node: "general-purpose" + concurrent: true + logrotate: + daysToKeep: 30 + numToKeep: 90 + artifactDaysToKeep: 7 + scm: + - git: + name: origin + # master node jenkins user ~/.ssh + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git" + branches: + - "%BRANCH%" + url: "https://github.com/atorok/elasticsearch.git" + basedir: "" + wipe-workspace: "yes" + wrappers: + - timeout: + type: absolute + timeout: "120" + fail: true + - ansicolor + - timestamps + # TODO: No support un JJBB ? + # - gradle-build-scan + properties: + - github: + url: https://github.com/elastic/elasticsearch/ + - inject: + properties-content: | + HOME=$JENKINS_HOME + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA7_HOME=$HOME/.java/java7 + JAVA8_HOME=$HOME/.java/java8 + JAVA9_HOME=$HOME/.java/java9 + JAVA10_HOME=$HOME/.java/java10 + JAVA11_HOME=$HOME/.java/java11 + JAVA12_HOME=$HOME/.java/openjdk12 + JAVA13_HOME=$HOME/.java/openjdk13 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr {runbld_args} + .ci/build.sh check + publishers: + - email: + recipients: infra-root+build@elastic.co + # Upload additional logs + - google-cloud-storage: + credentials-id: 'elasticsearch-ci-gcs-plugin' + uploads: + - classic: + file-pattern: 'build/*.tar.bz2' + storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME' + share-publicly: false + upload-for-failed-jobs: true + show-inline: true + # Notify homer + - postbuildscript: + builders: + - role: SLAVE + build-on: + - SUCCESS + - FAILURE + - UNSTABLE + build-steps: + - shell: | + curl -sS -X POST \ + -H "Content-Type: text/plain" \ + --data "$BUILD_URL" \ + "https://homer.app.elstc.co/webhook/jenkins/buil + + + + diff --git a/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml new file mode 100644 index 0000000000000..47e39939cdc8a --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml @@ -0,0 +1,11 @@ +- job: + name: elastic+elasticsearch+%BRANCH%+periodic-next + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic + display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)" + description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n" + triggers: + - timed: "H H/1 * * *" + builders: + - shell: | + #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+{branch}+git+push + .ci/build.sh -Dbwc.checkout.align=true check diff --git a/.ci/make-branch-config.sh b/.ci/make-branch-config.sh new file mode 100755 index 0000000000000..ecbdfe3f3a9c6 --- /dev/null +++ b/.ci/make-branch-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z "$BRANCH" ] ; then + echo "BRANCH is unset" + exit 1 +fi + +rm -Rf .ci/jobs +cp -r .ci/jobs.t .ci/jobs + +sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml diff --git a/.gitignore b/.gitignore index b1a78d832e924..e3f75ba3d1fb4 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ eclipse-build # projects using testfixtures testfixtures_shared/ + +# These are generated from .ci/jobs.t +.ci/jobs/ From 4d0b7eb575959b600e03189009f6c0045f3b506d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 29 Oct 2019 08:01:00 +0200 Subject: [PATCH 2/7] Added a minimal readme to provide pointers to the documentation --- .ci/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .ci/README.md diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 0000000000000..f32345a775759 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,14 @@ +CI configuration for Elasticsearch +================================== + +Layout and Syntax +----------------- + +CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/). +Jobs live in the [jobs.t](jobs.t) directory, these are defiend in YML using a syntax +simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/). +Macros are not allowed, and each job must be defined in it's own file. +Merging of the default configuration is customized so unlike in standard JJB, +it recurses into YML objects. + + From 86049af0dcd138708f4eef93a4186e65289b852d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 29 Oct 2019 14:31:40 +0200 Subject: [PATCH 3/7] Update .ci/README.md Co-Authored-By: Rory Hunter --- .ci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/README.md b/.ci/README.md index f32345a775759..16b9354cdd9cf 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -5,7 +5,7 @@ Layout and Syntax ----------------- CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/). -Jobs live in the [jobs.t](jobs.t) directory, these are defiend in YML using a syntax +Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/). Macros are not allowed, and each job must be defined in it's own file. Merging of the default configuration is customized so unlike in standard JJB, From 811b707209894132a421a22475475a622c973351 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 29 Oct 2019 14:31:53 +0200 Subject: [PATCH 4/7] Update .ci/README.md Co-Authored-By: Rory Hunter --- .ci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/README.md b/.ci/README.md index 16b9354cdd9cf..4ffb25a5e4909 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -7,7 +7,7 @@ Layout and Syntax CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/). Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/). -Macros are not allowed, and each job must be defined in it's own file. +Macros are not allowed, and each job must be defined in its own file. Merging of the default configuration is customized so unlike in standard JJB, it recurses into YML objects. From cc042d1d33e25a293d4688bb6049606ee5f4c77e Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 29 Oct 2019 14:33:04 +0200 Subject: [PATCH 5/7] point to main repo --- .ci/jobs.t/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml index 26b9c615c728d..afe69f7022e37 100644 --- a/.ci/jobs.t/defaults.yml +++ b/.ci/jobs.t/defaults.yml @@ -25,7 +25,7 @@ reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git" branches: - "%BRANCH%" - url: "https://github.com/atorok/elasticsearch.git" + url: "https://github.com/elastic/elasticsearch.git" basedir: "" wipe-workspace: "yes" wrappers: From b6342ccd9ff894a02985ea7eaafef0f36bf13555 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 30 Oct 2019 09:10:39 +0200 Subject: [PATCH 6/7] PR review --- .ci/jobs.t/defaults.yml | 10 +++++----- .../elastic+elasticsearch+{branch}+periodic-next.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml index afe69f7022e37..d0f13ac54000b 100644 --- a/.ci/jobs.t/defaults.yml +++ b/.ci/jobs.t/defaults.yml @@ -57,7 +57,7 @@ JAVA12_HOME=$HOME/.java/openjdk12 JAVA13_HOME=$HOME/.java/openjdk13 - shell: | - #!/usr/local/bin/runbld --redirect-stderr {runbld_args} + #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push .ci/build.sh check publishers: - email: @@ -82,10 +82,10 @@ - UNSTABLE build-steps: - shell: | - curl -sS -X POST \ - -H "Content-Type: text/plain" \ - --data "$BUILD_URL" \ - "https://homer.app.elstc.co/webhook/jenkins/buil + curl -sS -X POST \ + -H "Content-Type: text/plain" \ + --data "$BUILD_URL" \ + "https://homer.app.elstc.co/webhook/jenkins/build-finished" || true diff --git a/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml index 47e39939cdc8a..d4c408cb588c8 100644 --- a/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml +++ b/.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml @@ -7,5 +7,5 @@ - timed: "H H/1 * * *" builders: - shell: | - #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+{branch}+git+push + #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push .ci/build.sh -Dbwc.checkout.align=true check From 2d52f587150fea0abfe35334ab7ff7a5cd02b6e7 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Sun, 3 Nov 2019 11:13:57 +0200 Subject: [PATCH 7/7] Add link to JJBB --- .ci/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/README.md b/.ci/README.md index 4ffb25a5e4909..8f63884462a15 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -10,5 +10,7 @@ simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20maste Macros are not allowed, and each job must be defined in its own file. Merging of the default configuration is customized so unlike in standard JJB, it recurses into YML objects. - +Further (internal) documentation on the setup +[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md) +.