Skip to content

Commit 376c8ca

Browse files
michelvocksmichelvocks
authored
Distributed execution via worker (#166)
* First tests * A bit more work on the register api * Implemented registration process * little work * Removed trash from last merge conflict * Growing * Implemented load of client tls certificates * Some more work * Added new parameters * Tested worker registration process and fixed several bugs * WIP server grpc implementation * WIP memdb implementation * More work on memdb implementation * Added memdb to modules * More work done on getwork * Implemented binary streaming for pipeline binaries * More work done * Work work work * Implemented updatework function * Finalizing for first test * More work done * Massive work done. Refactored tons of stuff. Pray that it will not break something :-O * Implemented logs shipping * More work done and some testing * IT WORKS LOL * More bug fixing * Implemented authentication and deregister function * Implemented backend worker tags * More work on the frontend * Implemented status view of workers. Added worker permissions. Implemented automatic primary instance work scheduling when workers are busy. * Added worker data fetch * Implemented deregister process and some bug fixing * Implemented reset of worker secret. Implemented ignored specialized vault keys * Implemented pipeline worker tags. WIP backend testing * Implemented automatic worker status transission and a few more status checks * Added worker registration test and fixed a few tests * Fixed most tests * Upgraded circleci * Fixed last failing tests * circle ci * Fixed data race * fixed * fixed * Linter fixes * fix * fix tests * Fixed * Fixed * Added more workers handler tests * Fixed concurrent tests * Added memdb service test * Fixed failing test * Worked on tests * Added missing error check * Fixed racy test * More work on tests * fixed wrong fixtures path * Added more test * Added server worker tests * Fixed small test issues * More work on tests * Fixed small issues * Increased waiting time * Racy test fix * growing * fix * fix * Added missing piece * Added worker tags to UI. Added option to prevent work being scheduled on primary. * Improved test coverage * Added negative tags to allow users to remove automatically detected tags. Added tags info to overview view. Fixed add tags bug. * Removed prettify cpp tag since it might lead to confusion
1 parent c81a498 commit 376c8ca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+7870
-1078
lines changed

‎.circleci/config.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
test_and_coverage:
2121
working_directory: /go/src/github.com/gaia-pipeline/gaia
2222
docker:
23-
- image: circleci/golang:1.11.4
23+
- image: circleci/golang:1.12.4
2424
environment:
2525
GO111MODULE: "on"
2626
steps:
@@ -32,7 +32,7 @@ jobs:
3232
echo "" > coverage.txt
3333
3434
for d in $(go list ./... | grep -v vendor | grep -v /testacc); do
35-
go test -v -race -coverprofile=profile.out -covermode=atomic $d
35+
go test -v -timeout 20s -race -coverprofile=profile.out -covermode=atomic $d
3636
if [ -f profile.out ]; then
3737
cat profile.out >> coverage.txt
3838
rm profile.out
@@ -50,7 +50,7 @@ jobs:
5050
acceptance_tests:
5151
working_directory: /go/src/github.com/gaia-pipeline/gaia
5252
docker:
53-
- image: gaiapipeline/circleci:0.0.3
53+
- image: gaiapipeline/circleci:0.0.4
5454
environment:
5555
GO111MODULE: "on"
5656
steps:
@@ -64,7 +64,7 @@ jobs:
6464
compile:
6565
working_directory: /go/src/github.com/gaia-pipeline/gaia
6666
docker:
67-
- image: circleci/golang:1.11.4
67+
- image: circleci/golang:1.12.4
6868
environment:
6969
GO111MODULE: "on"
7070
steps:

‎.circleci/images/primary/Dockerfile

+56-48
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
2424
pkg-config \
2525
&& rm -rf /var/lib/apt/lists/*
2626

27-
ENV GOLANG_VERSION 1.11.4
27+
ENV GOLANG_VERSION 1.12.5
2828

2929
RUN set -eux; \
3030
\
3131
# this "case" statement is generated via "update.sh"
3232
dpkgArch="$(dpkg --print-architecture)"; \
3333
case "${dpkgArch##*-}" in \
34-
amd64) goRelArch='linux-amd64'; goRelSha256='fb26c30e6a04ad937bbc657a1b5bba92f80096af1e8ee6da6430c045a8db3a5b' ;; \
35-
armhf) goRelArch='linux-armv6l'; goRelSha256='9f7a71d27fef69f654a93e265560c8d9db1a2ca3f1dcdbe5288c46facfde5821' ;; \
36-
arm64) goRelArch='linux-arm64'; goRelSha256='b76df430ba8caff197b8558921deef782cdb20b62fa36fa93f81a8c08ab7c8e7' ;; \
37-
i386) goRelArch='linux-386'; goRelSha256='cecd2da1849043237d5f0756a93d601db6798fa3bb27a14563d201088aa415f3' ;; \
38-
ppc64el) goRelArch='linux-ppc64le'; goRelSha256='1f10146826acd56716b00b9188079af53823ddd79ceb6362e78e2f3aafb370ab' ;; \
39-
s390x) goRelArch='linux-s390x'; goRelSha256='4467442dacf89eb94c5d6f9f700204cb360be82db60e6296cc2ef8d0e890cd42' ;; \
40-
*) goRelArch='src'; goRelSha256='4cfd42720a6b1e79a8024895fa6607b69972e8e32446df76d6ce79801bbadb15'; \
34+
amd64) goRelArch='linux-amd64'; goRelSha256='aea86e3c73495f205929cfebba0d63f1382c8ac59be081b6351681415f4063cf' ;; \
35+
armhf) goRelArch='linux-armv6l'; goRelSha256='311f5e76c7cec1ec752474a61d837e474b8e750b8e3eed267911ab57c0e5da9a' ;; \
36+
arm64) goRelArch='linux-arm64'; goRelSha256='ff09f34935cd189a4912f3f308ec83e4683c309304144eae9cf60ebc552e7cd8' ;; \
37+
i386) goRelArch='linux-386'; goRelSha256='146605e13bf337ff3aacd941a816c5d97a8fef8b5817e07fcec4540632085980' ;; \
38+
ppc64el) goRelArch='linux-ppc64le'; goRelSha256='e88b2a2098bc79ad33912d1d27bc3282a7f3231b6f4672f306465bf46ff784ca' ;; \
39+
s390x) goRelArch='linux-s390x'; goRelSha256='168d297ec910cb446d1aea878baeb85f1387209f9eb55dde68bddcd4c006dcbb' ;; \
40+
*) goRelArch='src'; goRelSha256='2aa5f088cbb332e73fc3def546800616b38d3bfe6b8713b8a6404060f22503e8'; \
4141
echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \
4242
esac; \
4343
\
@@ -65,60 +65,68 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
6565
# --------------- End Go Part ---------------
6666

6767
# --------------- Start Java Part ---------------
68-
RUN apt-get update && apt-get install -y --no-install-recommends \
68+
RUN mkdir ~/.gnupg
69+
RUN echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
70+
RUN set -eux; \
71+
apt-get update; \
72+
apt-get install -y --no-install-recommends \
6973
bzip2 \
7074
unzip \
7175
xz-utils \
72-
&& rm -rf /var/lib/apt/lists/*
76+
\
77+
# java.lang.UnsatisfiedLinkError: /usr/local/openjdk-11/lib/libfontmanager.so: libfreetype.so.6: cannot open shared object file: No such file or directory
78+
# java.lang.NoClassDefFoundError: Could not initialize class sun.awt.X11FontManager
79+
# https://github.com/docker-library/openjdk/pull/235#issuecomment-424466077
80+
fontconfig libfreetype6 \
81+
; \
82+
rm -rf /var/lib/apt/lists/*
7383

7484
# Default to UTF-8 file.encoding
7585
ENV LANG C.UTF-8
7686

77-
# add a simple script that can auto-detect the appropriate JAVA_HOME value
78-
# based on whether the JDK or only the JRE is installed
79-
RUN { \
80-
echo '#!/bin/sh'; \
81-
echo 'set -e'; \
82-
echo; \
83-
echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \
84-
} > /usr/local/bin/docker-java-home \
85-
&& chmod +x /usr/local/bin/docker-java-home
86-
87-
# do some fancy footwork to create a JAVA_HOME that's cross-architecture-safe
88-
RUN ln -svT "/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)" /docker-java-home
89-
ENV JAVA_HOME /docker-java-home
87+
ENV JAVA_HOME /usr/local/openjdk-8
88+
ENV PATH $JAVA_HOME/bin:$PATH
9089

91-
ENV JAVA_VERSION 8u181
92-
ENV JAVA_DEBIAN_VERSION 8u181-b13-2~deb9u1
90+
# backwards compatibility shim
91+
RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ]
9392

94-
# see https://bugs.debian.org/775775
95-
# and https://github.com/docker-library/java/issues/19#issuecomment-70546872
96-
ENV CA_CERTIFICATES_JAVA_VERSION 20170531+nmu1
93+
# https://adoptopenjdk.net/upstream.html
94+
ENV JAVA_VERSION 8u212-b04
95+
ENV JAVA_BASE_URL https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u212-b04/OpenJDK8U-
96+
ENV JAVA_URL_VERSION 8u212b04
97+
# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246
9798

98-
RUN set -ex; \
99+
RUN set -eux; \
99100
\
100-
# deal with slim variants not having man page directories (which causes "update-alternatives" to fail)
101-
if [ ! -d /usr/share/man/man1 ]; then \
102-
mkdir -p /usr/share/man/man1; \
103-
fi; \
101+
dpkgArch="$(dpkg --print-architecture)"; \
102+
case "$dpkgArch" in \
103+
amd64) upstreamArch='x64' ;; \
104+
arm64) upstreamArch='aarch64' ;; \
105+
*) echo >&2 "error: unsupported architecture: $dpkgArch" ;; \
106+
esac; \
104107
\
105-
apt-get update; \
106-
apt-get install -y --no-install-recommends \
107-
openjdk-8-jdk="$JAVA_DEBIAN_VERSION" \
108-
ca-certificates-java="$CA_CERTIFICATES_JAVA_VERSION" \
109-
; \
110-
rm -rf /var/lib/apt/lists/*; \
108+
wget -O openjdk.tgz.asc "${JAVA_BASE_URL}${upstreamArch}_linux_${JAVA_URL_VERSION}.tar.gz.sign"; \
109+
wget -O openjdk.tgz "${JAVA_BASE_URL}${upstreamArch}_linux_${JAVA_URL_VERSION}.tar.gz" --progress=dot:giga; \
111110
\
112-
# verify that "docker-java-home" returns what we expect
113-
[ "$(readlink -f "$JAVA_HOME")" = "$(docker-java-home)" ]; \
111+
export GNUPGHOME="$(mktemp -d)"; \
112+
# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense?
113+
gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \
114+
# https://github.com/docker-library/openjdk/pull/322#discussion_r286839190
115+
gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \
116+
gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F | grep '0xA5CD6035332FA671' | grep 'Andrew Haley'; \
117+
gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \
118+
gpgconf --kill all; \
119+
rm -rf "$GNUPGHOME"; \
114120
\
115-
# update-alternatives so that future installs of other OpenJDK versions don't change /usr/bin/java
116-
update-alternatives --get-selections | awk -v home="$(readlink -f "$JAVA_HOME")" 'index($3, home) == 1 { $2 = "manual"; print | "update-alternatives --set-selections" }'; \
117-
# ... and verify that it actually worked for one of the alternatives we care about
118-
update-alternatives --query java | grep -q 'Status: manual'
119-
120-
# see CA_CERTIFICATES_JAVA_VERSION notes above
121-
RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
121+
mkdir -p "$JAVA_HOME"; \
122+
tar --extract --file openjdk.tgz --directory "$JAVA_HOME" --strip-components 1; \
123+
rm openjdk.tgz*; \
124+
\
125+
# TODO strip "demo" and "man" folders?
126+
\
127+
# basic smoke test
128+
javac -version; \
129+
java -version
122130
# --------------- End Java Part ---------------
123131

124132
# --------------- Start Maven Part ---------------

‎Makefile

+5-4
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ NAMESPACE=${NAME}
44
RELEASE_NAME=${NAME}
55
HELM_DIR=$(shell pwd)/helm
66
TEST=$$(go list ./... | grep -v /vendor/ | grep /testacc)
7-
TEST_TIMEOUT?=20m
7+
TEST_TIMEOUT_ACC?=20m
8+
TEST_TIMEOUT?=20s
89

910
default: dev
1011

@@ -34,13 +35,13 @@ get:
3435
go get ./...
3536

3637
test:
37-
go test -v ./...
38+
go test -v -race -timeout=$(TEST_TIMEOUT) ./...
3839

3940
test-cover:
40-
go test -v ./... --coverprofile=cover.out
41+
go test -v -timeout=$(TEST_TIMEOUT) ./... --coverprofile=cover.out
4142

4243
test-acc:
43-
GAIA_RUN_ACC=true GAIA_DEV=true go test -v $(TEST) -timeout=$(TEST_TIMEOUT)
44+
GAIA_RUN_ACC=true GAIA_DEV=true go test -v $(TEST) -timeout=$(TEST_TIMEOUT_ACC)
4445

4546
release: compile_frontend static_assets compile_backend
4647

‎frontend/client/app.js

+16-3
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ axiosInstance.interceptors.request.use(function (request) {
3232
Vue.config.devtools = true
3333
sync(store, router)
3434

35-
const nprogress = new NProgress({ parent: '.nprogress-container' })
35+
const nprogress = new NProgress({parent: '.nprogress-container'})
3636

37-
const { state } = store
37+
const {state} = store
3838

3939
Vue.directive('focus', {
4040
// When the bound element is inserted into the DOM...
@@ -98,6 +98,7 @@ function handleError (error) {
9898
console.log(error.response.data)
9999
}
100100
}
101+
101102
Vue.prototype.$onError = handleError
102103

103104
Vue.prototype.$onSuccess = (title, message) => {
@@ -109,6 +110,18 @@ Vue.prototype.$onSuccess = (title, message) => {
109110
})
110111
}
111112

113+
Vue.prototype.$prettifyTags = (tags) => {
114+
let prettyTags = ''
115+
for (let i = 0; i < tags.length; i++) {
116+
if (i === (tags.length - 1)) {
117+
prettyTags += tags[i]
118+
} else {
119+
prettyTags += tags[i] + ', '
120+
}
121+
}
122+
return prettyTags
123+
}
124+
112125
router.beforeEach((route, redirect, next) => {
113126
if (state.app.device.isMobile && state.app.sidebar.opened) {
114127
store.commit(TOGGLE_SIDEBAR, false)
@@ -130,4 +143,4 @@ const app = new Vue({
130143
// A simple event bus
131144
export const EventBus = new Vue()
132145

133-
export { app, router, store }
146+
export {app, router, store}

‎frontend/client/store/actions.js

-6
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,3 @@ export const toggleSidebar = ({ commit }, config) => {
77
}
88

99
export const toggleDevice = ({ commit }, device) => commit(types.TOGGLE_DEVICE, device)
10-
11-
export const switchEffect = ({ commit }, effectItem) => {
12-
if (effectItem) {
13-
commit(types.SWITCH_EFFECT, effectItem)
14-
}
15-
}

‎frontend/client/store/getters.js

-4
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@ const device = state => state.app.device
44
const sidebar = state => state.app.sidebar
55
const effect = state => state.app.effect
66
const menuitems = state => state.menu.items
7-
const componententry = state => {
8-
return state.menu.items.filter(c => c.meta && c.meta.label === 'Components')[0]
9-
}
107
const session = state => state.session
118
const intervals = state => state.intervals
129

@@ -17,7 +14,6 @@ export {
1714
sidebar,
1815
effect,
1916
menuitems,
20-
componententry,
2117
session,
2218
intervals
2319
}

‎frontend/client/views/overview/index.vue

+12-2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,16 @@
4747
<span v-else>
4848
unknown
4949
</span><br />
50+
<i class="fa fa-tags"></i>
51+
<span style="color: #b1adad;">
52+
Tags:
53+
</span>
54+
<span v-if="pipeline.p.tags">
55+
{{ $prettifyTags(pipeline.p.tags) }}
56+
</span>
57+
<span v-else>
58+
unknown
59+
</span><br />
5060
<div class="pipelinegrid-footer">
5161
<a class="button is-primary" @click="checkPipelineArgsAndStartPipeline(pipeline.p)" style="width: 100%;">
5262
<span class="icon">
@@ -132,10 +142,10 @@ export default {
132142
},
133143
134144
calculateDuration (startdate, finishdate) {
135-
if (!moment(startdate).millisecond()) {
145+
if (moment(startdate).valueOf() < 0) {
136146
startdate = moment()
137147
}
138-
if (!moment(finishdate).millisecond()) {
148+
if (moment(finishdate).valueOf() < 0) {
139149
finishdate = moment()
140150
}
141151

0 commit comments

Comments
 (0)