Skip to content

Commit 03f6ad9

Browse files
andre-marcos-perezandre-marcos-perez
committed
turned scala lang and kernel version a function of spark distro due to compatibility issues
1 parent 09fdcff commit 03f6ad9

File tree

15 files changed

+163
-132
lines changed

15 files changed

+163
-132
lines changed

‎.github/workflows/ci.yml

+63-34
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ jobs:
1515
name: Spark Images
1616
runs-on: ubuntu-latest
1717
env:
18-
HADOOP_VERSION: "2.7"
19-
SCALA_VERSION: "2.12.11"
2018
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
2119
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
2220

@@ -25,7 +23,9 @@ jobs:
2523
- name: Checkout Code
2624
uses: actions/checkout@master
2725

28-
- name: Build Base Image
26+
- name: Build Base Image - Scala v2.12.10
27+
env:
28+
SCALA_VERSION: "2.12.10"
2929
run: |
3030
cd ${GITHUB_WORKSPACE}/build
3131
docker build \
@@ -37,14 +37,15 @@ jobs:
3737
- name: Build Spark Base Image - Spark v3.0.0 (latest)
3838
env:
3939
SPARK_VERSION: "3.0.0"
40+
HADOOP_VERSION: "3.2"
4041
run: |
4142
cd ${GITHUB_WORKSPACE}/build
4243
docker build \
4344
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
4445
--build-arg spark_version="${SPARK_VERSION}" \
4546
--build-arg hadoop_version="${HADOOP_VERSION}" \
4647
-f docker/spark-base/Dockerfile \
47-
-t spark-base:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
48+
-t spark-base:${SPARK_VERSION} .
4849
4950
- name: Build & Push Spark Master Image - Spark v3.0.0 (latest)
5051
env:
@@ -54,13 +55,12 @@ jobs:
5455
docker build \
5556
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
5657
--build-arg spark_version="${SPARK_VERSION}" \
57-
--build-arg hadoop_version="${HADOOP_VERSION}" \
5858
-f docker/spark-master/Dockerfile \
59-
-t spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
59+
-t spark-master:${SPARK_VERSION} .
6060
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
61-
docker tag spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
62-
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
63-
docker tag spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-master:latest
61+
docker tag spark-master:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
62+
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
63+
docker tag spark-master:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-master:latest
6464
docker push ${DOCKERHUB_USR}/spark-master:latest
6565
6666
- name: Build & Push Spark Worker Image - Spark v3.0.0 (latest)
@@ -71,26 +71,37 @@ jobs:
7171
docker build \
7272
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
7373
--build-arg spark_version="${SPARK_VERSION}" \
74-
--build-arg hadoop_version="${HADOOP_VERSION}" \
7574
-f docker/spark-worker/Dockerfile \
76-
-t spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
75+
-t spark-worker:${SPARK_VERSION} .
7776
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
78-
docker tag spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
79-
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
80-
docker tag spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-worker:latest
77+
docker tag spark-worker:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
78+
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
79+
docker tag spark-worker:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-worker:latest
8180
docker push ${DOCKERHUB_USR}/spark-worker:latest
8281
82+
- name: Build Base Image - Scala v2.11.12
83+
env:
84+
SCALA_VERSION: "2.11.12"
85+
run: |
86+
cd ${GITHUB_WORKSPACE}/build
87+
docker build \
88+
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
89+
--build-arg scala_version="${SCALA_VERSION}" \
90+
-f docker/base/Dockerfile \
91+
-t base:latest .
92+
8393
- name: Build Spark Base Image - Spark v2.4.4
8494
env:
8595
SPARK_VERSION: "2.4.4"
96+
HADOOP_VERSION: "2.7"
8697
run: |
8798
cd ${GITHUB_WORKSPACE}/build
8899
docker build \
89100
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
90101
--build-arg spark_version="${SPARK_VERSION}" \
91102
--build-arg hadoop_version="${HADOOP_VERSION}" \
92103
-f docker/spark-base/Dockerfile \
93-
-t spark-base:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
104+
-t spark-base:${SPARK_VERSION} .
94105
95106
- name: Build & Push Spark Master Image - Spark v2.4.4
96107
env:
@@ -100,12 +111,11 @@ jobs:
100111
docker build \
101112
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
102113
--build-arg spark_version="${SPARK_VERSION}" \
103-
--build-arg hadoop_version="${HADOOP_VERSION}" \
104114
-f docker/spark-master/Dockerfile \
105-
-t spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
115+
-t spark-master:${SPARK_VERSION} .
106116
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
107-
docker tag spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
108-
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
117+
docker tag spark-master:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
118+
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
109119
110120
- name: Build & Push Spark Worker Image - Spark v2.4.4
111121
env:
@@ -115,24 +125,24 @@ jobs:
115125
docker build \
116126
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
117127
--build-arg spark_version="${SPARK_VERSION}" \
118-
--build-arg hadoop_version="${HADOOP_VERSION}" \
119128
-f docker/spark-worker/Dockerfile \
120-
-t spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
129+
-t spark-worker:${SPARK_VERSION} .
121130
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
122-
docker tag spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
123-
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
131+
docker tag spark-worker:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
132+
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
124133
125134
- name: Build Spark Base Image - Spark v2.4.0
126135
env:
127136
SPARK_VERSION: "2.4.0"
137+
HADOOP_VERSION: "2.7"
128138
run: |
129139
cd ${GITHUB_WORKSPACE}/build
130140
docker build \
131141
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
132142
--build-arg spark_version="${SPARK_VERSION}" \
133143
--build-arg hadoop_version="${HADOOP_VERSION}" \
134144
-f docker/spark-base/Dockerfile \
135-
-t spark-base:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
145+
-t spark-base:${SPARK_VERSION} .
136146
137147
- name: Build & Push Spark Master Image - Spark v2.4.0
138148
env:
@@ -142,12 +152,11 @@ jobs:
142152
docker build \
143153
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
144154
--build-arg spark_version="${SPARK_VERSION}" \
145-
--build-arg hadoop_version="${HADOOP_VERSION}" \
146155
-f docker/spark-master/Dockerfile \
147-
-t spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
156+
-t spark-master:${SPARK_VERSION} .
148157
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
149-
docker tag spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
150-
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
158+
docker tag spark-master:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
159+
docker push ${DOCKERHUB_USR}/spark-master:${SPARK_VERSION}
151160
152161
- name: Build & Push Spark Worker Image - Spark v2.4.0
153162
env:
@@ -157,19 +166,17 @@ jobs:
157166
docker build \
158167
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
159168
--build-arg spark_version="${SPARK_VERSION}" \
160-
--build-arg hadoop_version="${HADOOP_VERSION}" \
161169
-f docker/spark-worker/Dockerfile \
162-
-t spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} .
170+
-t spark-worker:${SPARK_VERSION} .
163171
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
164-
docker tag spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
165-
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}-hadoop-${HADOOP_VERSION}
172+
docker tag spark-worker:${SPARK_VERSION} ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
173+
docker push ${DOCKERHUB_USR}/spark-worker:${SPARK_VERSION}
166174
167175
jupyterlab-images:
168176

169177
name: JupyterLab Images
170178
runs-on: ubuntu-latest
171179
env:
172-
SCALA_VERSION: "2.12.11"
173180
JUPYTERLAB_VERSION: "2.1.4"
174181
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
175182
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
@@ -179,7 +186,9 @@ jobs:
179186
- name: Checkout Code
180187
uses: actions/checkout@master
181188

182-
- name: Build Base Image
189+
- name: Build Base Image - Scala v2.12.10
190+
env:
191+
SCALA_VERSION: "2.12.10"
183192
run: |
184193
cd ${GITHUB_WORKSPACE}/build
185194
docker build \
@@ -191,13 +200,16 @@ jobs:
191200
- name: Build & Push JupyterLab Image - Spark v3.0.0 (latest)
192201
env:
193202
SPARK_VERSION: "3.0.0"
203+
SCALA_VERSION: "2.12.10"
204+
SCALA_KERNEL_VERSION: "0.10.9"
194205
run: |
195206
cd ${GITHUB_WORKSPACE}/build
196207
docker build \
197208
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
198209
--build-arg scala_version="${SCALA_VERSION}" \
199210
--build-arg spark_version="${SPARK_VERSION}" \
200211
--build-arg jupyterlab_version="${JUPYTERLAB_VERSION}" \
212+
--build-arg scala_kernel_version="${SCALA_KERNEL_VERSION}" \
201213
-f docker/jupyterlab/Dockerfile \
202214
-t jupyterlab:${JUPYTERLAB_VERSION}-spark-${SPARK_VERSION} .
203215
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
@@ -206,16 +218,30 @@ jobs:
206218
docker tag jupyterlab:${JUPYTERLAB_VERSION}-spark-${SPARK_VERSION} ${DOCKERHUB_USR}/jupyterlab:latest
207219
docker push ${DOCKERHUB_USR}/jupyterlab:latest
208220
221+
- name: Build Base Image - Scala v2.11.12
222+
env:
223+
SCALA_VERSION: "2.11.12"
224+
run: |
225+
cd ${GITHUB_WORKSPACE}/build
226+
docker build \
227+
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
228+
--build-arg scala_version="${SCALA_VERSION}" \
229+
-f docker/base/Dockerfile \
230+
-t base:latest .
231+
209232
- name: Build & Push JupyterLab Image - Spark v2.4.4
210233
env:
211234
SPARK_VERSION: "2.4.4"
235+
SCALA_VERSION: "2.11.12"
236+
SCALA_KERNEL_VERSION: "0.6.0"
212237
run: |
213238
cd ${GITHUB_WORKSPACE}/build
214239
docker build \
215240
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
216241
--build-arg scala_version="${SCALA_VERSION}" \
217242
--build-arg spark_version="${SPARK_VERSION}" \
218243
--build-arg jupyterlab_version="${JUPYTERLAB_VERSION}" \
244+
--build-arg scala_kernel_version="${SCALA_KERNEL_VERSION}" \
219245
-f docker/jupyterlab/Dockerfile \
220246
-t jupyterlab:${JUPYTERLAB_VERSION}-spark-${SPARK_VERSION} .
221247
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin
@@ -225,13 +251,16 @@ jobs:
225251
- name: Build & Push JupyterLab Image - Spark v2.4.0
226252
env:
227253
SPARK_VERSION: "2.4.0"
254+
SCALA_VERSION: "2.11.12"
255+
SCALA_KERNEL_VERSION: "0.6.0"
228256
run: |
229257
cd ${GITHUB_WORKSPACE}/build
230258
docker build \
231259
--build-arg build_date="$(date -u +'%Y-%m-%d')" \
232260
--build-arg scala_version="${SCALA_VERSION}" \
233261
--build-arg spark_version="${SPARK_VERSION}" \
234262
--build-arg jupyterlab_version="${JUPYTERLAB_VERSION}" \
263+
--build-arg scala_kernel_version="${SCALA_KERNEL_VERSION}" \
235264
-f docker/jupyterlab/Dockerfile \
236265
-t jupyterlab:${JUPYTERLAB_VERSION}-spark-${SPARK_VERSION} .
237266
echo ${DOCKERHUB_PWD} | docker login -u ${DOCKERHUB_USR} --password-stdin

‎CHANGELOG.md

+11
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,17 @@
22

33
All notable changes to this project will be documented in this file.
44

5+
## [1.3.0](https://github.com/andre-marcos-perez/spark-standalone-cluster-on-docker/releases/tag/v1.3.0) ()
6+
7+
### Features
8+
9+
- Exposed Spark master WebUI 4040 port ([#41](https://github.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/pull/41)).
10+
- Made SparkR available for all Spark versions.
11+
12+
### Core
13+
14+
- Enhanced Spark compatibility with Scala kernel ([#35](https://github.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/issues/35)):
15+
516
## [1.2.0](https://github.com/andre-marcos-perez/spark-standalone-cluster-on-docker/releases/tag/v1.2.0) (2020-08-19)
617

718
### Features

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ parallel computing in distributed environments through our projects. :sparkles:
1111
4. Do your magic :rainbow:;
1212
5. Commit your changes;
1313
6. Push to your feature branch;
14-
7. Create a new pull request from your the **develop** branch.
14+
7. Create a new pull request from your feature branch to the **develop** branch.
1515

1616
### Contributions ideas
1717

‎CONTRIBUTORS.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Contributors
2+
3+
Hi there, thanks for helping the project! We are doing our best to help the community to learn and practice
4+
parallel computing in distributed environments through our projects. :sparkles:
5+
6+
- **André Perez** - [dekoperez](https://twitter.com/dekoperez) - [email protected]

0 commit comments

Comments
 (0)