Skip to content

Commit 6828052

Browse files
BYKBYK
committed
# This is a combination of 7 commits.
# This is the 1st commit message: build(docker): Split builder image out for better caching This is an experimental new approach which creates a separate builder image, that is highly cacheable (and allows caching for yarn install step) to generate the Python wheel. It should save us multiple minutes by leveraging our builds' incremental nature much better. # This is the commit message #2: remove redundant op, trigger subsequent build # This is the commit message #3: add SOURCE_COMMIT env to builder run # This is the commit message #4: oops # This is the commit message #5: add revision and license labels to builder # This is the commit message #6: use kaniko 0.22 as it should fix the weird failures # This is the commit message #7: Simpler and stronger .dockerignore
1 parent be31b59 commit 6828052

File tree

7 files changed

+203
-247
lines changed

7 files changed

+203
-247
lines changed

‎.dockerignore

+8-66
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,8 @@
1-
**/__pycache__
2-
.git
3-
.gitignore
4-
.gitattributes
5-
.dockerignore
6-
**/cloudbuild.yaml
7-
Gemfile.lock
8-
.idea/
9-
*.iml
10-
.pytest_cache/
11-
.vscode
12-
.tx
13-
.travis
14-
.github
15-
.mailmap
16-
.npmrc
17-
.travis.yml
18-
.pre-commit-config.yaml
19-
.eslint*
20-
.env
21-
.envrc
22-
coverage.xml
23-
conftest.py
24-
netlify.toml
25-
static
26-
junit.xml
27-
*.codestyle.xml
28-
package-lock.json
29-
.cache/
30-
.coverage
31-
.storybook-out/
32-
.DS_Store
33-
.venv
34-
*.egg-info
35-
*.pyc
36-
*.log
37-
*.egg
38-
*.db
39-
*.pid
40-
Brewfile
41-
Makefile
42-
MANIFEST
43-
test.conf
44-
pip-log.txt
45-
celerybeat-schedule
46-
sentry-package.json
47-
/.artifacts
48-
/coverage/
49-
/cover
50-
/build
51-
/env
52-
/tests
53-
/tmp
54-
/node_modules/
55-
/docs-ui/node_modules/
56-
/scripts
57-
/src/sentry/assets.json
58-
/src/sentry/static/version
59-
/src/sentry/static/sentry/dist/
60-
/src/sentry/static/sentry/vendor/
61-
/src/sentry/static/sentry/admin/
62-
/src/sentry/static/sentry/rest_framework/
63-
/src/sentry/integration-docs
64-
/src/sentry/loader/_registry.json
65-
/wheelhouse
66-
/test_cli/
1+
# Ignore everything
2+
*
3+
4+
!/docker
5+
!/package.json
6+
!/yarn.lock
7+
!/dist/requirements.txt
8+
!/dist/*.whl

‎MANIFEST.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ include setup.py README.md MANIFEST.in LICENSE AUTHORS
22
recursive-include ./ requirements*.txt
33
recursive-include ./config/relay *
44
graft src/sentry
5-
global-exclude *~
5+
global-exclude *.pyc

‎docker/Dockerfile

+108-168
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,3 @@
1-
FROM python:2.7.16-slim-buster as sdist
2-
3-
LABEL maintainer="[email protected]"
4-
LABEL org.opencontainers.image.title="Sentry PyPI Wheel"
5-
LABEL org.opencontainers.image.description="PyPI Wheel Builder for Sentry"
6-
LABEL org.opencontainers.image.url="https://sentry.io/"
7-
LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
8-
LABEL org.opencontainers.image.vendor="Functional Software, Inc."
9-
LABEL org.opencontainers.image.authors="[email protected]"
10-
11-
# Sane defaults for pip
12-
ENV PIP_NO_CACHE_DIR=off \
13-
PIP_DISABLE_PIP_VERSION_CHECK=1
14-
15-
RUN apt-get update && apt-get install -y --no-install-recommends \
16-
# Needed for GPG
17-
dirmngr \
18-
gnupg \
19-
# Needed for fetching stuff
20-
wget \
21-
&& rm -rf /var/lib/apt/lists/* \
22-
# Needed to extract final dependencies from the whl
23-
&& pip install pkginfo==1.5.0.1
24-
25-
# Fetch trusted keys
26-
RUN for key in \
27-
# gosu
28-
B42F6819007F00F88E364FD4036A9C25BF357DD4 \
29-
# tini
30-
595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
31-
# Node - gpg keys listed at https://github.com/nodejs/node
32-
94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
33-
FD3A5288F042B6850C66B31F09FE44734EB7990E \
34-
71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
35-
DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
36-
C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
37-
B9AE9905FFD7803F25714661B63B535A4C206CA9 \
38-
77984A986EBC2AA786BC0F66B01FBB92821C587A \
39-
8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
40-
4ED778F539E3634C779C87C6D7062848A1AB005C \
41-
A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
42-
B9E2F5981AA6E0CD28160D9FF13993A75599653C \
43-
; do \
44-
# TODO(byk): Replace the keyserver below w/ something owned by Sentry
45-
gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
46-
done
47-
48-
# grab gosu for easy step-down from root
49-
ENV GOSU_VERSION 1.11
50-
RUN set -x \
51-
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
52-
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
53-
&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
54-
&& rm -r /usr/local/bin/gosu.asc \
55-
&& chmod +x /usr/local/bin/gosu
56-
57-
# grab tini for signal processing and zombie killing
58-
ENV TINI_VERSION 0.18.0
59-
RUN set -x \
60-
&& wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
61-
&& wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
62-
&& gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
63-
&& rm /usr/local/bin/tini.asc \
64-
&& chmod +x /usr/local/bin/tini
65-
66-
# Get and set up Node for front-end asset building
67-
COPY .nvmrc /usr/src/sentry/
68-
RUN cd /usr/src/sentry \
69-
&& export NODE_VERSION="$(cat .nvmrc)" \
70-
&& wget "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
71-
&& wget "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
72-
&& gpg --batch --verify SHASUMS256.txt.asc \
73-
&& grep " node-v$NODE_VERSION-linux-x64.tar.gz\$" SHASUMS256.txt.asc | sha256sum -c - \
74-
&& tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
75-
&& rm -r "node-v$NODE_VERSION-linux-x64.tar.gz" SHASUMS256.txt.asc
76-
77-
ARG SOURCE_COMMIT
78-
ENV SENTRY_BUILD=${SOURCE_COMMIT:-unknown}
79-
LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
80-
LABEL org.opencontainers.image.licenses="https://github.com/getsentry/sentry/blob/${SOURCE_COMMIT:-master}/LICENSE"
81-
82-
COPY . /usr/src/sentry/
83-
RUN export YARN_CACHE_FOLDER="$(mktemp -d)" \
84-
&& cd /usr/src/sentry \
85-
&& python setup.py bdist_wheel \
86-
&& rm -r "$YARN_CACHE_FOLDER" \
87-
&& mv /usr/src/sentry/dist /dist \
88-
# Dump the dependencies of our wheel as a separate requirements.txt file
89-
# so we can install them first, leveraging Docker's caching when they
90-
# don't change across versions.
91-
&& pkginfo -f requires_dist --single --sequence-delim=! /dist/*.whl | tr ! \\n > /dist/requirements.txt
92-
931
# This is the image to be run
942
FROM python:2.7.16-slim-buster
953

@@ -102,91 +10,123 @@ LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
10210
LABEL org.opencontainers.image.vendor="Functional Software, Inc."
10311
LABEL org.opencontainers.image.authors="[email protected]"
10412

105-
10613
# add our user and group first to make sure their IDs get assigned consistently
10714
RUN groupadd -r sentry && useradd -r -m -g sentry sentry
10815

109-
COPY --from=sdist /usr/local/bin/gosu /usr/local/bin/tini /usr/local/bin/
16+
ENV GOSU_VERSION=1.11 \
17+
TINI_VERSION=0.18.0
18+
19+
RUN set -x \
20+
&& buildDeps=" \
21+
dirmngr \
22+
gnupg \
23+
wget \
24+
" \
25+
&& apt-get update && apt-get install -y --no-install-recommends $buildDeps \
26+
&& rm -rf /var/lib/apt/lists/* \
27+
# Fetch trusted keys
28+
&& for key in \
29+
# gosu
30+
B42F6819007F00F88E364FD4036A9C25BF357DD4 \
31+
# tini
32+
595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
33+
; do \
34+
# TODO(byk): Replace the keyserver below w/ something owned by Sentry
35+
gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
36+
done \
37+
# grab gosu for easy step-down from root
38+
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
39+
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
40+
&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
41+
&& rm -r /usr/local/bin/gosu.asc \
42+
&& chmod +x /usr/local/bin/gosu \
43+
# grab tini for signal processing and zombie killing
44+
&& wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
45+
&& wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
46+
&& gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
47+
&& rm /usr/local/bin/tini.asc \
48+
&& chmod +x /usr/local/bin/tini \
49+
&& apt-get purge -y --auto-remove $buildDeps
11050

11151
# Sane defaults for pip
11252
ENV PIP_NO_CACHE_DIR=off \
113-
PIP_DISABLE_PIP_VERSION_CHECK=1 \
114-
# Sentry config params
115-
SENTRY_CONF=/etc/sentry \
116-
# Disable some unused uWSGI features, saving dependencies
117-
# Thank to https://stackoverflow.com/a/25260588/90297
118-
UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
119-
# UWSGI dogstatsd plugin
120-
UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
53+
PIP_DISABLE_PIP_VERSION_CHECK=1 \
54+
# Sentry config params
55+
SENTRY_CONF=/etc/sentry \
56+
# Disable some unused uWSGI features, saving dependencies
57+
# Thank to https://stackoverflow.com/a/25260588/90297
58+
UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
59+
# UWSGI dogstatsd plugin
60+
UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
12161

12262
# Copy and install dependencies first to leverage Docker layer caching.
123-
COPY --from=sdist /dist/requirements.txt /tmp/dist/requirements.txt
63+
COPY /dist/requirements.txt /tmp/dist/requirements.txt
12464
RUN set -x \
125-
&& buildDeps="" \
126-
# uwsgi
127-
&& buildDeps="$buildDeps \
128-
gcc \
129-
g++ \
130-
wget \
131-
" \
132-
# maxminddb
133-
&& buildDeps="$buildDeps \
134-
libmaxminddb-dev \
135-
"\
136-
# librabbitmq
137-
&& buildDeps="$buildDeps \
138-
make \
139-
" \
140-
# xmlsec
141-
&& buildDeps="$buildDeps \
142-
libxmlsec1-dev \
143-
pkg-config \
144-
" \
145-
&& apt-get update \
146-
&& apt-get install -y --no-install-recommends $buildDeps \
147-
&& pip install -r /tmp/dist/requirements.txt \
148-
# Separate these due to https://git.io/fjyz6
149-
# Otherwise librabbitmq will install the latest amqp version,
150-
# violating kombu's amqp<2.0 constraint.
151-
&& pip install librabbitmq==1.6.1 \
152-
&& mkdir /tmp/uwsgi-dogstatsd \
153-
&& wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
154-
tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
155-
&& UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
156-
&& mkdir -p /var/lib/uwsgi \
157-
&& mv dogstatsd_plugin.so /var/lib/uwsgi/ \
158-
&& rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
159-
&& apt-get purge -y --auto-remove $buildDeps \
160-
# We install run-time dependencies strictly after
161-
# build dependencies to prevent accidental collusion.
162-
# These are also installed last as they are needed
163-
# during container run and can have the same deps w/
164-
# build deps such as maxminddb.
165-
&& apt-get install -y --no-install-recommends \
166-
# pillow
167-
libjpeg-dev \
168-
# rust bindings
169-
libffi-dev \
170-
# maxminddb bindings
171-
libmaxminddb-dev \
172-
# SAML needs these run-time
173-
libxmlsec1-dev \
174-
libxslt-dev \
175-
# pyyaml needs this run-time
176-
libyaml-dev \
177-
# other
178-
pkg-config \
179-
\
180-
&& apt-get clean \
181-
&& rm -rf /var/lib/apt/lists/* \
182-
&& python -c 'import librabbitmq' \
183-
# Fully verify that the C extension is correctly installed, it unfortunately
184-
# requires a full check into maxminddb.extension.Reader
185-
&& python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
186-
&& mkdir -p $SENTRY_CONF
187-
188-
COPY --from=sdist /dist/*.whl /tmp/dist/
189-
RUN pip install /tmp/dist/*.whl && pip check
65+
&& buildDeps="" \
66+
# uwsgi
67+
&& buildDeps="$buildDeps \
68+
gcc \
69+
g++ \
70+
wget \
71+
" \
72+
# maxminddb
73+
&& buildDeps="$buildDeps \
74+
libmaxminddb-dev \
75+
"\
76+
# librabbitmq
77+
&& buildDeps="$buildDeps \
78+
make \
79+
" \
80+
# xmlsec
81+
&& buildDeps="$buildDeps \
82+
libxmlsec1-dev \
83+
pkg-config \
84+
" \
85+
&& apt-get update \
86+
&& apt-get install -y --no-install-recommends $buildDeps \
87+
&& pip install -r /tmp/dist/requirements.txt \
88+
# Separate these due to https://git.io/fjyz6
89+
# Otherwise librabbitmq will install the latest amqp version,
90+
# violating kombu's amqp<2.0 constraint.
91+
&& pip install librabbitmq==1.6.1 \
92+
&& mkdir /tmp/uwsgi-dogstatsd \
93+
&& wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
94+
tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
95+
&& UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
96+
&& mkdir -p /var/lib/uwsgi \
97+
&& mv dogstatsd_plugin.so /var/lib/uwsgi/ \
98+
&& rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
99+
&& apt-get purge -y --auto-remove $buildDeps \
100+
# We install run-time dependencies strictly after
101+
# build dependencies to prevent accidental collusion.
102+
# These are also installed last as they are needed
103+
# during container run and can have the same deps w/
104+
# build deps such as maxminddb.
105+
&& apt-get install -y --no-install-recommends \
106+
# pillow
107+
libjpeg-dev \
108+
# rust bindings
109+
libffi-dev \
110+
# maxminddb bindings
111+
libmaxminddb-dev \
112+
# SAML needs these run-time
113+
libxmlsec1-dev \
114+
libxslt-dev \
115+
# pyyaml needs this run-time
116+
libyaml-dev \
117+
# other
118+
pkg-config \
119+
\
120+
&& apt-get clean \
121+
&& rm -rf /var/lib/apt/lists/* \
122+
&& python -c 'import librabbitmq' \
123+
# Fully verify that the C extension is correctly installed, it unfortunately
124+
# requires a full check into maxminddb.extension.Reader
125+
&& python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
126+
&& mkdir -p $SENTRY_CONF
127+
128+
COPY /dist/*.whl /tmp/dist/
129+
RUN pip install /tmp/dist/*.whl && pip check && rm -rf /tmp/dist
190130
RUN sentry help | sed '1,/Commands:/d' | awk '{print $1}' > /sentry-commands.txt
191131

192132
COPY ./docker/sentry.conf.py ./docker/config.yml $SENTRY_CONF/

0 commit comments

Comments
 (0)