diff --git a/buildbot/docker-builders/buster-32/Dockerfile b/buildbot/docker-builders/buster-32/Dockerfile new file mode 100644 index 0000000..ca88c7e --- /dev/null +++ b/buildbot/docker-builders/buster-32/Dockerfile @@ -0,0 +1,15 @@ +FROM i386/debian:buster + +RUN apt-get update && apt-get -y install \ +jq git curl wget devscripts \ +python3.5 python3-all-dev python3-numpy python3-pil python3-pip \ +build-essential libssl-dev libffi-dev rustc cargo + +RUN python3 -m pip install -U pip +RUN python3 -m pip install "buildbot[bundle]==2.*" + +ENV WORKER_NAME diax-buster-32 +ENV WORKER_ADMIN Nicolas Frattaroli <ovdev@fratti.ch> +ENV WORKER_INFO Debian (buster) 32-bit +COPY worker-start.sh /root/ +CMD /root/worker-start.sh diff --git a/buildbot/docker-builders/buster-32/worker-start.sh b/buildbot/docker-builders/buster-32/worker-start.sh new file mode 100755 index 0000000..d66bad4 --- /dev/null +++ b/buildbot/docker-builders/buster-32/worker-start.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +cd /root/ +if [ ! -d "worker" ]; then + echo "creating worker..." + buildbot-worker create-worker --umask=0o22 --no-logrotate worker bbmaster:9989 $WORKER_NAME $(python3 -c "import hmac, hashlib; print(hmac.new(bytes('$BUILDBOT_SECRET', 'utf-8'), bytes('$WORKER_NAME', 'utf-8'), hashlib.sha512).hexdigest())") + echo $WORKER_ADMIN > worker/info/admin + echo $WORKER_INFO > worker/info/host + uname -a >> worker/info/host +fi + +rm -f worker/twistd.pid worker/twistd.log + +# run in a clean env, don't leak secrets +exec env -i -- /bin/bash -c "source /etc/profile; export HOME=`pwd`; exec buildbot-worker start --nodaemon worker" diff --git a/buildbot/docker-builders/buster-64/Dockerfile b/buildbot/docker-builders/buster-64/Dockerfile new file mode 100644 index 0000000..1a463b7 --- /dev/null +++ b/buildbot/docker-builders/buster-64/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:buster + +RUN apt-get update && apt-get -y install \ +jq git curl wget devscripts \ +python3.5 python3-all-dev python3-numpy python3-pil python3-pip \ +build-essential libssl-dev libffi-dev + +RUN python3 -m pip install -U pip +RUN python3 -m pip install "buildbot[bundle]==2.*" + +ENV WORKER_NAME diax-buster-64 +ENV WORKER_ADMIN Nicolas Frattaroli <ovdev@fratti.ch> +ENV WORKER_INFO Debian (buster) 64-bit +COPY worker-start.sh /root/ +CMD /root/worker-start.sh diff --git a/buildbot/docker-builders/buster-64/worker-start.sh b/buildbot/docker-builders/buster-64/worker-start.sh new file mode 100755 index 0000000..d66bad4 --- /dev/null +++ b/buildbot/docker-builders/buster-64/worker-start.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +cd /root/ +if [ ! -d "worker" ]; then + echo "creating worker..." + buildbot-worker create-worker --umask=0o22 --no-logrotate worker bbmaster:9989 $WORKER_NAME $(python3 -c "import hmac, hashlib; print(hmac.new(bytes('$BUILDBOT_SECRET', 'utf-8'), bytes('$WORKER_NAME', 'utf-8'), hashlib.sha512).hexdigest())") + echo $WORKER_ADMIN > worker/info/admin + echo $WORKER_INFO > worker/info/host + uname -a >> worker/info/host +fi + +rm -f worker/twistd.pid worker/twistd.log + +# run in a clean env, don't leak secrets +exec env -i -- /bin/bash -c "source /etc/profile; export HOME=`pwd`; exec buildbot-worker start --nodaemon worker" diff --git a/buildbot/master/master.cfg b/buildbot/master/master.cfg index 9272c30..2f36774 100644 --- a/buildbot/master/master.cfg +++ b/buildbot/master/master.cfg @@ -52,15 +52,18 @@ class BaseConfig: DEFAULT_CLIENT_JAR_VER = "1.16" ALL_BUILDERS = ['src', 'render', 'win32', - 'win64', 'deb32', 'deb64', 'centos7-64'] - ENABLED_WORKERS = ['diax-stretch-64', - 'diax-stretch-32', 'ec2-windows', 'diax-centos7-64'] + 'win64', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64'] + ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', + 'diax-buster-64', 'diax-buster-32', + 'ec2-windows', 'diax-centos7-64'] # high-level worker list # workers[name] = [... list of builders ...] WORKERS_TO_BUILDERS = { - 'diax-stretch-64': ['src', 'render', 'deb64'], - 'diax-stretch-32': ['src', 'render', 'deb32'], + 'diax-stretch-64': ['src', 'render', 'deb9-64'], + 'diax-stretch-32': ['src', 'render', 'deb9-32'], + 'diax-buster-64': ['src', 'render', 'deb10-64'], + 'diax-buster-32': ['src', 'render', 'deb10-32'], 'diax-centos7-64': ['centos7-64'], 'ec2-windows': ['win32', 'win64'], } @@ -79,8 +82,8 @@ class DevConfig(BaseConfig): BASE_URL = "http://localhost:8020" ENABLE_GITHUB_AUTH = os.environ.get("DISABLE_GITHUB_AUTH") is None - ALL_BUILDERS = ['src', 'render', 'deb32', 'deb64', 'centos7-64'] - ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-centos7-64'] + ALL_BUILDERS = ['src', 'render', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64'] + ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-buster-32', 'diax-buster-64', 'diax-centos7-64'] ENV = os.environ.get("ENV", "production") @@ -489,7 +492,8 @@ def debian_build_changelog(props): deb_repo_lock = locks.MasterLock("deb_repo_lock") -def debian(): +def debian(deb_codename): + "deb_codename should be a string like 'stretch' or 'buster'" yield checkout() yield ShellCommand(command=["python3", "setup.py", "build"], name="build") @@ -511,18 +515,26 @@ def debian(): yield MasterShellCommand(command=["dpkg-sig", "-g", "--use-agent --batch --no-tty", "--sign", "builder", upload_dest(".deb")], name="sign package", description="signing", descriptionDone="signed") # repo nonsense, use symlinks - yield MasterShellCommand(command=["ln", "-f", upload_dest(".deb"), str(DEBIAN_REPO / "files")], name="link into repo", doStepIf=is_release_build, description="linking to repo", descriptionDone="linked to repo") - yield MasterShellCommand(command=["make", "-C", str(DEBIAN_REPO)], name="rebuild repo", doStepIf=is_release_build, locks=[deb_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo") + yield MasterShellCommand(command=["ln", "-f", upload_dest(".deb"), str(DEBIAN_REPO / str(deb_codename) / "files")], name="link into repo", doStepIf=is_release_build, description="linking to repo", descriptionDone="linked to repo") + yield MasterShellCommand(command=["make", "-C", str(DEBIAN_REPO / deb_codename)], name="rebuild repo", doStepIf=is_release_build, locks=[deb_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo") @builder() -def deb32(): - yield debian() +def deb9_32(): + yield debian("stretch") +@builder() +def deb10_32(): + yield debian("buster") + + +@builder() +def deb9_64(): + yield debian("stretch") @builder() -def deb64(): - yield debian() +def deb10_64(): + yield debian("buster") rpm_build_lock = locks.WorkerLock("rpm_build_lock", maxCount=1) diff --git a/buildbot/repos/debian/repo/Makefile b/buildbot/repos/debian/repo/buster/Makefile similarity index 100% rename from buildbot/repos/debian/repo/Makefile rename to buildbot/repos/debian/repo/buster/Makefile diff --git a/buildbot/repos/debian/repo/Release.header b/buildbot/repos/debian/repo/buster/Release.header similarity index 100% rename from buildbot/repos/debian/repo/Release.header rename to buildbot/repos/debian/repo/buster/Release.header diff --git a/buildbot/repos/debian/repo/gen_hashes.py b/buildbot/repos/debian/repo/buster/gen_hashes.py similarity index 100% rename from buildbot/repos/debian/repo/gen_hashes.py rename to buildbot/repos/debian/repo/buster/gen_hashes.py diff --git a/buildbot/repos/debian/repo/overviewer.gpg.asc b/buildbot/repos/debian/repo/buster/overviewer.gpg.asc similarity index 100% rename from buildbot/repos/debian/repo/overviewer.gpg.asc rename to buildbot/repos/debian/repo/buster/overviewer.gpg.asc diff --git a/buildbot/repos/debian/repo/stretch/Makefile b/buildbot/repos/debian/repo/stretch/Makefile new file mode 100644 index 0000000..d631544 --- /dev/null +++ b/buildbot/repos/debian/repo/stretch/Makefile @@ -0,0 +1,30 @@ +.PHONY : all clean Packages Sources + +GENERATED_FILES=Release Release.bz2 Release.gz Release.gpg +GENERATED_FILES+=Packages Packages.bz2 Packages.gz +GENERATED_FILES+=Sources Sources.bz2 Sources.gz + +all : ${GENERATED_FILES} + +clean : + rm -f ${GENERATED_FILES} + +Release : Release.header Packages Packages.bz2 Packages.gz Sources Sources.bz2 Sources.gz + ./gen_hashes.py Packages* Sources* | cat Release.header - > Release + +Release.gpg : Release + rm -f Release.gpg + gpg --batch --digest-algo SHA512 --output Release.gpg -a --detach-sig Release < /dev/null + chmod a+r Release.gpg + +Packages : + dpkg-scanpackages -m . /dev/null > Packages + +Sources : + dpkg-scansources . /dev/null > Sources + +%.bz2 : % + cat $< | bzip2 > $@ + +%.gz : % + cat $< | gzip > $@ diff --git a/buildbot/repos/debian/repo/stretch/Release.header b/buildbot/repos/debian/repo/stretch/Release.header new file mode 100644 index 0000000..630e32b --- /dev/null +++ b/buildbot/repos/debian/repo/stretch/Release.header @@ -0,0 +1,8 @@ +Origin: Overviewer.org (Minecraft Overviewer) +Label: overviewer.org +Suite: stable +Version: 1.0 +Codename: overviewer.org +Architectures: i386 amd64 +Components: main +Description: a repository for debian builds of Minecraft Overviewer diff --git a/buildbot/repos/debian/repo/stretch/gen_hashes.py b/buildbot/repos/debian/repo/stretch/gen_hashes.py new file mode 100755 index 0000000..7992b84 --- /dev/null +++ b/buildbot/repos/debian/repo/stretch/gen_hashes.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import sys +import time +import os.path +import hashlib + +args = sys.argv[1:] + +date = time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime()) +print("Date: {}".format(date)) + +def do_hashes(cls): + for arg in args: + with open(arg, 'rb') as f: + hash = cls(f.read()).hexdigest() + size = os.path.getsize(arg) + print(" {} {:>16} {}".format(hash, size, arg)) + +print("MD5Sum:") +do_hashes(hashlib.md5) +print("SHA1:") +do_hashes(hashlib.sha1) +print("SHA256:") +do_hashes(hashlib.sha256) +print("SHA512:") +do_hashes(hashlib.sha512) diff --git a/buildbot/repos/debian/repo/stretch/overviewer.gpg.asc b/buildbot/repos/debian/repo/stretch/overviewer.gpg.asc new file mode 100644 index 0000000..7b43e2a --- /dev/null +++ b/buildbot/repos/debian/repo/stretch/overviewer.gpg.asc @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.17 (GNU/Linux) + +mQINBFLApIgBEADoRbg82NU1q2x62lrHIhCbxX8ZV5trWU7SjMc4rhtkFSWqtHea +3Scs3wfoSDTWuIhUWz0IlN6hVlgAV9JHydLHoIPG37H6hKZ1m2OlaWznrb1D/80S +gNmlTZpSc8gOH3fm8QOrNvc+Njsfaji40pjzcZx/SE0XdGFoz+SjdSks4+UYc76s +W0V20k47nsLMv7FUdHV95FLLy0Ye/K4aiSLVg6GK6lsBl30kPMfuAK8U6TIjFKnt +AfCNTVxr5dr3EmHB9kLLtS9g8nqPCP/bqWG4T/fmnox65PlkogcME4+OlMnsVhk+ +XRx4jbbLpiUB2mObioZQT+Rvak4xcI2TV82QwvMEWn+XXV/WjzIqfa22hEGMbJ6J +fSoIodXC2sir0WWw/2kp81Q5hV1nFioy44SZZqKqtshdnRIpUJFrkube8FmK3YHM +pSv1YzfEzbDMwCtvBrZWi6CSUr4UFX8n+wNi/DyEApW/cpY9ucTCGzG+u/o8CJL6 +hY8J7A4UB5TlKjduPmC1ACL5Y2tVNUL1nwOM7GUATMrvVBDZeqZanjf1+sDIJ34e +6fMpHnYOmmh53ZfHTs8thetM0Wp0hVyYqzqN+uGxzG2aSdAwhztofG62KZ+EClt+ +nithDVoJ1cHUa2UiYCqpgqrCNFl3KWDDgvgdtUm1jwqoFVsQz8GvX7v5hQARAQAB +tDxvdmVydmlld2VyLm9yZyAoT3ZlcnZpZXdlciBCdWlsZCBLZXkpIDxhZG1pbkBv +dmVydmlld2VyLm9yZz6JAjgEEwECACIFAlLApIgCGwMGCwkIBwMCBhUIAgkKCwQW +AgMBAh4BAheAAAoJEKtur4cfTBvOtDgP/0ZVzdfYG6UyhLJjt6A6BgxT9vp1P6ks +tRxtPhLzAv7vCGuE5ONtXSIJR8rjdUjMRYwua/4oWgoBnOW4ASpi01T8pqJv8zIC +VnuY43XyW0lVjzj9Cyk6lpPsJfb5BkfMdqK5j5po3k/876PvJpDua5oIWmZKNL6V +8XRGheR7t52E1Bjppn7Yx9qWi7APUEMSbOsSLsnFM2zNDaLoP/H03iEy9BgWDREA +sZPZcYmztgjErOJ1c58rsPb5pmbZZV8Q2hvMZczY/gAXfMqEwFrfUKq+1Sp5LKvh +HnlOYirLs08NYZKV/qRg9+UYgVQsUTbz+Oxxp1LXjJALvawIi1t7e/1bP/l1dSaI +V4igaDBrQmEgKUQz2EtNlg86U7vwYT4EDSWUE5UeJqS2rJLc3gt1lzQ2clOk3xn7 +xkC02DqQfpryekSRPZo6DjRUU9WKwvft/WXcb5GY6y8P7kT1XpF6CXR1pYnsKOS3 +wkilU38jXtb+oYra7UEGCIYYzYjbseqgiVf+C1yfFgIencG7iTV8yHcAoNeiAhAu +Ui7LnIRmN/xl54MJlOQhFamyPfiPUZXlH42B0sI4FaGaZ7no9bAuSaBBswKACgJa +dsWl5o2uUbduUZJQSMwUZVtSyrguTeRx89eTGSLXenqoWfw7sZHhxXgS552jTV8z +Q+cOhN5STovQ +=PD3O +-----END PGP PUBLIC KEY BLOCK----- diff --git a/buildbot/start.sh b/buildbot/start.sh index 0ce24bf..673b471 100755 --- a/buildbot/start.sh +++ b/buildbot/start.sh @@ -18,9 +18,11 @@ unset CODESIGN_KEYGRIP mkdir -p uploads renders -mkdir -p repos/debian/repo/files +mkdir -p repos/debian/repo/buster/files +mkdir -p repos/debian/repo/stretch/files cp -r /root/repos/debian repos/ -make -C repos/debian/repo +make -C repos/debian/repo/buster +make -C repos/debian/repo/stretch cp -r /root/repos/rpm repos/ mkdir -p repos/rpm/repo/{6,7}/{i386,x86_64}/packages diff --git a/docker-compose.yml b/docker-compose.yml index 3a65443..d90619d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,3 +71,13 @@ services: env_file: confidential.env links: - bbmaster + bbbuster64: + build: buildbot/docker-builders/buster-64 + env_file: confidential.env + links: + - bbmaster + bbbuster32: + build: buildbot/docker-builders/buster-32 + env_file: confidential.env + links: + - bbmaster