WIP: add debian buster buildres

buildbot/docker-builders/buster-32/Dockerfile

@@ -0,0 +1,15 @@
+FROM i386/debian:buster
+
+RUN apt-get update && apt-get -y install \
+jq git curl wget devscripts \
+python3.5 python3-all-dev python3-numpy python3-pil python3-pip \
+build-essential libssl-dev libffi-dev rustc cargo
+
+RUN python3 -m pip install -U pip
+RUN python3 -m pip install "buildbot[bundle]==2.*"
+
+ENV WORKER_NAME diax-buster-32
+ENV WORKER_ADMIN Nicolas Frattaroli <[email protected]>
+ENV WORKER_INFO Debian (buster) 32-bit
+COPY worker-start.sh /root/
+CMD /root/worker-start.sh

buildbot/docker-builders/buster-32/worker-start.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cd /root/
+if [ ! -d "worker" ]; then
+    echo "creating worker..."
+    buildbot-worker create-worker --umask=0o22 --no-logrotate worker bbmaster:9989 $WORKER_NAME $(python3 -c "import hmac, hashlib; print(hmac.new(bytes('$BUILDBOT_SECRET', 'utf-8'), bytes('$WORKER_NAME', 'utf-8'), hashlib.sha512).hexdigest())")
+    echo $WORKER_ADMIN > worker/info/admin
+    echo $WORKER_INFO > worker/info/host
+    uname -a >> worker/info/host
+fi
+
+rm -f worker/twistd.pid worker/twistd.log
+
+# run in a clean env, don't leak secrets
+exec env -i -- /bin/bash -c "source /etc/profile; export HOME=`pwd`; exec buildbot-worker start --nodaemon worker"

buildbot/docker-builders/buster-64/Dockerfile

@@ -0,0 +1,15 @@
+FROM debian:buster
+
+RUN apt-get update && apt-get -y install \
+jq git curl wget devscripts \
+python3.5 python3-all-dev python3-numpy python3-pil python3-pip \
+build-essential libssl-dev libffi-dev
+
+RUN python3 -m pip install -U pip
+RUN python3 -m pip install "buildbot[bundle]==2.*"
+
+ENV WORKER_NAME diax-buster-64
+ENV WORKER_ADMIN Nicolas Frattaroli <[email protected]>
+ENV WORKER_INFO Debian (buster) 64-bit
+COPY worker-start.sh /root/
+CMD /root/worker-start.sh

buildbot/docker-builders/buster-64/worker-start.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cd /root/
+if [ ! -d "worker" ]; then
+    echo "creating worker..."
+    buildbot-worker create-worker --umask=0o22 --no-logrotate worker bbmaster:9989 $WORKER_NAME $(python3 -c "import hmac, hashlib; print(hmac.new(bytes('$BUILDBOT_SECRET', 'utf-8'), bytes('$WORKER_NAME', 'utf-8'), hashlib.sha512).hexdigest())")
+    echo $WORKER_ADMIN > worker/info/admin
+    echo $WORKER_INFO > worker/info/host
+    uname -a >> worker/info/host
+fi
+
+rm -f worker/twistd.pid worker/twistd.log
+
+# run in a clean env, don't leak secrets
+exec env -i -- /bin/bash -c "source /etc/profile; export HOME=`pwd`; exec buildbot-worker start --nodaemon worker"

buildbot/master/master.cfg

@@ -52,15 +52,18 @@ class BaseConfig:
     DEFAULT_CLIENT_JAR_VER = "1.16"
 
     ALL_BUILDERS = ['src', 'render', 'win32',
-                    'win64', 'deb32', 'deb64', 'centos7-64']
-    ENABLED_WORKERS = ['diax-stretch-64',
-                       'diax-stretch-32', 'ec2-windows', 'diax-centos7-64']
+                    'win64', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64']
+    ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32',
+                       'diax-buster-64', 'diax-buster-32',
+                       'ec2-windows', 'diax-centos7-64']
 
     # high-level worker list
     # workers[name] = [... list of builders ...]
     WORKERS_TO_BUILDERS = {
-        'diax-stretch-64': ['src', 'render', 'deb64'],
-        'diax-stretch-32': ['src', 'render', 'deb32'],
+        'diax-stretch-64': ['src', 'render', 'deb9-64'],
+        'diax-stretch-32': ['src', 'render', 'deb9-32'],
+        'diax-buster-64': ['src', 'render', 'deb10-64'],
+        'diax-buster-32': ['src', 'render', 'deb10-32'],
         'diax-centos7-64': ['centos7-64'],
         'ec2-windows': ['win32', 'win64'],
     }
@@ -79,8 +82,8 @@ class DevConfig(BaseConfig):
     BASE_URL = "http://localhost:8020"
     ENABLE_GITHUB_AUTH = os.environ.get("DISABLE_GITHUB_AUTH") is None
 
-    ALL_BUILDERS = ['src', 'render', 'deb32', 'deb64', 'centos7-64']
-    ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-centos7-64']
+    ALL_BUILDERS = ['src', 'render', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64']
+    ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-buster-32', 'diax-buster-64', 'diax-centos7-64']
 
 
 ENV = os.environ.get("ENV", "production")
@@ -489,7 +492,8 @@ def debian_build_changelog(props):
 deb_repo_lock = locks.MasterLock("deb_repo_lock")
 
 
-def debian():
+def debian(deb_codename):
+    "deb_codename should be a string like 'stretch' or 'buster'"
     yield checkout()
     yield ShellCommand(command=["python3", "setup.py", "build"], name="build")
 
@@ -511,18 +515,26 @@ def debian():
     yield MasterShellCommand(command=["dpkg-sig", "-g", "--use-agent --batch --no-tty", "--sign", "builder", upload_dest(".deb")], name="sign package", description="signing", descriptionDone="signed")
 
     # repo nonsense, use symlinks
-    yield MasterShellCommand(command=["ln", "-f", upload_dest(".deb"), str(DEBIAN_REPO / "files")], name="link into repo", doStepIf=is_release_build, description="linking to repo", descriptionDone="linked to repo")
-    yield MasterShellCommand(command=["make", "-C", str(DEBIAN_REPO)], name="rebuild repo", doStepIf=is_release_build, locks=[deb_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo")
+    yield MasterShellCommand(command=["ln", "-f", upload_dest(".deb"), str(DEBIAN_REPO / str(deb_codename) / "files")], name="link into repo", doStepIf=is_release_build, description="linking to repo", descriptionDone="linked to repo")
+    yield MasterShellCommand(command=["make", "-C", str(DEBIAN_REPO / deb_codename)], name="rebuild repo", doStepIf=is_release_build, locks=[deb_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo")
 
 
 @builder()
-def deb32():
-    yield debian()
+def deb9_32():
+    yield debian("stretch")
 
+@builder()
+def deb10_32():
+    yield debian("buster")
+
+
+@builder()
+def deb9_64():
+    yield debian("stretch")
 
 @builder()
-def deb64():
-    yield debian()
+def deb10_64():
+    yield debian("buster")
 
 
 rpm_build_lock = locks.WorkerLock("rpm_build_lock", maxCount=1)

buildbot/repos/debian/repo/stretch/Makefile

@@ -0,0 +1,30 @@
+.PHONY : all clean Packages Sources
+
+GENERATED_FILES=Release Release.bz2 Release.gz Release.gpg
+GENERATED_FILES+=Packages Packages.bz2 Packages.gz
+GENERATED_FILES+=Sources Sources.bz2 Sources.gz
+
+all : ${GENERATED_FILES}
+
+clean :
+	rm -f ${GENERATED_FILES}
+
+Release : Release.header Packages Packages.bz2 Packages.gz Sources Sources.bz2 Sources.gz
+	./gen_hashes.py Packages* Sources* | cat Release.header - > Release
+
+Release.gpg : Release
+	rm -f Release.gpg
+	gpg --batch --digest-algo SHA512 --output Release.gpg -a --detach-sig Release < /dev/null
+	chmod a+r Release.gpg
+
+Packages :
+	dpkg-scanpackages -m . /dev/null > Packages
+
+Sources :
+	dpkg-scansources . /dev/null > Sources
+
+%.bz2 : %
+	cat $< | bzip2 > $@
+
+%.gz : %
+	cat $< | gzip > $@

buildbot/repos/debian/repo/stretch/Release.header

@@ -0,0 +1,8 @@
+Origin: Overviewer.org (Minecraft Overviewer)
+Label: overviewer.org
+Suite: stable
+Version: 1.0
+Codename: overviewer.org
+Architectures: i386 amd64
+Components: main
+Description: a repository for debian builds of Minecraft Overviewer

buildbot/repos/debian/repo/stretch/gen_hashes.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import os.path
+import hashlib
+
+args = sys.argv[1:]
+
+date = time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime())
+print("Date: {}".format(date))
+
+def do_hashes(cls):
+    for arg in args:
+        with open(arg, 'rb') as f:
+            hash = cls(f.read()).hexdigest()
+        size = os.path.getsize(arg)
+        print(" {} {:>16} {}".format(hash, size, arg))
+
+print("MD5Sum:")
+do_hashes(hashlib.md5)
+print("SHA1:")
+do_hashes(hashlib.sha1)
+print("SHA256:")
+do_hashes(hashlib.sha256)
+print("SHA512:")
+do_hashes(hashlib.sha512)

buildbot/repos/debian/repo/stretch/overviewer.gpg.asc

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.17 (GNU/Linux)
+
+mQINBFLApIgBEADoRbg82NU1q2x62lrHIhCbxX8ZV5trWU7SjMc4rhtkFSWqtHea
+3Scs3wfoSDTWuIhUWz0IlN6hVlgAV9JHydLHoIPG37H6hKZ1m2OlaWznrb1D/80S
+gNmlTZpSc8gOH3fm8QOrNvc+Njsfaji40pjzcZx/SE0XdGFoz+SjdSks4+UYc76s
+W0V20k47nsLMv7FUdHV95FLLy0Ye/K4aiSLVg6GK6lsBl30kPMfuAK8U6TIjFKnt
+AfCNTVxr5dr3EmHB9kLLtS9g8nqPCP/bqWG4T/fmnox65PlkogcME4+OlMnsVhk+
+XRx4jbbLpiUB2mObioZQT+Rvak4xcI2TV82QwvMEWn+XXV/WjzIqfa22hEGMbJ6J
+fSoIodXC2sir0WWw/2kp81Q5hV1nFioy44SZZqKqtshdnRIpUJFrkube8FmK3YHM
+pSv1YzfEzbDMwCtvBrZWi6CSUr4UFX8n+wNi/DyEApW/cpY9ucTCGzG+u/o8CJL6
+hY8J7A4UB5TlKjduPmC1ACL5Y2tVNUL1nwOM7GUATMrvVBDZeqZanjf1+sDIJ34e
+6fMpHnYOmmh53ZfHTs8thetM0Wp0hVyYqzqN+uGxzG2aSdAwhztofG62KZ+EClt+
+nithDVoJ1cHUa2UiYCqpgqrCNFl3KWDDgvgdtUm1jwqoFVsQz8GvX7v5hQARAQAB
+tDxvdmVydmlld2VyLm9yZyAoT3ZlcnZpZXdlciBCdWlsZCBLZXkpIDxhZG1pbkBv
+dmVydmlld2VyLm9yZz6JAjgEEwECACIFAlLApIgCGwMGCwkIBwMCBhUIAgkKCwQW
+AgMBAh4BAheAAAoJEKtur4cfTBvOtDgP/0ZVzdfYG6UyhLJjt6A6BgxT9vp1P6ks
+tRxtPhLzAv7vCGuE5ONtXSIJR8rjdUjMRYwua/4oWgoBnOW4ASpi01T8pqJv8zIC
+VnuY43XyW0lVjzj9Cyk6lpPsJfb5BkfMdqK5j5po3k/876PvJpDua5oIWmZKNL6V
+8XRGheR7t52E1Bjppn7Yx9qWi7APUEMSbOsSLsnFM2zNDaLoP/H03iEy9BgWDREA
+sZPZcYmztgjErOJ1c58rsPb5pmbZZV8Q2hvMZczY/gAXfMqEwFrfUKq+1Sp5LKvh
+HnlOYirLs08NYZKV/qRg9+UYgVQsUTbz+Oxxp1LXjJALvawIi1t7e/1bP/l1dSaI
+V4igaDBrQmEgKUQz2EtNlg86U7vwYT4EDSWUE5UeJqS2rJLc3gt1lzQ2clOk3xn7
+xkC02DqQfpryekSRPZo6DjRUU9WKwvft/WXcb5GY6y8P7kT1XpF6CXR1pYnsKOS3
+wkilU38jXtb+oYra7UEGCIYYzYjbseqgiVf+C1yfFgIencG7iTV8yHcAoNeiAhAu
+Ui7LnIRmN/xl54MJlOQhFamyPfiPUZXlH42B0sI4FaGaZ7no9bAuSaBBswKACgJa
+dsWl5o2uUbduUZJQSMwUZVtSyrguTeRx89eTGSLXenqoWfw7sZHhxXgS552jTV8z
+Q+cOhN5STovQ
+=PD3O
+-----END PGP PUBLIC KEY BLOCK-----

buildbot/start.sh

@@ -18,9 +18,11 @@ unset CODESIGN_KEYGRIP
 
 mkdir -p uploads renders
 
-mkdir -p repos/debian/repo/files
+mkdir -p repos/debian/repo/buster/files
+mkdir -p repos/debian/repo/stretch/files
 cp -r /root/repos/debian repos/
-make -C repos/debian/repo
+make -C repos/debian/repo/buster
+make -C repos/debian/repo/stretch
 
 cp -r /root/repos/rpm repos/
 mkdir -p repos/rpm/repo/{6,7}/{i386,x86_64}/packages

docker-compose.yml

@@ -71,3 +71,13 @@ services:
     env_file: confidential.env
     links:
       - bbmaster
+  bbbuster64:
+    build: buildbot/docker-builders/buster-64
+    env_file: confidential.env
+    links:
+      - bbmaster
+  bbbuster32:
+    build: buildbot/docker-builders/buster-32
+    env_file: confidential.env
+    links:
+      - bbmaster