master.cfg

# -*- python -*-
# ex: set syntax=python:

import hashlib
import hmac
import os
import os.path
import stat
import time
import uuid
from pathlib import Path
import urllib.parse
from urllib.parse import urljoin

import itsdangerous
from buildbot import locks
from buildbot.changes import filter
from buildbot.plugins import reporters, util
from buildbot.process.builder import enforceChosenWorker
from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Interpolate, Property, renderer
from buildbot.process.results import SUCCESS
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import *
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.shell import SetPropertyFromCommand, ShellCommand
from buildbot.steps.source.git import Git
from buildbot.steps.transfer import (DirectoryUpload, FileDownload, FileUpload,
                                     StringDownload)
from buildbot.steps.worker import SetPropertiesFromEnv
from buildbot.worker import Worker
from buildbot.worker.ec2 import EC2LatentWorker
from buildbot.www import auth, resource, avatar
from buildbot.util import bytes2unicode
from twisted.internet import defer
from twisted.python import log
from twisted.web.error import Error


class BaseConfig:

    DEFAULT_REPOSITORY = "https://github.com/overviewer/Minecraft-Overviewer"
    EXMAPLE_REPOSITORY = "https://github.com/overviewer/Minecraft-Overviewer-Addons"
    BASE_URL = "https://overviewer.org"
    ENABLE_GITHUB_AUTH = True
    GITHUB_CLIENT_ID = os.environ['GITHUB_CLIENT_ID']
    GITHUB_CLIENT_SECRET = os.environ['GITHUB_CLIENT_SECRET']
    POSTGRES_PASSWORD = os.environ['POSTGRES_PASSWORD']
    BUILDBOT_SECRET = os.environ['BUILDBOT_SECRET']
    FLASK_SECRET = os.environ['FLASK_SECRET']

    DEFAULT_CLIENT_JAR_VER = "1.16"

    ALL_BUILDERS = ['src', 'render', 'win32',
                    'win64', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64']
    ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32',
                       'diax-buster-64', 'diax-buster-32',
                       'ec2-windows', 'diax-centos7-64']

    # high-level worker list
    # workers[name] = [... list of builders ...]
    WORKERS_TO_BUILDERS = {
        'diax-stretch-64': ['src', 'render', 'deb9-64'],
        'diax-stretch-32': ['src', 'render', 'deb9-32'],
        'diax-buster-64': ['src', 'render', 'deb10-64'],
        'diax-buster-32': ['src', 'render', 'deb10-32'],
        'diax-centos7-64': ['centos7-64'],
        'ec2-windows': ['win32', 'win64'],
    }


class ProductionConfig(BaseConfig):

    AWS_ACCESS_KEY = os.environ['AWS_ACCESS_KEY']
    AWS_SECRET_KEY = os.environ['AWS_SECRET_KEY']
    EC2_WINDOWS_AMI = os.environ['EC2_WINDOWS_AMI']


class DevConfig(BaseConfig):
    DEFAULT_REPOSITORY = os.environ.get(
        "DEFAULT_REPOSITORY", BaseConfig.DEFAULT_REPOSITORY)
    BASE_URL = "http://localhost:8020"
    ENABLE_GITHUB_AUTH = os.environ.get("DISABLE_GITHUB_AUTH") is None

    ALL_BUILDERS = ['src', 'render', 'deb9-32', 'deb10-32', 'deb9-64', 'deb10-64', 'centos7-64']
    ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-buster-32', 'diax-buster-64', 'diax-centos7-64']


ENV = os.environ.get("ENV", "production")
IN_PRODUCTION = ENV == "production"

# our config
config = ProductionConfig if IN_PRODUCTION else DevConfig

BUILDBOT_URL = urljoin(config.BASE_URL, "build/")
BUILDBOT_PATH = Path("/data/buildbot")

UPLOAD_DEST = BUILDBOT_PATH / "uploads"
UPLOAD_URL = urljoin(config.BASE_URL, "builds/")

RENDER_UPLOAD_DEST = BUILDBOT_PATH / "renders"
RENDER_UPLOAD_URL = urljoin(config.BASE_URL, "renders/")

EXMAPLE_PATH = BUILDBOT_PATH / "example"
REPOS_PATH = BUILDBOT_PATH / "repos"
DEBIAN_PATH = REPOS_PATH / "debian"
RPM_PATH = REPOS_PATH / "rpm"
DEBIAN_REPO = DEBIAN_PATH / "repo"
RPM_REPO = RPM_PATH / "repo"

ENV_WHITELIST = {'PATH', 'PWD', 'SHLVL', 'GPG_AGENT_INFO', 'CODESIGN_NAME'}
# don't leak secrets, only leave whitelisted envs
os.environ = {k: v for k, v in os.environ.items() if k in ENV_WHITELIST}


def gen_pw(name):
    return hmac.new(
        bytes(config.BUILDBOT_SECRET, 'utf-8'),
        bytes(name, 'utf-8'),
        hashlib.sha512
    ).hexdigest()


# BuildBot config object
c = BuildmasterConfig = {}

c['title'] = "Minecraft Overviewer"
c['titleURL'] = config.BASE_URL
c['buildbotURL'] = BUILDBOT_URL

c['protocols'] = {'pb': {'port': 9989}}

c["www"] = {
    "port": 8010,
    "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
    "change_hook_dialects": {"github": dict(secret=config.BUILDBOT_SECRET, strict=True)},
}


class RedirectResource(resource.Resource):
    def __init__(self, make_url, *args, **kwargs):
        super(RedirectResource, self).__init__(*args, **kwargs)
        self.make_redirect_url = make_url

    def render_GET(self, request):
        url = request.args.get(b"redirect", [b"/"])[0]
        url = bytes2unicode(url)
        r = resource.Redirect(self.master.config.buildbotURL + "#" + url)
        request.redirect(self.make_redirect_url(r.url))
        return b''

class GitHubAvatar(avatar.AvatarBase):
    name = "github"
    def getUserAvatar(self, email, size, defaultAvatarUrl):
        raise resource.Redirect('https://avatars.githubusercontent.com/{}'.format(urllib.parse.quote(email)))

class OverviewerAuth(auth.AuthBase):
    flask_signer = itsdangerous.URLSafeTimedSerializer(
        config.FLASK_SECRET,
        salt='cookie-session',
        signer_kwargs=dict(
            key_derivation='hmac',
            digest_method=hashlib.sha1,
        ),
    )

    def __init__(self, httpLoginUrl, httpLogoutUrl, **kwargs):
        super(OverviewerAuth, self).__init__(**kwargs)
        self.httpLoginUrl = httpLoginUrl
        self.httpLogoutUrl = httpLogoutUrl
        if self.userInfoProvider is None:
            self.userInfoProvider = auth.UserInfoProviderBase()
    
    def getLoginResource(self):
        return RedirectResource(lambda u: self.httpLoginUrl.format(urllib.parse.quote(u)), self.master)

    def getLogoutResource(self):
        return RedirectResource(lambda u: self.httpLogoutUrl.format(urllib.parse.quote(u)), self.master)

    def get_flask_session(self, request):
        try:
            return request._flask_session
        except AttributeError:
            pass

        session = request.getCookie(b'session')
        if not session:
            request._flask_session = None
            return None
        try:
            # attempt to decode session
            s = self.flask_signer.loads(session)
            request._flask_session = s
            return s
        except Exception:
            request._flask_session = None
            return None

    @defer.inlineCallbacks
    def maybeAutoLogin(self, request):
        s = self.get_flask_session(request)
        session = request.getSession()
        user_info = {}
        if s and s.get('logged_in'):
            user_info['username'] = s.get('user')
            user_info['full_name'] = s.get('user')
            user_info['groups'] = ['developer'] if s.get('developer') else []
        else:
            raise Error(403, b'no session found')
        if session.user_info != user_info:
            session.user_info = user_info
            yield self.updateUserInfo(request)

if config.ENABLE_GITHUB_AUTH:
    c['www']['auth'] = OverviewerAuth(
        httpLoginUrl="https://overviewer.org/login?next={}",
        httpLogoutUrl="https://overviewer.org/logout?next={}",
    )
    c['www']['avatar_methods'] = [GitHubAvatar()]
    c['www']['authz'] = util.Authz(
        allowRules=[
            util.AnyControlEndpointMatcher(role="developer")
        ],
        roleMatchers=[
            util.RolesFromGroups()
        ]
    )

c["services"] = [
    reporters.IRC(
        "irc.freenode.net", "overviewer-build",
        useColors=True,
        allowForce=False,
        notify_events={
            'failure': 1,
            'exception': 1,
            'failureToSuccess': 1,
            'exceptionToSuccess': 1,
        },
        channels=[{'channel': '#em32'}],
    ),
]

c['db'] = {
    'db_url': 'postgresql://postgres:' + config.POSTGRES_PASSWORD + '@postgres/bbmaster',
}

c['workers'] = []
builders_to_workers = {}
workers = {k: v for k, v in config.WORKERS_TO_BUILDERS.items()
           if k in config.ENABLED_WORKERS}
for name, builders in workers.items():
    if name == 'ec2-windows':
        workerobj = EC2LatentWorker(name, gen_pw(name), 'm3.medium', ami=config.EC2_WINDOWS_AMI,
                                    identifier=config.AWS_ACCESS_KEY, secret_identifier=config.AWS_SECRET_KEY,
                                    keypair_name='overviewer_buildbot', security_name='overviewer_buildbot')
    else:
        workerobj = Worker(name, gen_pw(name))
    c['workers'].append(workerobj)
    for builder in builders:
        builders_to_workers.setdefault(builder, list()).append(name)

c['schedulers'] = [
    SingleBranchScheduler(
        name='Overviewer Master Scheduler',
        change_filter=filter.ChangeFilter(branch='master'),
        treeStableTimer=None,
        builderNames=config.ALL_BUILDERS,
    ),
    ForceScheduler(
        name="Overviewer-Force-Scheduler",
        builderNames=config.ALL_BUILDERS,
        codebases=[
            util.CodebaseParameter(
                "",
                branch=StringParameter(name="branch", default="master"),
                repository=StringParameter(
                    name="repository", default=config.DEFAULT_REPOSITORY, size=80),
                project=FixedParameter(name="project", choices=[
                                       "overviewer"], default="overviewer"),
            )
        ],
    ),
    ForceScheduler(
        name="Overviewer-Renderer",
        builderNames=["render"],
        codebases=[
            util.CodebaseParameter(
                "",
                branch=StringParameter(name="branch", default="master"),
                repository=StringParameter(
                    name="repository", default=config.DEFAULT_REPOSITORY, size=80),
                project=FixedParameter(name="project", choices=[
                                       "overviewer"], default="overviewer"),
                reason=FixedParameter(name="reason", default="Rendering"),
            )
        ],
        properties=[
            StringParameter(name="client_jar_version", label="Textures Version",
                            default=config.DEFAULT_CLIENT_JAR_VER),
            StringParameter(name="exmaple_commit",
                            label="Exmaple Commit", default="master"),
            ChoiceStringParameter(name="rendermode", label="Render Mode", choices=[
                "normal",
                "lighting",
                "smooth_lighting",
                "night",
                "smooth_night",
            ], default="smooth_lighting"),
            ChoiceStringParameter(name="procs", label="Processes", choices=[
                "1", "2", "4",
            ], default="2"),
        ],
    ),
]

c['builders'] = []

# high-level builder maker
# @builder(builder_config_key=...)
# def src():
#     yield BuildStep({kwargs})


def builder(**configExtra):
    def wrapper(f):
        name = f.__name__.replace('_', '-')
        if name not in config.ALL_BUILDERS:
            return f
        factory = util.BuildFactory()

        def runstep(step):
            try:
                steps = iter(step)
            except TypeError:
                info = step._factory
                stepkwargs = dict(
                    haltOnFailure=True,
                )
                stepkwargs.update(info.kwargs)
                step = info.factory(*info.args, **stepkwargs)
                factory.addStep(step)
                return
            for step in steps:
                runstep(step)

        runstep(f())

        builder = util.BuilderConfig(
            **{
                "name": name,
                "canStartBuild": enforceChosenWorker,
                "workernames": builders_to_workers.get(name, []),
                "factory": factory,
                **configExtra,
            }
        )
        c['builders'].append(builder)
        return f
    return wrapper


class SetBuildProperty(BuildStep):
    """ Set a build property as a step """

    renderables = ['property', 'value']

    def __init__(self, name='buildproperty', property=None, value=None, **kwargs):
        BuildStep.__init__(self, name=name, **kwargs)
        self.property = property
        self.value = value

    def start(self):
        self.setProperty(self.property, self.value, self.name)
        self.finished(SUCCESS)


def extract_stdout(key):
    def extract(rc, stdout, stderr):
        return {key: stdout.strip()}
    return extract


def upload_dest(fext):
    def _masterdest(props):
        project = props.getProperty('project')
        version = props.getProperty('version')
        builder = props.getProperty('buildername')
        buildnum = props.getProperty('buildnumber')
        revision = props.getProperty('got_revision')
        if any(x is None for x in [project, version, builder, buildnum, revision]):
            raise RuntimeError(
                "need project, version, buildername, buildnumber, and got_revision properties set")
        revision = revision[:7]
        project = 'overviewer' # push changes don't respect ?project=
        basename = '%s-%s-%s.%s-%s%s' % (project,
                                         version, builder, buildnum, revision, fext)
        return str(UPLOAD_DEST / basename)
    return renderer(_masterdest)


def upload_url(fext):
    return Interpolate(UPLOAD_URL + "%(prop:buildername)s/%(prop:buildnumber)s/overviewer-%(prop:version)s" + fext)


def checkout(python="python3", git="git", workdir=None):
    yield Git(repourl=Property('repository'), mode='full', method='fresh', name='checkout', workdir=workdir)
    yield ShellCommand(command=[git, "fetch", "--tags"], name="fetch tags", workdir=workdir)
    yield SetPropertyFromCommand(command=[python, "setup.py", "--version"], extract_fn=extract_stdout('version'), name="get version", workdir=workdir)
    yield SetPropertyFromCommand(command=[git, "describe", "--tags"], extract_fn=extract_stdout('desc'), name="get desc", workdir=workdir)

    @renderer
    def is_release_build(props):
        return props.getProperty("branch", "") == "master" and props.getProperty("repository", "") == config.DEFAULT_REPOSITORY
    yield SetBuildProperty(property='release_build', value=is_release_build, name="set release_build")


def is_release_build(step):
    return step.build.getProperty('release_build')


def upload(src, fext, **kwargs):
    yield FileUpload(workersrc=src, masterdest=upload_dest(fext), url=upload_url(fext), name="upload", mode=0o644, blocksize=524288, **kwargs)


@builder()
def src():
    yield checkout()
    yield ShellCommand(command=["python3", "setup.py", "sdist"], name="sdist")
    yield upload(Interpolate("dist/Minecraft-Overviewer-%(prop:version)s.tar.gz"), ".tar.gz")


def windows(arch):
    if arch == 'x86':
        pythonexe = r"C:\Python37 (x86)\python.exe"
    else:
        pythonexe = r"C:\Python37\python.exe"

    yield ShellCommand(command=[pythonexe, r"C:\devel\virtualenv\virtualenv.py", Interpolate("%(prop:builddir)s\\ci_env")], name="virtualenv")

    def script(s, interp=True):
        path = "%(prop:builddir)s\\ci_env\\Scripts\\" + s + ".exe"
        if interp:
            return Interpolate(path)
        return path
    del pythonexe
    yield ShellCommand(command=[script("pip"), "install", "numpy>=1.17.1", "Pillow==5.1.0", "pyinstaller", "sphinx"], name="dependencies")

    yield checkout(python=script("python"))
    yield FileDownload(mastersrc="pyinstaller-windows.spec", workerdest="overviewer.spec", name="transfer spec file")

    vcvarsall = r'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" ' + arch

    yield ShellCommand(command=Interpolate(vcvarsall + ' && "' + script("python", interp=False) + '" setup.py build'), name="build")
    yield ShellCommand(command=[script("pyinstaller"), "overviewer.spec"], name="pyinstaller")

    yield ShellCommand(command=[script("sphinx-build"), "docs", Interpolate(r"dist\overviewer-%(prop:version)s\docs")], name="sphinx")
    yield ShellCommand(command=[script("python"), "-m", "zipfile", "-c", "dist.zip", Interpolate("dist\\overviewer-%(prop:version)s\\")], name="create zip")
    yield upload("dist.zip", ".zip")


windows_env = {
    'DISTUTILS_USE_SDK': '1',
    'MSSdk': '1',
    'PIL_INCLUDE_DIR': r'C:\devel\pillow\src\libImaging',
}


@builder(env=windows_env)
def win32():
    yield windows('x86')


@builder(env=windows_env)
def win64():
    yield windows('amd64')


@renderer
def debian_build_changelog(props):
    version = props.getProperty("version")
    desc = props.getProperty("desc")
    now = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
    s = """minecraft-overviewer (%s-0~overviewer1) unstable; urgency=low

  * Automatically generated from Git: %s

 -- Aaron Griffith <agrif@overviewer.org>  %s""" % (version, desc, now)
    return s


deb_repo_lock = locks.MasterLock("deb_repo_lock")


def debian(deb_codename):
    "deb_codename should be a string like 'stretch' or 'buster'"
    yield checkout()
    yield ShellCommand(command=["python3", "setup.py", "build"], name="build")

    # move and untar our control files
    yield FileDownload(mastersrc=str(DEBIAN_PATH / "control/all.tar"), workerdest="debian/all.tar", name="transfer control files")
    yield ShellCommand(command=["tar", "--no-same-owner", "-xf", "all.tar"], workdir="build/debian", name="untar control files")

    # note all.tar does not contain the changelog, it's generated here
    yield StringDownload(debian_build_changelog, workerdest="debian/changelog", name="create changelog")

    # build and upload
    yield ShellCommand(command=['debuild', '-i', '-us', '-uc', '-b'], name="deb build", env={'DEB_BUILD_OPTIONS': 'nocheck'})
    yield SetPropertyFromCommand(command="ls -1 minecraft-overviewer_*.deb", workdir=".", property="debfile", name="get deb filename")
    yield upload(Property("debfile"), ".deb", workdir="")
    yield ShellCommand(command="rm -f *.deb *.changes *.build", workdir=".", name="clean up built files")

    # sign the uploaded package
    # note the argument to -g is *intentionally* one string
    yield MasterShellCommand(command=["dpkg-sig", "-g", "--use-agent --batch --no-tty", "--sign", "builder", upload_dest(".deb")], name="sign package", description="signing", descriptionDone="signed")

    # repo nonsense, use symlinks
    yield MasterShellCommand(command=["ln", "-f", upload_dest(".deb"), str(DEBIAN_REPO / str(deb_codename) / "files")], name="link into repo", doStepIf=is_release_build, description="linking to repo", descriptionDone="linked to repo")
    yield MasterShellCommand(command=["make", "-C", str(DEBIAN_REPO / deb_codename)], name="rebuild repo", doStepIf=is_release_build, locks=[deb_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo")


@builder()
def deb9_32():
    yield debian("stretch")

@builder()
def deb10_32():
    yield debian("buster")


@builder()
def deb9_64():
    yield debian("stretch")

@builder()
def deb10_64():
    yield debian("buster")


rpm_build_lock = locks.WorkerLock("rpm_build_lock", maxCount=1)
rpm_repo_lock = locks.MasterLock("rpm_repo_lock")


def rpm(rpmbase, mockbase, mockconfig, rpmarch, mockarch):
    yield checkout(workdir="build/Minecraft-Overviewer")

    # move and untar our control files
    yield FileDownload(mastersrc=str(RPM_PATH / "control/all.tar"), workerdest="rpm/all.tar", name="transfer control files")
    yield ShellCommand(command=["tar", "--no-same-owner", "-xf", "all.tar"], workdir="build/rpm", name="untar control files")

    # build source tarball
    yield SetPropertiesFromEnv(variables=["HOME"])
    yield ShellCommand(command=["mkdir", "-p", Interpolate("%(prop:HOME)s/rpmbuild/SOURCES/")], workdir="build", name="ensure SOURCES exists")
    yield ShellCommand(command=["tar", "-czf", Interpolate("%(prop:HOME)s/rpmbuild/SOURCES/Minecraft-Overviewer-%(prop:version)s.tar.gz"), "Minecraft-Overviewer"], workdir="build", name="create source tarball")

    # replace {VERSION} string in spec file
    yield ShellCommand(command=['sed', '-i', Interpolate('s/{VERSION}/%(prop:version)s/'), "rpm/" + rpmbase + "/Minecraft-Overviewer.spec"], name="replace version in spec file")

    # build source RPM
    if rpmbase == "el5":
        yield ShellCommand(command=['rpmbuild', '-bs', '--define', 'dist .' + rpmbase, '--define', '_source_filedigest_algorithm md5', 'rpm/' + rpmbase + '/Minecraft-Overviewer.spec'], name='create source rpm')
    else:
        yield ShellCommand(command=['rpmbuild', '-bs', '--define', 'dist .' + rpmbase, 'rpm/' + rpmbase + '/Minecraft-Overviewer.spec'], name="create source rpm")

    # build RPM
    srpm = Interpolate(
        "%(prop:HOME)s/rpmbuild/SRPMS/Minecraft-Overviewer-%(prop:version)s-1." + rpmbase + ".src.rpm")
    yield ShellCommand(command=['/usr/bin/mock', '--old-chroot', '-r', mockconfig + '-' + rpmarch, "--enable-network", srpm], name="build RPM")

    # upload
    yield upload(Interpolate('/var/lib/mock/' + mockconfig + '-' + rpmarch + '/result/Minecraft-Overviewer-%(prop:version)s-1.' + mockbase + '.' + mockarch + '.rpm'), '.rpm')
    yield ShellCommand(command="rm -rf rpm", workdir="build", name="clean up control files")

    # sign the uploaded package
    yield MasterShellCommand(command=["/root/rpmsign.sh", upload_dest(".rpm")], name="sign package", description="signing", descriptionDone="signed")

    # link the new rpm into the repo
    system_map = {'centos6': '6', 'centos7': '7'}
    rpm_package_area = RPM_REPO / system_map[rpmbase] / rpmarch / "packages"

    yield MasterShellCommand(command=["ln", "-f", upload_dest(".rpm"), str(rpm_package_area)], name="link into repo", doStepIf=is_release_build, description="linking into repo", descriptionDone="linked into repo")
    yield MasterShellCommand(command=["make", "-C", str(RPM_REPO)], name="rebuild repo", doStepIf=is_release_build, locks=[rpm_repo_lock.access('exclusive')], description="rebuilding repo", descriptionDone="rebuilt repo")


@builder(locks=[rpm_build_lock.access('exclusive')])
def centos6_32():
    yield rpm('centos6', 'el6', 'epel-6', 'i386', 'i686')


@builder(locks=[rpm_build_lock.access('exclusive')])
def centos6_64():
    yield rpm('centos6', 'el6', 'epel-6', 'x86_64', 'x86_64')


@builder(locks=[rpm_build_lock.access('exclusive')])
def centos7_64():
    yield rpm('centos7', 'el7', 'epel-7', 'x86_64', 'x86_64')


class RenderDirUpload(DirectoryUpload):
    def __init__(self, *args, **kwargs):
        kwargs.update({'masterdest': RENDER_UPLOAD_DEST,
                       'url': RENDER_UPLOAD_URL})
        super().__init__(*args, **kwargs)

    def start(self):
        uid = str(uuid.uuid4())
        self.masterdest_path = RENDER_UPLOAD_DEST / uid
        self.masterdest = str(self.masterdest_path)
        self.url = urljoin(RENDER_UPLOAD_URL, uid)
        self.setProperty("render_upload", self.masterdest, self.name)
        return super().start()

    def finished(self, result):
        r = super().finished(result)
        log.msg("RenderDirUpload correcting permissions on %r" %
                self.masterdest)

        DIRPERM = (stat.S_IRUSR | stat.IWUSR | stat.S_IXUSR | stat.S_IRGRP |
                   stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
        FILEPERM = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)

        self.masterdest_path.chmod(DIRPERM)
        for root, dirs, files in os.walk(self.masterdest):
            root = Path(root)
            for d in [root / dd for dd in dirs]:
                root.chmod(DIRPERM)
            for d in [root / dd for dd in files]:
                root.chmod(FILEPERM)

        return r


exmaple_lock = locks.MasterLock("exmaple_lock")


@builder()
def render():
    EXMAPLE_COMMIT = Property('exmaple_commit', default='master')
    CLIENT_JAR_VERSION = Property(
        'client_jar_version', default=config.DEFAULT_CLIENT_JAR_VER)
    CLIENT_JAR_MKDIR = Interpolate(
        '''mkdir -p ~/.minecraft/versions/%(kw:ver)s/''', ver=CLIENT_JAR_VERSION)
    CLIENT_JAR_FETCH = Interpolate(
        '''curl 'https://launchermeta.mojang.com/mc/game/version_manifest.json' | jq -r '.versions[] | select(.id == "%(kw:ver)s") | .url' | xargs curl | jq -r '.downloads.client.url' | xargs wget -N -O ~/.minecraft/versions/%(kw:ver)s/%(kw:ver)s.jar''', ver=CLIENT_JAR_VERSION)
    RENDERMODE = Property('rendermode', default='smooth_lighting')

    yield checkout()
    yield ShellCommand(command=["git", "clone", config.EXMAPLE_REPOSITORY, "maprepo"], name='get exmaple')
    yield ShellCommand(command=["git", "checkout", EXMAPLE_COMMIT], workdir='build/maprepo', name='checkout exmaple commit')
    yield ShellCommand(command=CLIENT_JAR_MKDIR, name="make jar directory")
    yield ShellCommand(command=CLIENT_JAR_FETCH, name="fetch client jar")
    yield ShellCommand(command=["python3", "setup.py", "build"], name="build")
    yield ShellCommand(command=["python3", "overviewer.py", "--verbose", "--check-terrain"], name="check terrain", haltOnFailure=False)
    yield ShellCommand(command=["python3", "overviewer.py", "--verbose", "-p", Property("procs", default="2"), Interpolate("--rendermode=%(kw:rmode)s", rmode=RENDERMODE), "maprepo/exmaple", "output_render"], name="render")
    yield RenderDirUpload(workersrc="output_render", compress="bz2", name="upload")

    def is_exmaple(step):
        def has_val(name, val):
            return step.build.getProperty(name) in [None, val]
        return step.build.getProperty('release_build') and has_val('rendermode', 'smooth_lighting') and has_val('client_jar_version', config.DEFAULT_CLIENT_JAR_VER) and has_val('exmaple_commit', 'master')
    yield MasterShellCommand(command=Interpolate("rm -f " + EXMAPLE_PATH.as_posix() + " && ln -s %(prop:render_upload)s " + EXMAPLE_PATH.as_posix()), name="link exmaple", doStepIf=is_exmaple, locks=[exmaple_lock.access('exclusive')], description="linking exmaple", descriptionDone="linked exmaple")