services/nomad/build: add buildbot nomad job

services/nomad/build/buildbot.cfg

@@ -0,0 +1,323 @@
+# vim: set filetype=python:
+
+import configparser
+import json
+import shlex
+
+from twisted.internet import defer
+
+from buildbot.plugins import util, secrets, reporters, worker, schedulers
+from buildbot.plugins import steps
+
+ini = configparser.ConfigParser()
+ini.read('/local/config.ini')
+
+with open(ini['buildbot'].get('workers', '/local/workers.json')) as f:
+    js = json.load(f)
+    workers = js.get("workers", [])
+    builders = js.get("builders", [])
+
+# TODO
+userInfoProvider = util.ldapUserInfo(
+    uri='ldap://localhost:389',
+    accountBase='ou=entities,dc=netauth,dc=voidlinux,dc=org',
+    accountPattern='(uid=%(username)s)',
+    accountFullName='displayname',
+    groupBase='ou=groups,dc=netauth,dc=voidlinux,dc=org',
+    groupName='',
+)
+auth = util.RemoteUserAuth(userInfoProvider=userInfoProvider)
+authz = util.Authz(
+    allowRules=[
+        util.AnyEndpointMatcher(role="ops", defaultDeny=False),
+        util.AnyControlEndpointMatcher(role="ops"),
+    ],
+    roleMatchers=[
+        util.RolesFromGroups(groupPrefix="build-"),
+    ]
+)
+
+c = BuildmasterConfig = {
+    'buildbotNetUsageData': None,
+    'protocols': {'pb': {'port': ini['buildbot'].getint('worker-port', 9989)}},
+    'secretsProviders': [secrets.SecretInAFile(dirname="/local/secrets")],
+    'workers': [],
+    'change_source': [],
+    'collapseRequests': True,
+    'schedulers': [],
+    'builders': [],
+    'services': [],
+    'title': ini['buildbot'].get('title', 'Void Linux'),
+    'titleURL': ini['buildbot'].get('title-url', 'https://voidlinux.org/'),
+    'buildbotURL': ini['buildbot'].get('url', 'http://localhost:8010/'),
+    'www': {
+        'port': ini['buildbot'].getint('www-port', 8010),
+        'plugins': {'waterfall_view': {}, 'console_view': {}, 'grid_view': {}},
+        'change_hook_dialects': {
+            'github': {
+                'secret': util.Secret('github-webhook'),
+                'strict': True,
+            },
+        },
+        'avatar_methods': [
+            userInfoProvider, util.AvatarGitHub(), util.AvatarGravatar(),
+        ],
+        'authz': authz,
+        'auth': auth,
+        'ui_default_config': {
+            'Waterfall.number_background_waterfall': True,
+            'Waterfall.show_builders_without_builds': True,
+            'Builders.show_workers_name': True,
+            'Workers.showWorkerBuilders': True,
+        },
+    },
+    'db': {
+        'db_url': ini['buildbot'].get('db-url', 'sqlite:////db/state.sqlite'),
+    },
+}
+
+if 'irc' in ini:
+    c['services'].append(reporters.IRC(
+        host=ini['irc']['host'],
+        port=ini['irc'].getint('port', 6697),
+        nick=ini['irc']['nick'],
+        password=util.Secret('irc-password'),
+        channels=[ini['irc']['channel']],
+        authz={'!': ini['irc'].get('authz-users', '').split(' ')},
+        notify_events=ini['irc'].get(
+            'notify-events', 'failure exception cancelled worker'
+        ).split(' '),
+        noticeOnChannel=ini['irc'].getboolean('notice', True),
+        useRevisions=ini['irc'].getboolean('use-revisions', True),
+        showBlameList=ini['irc'].getboolean('show-blame', True),
+        useSSL=ini['irc'].getboolean('use-ssl', True),
+        useColors=ini['irc'].getboolean('use-colors', True),
+    ))
+
+# ###### WORKERS
+
+for w in workers:
+    name = 'worker-' + w['name']
+    passwd = util.Secret('worker-password')
+    max_builds = w.get('max-builds', 1)
+    c['workers'].append(worker.Worker(name, passwd, max_builds=max_builds))
+
+
+# ###### SCHEDULERS
+
+builder_names = []
+for b in builders:
+    name = b['name'] + '_builder'
+    builder_names.append(name)
+
+c['schedulers'].append(schedulers.SingleBranchScheduler(
+    name="all",
+    change_filter=util.ChangeFilter(branch='master'),
+    treeStableTimer=None,
+    builderNames=builder_names))
+
+c['schedulers'].append(schedulers.ForceScheduler(
+    name="force",
+    builderNames=builder_names))
+
+
+# ###### BUILDERS
+
+distdir = 'void-packages'
+hostdir = '/hostdir'
+buildroot = 'buildroot'
+builddir = util.Property('buildnumber')
+
+
+factory = util.BuildFactory()
+
+
+@util.renderer
+def make_xbps_src_cmd(props, cmd):
+    command = [
+        f'{distdir}/xbps-src',
+        '-H', hostdir,
+        '-m', buildroot,
+        '-A', props.getProperty('host'),
+    ]
+
+    if cmd == 'binary-bootstrap':
+        command += shlex.split(str(props.getProperty('bootstrap_args')))
+
+    if props.getProperty('cross') == 'True':
+        command += ['-a', props.getProperty('target')]
+
+    command += [cmd]
+
+    return command
+
+
+@util.renderer
+def make_xbps_src_make_cmd(props):
+    command = [
+        'xbps-src-make',
+        '-hostdir', hostdir,
+        '-distdir', f'../{distdir}',
+        '-masterdir', f'../{buildroot}',
+        '-arch', props.getProperty('host'),
+    ]
+
+    if props.getProperty('cross') == 'True':
+        command += ['-cross', props.getProperty('target')]
+
+    return command
+
+
+class ShellCommandWithChanges(steps.ShellCommand):
+    @defer.inlineCallbacks
+    def run(self):
+        cmd = yield self.makeRemoteShellCommand()
+        cmd.command += self.build.allFiles()
+        yield self.runCommand(cmd)
+        return cmd.results()
+
+
+@util.renderer
+def build_packages(props):
+    # TODO: when a better solver is in place
+    # cmds = []
+    # for p in str(props.getProperty('packages')).strip().split():
+    #     cmds.append(util.ShellArg(
+    #         command=['make', f'built/{p}'],
+    #         logname=f'pkg:{p}',
+    #         haltOnFailure=True,
+    #     ))
+    cmds = [util.ShellArg(
+        command=['make'],
+        logname='build',
+        haltOnFailure=True,
+    )]
+    if cmds:
+        cmds.append(util.ShellArg(
+            command=['make', 'clean'],
+            logname='cleanup',
+            haltOnFailure=True,
+        ))
+    return cmds
+
+
+@util.renderer
+def make_prune_cmd(props):
+    return ['bash', '-c',
+            util.Interpolate(f"""
+export XBPS_TARGET_ARCH="%(prop:target)s"
+for repo in / /debug /nonfree /bootstrap; do
+    xbps-rindex -r "{hostdir}/binpkgs/$repo"
+done
+if [ "$XBPS_TARGET_ARCH" = i686 ]; then
+    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
+        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
+    done
+fi
+""")]
+
+
+factory.addStep(steps.Git(
+    # TODO
+    # repourl='https://github.com/void-linux/void-packages.git',
+    repourl='https://github.com/classabbyamp/void-packages.git',
+    mode='incremental',
+    workdir=distdir,
+    progress=True,
+    alwaysUseLatest=True,
+    name='update_void_packages',
+    description='updating void-packages from git',
+    descriptionDone='void-packages updated',
+    haltOnFailure=True,
+    logEnviron=False,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_xbps_src_cmd.withArgs('binary-bootstrap'),
+    name='bootstrap',
+    description='running xbps-src binary-bootstrap',
+    descriptionDone='xbps-src binary-bootstrap done',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_xbps_src_cmd.withArgs('bootstrap-update'),
+    name='bootstrap_update',
+    description='updating xbps-src bootstrap packages',
+    descriptionDone='xbps-src bootstrap-update done',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+))
+
+factory.addStep(ShellCommandWithChanges(
+    command=make_xbps_src_make_cmd,
+    name='find_packages',
+    description='finding packages to build',
+    descriptionDone='found packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=str(builddir),
+))
+
+factory.addStep(steps.SetPropertyFromCommand(
+    command=['make', 'SORTONLY=1', 'print_pkgs'],
+    property='packages',
+    name='get_packages',
+    description='collecting packages to build',
+    descriptionDone='collected packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    workdir=str(builddir),
+))
+
+factory.addStep(steps.ShellSequence(
+    commands=build_packages,
+    name='build_packages',
+    description='building packages',
+    descriptionDone='built packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=str(builddir),
+    timeout=14400,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_prune_cmd,
+    name='prune_packages',
+    description='removing obsolete packages',
+    descriptionDone='removed obsolete packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+    timeout=14400,
+))
+
+for b in builders:
+    workernames = ["worker-" + b['worker']]
+
+    name = b['name'] + '_builder'
+    hostarch = b['host']
+    targetarch = b.get('target', hostarch)
+    props = {
+        'name': name,
+        'host': hostarch,
+        'target': targetarch,
+        'cross': str(hostarch != targetarch),
+        'worker': b['worker'],
+        'bootstrap_args': b.get('bootstrap_args', '-N'),
+    }
+
+    c['builders'].append(util.BuilderConfig(
+        name=name,
+        workernames=workernames,
+        factory=factory,
+        properties=props,
+    ))