From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 00:02:59 +0100 Subject: [PATCH] k 3 buildbot.master: refactor see buildbot-standalone.nix in shared/2configs for the current buildbot config --- krebs/3modules/buildbot/master.nix | 325 +++++++++++++++--------- shared/2configs/buildbot-standalone.nix | 107 +++++++- 2 files changed, 309 insertions(+), 123 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 6bf3fda2c..7078000fe 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -7,134 +7,81 @@ let # -*- python -*- from buildbot.plugins import * import re - + import json c = BuildmasterConfig = {} c['slaves'] = [] - # TODO: template potential buildslaves - # TODO: set password? - slavenames= [ 'testslave' ] - for i in slavenames: - c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) + slaves = json.loads('${builtins.toJSON cfg.slaves}') + slavenames = [ s for s in slaves ] + for k,v in slaves.items(): + c['slaves'].append(buildslave.BuildSlave(k, v)) + # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} ####### Build Inputs - stockholm_repo = 'http://cgit.gum/stockholm' - c['change_source'] = [] - c['change_source'].append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) + c['change_source'] = cs = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Change_Source: Begin of ${n} + ${v} + #### Change_Source: End of ${n} + '') cfg.change_source )} ####### Build Scheduler - # TODO: configure scheduler - c['schedulers'] = [] + c['schedulers'] = sched = [] - # test the master real quick - fast = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - name="fast-master-test", - builderNames=["fast-tests"]) + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Schedulers: Begin of ${n} + ${v} + #### Schedulers: End of ${n} + '') cfg.scheduler )} - force = schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"]) + ###### Builder + c['builders'] = bu = [] + + # Builder Pre: Begin + ${cfg.builder_pre} + # Builder Pre: End - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Builder: Begin of ${n} + ${v} + #### Builder: End of ${n} + '') cfg.builder )} - full = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - name="full-master-test", - builderNames=["full-tests"]) - c['schedulers'] = [ fast, force, full ] - ###### The actual build - # couple of fast steps: - f = util.BuildFactory() - # some slow steps - s = util.BuildFactory() - ## fetch repo - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - f.addStep(grab_repo) - s.addStep(grab_repo) - # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix", - "(import {}).pkgs.test.infest-cac-centos7", - "rsync" ] - # TODO: --pure , prepare ENV in nix-shell command: - # SSL_CERT_FILE,LOGNAME,NIX_REMOTE - nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] - env = {"LOGNAME": "shared", - "NIX_REMOTE": "daemon"} - def addShell(f,**kwargs): - f.addStep(steps.ShellCommand(**kwargs)) + ####### Status + c['status'] = st = [] - addShell(f,name="centos7-eval",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + # If you want to configure this url, override with extraConfig + c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" - addShell(f,name="wolf-eval",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) + ${optionalString (cfg.web.enable) '' + from buildbot.status import html + from buildbot.status.web import authz, auth + authz_cfg=authz.Authz( + auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), + # TODO: configure harder + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = 'auth', + cancelPendingBuild = 'auth' + ) + # TODO: configure krebs.nginx + st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + ''} - addShell(f,name="eval-cross-check",env=env, - command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - - c['builders'] = [] - c['builders'].append( - util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - - # slave needs 2 files: - # * cac.json - # * retiolum - for file in ["cac.json", "retiolum.rsa_key.priv"]: - s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), - slavedest=file)) - - addShell(s, name="infest-cac-centos7",env=env, - sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=5400, # 1.5h timeout - command=nixshell + ["infest-cac-centos7"]) - - c['builders'].append( - util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=s)) - - ####### Status of Builds - c['status'] = [] - - from buildbot.status import html - from buildbot.status.web import authz, auth - # TODO: configure if http is wanted - authz_cfg=authz.Authz( - # TODO: configure user/pw - auth=auth.BasicAuth([("krebs","bob")]), - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = False, - cancelPendingBuild = False, - ) - # TODO: configure nginx - c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - - from buildbot.status import words ${optionalString (cfg.irc.enable) '' - irc = words.IRC("${cfg.irc.server}", "krebsbuild", - # TODO: multiple channels - channels=["${cfg.irc.channel}"], + from buildbot.status import words + irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, 'failure': 1, @@ -145,15 +92,20 @@ let c['status'].append(irc) ''} + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Status: Begin of ${n} + ${v} + #### Status: End of ${n} + '') cfg.status )} + ####### PROJECT IDENTITY - c['title'] = "Stockholm" + c['title'] = "${cfg.title}" c['titleURL'] = "http://krebsco.de" - #c['buildbotURL'] = "http://buildbot.krebsco.de/" - # TODO: configure url - c['buildbotURL'] = "http://vbob:8010/" ####### DB URL + # TODO: configure c['db'] = { 'db_url' : "sqlite:///state.sqlite", } @@ -164,6 +116,13 @@ let api = { enable = mkEnableOption "Buildbot Master"; + title = mkOption { + default = "Buildbot CI"; + type = types.str; + description = '' + Title of the Buildbot Installation + ''; + }; workDir = mkOption { default = "/var/lib/buildbot/master"; type = types.str; @@ -172,16 +131,144 @@ let Will be created on startup. ''; }; + + slaves = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of slavenames with their passwords + slavename = slavepassword + ''; + }; + + change_source = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + stockholm = '' + cs.append(changes.GitPoller( + 'http://cgit.gum/stockholm', + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + }; + description = '' + Attrset of all the change_sources which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to cs + ''; + }; + + scheduler = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + }; + description = '' + Attrset of all the schedulers which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to sched + ''; + }; + + builder_pre = mkOption { + default = ""; + type = types.lines; + example = '' + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + ''; + description = '' + some code before the builders are being assembled. + can be used to define functions used by multiple builders + ''; + }; + + builder = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + fast-test = '' + ''; + }; + description = '' + Attrset of all the builder which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to bu + ''; + }; + + status = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of all the extra status which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to st + + Right now IRC and Web status can be configured by setting + buildbot.master.irc.enable and + buildbot.master.web.enable + ''; + }; + + # Configurable Stati + web = mkOption { + default = {}; + type = types.submodule ({ config2, ... }: { + options = { + enable = mkEnableOption "Buildbot Master Web Status"; + username = mkOption { + default = "krebs"; + type = types.str; + description = '' + username for web authentication + ''; + }; + hostname = mkOption { + default = config.networking.hostName; + type = types.str; + description = '' + web interface Hostname + ''; + }; + password = mkOption { + default = "bob"; + type = types.str; + description = '' + password for web authentication + ''; + }; + port = mkOption { + default = 8010; + type = types.int; + description = '' + port for buildbot web status + ''; + }; + }; + }); + }; + irc = mkOption { default = {}; type = types.submodule ({ config, ... }: { options = { enable = mkEnableOption "Buildbot Master IRC Status"; - channel = mkOption { - default = "nix-buildbot-meetup"; - type = types.str; + channels = mkOption { + default = [ "nix-buildbot-meetup" ]; + type = with types; listOf str; description = '' - irc channel the bot should connect to + irc channels the bot should connect to ''; }; allowForce = mkOption { @@ -235,6 +322,7 @@ let description = "Buildbot Master"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; + # TODO: add extra dependencies to master like svn and cvs path = [ pkgs.git ]; environment = { SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; @@ -242,7 +330,6 @@ let serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; secretsdir="${lib.shell.escape (toString )}"; - # TODO: check if git is the only dep in { PermissionsStartOnly = true; Type = "forking"; diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index adf44cada..baab059c9 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,16 +8,115 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 ]; krebs.buildbot.master = { + slaves = { + testslave = "krebspass"; + testslave2 = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + full-master-scheduler = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + addShell(f,name="centos7-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) + + addShell(f,name="eval-cross-check",env=env, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file), + slavedest=file)) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=5400, # 1.5h timeout + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; enable = true; + web = { + enable = true; + }; irc = { enable = true; + nick = "shared-buildbot"; server = "cd.retiolum"; - channel = "retiolum"; + channels = [ "retiolum" ]; allowForce = true; }; - extraConfig = '' - c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/" - ''; }; krebs.buildbot.slave = {