stockholm/krebs/3modules/buildbot/master.nix

403 lines
11 KiB
Nix
Raw Normal View History

2015-12-15 17:43:40 +00:00
{ config, pkgs, lib, ... }:
2016-10-20 18:54:38 +00:00
with import <stockholm/lib>;
2015-12-15 17:43:40 +00:00
let
2016-07-03 22:13:25 +00:00
2016-11-11 00:34:18 +00:00
buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{
patches = [ ./buildbot.patch ];
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
});
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
2015-12-15 17:43:40 +00:00
# -*- python -*-
from buildbot.plugins import *
import re
import json
2015-12-15 17:43:40 +00:00
c = BuildmasterConfig = {}
2016-11-11 00:34:18 +00:00
c['workers'] = []
workers = json.loads('${builtins.toJSON cfg.workers}')
workernames = [ s for s in workers ]
for k,v in workers.items():
c['workers'].append(worker.Worker(k, v))
2015-12-15 17:43:40 +00:00
# TODO: configure protocols?
2015-12-15 17:43:40 +00:00
c['protocols'] = {'pb': {'port': 9989}}
####### Build Inputs
c['change_source'] = cs = []
${ concatStringsSep "\n"
(mapAttrsToList (n: v: ''
#### Change_Source: Begin of ${n}
${v}
#### Change_Source: End of ${n}
'') cfg.change_source )}
2015-12-15 17:43:40 +00:00
####### Build Scheduler
c['schedulers'] = sched = []
${ concatStringsSep "\n"
(mapAttrsToList (n: v: ''
#### Schedulers: Begin of ${n}
${v}
#### Schedulers: End of ${n}
'') cfg.scheduler )}
###### Builder
c['builders'] = bu = []
# Builder Pre: Begin
${cfg.builder_pre}
# Builder Pre: End
${ concatStringsSep "\n"
(mapAttrsToList (n: v: ''
#### Builder: Begin of ${n}
${v}
#### Builder: End of ${n}
'') cfg.builder )}
####### Status
2016-11-11 00:34:18 +00:00
c['services'] = []
# If you want to configure this url, override with extraConfig
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
${optionalString (cfg.web.enable) ''
2016-11-11 00:34:18 +00:00
from buildbot.plugins import util
#authz_cfg=authz.Authz(
# auth=auth.BasicAuth([ ]),
# # TODO: configure harder
# gracefulShutdown = False,
# forceBuild = 'auth',
# forceAllBuilds = 'auth',
# pingBuilder = False,
# stopBuild = 'auth',
# stopAllBuilds = 'auth',
# cancelPendingBuild = 'auth'
#)
c['www'] = dict(
port = ${toString cfg.web.port},
plugins = { 'waterfall_view':{}, 'console_view':{} }
)
c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"})
c['www']['authz'] = util.Authz(
allowRules = [
util.StopBuildEndpointMatcher(role="admins"),
util.ForceBuildEndpointMatcher(role="admins"),
util.RebuildBuildEndpointMatcher(role="admins")
],
roleMatchers = [
util.RolesFromEmails(admins=["${cfg.web.username}"])
]
)
''}
${optionalString (cfg.irc.enable) ''
2016-11-11 00:34:18 +00:00
from buildbot.plugins import reporters
irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
channels=${builtins.toJSON cfg.irc.channels},
notify_events={
'success': 1,
'failure': 1,
'exception': 1,
'successToFailure': 1,
'failureToSuccess': 1,
}${optionalString cfg.irc.allowForce ",allowForce=True"})
2016-11-11 00:34:18 +00:00
c['services'].append(irc)
''}
2015-12-15 17:43:40 +00:00
${ concatStringsSep "\n"
(mapAttrsToList (n: v: ''
#### Status: Begin of ${n}
${v}
#### Status: End of ${n}
'') cfg.status )}
2015-12-15 17:43:40 +00:00
####### PROJECT IDENTITY
c['title'] = "${cfg.title}"
2015-12-15 17:43:40 +00:00
c['titleURL'] = "http://krebsco.de"
####### DB URL
# TODO: configure
2015-12-15 17:43:40 +00:00
c['db'] = {
'db_url' : "sqlite:///state.sqlite",
}
${cfg.extraConfig}
'';
2015-12-22 18:36:19 +00:00
cfg = config.krebs.buildbot.master;
2015-12-15 17:43:40 +00:00
api = {
enable = mkEnableOption "Buildbot Master";
title = mkOption {
default = "Buildbot CI";
type = types.str;
description = ''
Title of the Buildbot Installation
'';
};
2015-12-15 17:43:40 +00:00
workDir = mkOption {
default = "/var/lib/buildbot/master";
type = types.str;
description = ''
Path to build bot master directory.
Will be created on startup.
'';
};
2015-12-30 00:38:33 +00:00
secrets = mkOption {
default = [];
type = types.listOf types.str;
example = [ "cac.json" ];
description = ''
List of all the secrets in <secrets> which should be copied into the
buildbot master directory.
'';
};
2016-11-11 00:34:18 +00:00
workers = mkOption {
default = {};
type = types.attrsOf types.str;
description = ''
2016-11-11 00:34:18 +00:00
Attrset of workernames with their passwords
workername = workerpassword
'';
};
change_source = mkOption {
default = {};
type = types.attrsOf types.str;
example = {
stockholm = ''
cs.append(changes.GitPoller(
'http://cgit.gum/stockholm',
workdir='stockholm-poller', branch='master',
project='stockholm',
pollinterval=120))
'';
};
description = ''
Attrset of all the change_sources which should be configured.
It will be directly included into the master configuration.
At the end an change object should be appended to <literal>cs</literal>
'';
};
scheduler = mkOption {
default = {};
type = types.attrsOf types.str;
example = {
force-scheduler = ''
sched.append(schedulers.ForceScheduler(
name="force",
builderNames=["full-tests"]))
'';
};
description = ''
Attrset of all the schedulers which should be configured.
It will be directly included into the master configuration.
At the end an change object should be appended to <literal>sched</literal>
'';
};
builder_pre = mkOption {
default = "";
type = types.lines;
example = ''
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
'';
description = ''
some code before the builders are being assembled.
can be used to define functions used by multiple builders
'';
};
builder = mkOption {
default = {};
type = types.attrsOf types.str;
example = {
fast-test = ''
'';
};
description = ''
Attrset of all the builder which should be configured.
It will be directly included into the master configuration.
At the end an change object should be appended to <literal>bu</literal>
'';
};
status = mkOption {
default = {};
type = types.attrsOf types.str;
description = ''
Attrset of all the extra status which should be configured.
It will be directly included into the master configuration.
At the end an change object should be appended to <literal>st</literal>
Right now IRC and Web status can be configured by setting
<literal>buildbot.master.irc.enable</literal> and
<literal>buildbot.master.web.enable</literal>
'';
};
# Configurable Stati
web = mkOption {
default = {};
type = types.submodule ({ config2, ... }: {
options = {
enable = mkEnableOption "Buildbot Master Web Status";
username = mkOption {
default = "krebs";
type = types.str;
description = ''
username for web authentication
'';
};
hostname = mkOption {
default = config.networking.hostName;
type = types.str;
description = ''
web interface Hostname
'';
};
password = mkOption {
default = "bob";
type = types.str;
description = ''
password for web authentication
'';
};
port = mkOption {
default = 8010;
type = types.int;
description = ''
port for buildbot web status
'';
};
};
});
};
irc = mkOption {
default = {};
type = types.submodule ({ config, ... }: {
options = {
enable = mkEnableOption "Buildbot Master IRC Status";
channels = mkOption {
2016-11-11 00:34:18 +00:00
default = [ { channel = "nix-buildbot-meetup";} ];
example = literalExample ''[
{channel = "nix-buildbot-meetup";}
{channel = "nix-buildbot-lol"; "password" = "lol";}
]'';
type = with types; listOf (attrsOf str);
description = ''
irc channels the bot should connect to
'';
};
allowForce = mkOption {
default = false;
type = types.bool;
description = ''
Determines if builds can be forced via IRC
'';
};
nick = mkOption {
default = "nix-buildbot";
type = types.str;
description = ''
nickname for IRC
'';
};
server = mkOption {
default = "irc.freenode.net";
type = types.str;
description = ''
Buildbot Status IRC Server to connect to
'';
};
};
});
};
2015-12-15 17:43:40 +00:00
extraConfig = mkOption {
default = "";
type = types.lines;
description = ''
extra config appended to the generated master.cfg
'';
};
};
imp = {
users.extraUsers.buildbotMaster = {
2015-12-26 09:54:02 +00:00
uid = genid "buildbotMaster";
2015-12-15 17:43:40 +00:00
description = "Buildbot Master";
home = cfg.workDir;
createHome = false;
};
users.extraGroups.buildbotMaster = {
2016-11-11 00:34:18 +00:00
gid = genid "buildbotMaster";
2015-12-15 17:43:40 +00:00
};
systemd.services.buildbotMaster = {
description = "Buildbot Master";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
# TODO: add extra dependencies to master like svn and cvs
path = [ pkgs.git ];
environment = {
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
};
serviceConfig = let
2016-02-15 17:46:19 +00:00
workdir = shell.escape cfg.workDir;
secretsdir = shell.escape (toString <secrets>);
in {
2015-12-15 17:43:40 +00:00
PermissionsStartOnly = true;
# TODO: maybe also prepare buildbot.tac?
2016-06-13 00:04:22 +00:00
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
2015-12-15 17:43:40 +00:00
set -efux
if [ ! -e ${workdir} ];then
mkdir -p ${workdir}
${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
2015-12-15 17:43:40 +00:00
fi
# always override the master.cfg
cp ${buildbot-master-config} ${workdir}/master.cfg
2015-12-30 00:38:33 +00:00
# copy secrets
2015-12-30 00:38:33 +00:00
${ concatMapStringsSep "\n"
(f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
2015-12-15 17:43:40 +00:00
# sanity
${buildbot}/bin/buildbot checkconfig ${workdir}
# TODO: maybe upgrade? not sure about this
# normally we should write buildbot.tac by our own
# ${buildbot}/bin/buildbot upgrade-master ${workdir}
chmod 700 -R ${workdir}
chown buildbotMaster:buildbotMaster -R ${workdir}
2015-12-15 17:43:40 +00:00
'';
2016-11-11 00:34:18 +00:00
ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}";
# ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
2015-12-15 17:43:40 +00:00
PrivateTmp = "true";
User = "buildbotMaster";
Restart = "always";
RestartSec = "10";
};
};
};
in
{
2015-12-22 18:36:19 +00:00
options.krebs.buildbot.master = api;
2016-02-14 15:43:44 +00:00
config = lib.mkIf cfg.enable imp;
2015-12-15 17:43:40 +00:00
}