Mic92/stockholm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

buildbot: 0.8.4 -> 0.9.0rc2

Browse Source
This commit is contained in:
makefu 2016-11-11 01:34:18 +01:00
parent 8161fcfe3a
commit 6c092cd20c
No known key found for this signature in database
GPG Key ID: 36F7711F3FC0F225
6 changed files with 122 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
krebs/3modules/buildbot/buildbot-worker.patch Normal file
View File

@ -0,0 +1,11 @@
--- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100
+++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100
@@ -76,7 +76,7 @@
if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
tailBin = "/usr/xpg4/bin/tail"
else:
- tailBin = "/usr/bin/tail"
+ tailBin = "tail"
self.p = reactor.spawnProcess(self.pp, tailBin,
("tail", "-f", "-n", "0", self.logfile),
env=os.environ,

11
krebs/3modules/buildbot/buildbot.patch Normal file
View File

@ -0,0 +1,11 @@
--- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100
+++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100
@@ -76,7 +76,7 @@
if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
tailBin = "/usr/xpg4/bin/tail"
else:
- tailBin = "/usr/bin/tail"
+ tailBin = "tail"
self.p = reactor.spawnProcess(self.pp, tailBin,
("tail", "-f", "-n", "0", self.logfile),
env=os.environ,

95
krebs/3modules/buildbot/master.nix
View File

@ -3,14 +3,10 @@
with import <stockholm/lib>;
let
# https://github.com/NixOS/nixpkgs/issues/14026
nixpkgs-fix = import (pkgs.fetchgit {
url = https://github.com/nixos/nixpkgs;
rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632";
}) {};
buildbot = nixpkgs-fix.buildbot;
buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{
patches = [ ./buildbot.patch ];
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
});
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
# -*- python -*-
from buildbot.plugins import *
@ -18,11 +14,11 @@ let
import json
c = BuildmasterConfig = {}
c['slaves'] = []
slaves = json.loads('${builtins.toJSON cfg.slaves}')
slavenames = [ s for s in slaves ]
for k,v in slaves.items():
c['slaves'].append(buildslave.BuildSlave(k, v))
c['workers'] = []
workers = json.loads('${builtins.toJSON cfg.workers}')
workernames = [ s for s in workers ]
for k,v in workers.items():
c['workers'].append(worker.Worker(k, v))
# TODO: configure protocols?
c['protocols'] = {'pb': {'port': 9989}}
@ -63,32 +59,46 @@ let
####### Status
c['status'] = st = []
c['services'] = []
# If you want to configure this url, override with extraConfig
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
${optionalString (cfg.web.enable) ''
from buildbot.status import html
from buildbot.status.web import authz, auth
authz_cfg=authz.Authz(
auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
# TODO: configure harder
gracefulShutdown = False,
forceBuild = 'auth',
forceAllBuilds = 'auth',
pingBuilder = False,
stopBuild = 'auth',
stopAllBuilds = 'auth',
cancelPendingBuild = 'auth'
)
from buildbot.plugins import util
#authz_cfg=authz.Authz(
# auth=auth.BasicAuth([ ]),
# # TODO: configure harder
# gracefulShutdown = False,
# forceBuild = 'auth',
# forceAllBuilds = 'auth',
# pingBuilder = False,
# stopBuild = 'auth',
# stopAllBuilds = 'auth',
# cancelPendingBuild = 'auth'
#)
# TODO: configure krebs.nginx
st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
c['www'] = dict(
port = ${toString cfg.web.port},
plugins = { 'waterfall_view':{}, 'console_view':{} }
)
c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"})
c['www']['authz'] = util.Authz(
allowRules = [
util.StopBuildEndpointMatcher(role="admins"),
util.ForceBuildEndpointMatcher(role="admins"),
util.RebuildBuildEndpointMatcher(role="admins")
],
roleMatchers = [
util.RolesFromEmails(admins=["${cfg.web.username}"])
]
)
''}
${optionalString (cfg.irc.enable) ''
from buildbot.status import words
irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
from buildbot.plugins import reporters
irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
channels=${builtins.toJSON cfg.irc.channels},
notify_events={
'success': 1,
@ -97,7 +107,7 @@ let
'successToFailure': 1,
'failureToSuccess': 1,
}${optionalString cfg.irc.allowForce ",allowForce=True"})
c['status'].append(irc)
c['services'].append(irc)
''}
${ concatStringsSep "\n"
@ -150,12 +160,12 @@ let
'';
};
slaves = mkOption {
workers = mkOption {
default = {};
type = types.attrsOf types.str;
description = ''
Attrset of slavenames with their passwords
slavename = slavepassword
Attrset of workernames with their passwords
workername = workerpassword
'';
};
@ -283,8 +293,12 @@ let
options = {
enable = mkEnableOption "Buildbot Master IRC Status";
channels = mkOption {
default = [ "nix-buildbot-meetup" ];
type = with types; listOf str;
default = [ { channel = "nix-buildbot-meetup";} ];
example = literalExample ''[
{channel = "nix-buildbot-meetup";}
{channel = "nix-buildbot-lol"; "password" = "lol";}
]'';
type = with types; listOf (attrsOf str);
description = ''
irc channels the bot should connect to
'';
@ -333,7 +347,7 @@ let
};
users.extraGroups.buildbotMaster = {
gid = 672626386;
gid = genid "buildbotMaster";
};
systemd.services.buildbotMaster = {
@ -350,8 +364,6 @@ let
secretsdir = shell.escape (toString <secrets>);
in {
PermissionsStartOnly = true;
Type = "forking";
PIDFile = "${workdir}/twistd.pid";
# TODO: maybe also prepare buildbot.tac?
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
set -efux
@ -375,9 +387,8 @@ let
chmod 700 -R ${workdir}
chown buildbotMaster:buildbotMaster -R ${workdir}
'';
ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}";
# ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
PrivateTmp = "true";
User = "buildbotMaster";
Restart = "always";

101
krebs/3modules/buildbot/slave.nix
View File

@ -2,53 +2,21 @@
with import <stockholm/lib>;
let
buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
import os
from buildslave.bot import BuildSlave
from twisted.application import service
basedir = '${cfg.workDir}'
rotateLength = 10000000
maxRotatedFiles = 10
application = service.Application('buildslave')
from twisted.python.logfile import LogFile
from twisted.python.log import ILogObserver, FileLogObserver
logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
maxRotatedFiles=maxRotatedFiles)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
buildmaster_host = '${cfg.masterhost}'
# TODO: masterport?
port = 9989
slavename = '${cfg.username}'
passwd = '${cfg.password}'
keepalive = 600
usepty = 0
umask = None
maxdelay = 300
allow_shutdown = None
${cfg.extraConfig}
s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
keepalive, usepty, umask=umask, maxdelay=maxdelay,
allow_shutdown=allow_shutdown)
s.setServiceParent(application)
'';
default-packages = [ pkgs.git pkgs.bash ];
cfg = config.krebs.buildbot.slave;
buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{
patches = [ ./buildbot-worker.patch ];
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
});
cfg = config.krebs.buildbot.worker;
api = {
enable = mkEnableOption "Buildbot Slave";
enable = mkEnableOption "Buildbot worker";
workDir = mkOption {
default = "/var/lib/buildbot/slave";
default = "/var/lib/buildbot/worker";
type = types.str;
description = ''
Path to build bot slave directory.
Path to build bot worker directory.
Will be created on startup.
'';
};
@ -64,30 +32,30 @@ let
username = mkOption {
type = types.str;
description = ''
slavename used to authenticate with master
workername used to authenticate with master
'';
};
password = mkOption {
type = types.str;
description = ''
slave password used to authenticate with master
worker password used to authenticate with master
'';
};
contact = mkOption {
default = "nix slave <buildslave@${config.networking.hostName}>";
default = "nix worker <buildworker@${config.networking.hostName}>";
type = types.str;
description = ''
contact to be announced by buildslave
contact to be announced by buildworker
'';
};
description = mkOption {
default = "Nix Generated BuildSlave";
default = "Nix Generated Buildworker";
type = types.str;
description = ''
description for hostto be announced by buildslave
description for hostto be announced by buildworker
'';
};
@ -95,7 +63,7 @@ let
default = [ pkgs.git ];
type = with types; listOf package;
description = ''
packages which should be in path for buildslave
packages which should be in path for buildworker
'';
};
@ -106,7 +74,7 @@ let
};
type = types.attrsOf types.str;
description = ''
extra environment variables to be provided to the buildslave service
extra environment variables to be provided to the buildworker service
if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
'';
};
@ -119,26 +87,26 @@ let
keepalive = 600
'';
description = ''
extra config evaluated before calling BuildSlave init in .tac file
extra config evaluated before calling Buildworker init in .tac file
'';
};
};
imp = {
users.extraUsers.buildbotSlave = {
uid = genid "buildbotSlave";
description = "Buildbot Slave";
users.extraUsers.buildbotworker = {
uid = genid "buildbotworker";
description = "Buildbot worker";
home = cfg.workDir;
createHome = false;
};
users.extraGroups.buildbotSlave = {
gid = 1408105834;
users.extraGroups.buildbotworker = {
gid = genid "buildbotworker";
};
systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = {
description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = default-packages ++ cfg.packages;
@ -152,27 +120,28 @@ let
workdir = shell.escape cfg.workDir;
contact = shell.escape cfg.contact;
description = shell.escape cfg.description;
buildbot = pkgs.buildbot-slave;
# TODO:make this
masterhost = shell.escape cfg.masterhost;
username = shell.escape cfg.username;
password = shell.escape cfg.password;
in {
PermissionsStartOnly = true;
Type = "forking";
PIDFile = "${workdir}/twistd.pid";
# TODO: maybe also prepare buildbot.tac?
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
ExecStartPre = pkgs.writeDash "buildbot-slave-init" ''
set -efux
mkdir -p ${workdir}/info
cp ${buildbot-slave-init} ${workdir}/buildbot.tac
# TODO: cleanup .tac file?
${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password}
echo ${contact} > ${workdir}/info/admin
echo ${description} > ${workdir}/info/host
chown buildbotSlave:buildbotSlave -R ${workdir}
chown buildbotworker:buildbotworker -R ${workdir}
chmod 700 -R ${workdir}
'';
ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}";
ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}";
PrivateTmp = "true";
User = "buildbotSlave";
User = "buildbotworker";
Restart = "always";
RestartSec = "10";
};
@ -180,6 +149,6 @@ let
};
in
{
options.krebs.buildbot.slave = api;
options.krebs.buildbot.worker = api;
config = lib.mkIf cfg.enable imp;
}

2
lass/2configs/buildbot-standalone.nix
View File

@ -213,7 +213,7 @@ in {
enable = true;
nick = "buildbot-lass";
server = "cd.retiolum";
channels = [ "retiolum" ];
channels = [ { channels = "retiolum"; } ];
allowForce = true;
};
};

23
shared/2configs/shared-buildbot.nix
View File

@ -11,14 +11,13 @@
# /nix/store should be cleaned up automatically as well
nix.gc.automatic = true;
nix.gc.dates = "05:23";
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = let
stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ;
in {
secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
slaves = {
testslave = "krebspass";
workers = {
testworker = "krebspass";
};
change_source.stockholm = ''
stockholm_repo = '${stockholm-mirror-url}'
@ -109,7 +108,7 @@
system={}".format(i)])
bu.append(util.BuilderConfig(name="fast-tests",
slavenames=slavenames,
workernames=workernames,
factory=f))
'';
@ -130,25 +129,25 @@
system={}".format(i)])
bu.append(util.BuilderConfig(name="build-local",
slavenames=slavenames,
workernames=workernames,
factory=f))
'';
# slow-tests = ''
# s = util.BuildFactory()
# s.addStep(grab_repo)
#
# # slave needs 2 files:
# # worker needs 2 files:
# # * cac.json
# # * retiolum
# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", workerdest="cac.json"))
# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", workerdest="retiolum.rsa_key.priv"))
# addShell(s, name="infest-cac-centos7",env=env,
# sigtermTime=60, # SIGTERM 1 minute before SIGKILL
# timeout=10800, # 3h
# command=nixshell + ["infest-cac-centos7"])
#
# bu.append(util.BuilderConfig(name="full-tests",
# slavenames=slavenames,
# workernames=workernames,
# factory=s))
# '';
};
@ -160,15 +159,15 @@
enable = true;
nick = "wolfbot";
server = "cd.retiolum";
channels = [ "retiolum" ];
channels = [ { channel = "retiolum"; } ];
allowForce = true;
};
};
krebs.buildbot.slave = {
krebs.buildbot.worker = {
enable = true;
masterhost = "localhost";
username = "testslave";
username = "testworker";
password = "krebspass";
packages = with pkgs; [ gnumake jq nix populate ];
# all nix commands will need a working nixpkgs installation