ci: buildbot-classic -> buildbot; cleanup
This commit is contained in:
lassulus
parent
8ce4141683
commit
b48f08ea8e
@ -6,11 +6,13 @@
|
|||||||
enable = true;
|
enable = true;
|
||||||
virtualHosts.build = {
|
virtualHosts.build = {
|
||||||
serverAliases = [ "build.r" "build.${config.networking.hostName}.r" ];
|
serverAliases = [ "build.r" "build.${config.networking.hostName}.r" ];
|
||||||
locations."/".extraConfig = ''
|
locations."/" = {
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
proxyPass = "http://127.0.0.1:${toString config.services.buildbot-master.port}";
|
||||||
proxy_set_header Connection "upgrade";
|
proxyWebsockets = true;
|
||||||
proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
|
extraConfig = ''
|
||||||
'';
|
proxy_read_timeout 3600s;
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
krebs.ci = {
|
krebs.ci = {
|
||||||
@ -18,25 +20,20 @@
|
|||||||
repos = {
|
repos = {
|
||||||
disko.urls = [
|
disko.urls = [
|
||||||
"http://cgit.gum.r/disko"
|
"http://cgit.gum.r/disko"
|
||||||
"http://cgit.hotdog.r/disko"
|
|
||||||
"http://cgit.ni.r/disko"
|
"http://cgit.ni.r/disko"
|
||||||
"http://cgit.prism.r/disko"
|
"http://cgit.prism.r/disko"
|
||||||
];
|
];
|
||||||
krops.urls = [
|
krops.urls = [
|
||||||
"http://cgit.hotdog.r/krops"
|
|
||||||
"http://cgit.ni.r/krops"
|
"http://cgit.ni.r/krops"
|
||||||
"http://cgit.prism.r/krops"
|
"http://cgit.prism.r/krops"
|
||||||
"https://github.com/krebs/krops.git"
|
"https://github.com/krebs/krops.git"
|
||||||
];
|
];
|
||||||
nix_writers.urls = [
|
nix_writers.urls = [
|
||||||
"http://cgit.hotdog.r/nix-writers"
|
|
||||||
"http://cgit.ni.r/nix-writers"
|
"http://cgit.ni.r/nix-writers"
|
||||||
"http://cgit.prism.r/nix-writers"
|
"http://cgit.prism.r/nix-writers"
|
||||||
];
|
];
|
||||||
stockholm.urls = [
|
stockholm.urls = [
|
||||||
"http://cgit.enklave.r/stockholm"
|
|
||||||
"http://cgit.gum.r/stockholm"
|
"http://cgit.gum.r/stockholm"
|
||||||
"http://cgit.hotdog.r/stockholm"
|
|
||||||
"http://cgit.ni.r/stockholm"
|
"http://cgit.ni.r/stockholm"
|
||||||
"http://cgit.prism.r/stockholm"
|
"http://cgit.prism.r/stockholm"
|
||||||
];
|
];
|
||||||
|
|||||||
@ -1,382 +0,0 @@
|
|||||||
{ config, pkgs, lib, ... }:
|
|
||||||
|
|
||||||
with import <stockholm/lib>;
|
|
||||||
let
|
|
||||||
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
|
|
||||||
# -*- python -*-
|
|
||||||
from buildbot.plugins import *
|
|
||||||
import re
|
|
||||||
import json
|
|
||||||
c = BuildmasterConfig = {}
|
|
||||||
|
|
||||||
c['slaves'] = []
|
|
||||||
slaves = json.loads('${builtins.toJSON cfg.slaves}')
|
|
||||||
slavenames = [ s for s in slaves ]
|
|
||||||
for k,v in slaves.items():
|
|
||||||
c['slaves'].append(buildslave.BuildSlave(k, v))
|
|
||||||
|
|
||||||
# TODO: configure protocols?
|
|
||||||
c['protocols'] = {'pb': {'port': 9989}}
|
|
||||||
|
|
||||||
####### Build Inputs
|
|
||||||
c['change_source'] = cs = []
|
|
||||||
|
|
||||||
${ concatStringsSep "\n"
|
|
||||||
(mapAttrsToList (n: v: ''
|
|
||||||
#### Change_Source: Begin of ${n}
|
|
||||||
${v}
|
|
||||||
#### Change_Source: End of ${n}
|
|
||||||
'') cfg.change_source )}
|
|
||||||
|
|
||||||
####### Build Scheduler
|
|
||||||
c['schedulers'] = sched = []
|
|
||||||
|
|
||||||
${ concatStringsSep "\n"
|
|
||||||
(mapAttrsToList (n: v: ''
|
|
||||||
#### Schedulers: Begin of ${n}
|
|
||||||
${v}
|
|
||||||
#### Schedulers: End of ${n}
|
|
||||||
'') cfg.scheduler )}
|
|
||||||
|
|
||||||
###### Builder
|
|
||||||
c['builders'] = bu = []
|
|
||||||
|
|
||||||
# Builder Pre: Begin
|
|
||||||
${cfg.builder_pre}
|
|
||||||
# Builder Pre: End
|
|
||||||
|
|
||||||
${ concatStringsSep "\n"
|
|
||||||
(mapAttrsToList (n: v: ''
|
|
||||||
#### Builder: Begin of ${n}
|
|
||||||
${v}
|
|
||||||
#### Builder: End of ${n}
|
|
||||||
'') cfg.builder )}
|
|
||||||
|
|
||||||
|
|
||||||
####### Status
|
|
||||||
c['status'] = st = []
|
|
||||||
|
|
||||||
# If you want to configure this url, override with extraConfig
|
|
||||||
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
|
|
||||||
|
|
||||||
${optionalString (cfg.web.enable) ''
|
|
||||||
from buildbot.status import html
|
|
||||||
from buildbot.status.web import authz, auth
|
|
||||||
authz_cfg=authz.Authz(
|
|
||||||
auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
|
|
||||||
# TODO: configure harder
|
|
||||||
gracefulShutdown = False,
|
|
||||||
forceBuild = 'auth',
|
|
||||||
forceAllBuilds = 'auth',
|
|
||||||
pingBuilder = False,
|
|
||||||
stopBuild = 'auth',
|
|
||||||
stopAllBuilds = 'auth',
|
|
||||||
cancelPendingBuild = 'auth'
|
|
||||||
)
|
|
||||||
# TODO: configure krebs.nginx
|
|
||||||
st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
|
|
||||||
''}
|
|
||||||
|
|
||||||
${optionalString (cfg.irc.enable) ''
|
|
||||||
from buildbot.status import words
|
|
||||||
irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
|
|
||||||
channels=${builtins.toJSON cfg.irc.channels},
|
|
||||||
notify_events={
|
|
||||||
'started': 1,
|
|
||||||
'success': 1,
|
|
||||||
'failure': 1,
|
|
||||||
'exception': 1,
|
|
||||||
'successToFailure': 1,
|
|
||||||
'failureToSuccess': 1,
|
|
||||||
}${optionalString cfg.irc.allowForce ",allowForce=True"})
|
|
||||||
c['status'].append(irc)
|
|
||||||
''}
|
|
||||||
|
|
||||||
${ concatStringsSep "\n"
|
|
||||||
(mapAttrsToList (n: v: ''
|
|
||||||
#### Status: Begin of ${n}
|
|
||||||
${v}
|
|
||||||
#### Status: End of ${n}
|
|
||||||
'') cfg.status )}
|
|
||||||
|
|
||||||
####### PROJECT IDENTITY
|
|
||||||
c['title'] = "${cfg.title}"
|
|
||||||
c['titleURL'] = "http://krebsco.de"
|
|
||||||
|
|
||||||
|
|
||||||
####### DB URL
|
|
||||||
# TODO: configure
|
|
||||||
c['db'] = {
|
|
||||||
'db_url' : "sqlite:///state.sqlite",
|
|
||||||
}
|
|
||||||
${cfg.extraConfig}
|
|
||||||
'';
|
|
||||||
|
|
||||||
cfg = config.krebs.buildbot.master;
|
|
||||||
|
|
||||||
api = {
|
|
||||||
enable = mkEnableOption "Buildbot Master";
|
|
||||||
title = mkOption {
|
|
||||||
default = "Buildbot CI";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
Title of the Buildbot Installation
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
workDir = mkOption {
|
|
||||||
default = "/var/lib/buildbot/master";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
Path to build bot master directory.
|
|
||||||
Will be created on startup.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
secrets = mkOption {
|
|
||||||
default = [];
|
|
||||||
type = types.listOf types.str;
|
|
||||||
example = [ "cac.json" ];
|
|
||||||
description = ''
|
|
||||||
List of all the secrets in ‹secrets› which should be copied into the
|
|
||||||
buildbot master directory.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
slaves = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
description = ''
|
|
||||||
Attrset of slavenames with their passwords
|
|
||||||
slavename = slavepassword
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
change_source = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
example = {
|
|
||||||
stockholm = ''
|
|
||||||
cs.append(changes.GitPoller(
|
|
||||||
'http://cgit.gum/stockholm',
|
|
||||||
workdir='stockholm-poller', branch='master',
|
|
||||||
project='stockholm',
|
|
||||||
pollinterval=120))
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
description = ''
|
|
||||||
Attrset of all the change_sources which should be configured.
|
|
||||||
It will be directly included into the master configuration.
|
|
||||||
|
|
||||||
At the end an change object should be appended to <literal>cs</literal>
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
scheduler = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
example = {
|
|
||||||
force-scheduler = ''
|
|
||||||
sched.append(schedulers.ForceScheduler(
|
|
||||||
name="force",
|
|
||||||
builderNames=["full-tests"]))
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
description = ''
|
|
||||||
Attrset of all the schedulers which should be configured.
|
|
||||||
It will be directly included into the master configuration.
|
|
||||||
|
|
||||||
At the end an change object should be appended to <literal>sched</literal>
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
builder_pre = mkOption {
|
|
||||||
default = "";
|
|
||||||
type = types.lines;
|
|
||||||
example = ''
|
|
||||||
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
|
|
||||||
'';
|
|
||||||
description = ''
|
|
||||||
some code before the builders are being assembled.
|
|
||||||
can be used to define functions used by multiple builders
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
builder = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
example = {
|
|
||||||
fast-test = ''
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
description = ''
|
|
||||||
Attrset of all the builder which should be configured.
|
|
||||||
It will be directly included into the master configuration.
|
|
||||||
|
|
||||||
At the end an change object should be appended to <literal>bu</literal>
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
status = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
description = ''
|
|
||||||
Attrset of all the extra status which should be configured.
|
|
||||||
It will be directly included into the master configuration.
|
|
||||||
|
|
||||||
At the end an change object should be appended to <literal>st</literal>
|
|
||||||
|
|
||||||
Right now IRC and Web status can be configured by setting
|
|
||||||
<literal>buildbot.master.irc.enable</literal> and
|
|
||||||
<literal>buildbot.master.web.enable</literal>
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
# Configurable Stati
|
|
||||||
web = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.submodule ({ config2, ... }: {
|
|
||||||
options = {
|
|
||||||
enable = mkEnableOption "Buildbot Master Web Status";
|
|
||||||
username = mkOption {
|
|
||||||
default = "krebs";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
username for web authentication
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
hostname = mkOption {
|
|
||||||
default = config.networking.hostName;
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
web interface Hostname
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
password = mkOption {
|
|
||||||
default = "bob";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
password for web authentication
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
port = mkOption {
|
|
||||||
default = 8010;
|
|
||||||
type = types.int;
|
|
||||||
description = ''
|
|
||||||
port for buildbot web status
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
irc = mkOption {
|
|
||||||
default = {};
|
|
||||||
type = types.submodule ({ config, ... }: {
|
|
||||||
options = {
|
|
||||||
enable = mkEnableOption "Buildbot Master IRC Status";
|
|
||||||
channels = mkOption {
|
|
||||||
default = [ "nix-buildbot-meetup" ];
|
|
||||||
type = with types; listOf str;
|
|
||||||
description = ''
|
|
||||||
irc channels the bot should connect to
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
allowForce = mkOption {
|
|
||||||
default = false;
|
|
||||||
type = types.bool;
|
|
||||||
description = ''
|
|
||||||
Determines if builds can be forced via IRC
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
nick = mkOption {
|
|
||||||
default = "nix-buildbot";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
nickname for IRC
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
server = mkOption {
|
|
||||||
default = "irc.freenode.net";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
Buildbot Status IRC Server to connect to
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
extraConfig = mkOption {
|
|
||||||
default = "";
|
|
||||||
type = types.lines;
|
|
||||||
description = ''
|
|
||||||
extra config appended to the generated master.cfg
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
imp = {
|
|
||||||
|
|
||||||
users.extraUsers.buildbotMaster = {
|
|
||||||
uid = genid "buildbotMaster";
|
|
||||||
group = "buildbotMaster";
|
|
||||||
description = "Buildbot Master";
|
|
||||||
home = cfg.workDir;
|
|
||||||
createHome = false;
|
|
||||||
isSystemUser = true;
|
|
||||||
};
|
|
||||||
|
|
||||||
users.extraGroups.buildbotMaster = {
|
|
||||||
gid = 672626386;
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services.buildbotMaster = {
|
|
||||||
description = "Buildbot Master";
|
|
||||||
after = [ "network.target" ];
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
# TODO: add extra dependencies to master like svn and cvs
|
|
||||||
path = [ pkgs.git ];
|
|
||||||
environment = {
|
|
||||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
|
||||||
};
|
|
||||||
serviceConfig = let
|
|
||||||
workdir = shell.escape cfg.workDir;
|
|
||||||
secretsdir = shell.escape (toString <secrets>);
|
|
||||||
in {
|
|
||||||
PermissionsStartOnly = true;
|
|
||||||
# TODO: maybe also prepare buildbot.tac?
|
|
||||||
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
|
|
||||||
set -efux
|
|
||||||
if [ ! -e ${workdir} ];then
|
|
||||||
mkdir -p ${workdir}
|
|
||||||
${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir}
|
|
||||||
fi
|
|
||||||
# always override the master.cfg
|
|
||||||
cp ${buildbot-master-config} ${workdir}/master.cfg
|
|
||||||
|
|
||||||
# copy secrets
|
|
||||||
${ concatMapStringsSep "\n"
|
|
||||||
(f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
|
|
||||||
# sanity
|
|
||||||
${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir}
|
|
||||||
|
|
||||||
# TODO: maybe upgrade? not sure about this
|
|
||||||
# normally we should write buildbot.tac by our own
|
|
||||||
# ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir}
|
|
||||||
|
|
||||||
chmod 700 ${workdir}
|
|
||||||
chown buildbotMaster:buildbotMaster -R ${workdir}
|
|
||||||
'';
|
|
||||||
ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start --nodaemon ${workdir}";
|
|
||||||
PrivateTmp = "true";
|
|
||||||
User = "buildbotMaster";
|
|
||||||
Restart = "always";
|
|
||||||
RestartSec = "10";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.krebs.buildbot.master = api;
|
|
||||||
config = lib.mkIf cfg.enable imp;
|
|
||||||
}
|
|
||||||
@ -1,186 +0,0 @@
|
|||||||
{ config, pkgs, lib, ... }:
|
|
||||||
|
|
||||||
with import <stockholm/lib>;
|
|
||||||
let
|
|
||||||
buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
|
|
||||||
import os
|
|
||||||
|
|
||||||
from buildslave.bot import BuildSlave
|
|
||||||
from twisted.application import service
|
|
||||||
|
|
||||||
basedir = '${cfg.workDir}'
|
|
||||||
rotateLength = 10000000
|
|
||||||
maxRotatedFiles = 10
|
|
||||||
|
|
||||||
application = service.Application('buildslave')
|
|
||||||
|
|
||||||
from twisted.python.logfile import LogFile
|
|
||||||
from twisted.python.log import ILogObserver, FileLogObserver
|
|
||||||
logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
|
|
||||||
maxRotatedFiles=maxRotatedFiles)
|
|
||||||
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
|
|
||||||
|
|
||||||
buildmaster_host = '${cfg.masterhost}'
|
|
||||||
# TODO: masterport?
|
|
||||||
port = 9989
|
|
||||||
slavename = '${cfg.username}'
|
|
||||||
passwd = '${cfg.password}'
|
|
||||||
keepalive = 600
|
|
||||||
usepty = 0
|
|
||||||
umask = None
|
|
||||||
maxdelay = 300
|
|
||||||
allow_shutdown = None
|
|
||||||
|
|
||||||
${cfg.extraConfig}
|
|
||||||
|
|
||||||
s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
|
|
||||||
keepalive, usepty, umask=umask, maxdelay=maxdelay,
|
|
||||||
allow_shutdown=allow_shutdown)
|
|
||||||
s.setServiceParent(application)
|
|
||||||
'';
|
|
||||||
default-packages = [ pkgs.git pkgs.bash ];
|
|
||||||
cfg = config.krebs.buildbot.slave;
|
|
||||||
|
|
||||||
api = {
|
|
||||||
enable = mkEnableOption "Buildbot Slave";
|
|
||||||
|
|
||||||
workDir = mkOption {
|
|
||||||
default = "/var/lib/buildbot/slave";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
Path to build bot slave directory.
|
|
||||||
Will be created on startup.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
masterhost = mkOption {
|
|
||||||
default = "localhost";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
Hostname/IP of the buildbot master
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
username = mkOption {
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
slavename used to authenticate with master
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
password = mkOption {
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
slave password used to authenticate with master
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
contact = mkOption {
|
|
||||||
default = "nix slave <buildslave@${config.networking.hostName}>";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
contact to be announced by buildslave
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
description = mkOption {
|
|
||||||
default = "Nix Generated BuildSlave";
|
|
||||||
type = types.str;
|
|
||||||
description = ''
|
|
||||||
description for hostto be announced by buildslave
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
packages = mkOption {
|
|
||||||
default = [ pkgs.git ];
|
|
||||||
type = with types; listOf package;
|
|
||||||
description = ''
|
|
||||||
packages which should be in path for buildslave
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
extraEnviron = mkOption {
|
|
||||||
default = {};
|
|
||||||
example = {
|
|
||||||
NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
|
|
||||||
};
|
|
||||||
type = types.attrsOf types.str;
|
|
||||||
description = ''
|
|
||||||
extra environment variables to be provided to the buildslave service
|
|
||||||
if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
extraConfig = mkOption {
|
|
||||||
default = "";
|
|
||||||
type = types.lines;
|
|
||||||
example = ''
|
|
||||||
port = 443
|
|
||||||
keepalive = 600
|
|
||||||
'';
|
|
||||||
description = ''
|
|
||||||
extra config evaluated before calling BuildSlave init in .tac file
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
imp = {
|
|
||||||
|
|
||||||
users.extraUsers.buildbotSlave = {
|
|
||||||
uid = genid "buildbotSlave";
|
|
||||||
group = "buildbotSlave";
|
|
||||||
description = "Buildbot Slave";
|
|
||||||
home = cfg.workDir;
|
|
||||||
createHome = false;
|
|
||||||
isSystemUser = true;
|
|
||||||
};
|
|
||||||
|
|
||||||
users.extraGroups.buildbotSlave = {
|
|
||||||
gid = 1408105834;
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
|
|
||||||
description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
|
|
||||||
after = [ "network.target" ];
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
path = default-packages ++ cfg.packages;
|
|
||||||
|
|
||||||
environment = {
|
|
||||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
|
||||||
NIX_REMOTE="daemon";
|
|
||||||
} // cfg.extraEnviron;
|
|
||||||
|
|
||||||
serviceConfig = let
|
|
||||||
workdir = shell.escape cfg.workDir;
|
|
||||||
contact = shell.escape cfg.contact;
|
|
||||||
description = shell.escape cfg.description;
|
|
||||||
# TODO:make this
|
|
||||||
in {
|
|
||||||
PermissionsStartOnly = true;
|
|
||||||
Type = "forking";
|
|
||||||
PIDFile = "${workdir}/twistd.pid";
|
|
||||||
# TODO: maybe also prepare buildbot.tac?
|
|
||||||
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
|
|
||||||
set -efux
|
|
||||||
mkdir -p ${workdir}/info
|
|
||||||
cp ${buildbot-slave-init} ${workdir}/buildbot.tac
|
|
||||||
echo ${contact} > ${workdir}/info/admin
|
|
||||||
echo ${description} > ${workdir}/info/host
|
|
||||||
|
|
||||||
chown buildbotSlave:buildbotSlave -R ${workdir}
|
|
||||||
chmod 700 ${workdir}
|
|
||||||
'';
|
|
||||||
ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}";
|
|
||||||
ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}";
|
|
||||||
PrivateTmp = "true";
|
|
||||||
User = "buildbotSlave";
|
|
||||||
Restart = "always";
|
|
||||||
RestartSec = "10";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.krebs.buildbot.slave = api;
|
|
||||||
config = lib.mkIf cfg.enable imp;
|
|
||||||
}
|
|
||||||
@ -39,148 +39,131 @@ let
|
|||||||
|
|
||||||
profileRoot = "/nix/var/nix/profiles/ci";
|
profileRoot = "/nix/var/nix/profiles/ci";
|
||||||
|
|
||||||
|
bcfg = config.services.buildbot-master;
|
||||||
|
|
||||||
imp = {
|
imp = {
|
||||||
krebs.buildbot.master = {
|
services.buildbot-master = {
|
||||||
slaves = {
|
workers = [ "worker.Worker('testworker', 'pass')" ];
|
||||||
testslave = "lasspass";
|
|
||||||
};
|
|
||||||
|
|
||||||
change_source = mapAttrs' (name: repo:
|
changeSource = mapAttrsToList (name: repo:
|
||||||
nameValuePair name (concatMapStrings (url: ''
|
concatMapStringsSep "," (url: ''
|
||||||
cs.append(
|
changes.GitPoller(
|
||||||
changes.GitPoller(
|
"${url}",
|
||||||
"${url}",
|
workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
|
||||||
workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
|
project='${name}',
|
||||||
project='${name}',
|
pollinterval=100
|
||||||
pollinterval=100
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
'') repo.urls)
|
'') repo.urls
|
||||||
) cfg.repos;
|
) cfg.repos;
|
||||||
|
|
||||||
scheduler = mapAttrs' (name: repo:
|
schedulers = mapAttrsToList (name: repo: ''
|
||||||
nameValuePair name ''
|
schedulers.SingleBranchScheduler(
|
||||||
sched.append(
|
change_filter=util.ChangeFilter(
|
||||||
schedulers.SingleBranchScheduler(
|
branch_re=".*",
|
||||||
change_filter=util.ChangeFilter(
|
project='${name}',
|
||||||
branch_re=".*",
|
),
|
||||||
project='${name}',
|
treeStableTimer=60,
|
||||||
),
|
name="${name}-all-branches",
|
||||||
treeStableTimer=60,
|
builderNames=[
|
||||||
name="${name}-all-branches",
|
"${name}",
|
||||||
builderNames=[
|
]
|
||||||
"${name}",
|
),
|
||||||
]
|
schedulers.ForceScheduler(
|
||||||
)
|
name="${name}",
|
||||||
)
|
builderNames=[
|
||||||
sched.append(
|
"${name}",
|
||||||
schedulers.ForceScheduler(
|
]
|
||||||
name="${name}",
|
)
|
||||||
builderNames=[
|
'') cfg.repos;
|
||||||
"${name}",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
''
|
|
||||||
) cfg.repos;
|
|
||||||
builder_pre = ''
|
|
||||||
from buildbot import interfaces
|
|
||||||
from buildbot.steps.shell import ShellCommand
|
|
||||||
|
|
||||||
class StepToStartMoreSteps(ShellCommand):
|
builders = [];
|
||||||
|
|
||||||
|
extraConfig = ''
|
||||||
|
# https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
|
||||||
|
from buildbot.plugins import util, steps
|
||||||
|
from buildbot.process import buildstep, logobserver
|
||||||
|
from twisted.internet import defer
|
||||||
|
import json
|
||||||
|
|
||||||
|
class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
ShellCommand.__init__(self, **kwargs)
|
kwargs = self.setupShellMixin(kwargs)
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
self.observer = logobserver.BufferLogObserver()
|
||||||
|
self.addLogObserver('stdio', self.observer)
|
||||||
|
|
||||||
def addBuildSteps(self, steps_factories):
|
def extract_stages(self, stdout):
|
||||||
for sf in steps_factories:
|
stages = json.loads(stdout)
|
||||||
step = interfaces.IBuildStepFactory(sf).buildStep()
|
return stages
|
||||||
step.setBuild(self.build)
|
|
||||||
step.setBuildSlave(self.build.slavebuilder.slave)
|
|
||||||
step_status = self.build.build_status.addStepWithName(step.name)
|
|
||||||
step.setStepStatus(step_status)
|
|
||||||
self.build.steps.append(step)
|
|
||||||
|
|
||||||
def start(self):
|
@defer.inlineCallbacks
|
||||||
props = self.build.getProperties()
|
def run(self):
|
||||||
new_steps = json.loads(props.getProperty('steps_json'))
|
# run nix-instanstiate to generate the dict of stages
|
||||||
for new_step in new_steps:
|
cmd = yield self.makeRemoteShellCommand()
|
||||||
self.addBuildSteps([steps.ShellCommand(
|
yield self.runCommand(cmd)
|
||||||
name=str(new_step),
|
|
||||||
command=[
|
|
||||||
"${pkgs.writeDash "build-stepper.sh" ''
|
|
||||||
set -xefu
|
|
||||||
profile=${shell.escape profileRoot}/$build_name
|
|
||||||
result=$("$build_script")
|
|
||||||
if [ -n "$result" ]; then
|
|
||||||
${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
|
|
||||||
fi
|
|
||||||
''}"
|
|
||||||
],
|
|
||||||
env={
|
|
||||||
"build_name": new_step,
|
|
||||||
"build_script": new_steps[new_step],
|
|
||||||
"NIX_REMOTE": "daemon",
|
|
||||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
|
||||||
},
|
|
||||||
timeout=90001,
|
|
||||||
workdir='build', # TODO figure out why we need this?
|
|
||||||
)])
|
|
||||||
|
|
||||||
ShellCommand.start(self)
|
# if the command passes extract the list of stages
|
||||||
|
result = cmd.results()
|
||||||
|
if result == util.SUCCESS:
|
||||||
|
# create a ShellCommand for each stage and add them to the build
|
||||||
|
stages = self.extract_stages(self.observer.getStdout())
|
||||||
|
self.build.addStepsAfterCurrentStep([
|
||||||
|
steps.ShellCommand(name=stage, command=[stages[stage]])
|
||||||
|
for stage in stages
|
||||||
|
])
|
||||||
|
|
||||||
'';
|
return result
|
||||||
|
|
||||||
builder = mapAttrs' (name: repo:
|
|
||||||
nameValuePair name ''
|
${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
|
||||||
f_${name} = util.BuildFactory()
|
factory_${name} = util.BuildFactory()
|
||||||
f_${name}.addStep(steps.Git(
|
factory_${name}.addStep(steps.Git(
|
||||||
repourl=util.Property('repository', '${head repo.urls}'),
|
repourl=util.Property('repository', '${head repo.urls}'),
|
||||||
method='clobber',
|
method='clobber',
|
||||||
mode='full',
|
mode='full',
|
||||||
submodules=True,
|
submodules=True,
|
||||||
))
|
))
|
||||||
|
|
||||||
f_${name}.addStep(steps.SetPropertyFromCommand(
|
factory_${name}.addStep(GenerateStagesCommand(
|
||||||
env={
|
env={
|
||||||
"NIX_REMOTE": "daemon",
|
"NIX_REMOTE": "daemon",
|
||||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||||
},
|
},
|
||||||
name="get_steps",
|
name="Generate build stages",
|
||||||
command=["${getJobs}"],
|
command=[
|
||||||
extract_fn=lambda rc, stdout, stderr: { 'steps_json': stdout },
|
"${getJobs}"
|
||||||
|
],
|
||||||
|
haltOnFailure=True,
|
||||||
))
|
))
|
||||||
f_${name}.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
|
|
||||||
|
|
||||||
bu.append(
|
c['builders'].append(
|
||||||
util.BuilderConfig(
|
util.BuilderConfig(
|
||||||
name="${name}",
|
name="${name}",
|
||||||
slavenames=slavenames,
|
workernames=['testworker'],
|
||||||
factory=f_${name}
|
factory=factory_${name}
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
''
|
'') cfg.repos)}
|
||||||
) cfg.repos;
|
'';
|
||||||
|
|
||||||
enable = true;
|
enable = true;
|
||||||
web.enable = true;
|
reporters = [''
|
||||||
irc = {
|
reporters.IRC(
|
||||||
enable = true;
|
host = "irc.r",
|
||||||
nick = "build|${hostname}";
|
nick = "buildbot|${hostname}",
|
||||||
server = "irc.r";
|
notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
|
||||||
channels = [ "xxx" "noise" ];
|
channels = [{"channel": "#xxx"}],
|
||||||
allowForce = true;
|
)
|
||||||
};
|
''];
|
||||||
extraConfig = ''
|
|
||||||
c['buildbotURL'] = "http://build.${hostname}.r/"
|
buildbotUrl = "http://build.${hostname}.r/";
|
||||||
'';
|
|
||||||
};
|
};
|
||||||
|
|
||||||
krebs.buildbot.slave = {
|
services.buildbot-worker = {
|
||||||
enable = true;
|
enable = true;
|
||||||
masterhost = "localhost";
|
workerUser = "testworker";
|
||||||
username = "testslave";
|
workerPass = "pass";
|
||||||
password = "lasspass";
|
packages = with pkgs; [ git gnutar gzip jq nix populate ];
|
||||||
packages = with pkgs; [ gnumake jq nix populate gnutar lzma gzip ];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
system.activationScripts.buildbots-nix-profile = ''
|
system.activationScripts.buildbots-nix-profile = ''
|
||||||
@ -192,11 +175,10 @@ let
|
|||||||
users = {
|
users = {
|
||||||
groups.buildbots.gid = genid "buildbots";
|
groups.buildbots.gid = genid "buildbots";
|
||||||
users = {
|
users = {
|
||||||
buildbotMaster.extraGroups = [ "buildbots" ];
|
buildbot.extraGroups = [ "buildbots" ];
|
||||||
buildbotSlave.extraGroups = [ "buildbots" ];
|
bbworker.extraGroups = [ "buildbots" ];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
in out
|
in out
|
||||||
|
|
||||||
|
|||||||
@ -13,8 +13,6 @@ let
|
|||||||
./bepasty-server.nix
|
./bepasty-server.nix
|
||||||
./bindfs.nix
|
./bindfs.nix
|
||||||
./brockman.nix
|
./brockman.nix
|
||||||
./buildbot/master.nix
|
|
||||||
./buildbot/slave.nix
|
|
||||||
./build.nix
|
./build.nix
|
||||||
./cachecache.nix
|
./cachecache.nix
|
||||||
./ci.nix
|
./ci.nix
|
||||||
|
|||||||
@ -1,34 +0,0 @@
|
|||||||
{ pkgs, fetchFromGitHub, python2Packages, git, ... }: let
|
|
||||||
|
|
||||||
# we need the old sqlparse since the new one is python2 incompatible
|
|
||||||
sqlparse = python2Packages.callPackage ./sqlparse.nix {};
|
|
||||||
|
|
||||||
in python2Packages.buildPythonApplication rec {
|
|
||||||
name = "buildbot-classic-${version}";
|
|
||||||
version = "0.8.18";
|
|
||||||
namePrefix = "";
|
|
||||||
patches = [];
|
|
||||||
|
|
||||||
src = fetchFromGitHub {
|
|
||||||
owner = "krebs";
|
|
||||||
repo = "buildbot-classic";
|
|
||||||
rev = version;
|
|
||||||
sha256 = "0b4y3n9zd2gdy8xwk1vpvs4n9fbg72vi8mx4ydgijwngcmdqkjmq";
|
|
||||||
};
|
|
||||||
postUnpack = "sourceRoot=\${sourceRoot}/master";
|
|
||||||
|
|
||||||
propagatedBuildInputs = [
|
|
||||||
python2Packages.jinja2
|
|
||||||
python2Packages.twisted
|
|
||||||
python2Packages.dateutil
|
|
||||||
(python2Packages.sqlalchemy_migrate.override { sqlparse = sqlparse; })
|
|
||||||
python2Packages.pysqlite
|
|
||||||
pkgs.coreutils
|
|
||||||
];
|
|
||||||
doCheck = false;
|
|
||||||
postInstall = ''
|
|
||||||
mkdir -p "$out/share/man/man1"
|
|
||||||
cp docs/buildbot.1 "$out/share/man/man1"
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
{ lib
|
|
||||||
, buildPythonPackage
|
|
||||||
, fetchPypi
|
|
||||||
, pytest
|
|
||||||
, isPy3k
|
|
||||||
}:
|
|
||||||
|
|
||||||
buildPythonPackage rec {
|
|
||||||
pname = "sqlparse";
|
|
||||||
version = "0.3.1";
|
|
||||||
|
|
||||||
src = fetchPypi {
|
|
||||||
inherit pname version;
|
|
||||||
sha256 = "e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548";
|
|
||||||
};
|
|
||||||
|
|
||||||
checkInputs = [ pytest ];
|
|
||||||
checkPhase = ''
|
|
||||||
py.test
|
|
||||||
'';
|
|
||||||
|
|
||||||
# Package supports 3.x, but tests are clearly 2.x only.
|
|
||||||
doCheck = !isPy3k;
|
|
||||||
|
|
||||||
meta = with lib; {
|
|
||||||
description = "Non-validating SQL parser for Python";
|
|
||||||
longDescription = ''
|
|
||||||
Provides support for parsing, splitting and formatting SQL statements.
|
|
||||||
'';
|
|
||||||
homepage = "https://github.com/andialbrecht/sqlparse";
|
|
||||||
license = licenses.bsd3;
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user