ci: buildbot-classic -> buildbot; cleanup
This commit is contained in:
lassulus
parent
8ce4141683
commit
b48f08ea8e
|
|
@ -6,11 +6,13 @@
|
|||
enable = true;
|
||||
virtualHosts.build = {
|
||||
serverAliases = [ "build.r" "build.${config.networking.hostName}.r" ];
|
||||
locations."/".extraConfig = ''
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
|
||||
'';
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString config.services.buildbot-master.port}";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = ''
|
||||
proxy_read_timeout 3600s;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
krebs.ci = {
|
||||
|
|
@ -18,25 +20,20 @@
|
|||
repos = {
|
||||
disko.urls = [
|
||||
"http://cgit.gum.r/disko"
|
||||
"http://cgit.hotdog.r/disko"
|
||||
"http://cgit.ni.r/disko"
|
||||
"http://cgit.prism.r/disko"
|
||||
];
|
||||
krops.urls = [
|
||||
"http://cgit.hotdog.r/krops"
|
||||
"http://cgit.ni.r/krops"
|
||||
"http://cgit.prism.r/krops"
|
||||
"https://github.com/krebs/krops.git"
|
||||
];
|
||||
nix_writers.urls = [
|
||||
"http://cgit.hotdog.r/nix-writers"
|
||||
"http://cgit.ni.r/nix-writers"
|
||||
"http://cgit.prism.r/nix-writers"
|
||||
];
|
||||
stockholm.urls = [
|
||||
"http://cgit.enklave.r/stockholm"
|
||||
"http://cgit.gum.r/stockholm"
|
||||
"http://cgit.hotdog.r/stockholm"
|
||||
"http://cgit.ni.r/stockholm"
|
||||
"http://cgit.prism.r/stockholm"
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,382 +0,0 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with import <stockholm/lib>;
|
||||
let
|
||||
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
|
||||
# -*- python -*-
|
||||
from buildbot.plugins import *
|
||||
import re
|
||||
import json
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
c['slaves'] = []
|
||||
slaves = json.loads('${builtins.toJSON cfg.slaves}')
|
||||
slavenames = [ s for s in slaves ]
|
||||
for k,v in slaves.items():
|
||||
c['slaves'].append(buildslave.BuildSlave(k, v))
|
||||
|
||||
# TODO: configure protocols?
|
||||
c['protocols'] = {'pb': {'port': 9989}}
|
||||
|
||||
####### Build Inputs
|
||||
c['change_source'] = cs = []
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Change_Source: Begin of ${n}
|
||||
${v}
|
||||
#### Change_Source: End of ${n}
|
||||
'') cfg.change_source )}
|
||||
|
||||
####### Build Scheduler
|
||||
c['schedulers'] = sched = []
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Schedulers: Begin of ${n}
|
||||
${v}
|
||||
#### Schedulers: End of ${n}
|
||||
'') cfg.scheduler )}
|
||||
|
||||
###### Builder
|
||||
c['builders'] = bu = []
|
||||
|
||||
# Builder Pre: Begin
|
||||
${cfg.builder_pre}
|
||||
# Builder Pre: End
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Builder: Begin of ${n}
|
||||
${v}
|
||||
#### Builder: End of ${n}
|
||||
'') cfg.builder )}
|
||||
|
||||
|
||||
####### Status
|
||||
c['status'] = st = []
|
||||
|
||||
# If you want to configure this url, override with extraConfig
|
||||
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
|
||||
|
||||
${optionalString (cfg.web.enable) ''
|
||||
from buildbot.status import html
|
||||
from buildbot.status.web import authz, auth
|
||||
authz_cfg=authz.Authz(
|
||||
auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
|
||||
# TODO: configure harder
|
||||
gracefulShutdown = False,
|
||||
forceBuild = 'auth',
|
||||
forceAllBuilds = 'auth',
|
||||
pingBuilder = False,
|
||||
stopBuild = 'auth',
|
||||
stopAllBuilds = 'auth',
|
||||
cancelPendingBuild = 'auth'
|
||||
)
|
||||
# TODO: configure krebs.nginx
|
||||
st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
|
||||
''}
|
||||
|
||||
${optionalString (cfg.irc.enable) ''
|
||||
from buildbot.status import words
|
||||
irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
|
||||
channels=${builtins.toJSON cfg.irc.channels},
|
||||
notify_events={
|
||||
'started': 1,
|
||||
'success': 1,
|
||||
'failure': 1,
|
||||
'exception': 1,
|
||||
'successToFailure': 1,
|
||||
'failureToSuccess': 1,
|
||||
}${optionalString cfg.irc.allowForce ",allowForce=True"})
|
||||
c['status'].append(irc)
|
||||
''}
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Status: Begin of ${n}
|
||||
${v}
|
||||
#### Status: End of ${n}
|
||||
'') cfg.status )}
|
||||
|
||||
####### PROJECT IDENTITY
|
||||
c['title'] = "${cfg.title}"
|
||||
c['titleURL'] = "http://krebsco.de"
|
||||
|
||||
|
||||
####### DB URL
|
||||
# TODO: configure
|
||||
c['db'] = {
|
||||
'db_url' : "sqlite:///state.sqlite",
|
||||
}
|
||||
${cfg.extraConfig}
|
||||
'';
|
||||
|
||||
cfg = config.krebs.buildbot.master;
|
||||
|
||||
api = {
|
||||
enable = mkEnableOption "Buildbot Master";
|
||||
title = mkOption {
|
||||
default = "Buildbot CI";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Title of the Buildbot Installation
|
||||
'';
|
||||
};
|
||||
workDir = mkOption {
|
||||
default = "/var/lib/buildbot/master";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Path to build bot master directory.
|
||||
Will be created on startup.
|
||||
'';
|
||||
};
|
||||
|
||||
secrets = mkOption {
|
||||
default = [];
|
||||
type = types.listOf types.str;
|
||||
example = [ "cac.json" ];
|
||||
description = ''
|
||||
List of all the secrets in ‹secrets› which should be copied into the
|
||||
buildbot master directory.
|
||||
'';
|
||||
};
|
||||
|
||||
slaves = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
description = ''
|
||||
Attrset of slavenames with their passwords
|
||||
slavename = slavepassword
|
||||
'';
|
||||
};
|
||||
|
||||
change_source = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
stockholm = ''
|
||||
cs.append(changes.GitPoller(
|
||||
'http://cgit.gum/stockholm',
|
||||
workdir='stockholm-poller', branch='master',
|
||||
project='stockholm',
|
||||
pollinterval=120))
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the change_sources which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>cs</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
scheduler = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
force-scheduler = ''
|
||||
sched.append(schedulers.ForceScheduler(
|
||||
name="force",
|
||||
builderNames=["full-tests"]))
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the schedulers which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>sched</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
builder_pre = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
example = ''
|
||||
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
|
||||
'';
|
||||
description = ''
|
||||
some code before the builders are being assembled.
|
||||
can be used to define functions used by multiple builders
|
||||
'';
|
||||
};
|
||||
|
||||
builder = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
fast-test = ''
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the builder which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>bu</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
status = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
description = ''
|
||||
Attrset of all the extra status which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>st</literal>
|
||||
|
||||
Right now IRC and Web status can be configured by setting
|
||||
<literal>buildbot.master.irc.enable</literal> and
|
||||
<literal>buildbot.master.web.enable</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
# Configurable Stati
|
||||
web = mkOption {
|
||||
default = {};
|
||||
type = types.submodule ({ config2, ... }: {
|
||||
options = {
|
||||
enable = mkEnableOption "Buildbot Master Web Status";
|
||||
username = mkOption {
|
||||
default = "krebs";
|
||||
type = types.str;
|
||||
description = ''
|
||||
username for web authentication
|
||||
'';
|
||||
};
|
||||
hostname = mkOption {
|
||||
default = config.networking.hostName;
|
||||
type = types.str;
|
||||
description = ''
|
||||
web interface Hostname
|
||||
'';
|
||||
};
|
||||
password = mkOption {
|
||||
default = "bob";
|
||||
type = types.str;
|
||||
description = ''
|
||||
password for web authentication
|
||||
'';
|
||||
};
|
||||
port = mkOption {
|
||||
default = 8010;
|
||||
type = types.int;
|
||||
description = ''
|
||||
port for buildbot web status
|
||||
'';
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
irc = mkOption {
|
||||
default = {};
|
||||
type = types.submodule ({ config, ... }: {
|
||||
options = {
|
||||
enable = mkEnableOption "Buildbot Master IRC Status";
|
||||
channels = mkOption {
|
||||
default = [ "nix-buildbot-meetup" ];
|
||||
type = with types; listOf str;
|
||||
description = ''
|
||||
irc channels the bot should connect to
|
||||
'';
|
||||
};
|
||||
allowForce = mkOption {
|
||||
default = false;
|
||||
type = types.bool;
|
||||
description = ''
|
||||
Determines if builds can be forced via IRC
|
||||
'';
|
||||
};
|
||||
nick = mkOption {
|
||||
default = "nix-buildbot";
|
||||
type = types.str;
|
||||
description = ''
|
||||
nickname for IRC
|
||||
'';
|
||||
};
|
||||
server = mkOption {
|
||||
default = "irc.freenode.net";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Buildbot Status IRC Server to connect to
|
||||
'';
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
description = ''
|
||||
extra config appended to the generated master.cfg
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
imp = {
|
||||
|
||||
users.extraUsers.buildbotMaster = {
|
||||
uid = genid "buildbotMaster";
|
||||
group = "buildbotMaster";
|
||||
description = "Buildbot Master";
|
||||
home = cfg.workDir;
|
||||
createHome = false;
|
||||
isSystemUser = true;
|
||||
};
|
||||
|
||||
users.extraGroups.buildbotMaster = {
|
||||
gid = 672626386;
|
||||
};
|
||||
|
||||
systemd.services.buildbotMaster = {
|
||||
description = "Buildbot Master";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# TODO: add extra dependencies to master like svn and cvs
|
||||
path = [ pkgs.git ];
|
||||
environment = {
|
||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
};
|
||||
serviceConfig = let
|
||||
workdir = shell.escape cfg.workDir;
|
||||
secretsdir = shell.escape (toString <secrets>);
|
||||
in {
|
||||
PermissionsStartOnly = true;
|
||||
# TODO: maybe also prepare buildbot.tac?
|
||||
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
|
||||
set -efux
|
||||
if [ ! -e ${workdir} ];then
|
||||
mkdir -p ${workdir}
|
||||
${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir}
|
||||
fi
|
||||
# always override the master.cfg
|
||||
cp ${buildbot-master-config} ${workdir}/master.cfg
|
||||
|
||||
# copy secrets
|
||||
${ concatMapStringsSep "\n"
|
||||
(f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
|
||||
# sanity
|
||||
${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir}
|
||||
|
||||
# TODO: maybe upgrade? not sure about this
|
||||
# normally we should write buildbot.tac by our own
|
||||
# ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir}
|
||||
|
||||
chmod 700 ${workdir}
|
||||
chown buildbotMaster:buildbotMaster -R ${workdir}
|
||||
'';
|
||||
ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start --nodaemon ${workdir}";
|
||||
PrivateTmp = "true";
|
||||
User = "buildbotMaster";
|
||||
Restart = "always";
|
||||
RestartSec = "10";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.krebs.buildbot.master = api;
|
||||
config = lib.mkIf cfg.enable imp;
|
||||
}
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with import <stockholm/lib>;
|
||||
let
|
||||
buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
|
||||
import os
|
||||
|
||||
from buildslave.bot import BuildSlave
|
||||
from twisted.application import service
|
||||
|
||||
basedir = '${cfg.workDir}'
|
||||
rotateLength = 10000000
|
||||
maxRotatedFiles = 10
|
||||
|
||||
application = service.Application('buildslave')
|
||||
|
||||
from twisted.python.logfile import LogFile
|
||||
from twisted.python.log import ILogObserver, FileLogObserver
|
||||
logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
|
||||
maxRotatedFiles=maxRotatedFiles)
|
||||
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
|
||||
|
||||
buildmaster_host = '${cfg.masterhost}'
|
||||
# TODO: masterport?
|
||||
port = 9989
|
||||
slavename = '${cfg.username}'
|
||||
passwd = '${cfg.password}'
|
||||
keepalive = 600
|
||||
usepty = 0
|
||||
umask = None
|
||||
maxdelay = 300
|
||||
allow_shutdown = None
|
||||
|
||||
${cfg.extraConfig}
|
||||
|
||||
s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
|
||||
keepalive, usepty, umask=umask, maxdelay=maxdelay,
|
||||
allow_shutdown=allow_shutdown)
|
||||
s.setServiceParent(application)
|
||||
'';
|
||||
default-packages = [ pkgs.git pkgs.bash ];
|
||||
cfg = config.krebs.buildbot.slave;
|
||||
|
||||
api = {
|
||||
enable = mkEnableOption "Buildbot Slave";
|
||||
|
||||
workDir = mkOption {
|
||||
default = "/var/lib/buildbot/slave";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Path to build bot slave directory.
|
||||
Will be created on startup.
|
||||
'';
|
||||
};
|
||||
|
||||
masterhost = mkOption {
|
||||
default = "localhost";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Hostname/IP of the buildbot master
|
||||
'';
|
||||
};
|
||||
|
||||
username = mkOption {
|
||||
type = types.str;
|
||||
description = ''
|
||||
slavename used to authenticate with master
|
||||
'';
|
||||
};
|
||||
|
||||
password = mkOption {
|
||||
type = types.str;
|
||||
description = ''
|
||||
slave password used to authenticate with master
|
||||
'';
|
||||
};
|
||||
|
||||
contact = mkOption {
|
||||
default = "nix slave <buildslave@${config.networking.hostName}>";
|
||||
type = types.str;
|
||||
description = ''
|
||||
contact to be announced by buildslave
|
||||
'';
|
||||
};
|
||||
|
||||
description = mkOption {
|
||||
default = "Nix Generated BuildSlave";
|
||||
type = types.str;
|
||||
description = ''
|
||||
description for hostto be announced by buildslave
|
||||
'';
|
||||
};
|
||||
|
||||
packages = mkOption {
|
||||
default = [ pkgs.git ];
|
||||
type = with types; listOf package;
|
||||
description = ''
|
||||
packages which should be in path for buildslave
|
||||
'';
|
||||
};
|
||||
|
||||
extraEnviron = mkOption {
|
||||
default = {};
|
||||
example = {
|
||||
NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
|
||||
};
|
||||
type = types.attrsOf types.str;
|
||||
description = ''
|
||||
extra environment variables to be provided to the buildslave service
|
||||
if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
example = ''
|
||||
port = 443
|
||||
keepalive = 600
|
||||
'';
|
||||
description = ''
|
||||
extra config evaluated before calling BuildSlave init in .tac file
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
imp = {
|
||||
|
||||
users.extraUsers.buildbotSlave = {
|
||||
uid = genid "buildbotSlave";
|
||||
group = "buildbotSlave";
|
||||
description = "Buildbot Slave";
|
||||
home = cfg.workDir;
|
||||
createHome = false;
|
||||
isSystemUser = true;
|
||||
};
|
||||
|
||||
users.extraGroups.buildbotSlave = {
|
||||
gid = 1408105834;
|
||||
};
|
||||
|
||||
systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
|
||||
description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = default-packages ++ cfg.packages;
|
||||
|
||||
environment = {
|
||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
NIX_REMOTE="daemon";
|
||||
} // cfg.extraEnviron;
|
||||
|
||||
serviceConfig = let
|
||||
workdir = shell.escape cfg.workDir;
|
||||
contact = shell.escape cfg.contact;
|
||||
description = shell.escape cfg.description;
|
||||
# TODO:make this
|
||||
in {
|
||||
PermissionsStartOnly = true;
|
||||
Type = "forking";
|
||||
PIDFile = "${workdir}/twistd.pid";
|
||||
# TODO: maybe also prepare buildbot.tac?
|
||||
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
|
||||
set -efux
|
||||
mkdir -p ${workdir}/info
|
||||
cp ${buildbot-slave-init} ${workdir}/buildbot.tac
|
||||
echo ${contact} > ${workdir}/info/admin
|
||||
echo ${description} > ${workdir}/info/host
|
||||
|
||||
chown buildbotSlave:buildbotSlave -R ${workdir}
|
||||
chmod 700 ${workdir}
|
||||
'';
|
||||
ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}";
|
||||
ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}";
|
||||
PrivateTmp = "true";
|
||||
User = "buildbotSlave";
|
||||
Restart = "always";
|
||||
RestartSec = "10";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.krebs.buildbot.slave = api;
|
||||
config = lib.mkIf cfg.enable imp;
|
||||
}
|
||||
|
|
@ -39,148 +39,131 @@ let
|
|||
|
||||
profileRoot = "/nix/var/nix/profiles/ci";
|
||||
|
||||
bcfg = config.services.buildbot-master;
|
||||
|
||||
imp = {
|
||||
krebs.buildbot.master = {
|
||||
slaves = {
|
||||
testslave = "lasspass";
|
||||
};
|
||||
services.buildbot-master = {
|
||||
workers = [ "worker.Worker('testworker', 'pass')" ];
|
||||
|
||||
change_source = mapAttrs' (name: repo:
|
||||
nameValuePair name (concatMapStrings (url: ''
|
||||
cs.append(
|
||||
changes.GitPoller(
|
||||
"${url}",
|
||||
workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
|
||||
project='${name}',
|
||||
pollinterval=100
|
||||
)
|
||||
changeSource = mapAttrsToList (name: repo:
|
||||
concatMapStringsSep "," (url: ''
|
||||
changes.GitPoller(
|
||||
"${url}",
|
||||
workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
|
||||
project='${name}',
|
||||
pollinterval=100
|
||||
)
|
||||
'') repo.urls)
|
||||
'') repo.urls
|
||||
) cfg.repos;
|
||||
|
||||
scheduler = mapAttrs' (name: repo:
|
||||
nameValuePair name ''
|
||||
sched.append(
|
||||
schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(
|
||||
branch_re=".*",
|
||||
project='${name}',
|
||||
),
|
||||
treeStableTimer=60,
|
||||
name="${name}-all-branches",
|
||||
builderNames=[
|
||||
"${name}",
|
||||
]
|
||||
)
|
||||
)
|
||||
sched.append(
|
||||
schedulers.ForceScheduler(
|
||||
name="${name}",
|
||||
builderNames=[
|
||||
"${name}",
|
||||
]
|
||||
)
|
||||
)
|
||||
''
|
||||
) cfg.repos;
|
||||
builder_pre = ''
|
||||
from buildbot import interfaces
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
schedulers = mapAttrsToList (name: repo: ''
|
||||
schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(
|
||||
branch_re=".*",
|
||||
project='${name}',
|
||||
),
|
||||
treeStableTimer=60,
|
||||
name="${name}-all-branches",
|
||||
builderNames=[
|
||||
"${name}",
|
||||
]
|
||||
),
|
||||
schedulers.ForceScheduler(
|
||||
name="${name}",
|
||||
builderNames=[
|
||||
"${name}",
|
||||
]
|
||||
)
|
||||
'') cfg.repos;
|
||||
|
||||
class StepToStartMoreSteps(ShellCommand):
|
||||
builders = [];
|
||||
|
||||
extraConfig = ''
|
||||
# https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
|
||||
from buildbot.plugins import util, steps
|
||||
from buildbot.process import buildstep, logobserver
|
||||
from twisted.internet import defer
|
||||
import json
|
||||
|
||||
class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||
def __init__(self, **kwargs):
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
kwargs = self.setupShellMixin(kwargs)
|
||||
super().__init__(**kwargs)
|
||||
self.observer = logobserver.BufferLogObserver()
|
||||
self.addLogObserver('stdio', self.observer)
|
||||
|
||||
def addBuildSteps(self, steps_factories):
|
||||
for sf in steps_factories:
|
||||
step = interfaces.IBuildStepFactory(sf).buildStep()
|
||||
step.setBuild(self.build)
|
||||
step.setBuildSlave(self.build.slavebuilder.slave)
|
||||
step_status = self.build.build_status.addStepWithName(step.name)
|
||||
step.setStepStatus(step_status)
|
||||
self.build.steps.append(step)
|
||||
def extract_stages(self, stdout):
|
||||
stages = json.loads(stdout)
|
||||
return stages
|
||||
|
||||
def start(self):
|
||||
props = self.build.getProperties()
|
||||
new_steps = json.loads(props.getProperty('steps_json'))
|
||||
for new_step in new_steps:
|
||||
self.addBuildSteps([steps.ShellCommand(
|
||||
name=str(new_step),
|
||||
command=[
|
||||
"${pkgs.writeDash "build-stepper.sh" ''
|
||||
set -xefu
|
||||
profile=${shell.escape profileRoot}/$build_name
|
||||
result=$("$build_script")
|
||||
if [ -n "$result" ]; then
|
||||
${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
|
||||
fi
|
||||
''}"
|
||||
],
|
||||
env={
|
||||
"build_name": new_step,
|
||||
"build_script": new_steps[new_step],
|
||||
"NIX_REMOTE": "daemon",
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||
},
|
||||
timeout=90001,
|
||||
workdir='build', # TODO figure out why we need this?
|
||||
)])
|
||||
@defer.inlineCallbacks
|
||||
def run(self):
|
||||
# run nix-instanstiate to generate the dict of stages
|
||||
cmd = yield self.makeRemoteShellCommand()
|
||||
yield self.runCommand(cmd)
|
||||
|
||||
ShellCommand.start(self)
|
||||
# if the command passes extract the list of stages
|
||||
result = cmd.results()
|
||||
if result == util.SUCCESS:
|
||||
# create a ShellCommand for each stage and add them to the build
|
||||
stages = self.extract_stages(self.observer.getStdout())
|
||||
self.build.addStepsAfterCurrentStep([
|
||||
steps.ShellCommand(name=stage, command=[stages[stage]])
|
||||
for stage in stages
|
||||
])
|
||||
|
||||
'';
|
||||
return result
|
||||
|
||||
builder = mapAttrs' (name: repo:
|
||||
nameValuePair name ''
|
||||
f_${name} = util.BuildFactory()
|
||||
f_${name}.addStep(steps.Git(
|
||||
|
||||
${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
|
||||
factory_${name} = util.BuildFactory()
|
||||
factory_${name}.addStep(steps.Git(
|
||||
repourl=util.Property('repository', '${head repo.urls}'),
|
||||
method='clobber',
|
||||
mode='full',
|
||||
submodules=True,
|
||||
))
|
||||
|
||||
f_${name}.addStep(steps.SetPropertyFromCommand(
|
||||
factory_${name}.addStep(GenerateStagesCommand(
|
||||
env={
|
||||
"NIX_REMOTE": "daemon",
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||
"NIX_REMOTE": "daemon",
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||
},
|
||||
name="get_steps",
|
||||
command=["${getJobs}"],
|
||||
extract_fn=lambda rc, stdout, stderr: { 'steps_json': stdout },
|
||||
name="Generate build stages",
|
||||
command=[
|
||||
"${getJobs}"
|
||||
],
|
||||
haltOnFailure=True,
|
||||
))
|
||||
f_${name}.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
|
||||
|
||||
bu.append(
|
||||
c['builders'].append(
|
||||
util.BuilderConfig(
|
||||
name="${name}",
|
||||
slavenames=slavenames,
|
||||
factory=f_${name}
|
||||
workernames=['testworker'],
|
||||
factory=factory_${name}
|
||||
)
|
||||
)
|
||||
''
|
||||
) cfg.repos;
|
||||
'') cfg.repos)}
|
||||
'';
|
||||
|
||||
enable = true;
|
||||
web.enable = true;
|
||||
irc = {
|
||||
enable = true;
|
||||
nick = "build|${hostname}";
|
||||
server = "irc.r";
|
||||
channels = [ "xxx" "noise" ];
|
||||
allowForce = true;
|
||||
};
|
||||
extraConfig = ''
|
||||
c['buildbotURL'] = "http://build.${hostname}.r/"
|
||||
'';
|
||||
reporters = [''
|
||||
reporters.IRC(
|
||||
host = "irc.r",
|
||||
nick = "buildbot|${hostname}",
|
||||
notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
|
||||
channels = [{"channel": "#xxx"}],
|
||||
)
|
||||
''];
|
||||
|
||||
buildbotUrl = "http://build.${hostname}.r/";
|
||||
};
|
||||
|
||||
krebs.buildbot.slave = {
|
||||
services.buildbot-worker = {
|
||||
enable = true;
|
||||
masterhost = "localhost";
|
||||
username = "testslave";
|
||||
password = "lasspass";
|
||||
packages = with pkgs; [ gnumake jq nix populate gnutar lzma gzip ];
|
||||
workerUser = "testworker";
|
||||
workerPass = "pass";
|
||||
packages = with pkgs; [ git gnutar gzip jq nix populate ];
|
||||
};
|
||||
|
||||
system.activationScripts.buildbots-nix-profile = ''
|
||||
|
|
@ -192,11 +175,10 @@ let
|
|||
users = {
|
||||
groups.buildbots.gid = genid "buildbots";
|
||||
users = {
|
||||
buildbotMaster.extraGroups = [ "buildbots" ];
|
||||
buildbotSlave.extraGroups = [ "buildbots" ];
|
||||
buildbot.extraGroups = [ "buildbots" ];
|
||||
bbworker.extraGroups = [ "buildbots" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
in out
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ let
|
|||
./bepasty-server.nix
|
||||
./bindfs.nix
|
||||
./brockman.nix
|
||||
./buildbot/master.nix
|
||||
./buildbot/slave.nix
|
||||
./build.nix
|
||||
./cachecache.nix
|
||||
./ci.nix
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
{ pkgs, fetchFromGitHub, python2Packages, git, ... }: let
|
||||
|
||||
# we need the old sqlparse since the new one is python2 incompatible
|
||||
sqlparse = python2Packages.callPackage ./sqlparse.nix {};
|
||||
|
||||
in python2Packages.buildPythonApplication rec {
|
||||
name = "buildbot-classic-${version}";
|
||||
version = "0.8.18";
|
||||
namePrefix = "";
|
||||
patches = [];
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "krebs";
|
||||
repo = "buildbot-classic";
|
||||
rev = version;
|
||||
sha256 = "0b4y3n9zd2gdy8xwk1vpvs4n9fbg72vi8mx4ydgijwngcmdqkjmq";
|
||||
};
|
||||
postUnpack = "sourceRoot=\${sourceRoot}/master";
|
||||
|
||||
propagatedBuildInputs = [
|
||||
python2Packages.jinja2
|
||||
python2Packages.twisted
|
||||
python2Packages.dateutil
|
||||
(python2Packages.sqlalchemy_migrate.override { sqlparse = sqlparse; })
|
||||
python2Packages.pysqlite
|
||||
pkgs.coreutils
|
||||
];
|
||||
doCheck = false;
|
||||
postInstall = ''
|
||||
mkdir -p "$out/share/man/man1"
|
||||
cp docs/buildbot.1 "$out/share/man/man1"
|
||||
'';
|
||||
}
|
||||
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
{ lib
|
||||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
, pytest
|
||||
, isPy3k
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "sqlparse";
|
||||
version = "0.3.1";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548";
|
||||
};
|
||||
|
||||
checkInputs = [ pytest ];
|
||||
checkPhase = ''
|
||||
py.test
|
||||
'';
|
||||
|
||||
# Package supports 3.x, but tests are clearly 2.x only.
|
||||
doCheck = !isPy3k;
|
||||
|
||||
meta = with lib; {
|
||||
description = "Non-validating SQL parser for Python";
|
||||
longDescription = ''
|
||||
Provides support for parsing, splitting and formatting SQL statements.
|
||||
'';
|
||||
homepage = "https://github.com/andialbrecht/sqlparse";
|
||||
license = licenses.bsd3;
|
||||
};
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user