Merge remote-tracking branch 'gum/master'
This commit is contained in:
lassulus
commit
d7741ca83f
7
Makefile
7
Makefile
@ -35,7 +35,7 @@ ifeq ($(filter),json)
|
||||
else
|
||||
filter() { cat; }
|
||||
endif
|
||||
nix-instantiate \
|
||||
result=$$(nix-instantiate \
|
||||
$${extraArgs-} \
|
||||
--eval \
|
||||
-A "$$get" \
|
||||
@ -45,8 +45,9 @@ endif
|
||||
--argstr current-host-name "$$HOSTNAME" \
|
||||
--argstr current-user-name "$$LOGNAME" \
|
||||
$${system+--argstr system "$$system"} \
|
||||
$${target+--argstr target "$$target"} \
|
||||
| filter
|
||||
$${target+--argstr target "$$target"})
|
||||
echo "$$result" | filter
|
||||
|
||||
else
|
||||
$(error unbound variable: system[s])
|
||||
endif
|
||||
|
||||
375
krebs/3modules/buildbot/master.nix
Normal file
375
krebs/3modules/buildbot/master.nix
Normal file
@ -0,0 +1,375 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with lib;
|
||||
let
|
||||
buildbot = pkgs.buildbot;
|
||||
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
|
||||
# -*- python -*-
|
||||
from buildbot.plugins import *
|
||||
import re
|
||||
import json
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
c['slaves'] = []
|
||||
slaves = json.loads('${builtins.toJSON cfg.slaves}')
|
||||
slavenames = [ s for s in slaves ]
|
||||
for k,v in slaves.items():
|
||||
c['slaves'].append(buildslave.BuildSlave(k, v))
|
||||
|
||||
# TODO: configure protocols?
|
||||
c['protocols'] = {'pb': {'port': 9989}}
|
||||
|
||||
####### Build Inputs
|
||||
c['change_source'] = cs = []
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Change_Source: Begin of ${n}
|
||||
${v}
|
||||
#### Change_Source: End of ${n}
|
||||
'') cfg.change_source )}
|
||||
|
||||
####### Build Scheduler
|
||||
c['schedulers'] = sched = []
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Schedulers: Begin of ${n}
|
||||
${v}
|
||||
#### Schedulers: End of ${n}
|
||||
'') cfg.scheduler )}
|
||||
|
||||
###### Builder
|
||||
c['builders'] = bu = []
|
||||
|
||||
# Builder Pre: Begin
|
||||
${cfg.builder_pre}
|
||||
# Builder Pre: End
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Builder: Begin of ${n}
|
||||
${v}
|
||||
#### Builder: End of ${n}
|
||||
'') cfg.builder )}
|
||||
|
||||
|
||||
####### Status
|
||||
c['status'] = st = []
|
||||
|
||||
# If you want to configure this url, override with extraConfig
|
||||
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
|
||||
|
||||
${optionalString (cfg.web.enable) ''
|
||||
from buildbot.status import html
|
||||
from buildbot.status.web import authz, auth
|
||||
authz_cfg=authz.Authz(
|
||||
auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
|
||||
# TODO: configure harder
|
||||
gracefulShutdown = False,
|
||||
forceBuild = 'auth',
|
||||
forceAllBuilds = 'auth',
|
||||
pingBuilder = False,
|
||||
stopBuild = 'auth',
|
||||
stopAllBuilds = 'auth',
|
||||
cancelPendingBuild = 'auth'
|
||||
)
|
||||
# TODO: configure krebs.nginx
|
||||
st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
|
||||
''}
|
||||
|
||||
${optionalString (cfg.irc.enable) ''
|
||||
from buildbot.status import words
|
||||
irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
|
||||
channels=${builtins.toJSON cfg.irc.channels},
|
||||
notify_events={
|
||||
'success': 1,
|
||||
'failure': 1,
|
||||
'exception': 1,
|
||||
'successToFailure': 1,
|
||||
'failureToSuccess': 1,
|
||||
}${optionalString cfg.irc.allowForce ",allowForce=True"})
|
||||
c['status'].append(irc)
|
||||
''}
|
||||
|
||||
${ concatStringsSep "\n"
|
||||
(mapAttrsToList (n: v: ''
|
||||
#### Status: Begin of ${n}
|
||||
${v}
|
||||
#### Status: End of ${n}
|
||||
'') cfg.status )}
|
||||
|
||||
####### PROJECT IDENTITY
|
||||
c['title'] = "${cfg.title}"
|
||||
c['titleURL'] = "http://krebsco.de"
|
||||
|
||||
|
||||
####### DB URL
|
||||
# TODO: configure
|
||||
c['db'] = {
|
||||
'db_url' : "sqlite:///state.sqlite",
|
||||
}
|
||||
${cfg.extraConfig}
|
||||
'';
|
||||
|
||||
cfg = config.krebs.buildbot.master;
|
||||
|
||||
api = {
|
||||
enable = mkEnableOption "Buildbot Master";
|
||||
title = mkOption {
|
||||
default = "Buildbot CI";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Title of the Buildbot Installation
|
||||
'';
|
||||
};
|
||||
workDir = mkOption {
|
||||
default = "/var/lib/buildbot/master";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Path to build bot master directory.
|
||||
Will be created on startup.
|
||||
'';
|
||||
};
|
||||
|
||||
slaves = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
description = ''
|
||||
Attrset of slavenames with their passwords
|
||||
slavename = slavepassword
|
||||
'';
|
||||
};
|
||||
|
||||
change_source = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
stockholm = ''
|
||||
cs.append(changes.GitPoller(
|
||||
'http://cgit.gum/stockholm',
|
||||
workdir='stockholm-poller', branch='master',
|
||||
project='stockholm',
|
||||
pollinterval=120))
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the change_sources which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>cs</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
scheduler = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
force-scheduler = ''
|
||||
sched.append(schedulers.ForceScheduler(
|
||||
name="force",
|
||||
builderNames=["full-tests"]))
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the schedulers which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>sched</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
builder_pre = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
example = ''
|
||||
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
|
||||
'';
|
||||
description = ''
|
||||
some code before the builders are being assembled.
|
||||
can be used to define functions used by multiple builders
|
||||
'';
|
||||
};
|
||||
|
||||
builder = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
example = {
|
||||
fast-test = ''
|
||||
'';
|
||||
};
|
||||
description = ''
|
||||
Attrset of all the builder which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>bu</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
status = mkOption {
|
||||
default = {};
|
||||
type = types.attrsOf types.str;
|
||||
description = ''
|
||||
Attrset of all the extra status which should be configured.
|
||||
It will be directly included into the master configuration.
|
||||
|
||||
At the end an change object should be appended to <literal>st</literal>
|
||||
|
||||
Right now IRC and Web status can be configured by setting
|
||||
<literal>buildbot.master.irc.enable</literal> and
|
||||
<literal>buildbot.master.web.enable</literal>
|
||||
'';
|
||||
};
|
||||
|
||||
# Configurable Stati
|
||||
web = mkOption {
|
||||
default = {};
|
||||
type = types.submodule ({ config2, ... }: {
|
||||
options = {
|
||||
enable = mkEnableOption "Buildbot Master Web Status";
|
||||
username = mkOption {
|
||||
default = "krebs";
|
||||
type = types.str;
|
||||
description = ''
|
||||
username for web authentication
|
||||
'';
|
||||
};
|
||||
hostname = mkOption {
|
||||
default = config.networking.hostName;
|
||||
type = types.str;
|
||||
description = ''
|
||||
web interface Hostname
|
||||
'';
|
||||
};
|
||||
password = mkOption {
|
||||
default = "bob";
|
||||
type = types.str;
|
||||
description = ''
|
||||
password for web authentication
|
||||
'';
|
||||
};
|
||||
port = mkOption {
|
||||
default = 8010;
|
||||
type = types.int;
|
||||
description = ''
|
||||
port for buildbot web status
|
||||
'';
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
irc = mkOption {
|
||||
default = {};
|
||||
type = types.submodule ({ config, ... }: {
|
||||
options = {
|
||||
enable = mkEnableOption "Buildbot Master IRC Status";
|
||||
channels = mkOption {
|
||||
default = [ "nix-buildbot-meetup" ];
|
||||
type = with types; listOf str;
|
||||
description = ''
|
||||
irc channels the bot should connect to
|
||||
'';
|
||||
};
|
||||
allowForce = mkOption {
|
||||
default = false;
|
||||
type = types.bool;
|
||||
description = ''
|
||||
Determines if builds can be forced via IRC
|
||||
'';
|
||||
};
|
||||
nick = mkOption {
|
||||
default = "nix-buildbot";
|
||||
type = types.str;
|
||||
description = ''
|
||||
nickname for IRC
|
||||
'';
|
||||
};
|
||||
server = mkOption {
|
||||
default = "irc.freenode.net";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Buildbot Status IRC Server to connect to
|
||||
'';
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
description = ''
|
||||
extra config appended to the generated master.cfg
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
imp = {
|
||||
|
||||
users.extraUsers.buildbotMaster = {
|
||||
uid = 672626386; #genid buildbotMaster
|
||||
description = "Buildbot Master";
|
||||
home = cfg.workDir;
|
||||
createHome = false;
|
||||
};
|
||||
|
||||
users.extraGroups.buildbotMaster = {
|
||||
gid = 672626386;
|
||||
};
|
||||
|
||||
systemd.services.buildbotMaster = {
|
||||
description = "Buildbot Master";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# TODO: add extra dependencies to master like svn and cvs
|
||||
path = [ pkgs.git ];
|
||||
environment = {
|
||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
};
|
||||
serviceConfig = let
|
||||
workdir="${lib.shell.escape cfg.workDir}";
|
||||
secretsdir="${lib.shell.escape (toString <secrets>)}";
|
||||
in {
|
||||
PermissionsStartOnly = true;
|
||||
Type = "forking";
|
||||
PIDFile = "${workdir}/twistd.pid";
|
||||
# TODO: maybe also prepare buildbot.tac?
|
||||
ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
|
||||
#!/bin/sh
|
||||
set -efux
|
||||
if [ ! -e ${workdir} ];then
|
||||
mkdir -p ${workdir}
|
||||
${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
|
||||
fi
|
||||
# always override the master.cfg
|
||||
cp ${buildbot-master-config} ${workdir}/master.cfg
|
||||
# copy secrets
|
||||
cp ${secretsdir}/cac.json ${workdir}
|
||||
cp ${secretsdir}/retiolum-ci.rsa_key.priv \
|
||||
${workdir}/retiolum.rsa_key.priv
|
||||
# sanity
|
||||
${buildbot}/bin/buildbot checkconfig ${workdir}
|
||||
|
||||
# TODO: maybe upgrade? not sure about this
|
||||
# normally we should write buildbot.tac by our own
|
||||
# ${buildbot}/bin/buildbot upgrade-master ${workdir}
|
||||
|
||||
chmod 700 -R ${workdir}
|
||||
chown buildbotMaster:buildbotMaster -R ${workdir}
|
||||
'';
|
||||
ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
|
||||
ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
|
||||
ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
|
||||
PrivateTmp = "true";
|
||||
User = "buildbotMaster";
|
||||
Restart = "always";
|
||||
RestartSec = "10";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.krebs.buildbot.master = api;
|
||||
config = mkIf cfg.enable imp;
|
||||
}
|
||||
@ -39,7 +39,7 @@ let
|
||||
s.setServiceParent(application)
|
||||
'';
|
||||
default-packages = [ pkgs.git pkgs.bash ];
|
||||
cfg = config.makefu.buildbot.slave;
|
||||
cfg = config.krebs.buildbot.slave;
|
||||
|
||||
api = {
|
||||
enable = mkEnableOption "Buildbot Slave";
|
||||
@ -144,6 +144,7 @@ let
|
||||
path = default-packages ++ cfg.packages;
|
||||
|
||||
environment = {
|
||||
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
NIX_REMOTE="daemon";
|
||||
} // cfg.extraEnviron;
|
||||
|
||||
@ -180,6 +181,6 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
options.makefu.buildbot.slave = api;
|
||||
options.krebs.buildbot.slave = api;
|
||||
config = mkIf cfg.enable imp;
|
||||
}
|
||||
@ -9,6 +9,8 @@ let
|
||||
./apt-cacher-ng.nix
|
||||
./bepasty-server.nix
|
||||
./build.nix
|
||||
./buildbot/master.nix
|
||||
./buildbot/slave.nix
|
||||
./current.nix
|
||||
./exim-retiolum.nix
|
||||
./exim-smarthost.nix
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
{ stdenv, fetchgit, coreutils, curl, gnused, inotifyTools, jq, ncurses, sshpass, ... }:
|
||||
{ stdenv, fetchgit, bc, coreutils, curl, gnused, inotifyTools, jq, ncurses, sshpass, ... }:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "cac-1.0.0";
|
||||
name = "cac-1.0.3";
|
||||
|
||||
src = fetchgit {
|
||||
url = http://cgit.gum/cac;
|
||||
rev = "fe3b2ecb0aaf7d863842b896e18cd2b829f2297b";
|
||||
sha256 = "05bnd7wyjhqy8srmpnc8d234rv3jxdjgb4z0hlfb9kg7mb12w1ya";
|
||||
url = http://cgit.cd.retiolum/cac;
|
||||
rev = "22acc1b990ac7d97c16344fbcbc2621e24cdf915";
|
||||
sha256 = "135b740617c983b3f46a1983d4744be17340d5146a0a0de0dff4bb7a53688f2f";
|
||||
};
|
||||
|
||||
phases = [
|
||||
@ -17,6 +17,7 @@ stdenv.mkDerivation {
|
||||
installPhase =
|
||||
let
|
||||
path = stdenv.lib.makeSearchPath "bin" [
|
||||
bc
|
||||
coreutils
|
||||
curl
|
||||
gnused
|
||||
@ -29,10 +30,9 @@ stdenv.mkDerivation {
|
||||
''
|
||||
mkdir -p $out/bin
|
||||
|
||||
sed \
|
||||
's,^\( true) \)\(cac "$@";;\)$,\1 PATH=${path}${PATH+:$PATH} \2,' \
|
||||
< ./cac \
|
||||
> $out/bin/cac
|
||||
sed < ./cac > $out/bin/cac '
|
||||
s;^_cac_main .*;PATH=${path}''${PATH+:$PATH} &;
|
||||
'
|
||||
|
||||
chmod +x $out/bin/cac
|
||||
'';
|
||||
|
||||
@ -40,6 +40,10 @@ subdirs // rec {
|
||||
}
|
||||
'';
|
||||
|
||||
test = {
|
||||
infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {};
|
||||
};
|
||||
|
||||
execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; });
|
||||
|
||||
writeC = name: { destination ? "" }: src: pkgs.runCommand name {} ''
|
||||
|
||||
39
krebs/5pkgs/test/infest-cac-centos7/default.nix
Normal file
39
krebs/5pkgs/test/infest-cac-centos7/default.nix
Normal file
@ -0,0 +1,39 @@
|
||||
{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
|
||||
|
||||
stdenv.mkDerivation rec {
|
||||
name = "${shortname}-${version}";
|
||||
shortname = "infest-cac-centos7";
|
||||
version = "0.2.0";
|
||||
|
||||
src = ./notes;
|
||||
|
||||
phases = [
|
||||
"installPhase"
|
||||
];
|
||||
buildInputs = [ makeWrapper ];
|
||||
|
||||
path = stdenv.lib.makeSearchPath "bin" [
|
||||
coreutils
|
||||
cac
|
||||
cacpanel
|
||||
gnumake
|
||||
gnused
|
||||
jq
|
||||
openssh
|
||||
];
|
||||
|
||||
installPhase =
|
||||
''
|
||||
mkdir -p $out/bin
|
||||
cp ${src} $out/bin/${shortname}
|
||||
chmod +x $out/bin/${shortname}
|
||||
wrapProgram $out/bin/${shortname} \
|
||||
--prefix PATH : ${path}
|
||||
'';
|
||||
meta = with stdenv.lib; {
|
||||
homepage = http://krebsco.de;
|
||||
description = "Krebs CI Scripts";
|
||||
license = licenses.wtfpl;
|
||||
maintainers = [ maintainers.makefu ];
|
||||
};
|
||||
}
|
||||
116
krebs/5pkgs/test/infest-cac-centos7/notes
Executable file
116
krebs/5pkgs/test/infest-cac-centos7/notes
Executable file
@ -0,0 +1,116 @@
|
||||
#! /bin/sh
|
||||
|
||||
# nix-shell -p gnumake jq openssh cac cacpanel
|
||||
set -eufx
|
||||
|
||||
# 2 secrets are required:
|
||||
|
||||
krebs_cred=${krebs_cred-./cac.json}
|
||||
retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
|
||||
|
||||
# Sanity
|
||||
if test ! -r "$krebs_cred";then
|
||||
echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
|
||||
fi
|
||||
if test ! -r "$retiolum_key";then
|
||||
echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
|
||||
fi
|
||||
|
||||
krebs_secrets=$(mktemp -d)
|
||||
sec_file=$krebs_secrets/cac_config
|
||||
krebs_ssh=$krebs_secrets/tempssh
|
||||
export cac_resources_cache=$krebs_secrets/res_cache.json
|
||||
export cac_servers_cache=$krebs_secrets/servers_cache.json
|
||||
export cac_tasks_cache=$krebs_secrets/tasks_cache.json
|
||||
export cac_templates_cache=$krebs_secrets/templates_cache.json
|
||||
# we need to receive this key from buildmaster to speed up tinc bootstrap
|
||||
TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT"
|
||||
trap "$TRAP" INT TERM EXIT
|
||||
|
||||
cat > $sec_file <<EOF
|
||||
cac_login="$(jq -r .email $krebs_cred)"
|
||||
cac_key="$(cac-cli panel --config $krebs_cred settings | jq -r .apicode)"
|
||||
EOF
|
||||
|
||||
export cac_secrets=$sec_file
|
||||
cac-cli panel --config $krebs_cred update-api-ip
|
||||
|
||||
# test login:
|
||||
cac update
|
||||
cac servers
|
||||
|
||||
# Template 26: CentOS7
|
||||
# TODO: use cac templates to determine the real Centos7 template in case it changes
|
||||
name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
|
||||
| jq -r .servername)
|
||||
|
||||
id=servername:$name
|
||||
trap "cac delete $id;$TRAP;exit" INT TERM EXIT
|
||||
# TODO: timeout?
|
||||
|
||||
wait_login_cac(){
|
||||
# timeout
|
||||
for t in `seq 180`;do
|
||||
# now we have a working cac server
|
||||
if cac ssh $1 -o ConnectTimeout=10 \
|
||||
cat /etc/redhat-release | \
|
||||
grep CentOS ;then
|
||||
return 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
}
|
||||
# die on timeout
|
||||
wait_login_cac $id
|
||||
|
||||
mkdir -p shared/2configs/temp
|
||||
cac generatenetworking $id > \
|
||||
shared/2configs/temp/networking.nix
|
||||
# new temporary ssh key we will use to log in after infest
|
||||
ssh-keygen -f $krebs_ssh -N ""
|
||||
cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
|
||||
# we override the directories for secrets and stockholm
|
||||
# additionally we set the ssh key we generated
|
||||
ip=$(cac getserver $id | jq -r .ip)
|
||||
|
||||
cat > shared/2configs/temp/dirs.nix <<EOF
|
||||
_: {
|
||||
krebs.build.source.dir = {
|
||||
secrets.path = "$krebs_secrets";
|
||||
stockholm.path = "$(pwd)";
|
||||
};
|
||||
users.extraUsers.root.openssh.authorizedKeys.keys = [
|
||||
"$(cat ${krebs_ssh}.pub)"
|
||||
];
|
||||
krebs.build.target = "$ip";
|
||||
}
|
||||
EOF
|
||||
|
||||
LOGNAME=shared make eval get=krebs.infest \
|
||||
target=derp system=test-centos7 filter=json \
|
||||
| sed -e "s#^ssh.*<<#cac ssh $id<<#" \
|
||||
-e "/^rsync/a -e 'cac ssh $id' \\\\" \
|
||||
-e "s#root.derp:#:#" > $krebs_secrets/infest
|
||||
sh -x $krebs_secrets/infest
|
||||
|
||||
# TODO: generate secrets directory $krebs_secrets for nix import
|
||||
cac powerop $id reset
|
||||
|
||||
wait_login(){
|
||||
# timeout
|
||||
for t in `seq 90`;do
|
||||
# now we have a working cac server
|
||||
if ssh -o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-i $krebs_ssh \
|
||||
-o ConnectTimeout=10 \
|
||||
-o BatchMode=yes \
|
||||
root@$1 nixos-version ;then
|
||||
return 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
}
|
||||
wait_login $ip
|
||||
@ -24,7 +24,7 @@ with lib;
|
||||
git.nixpkgs = {
|
||||
#url = https://github.com/NixOS/nixpkgs;
|
||||
url = mkDefault https://github.com/makefu/nixpkgs;
|
||||
rev = mkDefault "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picking
|
||||
rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking
|
||||
target-path = "/var/src/nixpkgs";
|
||||
};
|
||||
|
||||
|
||||
@ -1,263 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with lib;
|
||||
let
|
||||
buildbot = pkgs.buildbot;
|
||||
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
|
||||
# -*- python -*-
|
||||
from buildbot.plugins import *
|
||||
import re
|
||||
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
c['slaves'] = []
|
||||
# TODO: template potential buildslaves
|
||||
# TODO: set password?
|
||||
slavenames= [ 'testslave' ]
|
||||
for i in slavenames:
|
||||
c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
|
||||
|
||||
c['protocols'] = {'pb': {'port': 9989}}
|
||||
|
||||
####### Build Inputs
|
||||
stockholm_repo = 'http://cgit.gum/stockholm'
|
||||
c['change_source'] = []
|
||||
c['change_source'].append(changes.GitPoller(
|
||||
stockholm_repo,
|
||||
workdir='stockholm-poller', branch='master',
|
||||
project='stockholm',
|
||||
pollinterval=120))
|
||||
|
||||
####### Build Scheduler
|
||||
# TODO: configure scheduler
|
||||
c['schedulers'] = []
|
||||
|
||||
# test the master real quick
|
||||
fast = schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch="master"),
|
||||
name="fast-master-test",
|
||||
builderNames=["fast-tests"])
|
||||
|
||||
force = schedulers.ForceScheduler(
|
||||
name="force",
|
||||
builderNames=["full-tests"])
|
||||
|
||||
# files everyone depends on or are part of the share branch
|
||||
def shared_files(change):
|
||||
r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
|
||||
for file in change.files:
|
||||
if r.match(file):
|
||||
return True
|
||||
return False
|
||||
|
||||
full = schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch="master"),
|
||||
fileIsImportant=shared_files,
|
||||
name="full-master-test",
|
||||
builderNames=["full-tests"])
|
||||
c['schedulers'] = [ fast, force, full ]
|
||||
###### The actual build
|
||||
# couple of fast steps:
|
||||
f = util.BuildFactory()
|
||||
## fetch repo
|
||||
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
|
||||
f.addStep(grab_repo)
|
||||
|
||||
# the dependencies which are used by the test script
|
||||
deps = [ "gnumake", "jq" ]
|
||||
nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
|
||||
def addShell(f,**kwargs):
|
||||
f.addStep(steps.ShellCommand(**kwargs))
|
||||
|
||||
addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
|
||||
"get" : "krebs.deploy",
|
||||
"filter" : "json"
|
||||
},
|
||||
command=nixshell + ["make -s eval system=test-centos7"])
|
||||
|
||||
addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
|
||||
"get" : "krebs.deploy",
|
||||
"filter" : "json"
|
||||
},
|
||||
command=nixshell + ["make -s eval system=wolf"])
|
||||
|
||||
c['builders'] = []
|
||||
c['builders'].append(
|
||||
util.BuilderConfig(name="fast-tests",
|
||||
slavenames=slavenames,
|
||||
factory=f))
|
||||
|
||||
# TODO slow build
|
||||
c['builders'].append(
|
||||
util.BuilderConfig(name="full-tests",
|
||||
slavenames=slavenames,
|
||||
factory=f))
|
||||
|
||||
####### Status of Builds
|
||||
c['status'] = []
|
||||
|
||||
from buildbot.status import html
|
||||
from buildbot.status.web import authz, auth
|
||||
# TODO: configure if http is wanted
|
||||
authz_cfg=authz.Authz(
|
||||
# TODO: configure user/pw
|
||||
auth=auth.BasicAuth([("krebs","bob")]),
|
||||
gracefulShutdown = False,
|
||||
forceBuild = 'auth',
|
||||
forceAllBuilds = 'auth',
|
||||
pingBuilder = False,
|
||||
stopBuild = False,
|
||||
stopAllBuilds = False,
|
||||
cancelPendingBuild = False,
|
||||
)
|
||||
# TODO: configure nginx
|
||||
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
||||
|
||||
from buildbot.status import words
|
||||
${optionalString (cfg.irc.enable) ''
|
||||
irc = words.IRC("${cfg.irc.server}", "krebsbuild",
|
||||
# TODO: multiple channels
|
||||
channels=["${cfg.irc.channel}"],
|
||||
notify_events={
|
||||
#'success': 1,
|
||||
#'failure': 1,
|
||||
'exception': 1,
|
||||
'successToFailure': 1,
|
||||
'failureToSuccess': 1,
|
||||
}${optionalString cfg.irc.allowForce ",allowForce=True"})
|
||||
c['status'].append(irc)
|
||||
''}
|
||||
|
||||
####### PROJECT IDENTITY
|
||||
c['title'] = "Stockholm"
|
||||
c['titleURL'] = "http://krebsco.de"
|
||||
|
||||
#c['buildbotURL'] = "http://buildbot.krebsco.de/"
|
||||
# TODO: configure url
|
||||
c['buildbotURL'] = "http://vbob:8010/"
|
||||
|
||||
####### DB URL
|
||||
c['db'] = {
|
||||
'db_url' : "sqlite:///state.sqlite",
|
||||
}
|
||||
${cfg.extraConfig}
|
||||
'';
|
||||
|
||||
cfg = config.makefu.buildbot.master;
|
||||
|
||||
api = {
|
||||
enable = mkEnableOption "Buildbot Master";
|
||||
workDir = mkOption {
|
||||
default = "/var/lib/buildbot/master";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Path to build bot master directory.
|
||||
Will be created on startup.
|
||||
'';
|
||||
};
|
||||
irc = mkOption {
|
||||
default = {};
|
||||
type = types.submodule ({ config, ... }: {
|
||||
options = {
|
||||
enable = mkEnableOption "Buildbot Master IRC Status";
|
||||
channel = mkOption {
|
||||
default = "nix-buildbot-meetup";
|
||||
type = types.str;
|
||||
description = ''
|
||||
irc channel the bot should connect to
|
||||
'';
|
||||
};
|
||||
allowForce = mkOption {
|
||||
default = false;
|
||||
type = types.bool;
|
||||
description = ''
|
||||
Determines if builds can be forced via IRC
|
||||
'';
|
||||
};
|
||||
nick = mkOption {
|
||||
default = "nix-buildbot";
|
||||
type = types.str;
|
||||
description = ''
|
||||
nickname for IRC
|
||||
'';
|
||||
};
|
||||
server = mkOption {
|
||||
default = "irc.freenode.net";
|
||||
type = types.str;
|
||||
description = ''
|
||||
Buildbot Status IRC Server to connect to
|
||||
'';
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
default = "";
|
||||
type = types.lines;
|
||||
description = ''
|
||||
extra config appended to the generated master.cfg
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
imp = {
|
||||
|
||||
users.extraUsers.buildbotMaster = {
|
||||
uid = 672626386; #genid buildbotMaster
|
||||
description = "Buildbot Master";
|
||||
home = cfg.workDir;
|
||||
createHome = false;
|
||||
};
|
||||
|
||||
users.extraGroups.buildbotMaster = {
|
||||
gid = 672626386;
|
||||
};
|
||||
|
||||
systemd.services.buildbotMaster = {
|
||||
description = "Buildbot Master";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [ pkgs.git ];
|
||||
serviceConfig = let
|
||||
workdir="${lib.shell.escape cfg.workDir}";
|
||||
# TODO: check if git is the only dep
|
||||
in {
|
||||
PermissionsStartOnly = true;
|
||||
Type = "forking";
|
||||
PIDFile = "${workdir}/twistd.pid";
|
||||
# TODO: maybe also prepare buildbot.tac?
|
||||
ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
|
||||
#!/bin/sh
|
||||
set -efux
|
||||
if [ ! -e ${workdir} ];then
|
||||
mkdir -p ${workdir}
|
||||
${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
|
||||
fi
|
||||
# always override the master.cfg
|
||||
cp ${buildbot-master-config} ${workdir}/master.cfg
|
||||
# sanity
|
||||
${buildbot}/bin/buildbot checkconfig ${workdir}
|
||||
|
||||
# TODO: maybe upgrade? not sure about this
|
||||
# normally we should write buildbot.tac by our own
|
||||
# ${buildbot}/bin/buildbot upgrade-master ${workdir}
|
||||
|
||||
chmod 700 -R ${workdir}
|
||||
chown buildbotMaster:buildbotMaster -R ${workdir}
|
||||
'';
|
||||
ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
|
||||
ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
|
||||
ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
|
||||
PrivateTmp = "true";
|
||||
User = "buildbotMaster";
|
||||
Restart = "always";
|
||||
RestartSec = "10";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.makefu.buildbot.master = api;
|
||||
config = mkIf cfg.enable imp;
|
||||
}
|
||||
@ -2,8 +2,6 @@ _:
|
||||
|
||||
{
|
||||
imports = [
|
||||
./buildbot/master.nix
|
||||
./buildbot/slave.nix
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,8 @@ in {
|
||||
imports = [
|
||||
../2configs/base.nix
|
||||
../2configs/os-templates/CAC-CentOS-7-64bit.nix
|
||||
../2configs/os-templates/temp-networking.nix
|
||||
../2configs/temp/networking.nix
|
||||
../2configs/temp/dirs.nix
|
||||
];
|
||||
|
||||
sound.enable = false;
|
||||
|
||||
6
shared/1systems/test-failing.nix
Normal file
6
shared/1systems/test-failing.nix
Normal file
@ -0,0 +1,6 @@
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
{
|
||||
programs.ssh.startAgent = true;
|
||||
programs.ssh.startAgent = false;
|
||||
}
|
||||
@ -11,7 +11,7 @@ in
|
||||
../2configs/collectd-base.nix
|
||||
../2configs/shack-nix-cacher.nix
|
||||
../2configs/shack-drivedroid.nix
|
||||
../2configs/cac-ci.nix
|
||||
../2configs/buildbot-standalone.nix
|
||||
../2configs/graphite.nix
|
||||
];
|
||||
# use your own binary cache, fallback use cache.nixos.org (which is used by
|
||||
@ -33,8 +33,6 @@ in
|
||||
# uninteresting stuff
|
||||
#####################
|
||||
krebs.build.host = config.krebs.hosts.wolf;
|
||||
# TODO rename shared user to "krebs"
|
||||
krebs.build.user = config.krebs.users.shared;
|
||||
krebs.build.target = "wolf";
|
||||
|
||||
boot.kernel.sysctl = {
|
||||
|
||||
@ -13,6 +13,8 @@ with lib;
|
||||
];
|
||||
};
|
||||
|
||||
# TODO rename shared user to "krebs"
|
||||
krebs.build.user = mkDefault config.krebs.users.shared;
|
||||
krebs.build.source = {
|
||||
git.nixpkgs = {
|
||||
url = https://github.com/NixOS/nixpkgs;
|
||||
@ -20,11 +22,11 @@ with lib;
|
||||
};
|
||||
dir.secrets = {
|
||||
host = config.krebs.current.host;
|
||||
path = "${getEnv "HOME"}/secrets/krebs/wolf";
|
||||
path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
|
||||
};
|
||||
dir.stockholm = {
|
||||
host = config.krebs.current.host;
|
||||
path = "${getEnv "HOME"}/stockholm";
|
||||
path = mkDefault "${getEnv "HOME"}/stockholm";
|
||||
};
|
||||
};
|
||||
|
||||
@ -65,7 +67,7 @@ with lib;
|
||||
config.krebs.users.lass.pubkey
|
||||
config.krebs.users.makefu.pubkey
|
||||
# TODO HARDER:
|
||||
(readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub)
|
||||
config.krebs.users.makefu-omo.pubkey
|
||||
config.krebs.users.tv.pubkey
|
||||
];
|
||||
|
||||
|
||||
130
shared/2configs/buildbot-standalone.nix
Normal file
130
shared/2configs/buildbot-standalone.nix
Normal file
@ -0,0 +1,130 @@
|
||||
{ lib, config, pkgs, ... }:
|
||||
let
|
||||
pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
|
||||
in {
|
||||
nixpkgs.config.packageOverrides = pkgs: {
|
||||
buildbot = pkgs-unst.buildbot;
|
||||
buildbot-slave = pkgs-unst.buildbot-slave;
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 8010 ];
|
||||
krebs.buildbot.master = {
|
||||
slaves = {
|
||||
testslave = "krebspass";
|
||||
testslave2 = "krebspass";
|
||||
};
|
||||
change_source.stockholm = ''
|
||||
stockholm_repo = 'http://cgit.gum/stockholm'
|
||||
cs.append(changes.GitPoller(
|
||||
stockholm_repo,
|
||||
workdir='stockholm-poller', branch='master',
|
||||
project='stockholm',
|
||||
pollinterval=120))
|
||||
'';
|
||||
scheduler = {
|
||||
force-scheduler = ''
|
||||
sched.append(schedulers.ForceScheduler(
|
||||
name="force",
|
||||
builderNames=["full-tests"]))
|
||||
'';
|
||||
fast-tests-scheduler = ''
|
||||
# test the master real quick
|
||||
sched.append(schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch="master"),
|
||||
name="fast-master-test",
|
||||
builderNames=["fast-tests"]))
|
||||
'';
|
||||
full-master-scheduler = ''
|
||||
# files everyone depends on or are part of the share branch
|
||||
def shared_files(change):
|
||||
r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
|
||||
for file in change.files:
|
||||
if r.match(file):
|
||||
return True
|
||||
return False
|
||||
|
||||
sched.append(schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch="master"),
|
||||
fileIsImportant=shared_files,
|
||||
name="full-master-test",
|
||||
builderNames=["full-tests"]))
|
||||
'';
|
||||
};
|
||||
builder_pre = ''
|
||||
# prepare grab_repo step for stockholm
|
||||
stockholm_repo = "http://cgit.gum.retiolum/stockholm"
|
||||
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
|
||||
|
||||
env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
|
||||
|
||||
# prepare nix-shell
|
||||
# the dependencies which are used by the test script
|
||||
deps = [ "gnumake", "jq","nix","rsync",
|
||||
"(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
|
||||
# TODO: --pure , prepare ENV in nix-shell command:
|
||||
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
|
||||
nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
|
||||
|
||||
# prepare addShell function
|
||||
def addShell(factory,**kwargs):
|
||||
factory.addStep(steps.ShellCommand(**kwargs))
|
||||
'';
|
||||
builder = {
|
||||
fast-tests = ''
|
||||
f = util.BuildFactory()
|
||||
f.addStep(grab_repo)
|
||||
addShell(f,name="centos7-eval",env=env,
|
||||
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
|
||||
|
||||
addShell(f,name="wolf-eval",env=env,
|
||||
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
|
||||
|
||||
addShell(f,name="eval-cross-check",env=env,
|
||||
command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
|
||||
|
||||
bu.append(util.BuilderConfig(name="fast-tests",
|
||||
slavenames=slavenames,
|
||||
factory=f))
|
||||
'';
|
||||
slow-tests = ''
|
||||
s = util.BuildFactory()
|
||||
s.addStep(grab_repo)
|
||||
|
||||
# slave needs 2 files:
|
||||
# * cac.json
|
||||
# * retiolum
|
||||
for file in ["cac.json", "retiolum.rsa_key.priv"]:
|
||||
s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
|
||||
slavedest=file))
|
||||
|
||||
addShell(s, name="infest-cac-centos7",env=env,
|
||||
sigtermTime=60, # SIGTERM 1 minute before SIGKILL
|
||||
timeout=5400, # 1.5h timeout
|
||||
command=nixshell + ["infest-cac-centos7"])
|
||||
|
||||
bu.append(util.BuilderConfig(name="full-tests",
|
||||
slavenames=slavenames,
|
||||
factory=s))
|
||||
'';
|
||||
};
|
||||
enable = true;
|
||||
web = {
|
||||
enable = true;
|
||||
};
|
||||
irc = {
|
||||
enable = true;
|
||||
nick = "shared-buildbot";
|
||||
server = "cd.retiolum";
|
||||
channels = [ "retiolum" ];
|
||||
allowForce = true;
|
||||
};
|
||||
};
|
||||
|
||||
krebs.buildbot.slave = {
|
||||
enable = true;
|
||||
masterhost = "localhost";
|
||||
username = "testslave";
|
||||
password = "krebspass";
|
||||
packages = with pkgs;[ git nix ];
|
||||
extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
|
||||
};
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
{
|
||||
environment.systemPackages = with pkgs;[
|
||||
get
|
||||
cac
|
||||
cacpanel
|
||||
jq
|
||||
];
|
||||
}
|
||||
1
shared/2configs/temp/dirs.nix
Normal file
1
shared/2configs/temp/dirs.nix
Normal file
@ -0,0 +1 @@
|
||||
_: { }
|
||||
1
shared/2configs/temp/networking.nix
Normal file
1
shared/2configs/temp/networking.nix
Normal file
@ -0,0 +1 @@
|
||||
_: { }
|
||||
@ -234,7 +234,12 @@ with lib;
|
||||
KERNEL=="hpet", GROUP="audio"
|
||||
'';
|
||||
|
||||
services.bitlbee.enable = true;
|
||||
services.bitlbee = {
|
||||
enable = true;
|
||||
plugins = [
|
||||
pkgs.bitlbee-facebook
|
||||
];
|
||||
};
|
||||
services.tor.client.enable = true;
|
||||
services.tor.enable = true;
|
||||
services.virtualboxHost.enable = true;
|
||||
|
||||
@ -33,6 +33,9 @@ let
|
||||
nixpkgs = {};
|
||||
push = {};
|
||||
regfish = {};
|
||||
soundcloud = {
|
||||
desc = "SoundCloud command line interface";
|
||||
};
|
||||
stockholm = {
|
||||
desc = "take all the computers hostage, they'll love you!";
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user