diff --git a/Makefile b/Makefile
index aefd17147..5b898c54c 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ ifeq ($(filter),json)
else
filter() { cat; }
endif
- nix-instantiate \
+ result=$$(nix-instantiate \
$${extraArgs-} \
--eval \
-A "$$get" \
@@ -45,8 +45,9 @@ endif
--argstr current-host-name "$$HOSTNAME" \
--argstr current-user-name "$$LOGNAME" \
$${system+--argstr system "$$system"} \
- $${target+--argstr target "$$target"} \
- | filter
+ $${target+--argstr target "$$target"})
+ echo "$$result" | filter
+
else
$(error unbound variable: system[s])
endif
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
new file mode 100644
index 000000000..7078000fe
--- /dev/null
+++ b/krebs/3modules/buildbot/master.nix
@@ -0,0 +1,375 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+ buildbot = pkgs.buildbot;
+ buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+ # -*- python -*-
+ from buildbot.plugins import *
+ import re
+ import json
+ c = BuildmasterConfig = {}
+
+ c['slaves'] = []
+ slaves = json.loads('${builtins.toJSON cfg.slaves}')
+ slavenames = [ s for s in slaves ]
+ for k,v in slaves.items():
+ c['slaves'].append(buildslave.BuildSlave(k, v))
+
+ # TODO: configure protocols?
+ c['protocols'] = {'pb': {'port': 9989}}
+
+ ####### Build Inputs
+ c['change_source'] = cs = []
+
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Change_Source: Begin of ${n}
+ ${v}
+ #### Change_Source: End of ${n}
+ '') cfg.change_source )}
+
+ ####### Build Scheduler
+ c['schedulers'] = sched = []
+
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Schedulers: Begin of ${n}
+ ${v}
+ #### Schedulers: End of ${n}
+ '') cfg.scheduler )}
+
+ ###### Builder
+ c['builders'] = bu = []
+
+ # Builder Pre: Begin
+ ${cfg.builder_pre}
+ # Builder Pre: End
+
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Builder: Begin of ${n}
+ ${v}
+ #### Builder: End of ${n}
+ '') cfg.builder )}
+
+
+ ####### Status
+ c['status'] = st = []
+
+ # If you want to configure this url, override with extraConfig
+ c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+ ${optionalString (cfg.web.enable) ''
+ from buildbot.status import html
+ from buildbot.status.web import authz, auth
+ authz_cfg=authz.Authz(
+ auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+ # TODO: configure harder
+ gracefulShutdown = False,
+ forceBuild = 'auth',
+ forceAllBuilds = 'auth',
+ pingBuilder = False,
+ stopBuild = 'auth',
+ stopAllBuilds = 'auth',
+ cancelPendingBuild = 'auth'
+ )
+ # TODO: configure krebs.nginx
+ st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+ ''}
+
+ ${optionalString (cfg.irc.enable) ''
+ from buildbot.status import words
+ irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+ channels=${builtins.toJSON cfg.irc.channels},
+ notify_events={
+ 'success': 1,
+ 'failure': 1,
+ 'exception': 1,
+ 'successToFailure': 1,
+ 'failureToSuccess': 1,
+ }${optionalString cfg.irc.allowForce ",allowForce=True"})
+ c['status'].append(irc)
+ ''}
+
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Status: Begin of ${n}
+ ${v}
+ #### Status: End of ${n}
+ '') cfg.status )}
+
+ ####### PROJECT IDENTITY
+ c['title'] = "${cfg.title}"
+ c['titleURL'] = "http://krebsco.de"
+
+
+ ####### DB URL
+ # TODO: configure
+ c['db'] = {
+ 'db_url' : "sqlite:///state.sqlite",
+ }
+ ${cfg.extraConfig}
+ '';
+
+ cfg = config.krebs.buildbot.master;
+
+ api = {
+ enable = mkEnableOption "Buildbot Master";
+ title = mkOption {
+ default = "Buildbot CI";
+ type = types.str;
+ description = ''
+ Title of the Buildbot Installation
+ '';
+ };
+ workDir = mkOption {
+ default = "/var/lib/buildbot/master";
+ type = types.str;
+ description = ''
+ Path to build bot master directory.
+ Will be created on startup.
+ '';
+ };
+
+ slaves = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ description = ''
+ Attrset of slavenames with their passwords
+ slavename = slavepassword
+ '';
+ };
+
+ change_source = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ stockholm = ''
+ cs.append(changes.GitPoller(
+ 'http://cgit.gum/stockholm',
+ workdir='stockholm-poller', branch='master',
+ project='stockholm',
+ pollinterval=120))
+ '';
+ };
+ description = ''
+ Attrset of all the change_sources which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to cs
+ '';
+ };
+
+ scheduler = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ force-scheduler = ''
+ sched.append(schedulers.ForceScheduler(
+ name="force",
+ builderNames=["full-tests"]))
+ '';
+ };
+ description = ''
+ Attrset of all the schedulers which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to sched
+ '';
+ };
+
+ builder_pre = mkOption {
+ default = "";
+ type = types.lines;
+ example = ''
+ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+ '';
+ description = ''
+ some code before the builders are being assembled.
+ can be used to define functions used by multiple builders
+ '';
+ };
+
+ builder = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ fast-test = ''
+ '';
+ };
+ description = ''
+ Attrset of all the builder which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to bu
+ '';
+ };
+
+ status = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ description = ''
+ Attrset of all the extra status which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to st
+
+ Right now IRC and Web status can be configured by setting
+ buildbot.master.irc.enable and
+ buildbot.master.web.enable
+ '';
+ };
+
+ # Configurable Stati
+ web = mkOption {
+ default = {};
+ type = types.submodule ({ config2, ... }: {
+ options = {
+ enable = mkEnableOption "Buildbot Master Web Status";
+ username = mkOption {
+ default = "krebs";
+ type = types.str;
+ description = ''
+ username for web authentication
+ '';
+ };
+ hostname = mkOption {
+ default = config.networking.hostName;
+ type = types.str;
+ description = ''
+ web interface Hostname
+ '';
+ };
+ password = mkOption {
+ default = "bob";
+ type = types.str;
+ description = ''
+ password for web authentication
+ '';
+ };
+ port = mkOption {
+ default = 8010;
+ type = types.int;
+ description = ''
+ port for buildbot web status
+ '';
+ };
+ };
+ });
+ };
+
+ irc = mkOption {
+ default = {};
+ type = types.submodule ({ config, ... }: {
+ options = {
+ enable = mkEnableOption "Buildbot Master IRC Status";
+ channels = mkOption {
+ default = [ "nix-buildbot-meetup" ];
+ type = with types; listOf str;
+ description = ''
+ irc channels the bot should connect to
+ '';
+ };
+ allowForce = mkOption {
+ default = false;
+ type = types.bool;
+ description = ''
+ Determines if builds can be forced via IRC
+ '';
+ };
+ nick = mkOption {
+ default = "nix-buildbot";
+ type = types.str;
+ description = ''
+ nickname for IRC
+ '';
+ };
+ server = mkOption {
+ default = "irc.freenode.net";
+ type = types.str;
+ description = ''
+ Buildbot Status IRC Server to connect to
+ '';
+ };
+ };
+ });
+ };
+
+ extraConfig = mkOption {
+ default = "";
+ type = types.lines;
+ description = ''
+ extra config appended to the generated master.cfg
+ '';
+ };
+ };
+
+ imp = {
+
+ users.extraUsers.buildbotMaster = {
+ uid = 672626386; #genid buildbotMaster
+ description = "Buildbot Master";
+ home = cfg.workDir;
+ createHome = false;
+ };
+
+ users.extraGroups.buildbotMaster = {
+ gid = 672626386;
+ };
+
+ systemd.services.buildbotMaster = {
+ description = "Buildbot Master";
+ after = [ "network.target" ];
+ wantedBy = [ "multi-user.target" ];
+ # TODO: add extra dependencies to master like svn and cvs
+ path = [ pkgs.git ];
+ environment = {
+ SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+ };
+ serviceConfig = let
+ workdir="${lib.shell.escape cfg.workDir}";
+ secretsdir="${lib.shell.escape (toString )}";
+ in {
+ PermissionsStartOnly = true;
+ Type = "forking";
+ PIDFile = "${workdir}/twistd.pid";
+ # TODO: maybe also prepare buildbot.tac?
+ ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+ #!/bin/sh
+ set -efux
+ if [ ! -e ${workdir} ];then
+ mkdir -p ${workdir}
+ ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+ fi
+ # always override the master.cfg
+ cp ${buildbot-master-config} ${workdir}/master.cfg
+ # copy secrets
+ cp ${secretsdir}/cac.json ${workdir}
+ cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+ ${workdir}/retiolum.rsa_key.priv
+ # sanity
+ ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+ # TODO: maybe upgrade? not sure about this
+ # normally we should write buildbot.tac by our own
+ # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+ chmod 700 -R ${workdir}
+ chown buildbotMaster:buildbotMaster -R ${workdir}
+ '';
+ ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+ ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+ ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+ PrivateTmp = "true";
+ User = "buildbotMaster";
+ Restart = "always";
+ RestartSec = "10";
+ };
+ };
+ };
+in
+{
+ options.krebs.buildbot.master = api;
+ config = mkIf cfg.enable imp;
+}
diff --git a/makefu/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
similarity index 97%
rename from makefu/3modules/buildbot/slave.nix
rename to krebs/3modules/buildbot/slave.nix
index 69d0361bf..8711a287a 100644
--- a/makefu/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -39,7 +39,7 @@ let
s.setServiceParent(application)
'';
default-packages = [ pkgs.git pkgs.bash ];
- cfg = config.makefu.buildbot.slave;
+ cfg = config.krebs.buildbot.slave;
api = {
enable = mkEnableOption "Buildbot Slave";
@@ -144,6 +144,7 @@ let
path = default-packages ++ cfg.packages;
environment = {
+ SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
NIX_REMOTE="daemon";
} // cfg.extraEnviron;
@@ -180,6 +181,6 @@ let
};
in
{
- options.makefu.buildbot.slave = api;
+ options.krebs.buildbot.slave = api;
config = mkIf cfg.enable imp;
}
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..cbc1291fa 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -9,6 +9,8 @@ let
./apt-cacher-ng.nix
./bepasty-server.nix
./build.nix
+ ./buildbot/master.nix
+ ./buildbot/slave.nix
./current.nix
./exim-retiolum.nix
./exim-smarthost.nix
diff --git a/krebs/5pkgs/cac/default.nix b/krebs/5pkgs/cac/default.nix
index 40dd56412..4d39ce2fb 100644
--- a/krebs/5pkgs/cac/default.nix
+++ b/krebs/5pkgs/cac/default.nix
@@ -1,12 +1,12 @@
-{ stdenv, fetchgit, coreutils, curl, gnused, inotifyTools, jq, ncurses, sshpass, ... }:
+{ stdenv, fetchgit, bc, coreutils, curl, gnused, inotifyTools, jq, ncurses, sshpass, ... }:
stdenv.mkDerivation {
- name = "cac-1.0.0";
+ name = "cac-1.0.3";
src = fetchgit {
- url = http://cgit.gum/cac;
- rev = "fe3b2ecb0aaf7d863842b896e18cd2b829f2297b";
- sha256 = "05bnd7wyjhqy8srmpnc8d234rv3jxdjgb4z0hlfb9kg7mb12w1ya";
+ url = http://cgit.cd.retiolum/cac;
+ rev = "22acc1b990ac7d97c16344fbcbc2621e24cdf915";
+ sha256 = "135b740617c983b3f46a1983d4744be17340d5146a0a0de0dff4bb7a53688f2f";
};
phases = [
@@ -17,6 +17,7 @@ stdenv.mkDerivation {
installPhase =
let
path = stdenv.lib.makeSearchPath "bin" [
+ bc
coreutils
curl
gnused
@@ -29,10 +30,9 @@ stdenv.mkDerivation {
''
mkdir -p $out/bin
- sed \
- 's,^\( true) \)\(cac "$@";;\)$,\1 PATH=${path}${PATH+:$PATH} \2,' \
- < ./cac \
- > $out/bin/cac
+ sed < ./cac > $out/bin/cac '
+ s;^_cac_main .*;PATH=${path}''${PATH+:$PATH} &;
+ '
chmod +x $out/bin/cac
'';
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 7df7b7d3c..0562fe836 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -40,6 +40,10 @@ subdirs // rec {
}
'';
+ test = {
+ infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {};
+ };
+
execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; });
writeC = name: { destination ? "" }: src: pkgs.runCommand name {} ''
diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix
new file mode 100644
index 000000000..7f2e3f231
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix
@@ -0,0 +1,39 @@
+{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
+
+stdenv.mkDerivation rec {
+ name = "${shortname}-${version}";
+ shortname = "infest-cac-centos7";
+ version = "0.2.0";
+
+ src = ./notes;
+
+ phases = [
+ "installPhase"
+ ];
+ buildInputs = [ makeWrapper ];
+
+ path = stdenv.lib.makeSearchPath "bin" [
+ coreutils
+ cac
+ cacpanel
+ gnumake
+ gnused
+ jq
+ openssh
+ ];
+
+ installPhase =
+ ''
+ mkdir -p $out/bin
+ cp ${src} $out/bin/${shortname}
+ chmod +x $out/bin/${shortname}
+ wrapProgram $out/bin/${shortname} \
+ --prefix PATH : ${path}
+ '';
+ meta = with stdenv.lib; {
+ homepage = http://krebsco.de;
+ description = "Krebs CI Scripts";
+ license = licenses.wtfpl;
+ maintainers = [ maintainers.makefu ];
+ };
+}
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
new file mode 100755
index 000000000..5bb5de2c4
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -0,0 +1,116 @@
+#! /bin/sh
+
+# nix-shell -p gnumake jq openssh cac cacpanel
+set -eufx
+
+# 2 secrets are required:
+
+krebs_cred=${krebs_cred-./cac.json}
+retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+
+# Sanity
+if test ! -r "$krebs_cred";then
+ echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
+fi
+if test ! -r "$retiolum_key";then
+ echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
+fi
+
+krebs_secrets=$(mktemp -d)
+sec_file=$krebs_secrets/cac_config
+krebs_ssh=$krebs_secrets/tempssh
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
+# we need to receive this key from buildmaster to speed up tinc bootstrap
+TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT"
+trap "$TRAP" INT TERM EXIT
+
+cat > $sec_file <&1\
+ | jq -r .servername)
+
+id=servername:$name
+trap "cac delete $id;$TRAP;exit" INT TERM EXIT
+# TODO: timeout?
+
+wait_login_cac(){
+ # timeout
+ for t in `seq 180`;do
+ # now we have a working cac server
+ if cac ssh $1 -o ConnectTimeout=10 \
+ cat /etc/redhat-release | \
+ grep CentOS ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+}
+# die on timeout
+wait_login_cac $id
+
+mkdir -p shared/2configs/temp
+cac generatenetworking $id > \
+ shared/2configs/temp/networking.nix
+# new temporary ssh key we will use to log in after infest
+ssh-keygen -f $krebs_ssh -N ""
+cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
+# we override the directories for secrets and stockholm
+# additionally we set the ssh key we generated
+ip=$(cac getserver $id | jq -r .ip)
+
+cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest
+sh -x $krebs_secrets/infest
+
+# TODO: generate secrets directory $krebs_secrets for nix import
+cac powerop $id reset
+
+wait_login(){
+ # timeout
+ for t in `seq 90`;do
+ # now we have a working cac server
+ if ssh -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ -i $krebs_ssh \
+ -o ConnectTimeout=10 \
+ -o BatchMode=yes \
+ root@$1 nixos-version ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+}
+wait_login $ip
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index c0d7685e3..a0b49edaf 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -24,7 +24,7 @@ with lib;
git.nixpkgs = {
#url = https://github.com/NixOS/nixpkgs;
url = mkDefault https://github.com/makefu/nixpkgs;
- rev = mkDefault "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picking
+ rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking
target-path = "/var/src/nixpkgs";
};
diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
deleted file mode 100644
index 58e2f8175..000000000
--- a/makefu/3modules/buildbot/master.nix
+++ /dev/null
@@ -1,263 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-let
- buildbot = pkgs.buildbot;
- buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
- # -*- python -*-
- from buildbot.plugins import *
- import re
-
- c = BuildmasterConfig = {}
-
- c['slaves'] = []
- # TODO: template potential buildslaves
- # TODO: set password?
- slavenames= [ 'testslave' ]
- for i in slavenames:
- c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
-
- c['protocols'] = {'pb': {'port': 9989}}
-
- ####### Build Inputs
- stockholm_repo = 'http://cgit.gum/stockholm'
- c['change_source'] = []
- c['change_source'].append(changes.GitPoller(
- stockholm_repo,
- workdir='stockholm-poller', branch='master',
- project='stockholm',
- pollinterval=120))
-
- ####### Build Scheduler
- # TODO: configure scheduler
- c['schedulers'] = []
-
- # test the master real quick
- fast = schedulers.SingleBranchScheduler(
- change_filter=util.ChangeFilter(branch="master"),
- name="fast-master-test",
- builderNames=["fast-tests"])
-
- force = schedulers.ForceScheduler(
- name="force",
- builderNames=["full-tests"])
-
- # files everyone depends on or are part of the share branch
- def shared_files(change):
- r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
- for file in change.files:
- if r.match(file):
- return True
- return False
-
- full = schedulers.SingleBranchScheduler(
- change_filter=util.ChangeFilter(branch="master"),
- fileIsImportant=shared_files,
- name="full-master-test",
- builderNames=["full-tests"])
- c['schedulers'] = [ fast, force, full ]
- ###### The actual build
- # couple of fast steps:
- f = util.BuildFactory()
- ## fetch repo
- grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
- f.addStep(grab_repo)
-
- # the dependencies which are used by the test script
- deps = [ "gnumake", "jq" ]
- nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
- def addShell(f,**kwargs):
- f.addStep(steps.ShellCommand(**kwargs))
-
- addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=test-centos7"])
-
- addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=wolf"])
-
- c['builders'] = []
- c['builders'].append(
- util.BuilderConfig(name="fast-tests",
- slavenames=slavenames,
- factory=f))
-
- # TODO slow build
- c['builders'].append(
- util.BuilderConfig(name="full-tests",
- slavenames=slavenames,
- factory=f))
-
- ####### Status of Builds
- c['status'] = []
-
- from buildbot.status import html
- from buildbot.status.web import authz, auth
- # TODO: configure if http is wanted
- authz_cfg=authz.Authz(
- # TODO: configure user/pw
- auth=auth.BasicAuth([("krebs","bob")]),
- gracefulShutdown = False,
- forceBuild = 'auth',
- forceAllBuilds = 'auth',
- pingBuilder = False,
- stopBuild = False,
- stopAllBuilds = False,
- cancelPendingBuild = False,
- )
- # TODO: configure nginx
- c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
- from buildbot.status import words
- ${optionalString (cfg.irc.enable) ''
- irc = words.IRC("${cfg.irc.server}", "krebsbuild",
- # TODO: multiple channels
- channels=["${cfg.irc.channel}"],
- notify_events={
- #'success': 1,
- #'failure': 1,
- 'exception': 1,
- 'successToFailure': 1,
- 'failureToSuccess': 1,
- }${optionalString cfg.irc.allowForce ",allowForce=True"})
- c['status'].append(irc)
- ''}
-
- ####### PROJECT IDENTITY
- c['title'] = "Stockholm"
- c['titleURL'] = "http://krebsco.de"
-
- #c['buildbotURL'] = "http://buildbot.krebsco.de/"
- # TODO: configure url
- c['buildbotURL'] = "http://vbob:8010/"
-
- ####### DB URL
- c['db'] = {
- 'db_url' : "sqlite:///state.sqlite",
- }
- ${cfg.extraConfig}
- '';
-
- cfg = config.makefu.buildbot.master;
-
- api = {
- enable = mkEnableOption "Buildbot Master";
- workDir = mkOption {
- default = "/var/lib/buildbot/master";
- type = types.str;
- description = ''
- Path to build bot master directory.
- Will be created on startup.
- '';
- };
- irc = mkOption {
- default = {};
- type = types.submodule ({ config, ... }: {
- options = {
- enable = mkEnableOption "Buildbot Master IRC Status";
- channel = mkOption {
- default = "nix-buildbot-meetup";
- type = types.str;
- description = ''
- irc channel the bot should connect to
- '';
- };
- allowForce = mkOption {
- default = false;
- type = types.bool;
- description = ''
- Determines if builds can be forced via IRC
- '';
- };
- nick = mkOption {
- default = "nix-buildbot";
- type = types.str;
- description = ''
- nickname for IRC
- '';
- };
- server = mkOption {
- default = "irc.freenode.net";
- type = types.str;
- description = ''
- Buildbot Status IRC Server to connect to
- '';
- };
- };
- });
- };
-
- extraConfig = mkOption {
- default = "";
- type = types.lines;
- description = ''
- extra config appended to the generated master.cfg
- '';
- };
- };
-
- imp = {
-
- users.extraUsers.buildbotMaster = {
- uid = 672626386; #genid buildbotMaster
- description = "Buildbot Master";
- home = cfg.workDir;
- createHome = false;
- };
-
- users.extraGroups.buildbotMaster = {
- gid = 672626386;
- };
-
- systemd.services.buildbotMaster = {
- description = "Buildbot Master";
- after = [ "network.target" ];
- wantedBy = [ "multi-user.target" ];
- path = [ pkgs.git ];
- serviceConfig = let
- workdir="${lib.shell.escape cfg.workDir}";
- # TODO: check if git is the only dep
- in {
- PermissionsStartOnly = true;
- Type = "forking";
- PIDFile = "${workdir}/twistd.pid";
- # TODO: maybe also prepare buildbot.tac?
- ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
- #!/bin/sh
- set -efux
- if [ ! -e ${workdir} ];then
- mkdir -p ${workdir}
- ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
- fi
- # always override the master.cfg
- cp ${buildbot-master-config} ${workdir}/master.cfg
- # sanity
- ${buildbot}/bin/buildbot checkconfig ${workdir}
-
- # TODO: maybe upgrade? not sure about this
- # normally we should write buildbot.tac by our own
- # ${buildbot}/bin/buildbot upgrade-master ${workdir}
-
- chmod 700 -R ${workdir}
- chown buildbotMaster:buildbotMaster -R ${workdir}
- '';
- ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
- ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
- ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
- PrivateTmp = "true";
- User = "buildbotMaster";
- Restart = "always";
- RestartSec = "10";
- };
- };
- };
-in
-{
- options.makefu.buildbot.master = api;
- config = mkIf cfg.enable imp;
-}
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index ffbf54cc0..a8a1f69d0 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,8 +2,6 @@ _:
{
imports = [
- ./buildbot/master.nix
- ./buildbot/slave.nix
];
}
diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix
index 077a5d61b..48cecc877 100644
--- a/shared/1systems/test-centos7.nix
+++ b/shared/1systems/test-centos7.nix
@@ -7,7 +7,8 @@ in {
imports = [
../2configs/base.nix
../2configs/os-templates/CAC-CentOS-7-64bit.nix
- ../2configs/os-templates/temp-networking.nix
+ ../2configs/temp/networking.nix
+ ../2configs/temp/dirs.nix
];
sound.enable = false;
diff --git a/shared/1systems/test-failing.nix b/shared/1systems/test-failing.nix
new file mode 100644
index 000000000..81a9e48d6
--- /dev/null
+++ b/shared/1systems/test-failing.nix
@@ -0,0 +1,6 @@
+{ config, pkgs, ... }:
+
+{
+ programs.ssh.startAgent = true;
+ programs.ssh.startAgent = false;
+}
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 2c51ac8fe..f05356f0f 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -11,7 +11,7 @@ in
../2configs/collectd-base.nix
../2configs/shack-nix-cacher.nix
../2configs/shack-drivedroid.nix
- ../2configs/cac-ci.nix
+ ../2configs/buildbot-standalone.nix
../2configs/graphite.nix
];
# use your own binary cache, fallback use cache.nixos.org (which is used by
@@ -33,8 +33,6 @@ in
# uninteresting stuff
#####################
krebs.build.host = config.krebs.hosts.wolf;
- # TODO rename shared user to "krebs"
- krebs.build.user = config.krebs.users.shared;
krebs.build.target = "wolf";
boot.kernel.sysctl = {
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index df41eae1a..0ce336558 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -13,6 +13,8 @@ with lib;
];
};
+ # TODO rename shared user to "krebs"
+ krebs.build.user = mkDefault config.krebs.users.shared;
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
@@ -20,11 +22,11 @@ with lib;
};
dir.secrets = {
host = config.krebs.current.host;
- path = "${getEnv "HOME"}/secrets/krebs/wolf";
+ path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
};
dir.stockholm = {
host = config.krebs.current.host;
- path = "${getEnv "HOME"}/stockholm";
+ path = mkDefault "${getEnv "HOME"}/stockholm";
};
};
@@ -65,7 +67,7 @@ with lib;
config.krebs.users.lass.pubkey
config.krebs.users.makefu.pubkey
# TODO HARDER:
- (readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub)
+ config.krebs.users.makefu-omo.pubkey
config.krebs.users.tv.pubkey
];
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
new file mode 100644
index 000000000..baab059c9
--- /dev/null
+++ b/shared/2configs/buildbot-standalone.nix
@@ -0,0 +1,130 @@
+{ lib, config, pkgs, ... }:
+let
+ pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
+in {
+ nixpkgs.config.packageOverrides = pkgs: {
+ buildbot = pkgs-unst.buildbot;
+ buildbot-slave = pkgs-unst.buildbot-slave;
+ };
+ networking.firewall.allowedTCPPorts = [ 8010 ];
+ krebs.buildbot.master = {
+ slaves = {
+ testslave = "krebspass";
+ testslave2 = "krebspass";
+ };
+ change_source.stockholm = ''
+ stockholm_repo = 'http://cgit.gum/stockholm'
+ cs.append(changes.GitPoller(
+ stockholm_repo,
+ workdir='stockholm-poller', branch='master',
+ project='stockholm',
+ pollinterval=120))
+ '';
+ scheduler = {
+ force-scheduler = ''
+ sched.append(schedulers.ForceScheduler(
+ name="force",
+ builderNames=["full-tests"]))
+ '';
+ fast-tests-scheduler = ''
+ # test the master real quick
+ sched.append(schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(branch="master"),
+ name="fast-master-test",
+ builderNames=["fast-tests"]))
+ '';
+ full-master-scheduler = ''
+ # files everyone depends on or are part of the share branch
+ def shared_files(change):
+ r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
+ for file in change.files:
+ if r.match(file):
+ return True
+ return False
+
+ sched.append(schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(branch="master"),
+ fileIsImportant=shared_files,
+ name="full-master-test",
+ builderNames=["full-tests"]))
+ '';
+ };
+ builder_pre = ''
+ # prepare grab_repo step for stockholm
+ stockholm_repo = "http://cgit.gum.retiolum/stockholm"
+ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+
+ env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+
+ # prepare nix-shell
+ # the dependencies which are used by the test script
+ deps = [ "gnumake", "jq","nix","rsync",
+ "(import {}).pkgs.test.infest-cac-centos7" ]
+ # TODO: --pure , prepare ENV in nix-shell command:
+ # SSL_CERT_FILE,LOGNAME,NIX_REMOTE
+ nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
+ # prepare addShell function
+ def addShell(factory,**kwargs):
+ factory.addStep(steps.ShellCommand(**kwargs))
+ '';
+ builder = {
+ fast-tests = ''
+ f = util.BuildFactory()
+ f.addStep(grab_repo)
+ addShell(f,name="centos7-eval",env=env,
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+ addShell(f,name="wolf-eval",env=env,
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+ addShell(f,name="eval-cross-check",env=env,
+ command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+
+ bu.append(util.BuilderConfig(name="fast-tests",
+ slavenames=slavenames,
+ factory=f))
+ '';
+ slow-tests = ''
+ s = util.BuildFactory()
+ s.addStep(grab_repo)
+
+ # slave needs 2 files:
+ # * cac.json
+ # * retiolum
+ for file in ["cac.json", "retiolum.rsa_key.priv"]:
+ s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
+ slavedest=file))
+
+ addShell(s, name="infest-cac-centos7",env=env,
+ sigtermTime=60, # SIGTERM 1 minute before SIGKILL
+ timeout=5400, # 1.5h timeout
+ command=nixshell + ["infest-cac-centos7"])
+
+ bu.append(util.BuilderConfig(name="full-tests",
+ slavenames=slavenames,
+ factory=s))
+ '';
+ };
+ enable = true;
+ web = {
+ enable = true;
+ };
+ irc = {
+ enable = true;
+ nick = "shared-buildbot";
+ server = "cd.retiolum";
+ channels = [ "retiolum" ];
+ allowForce = true;
+ };
+ };
+
+ krebs.buildbot.slave = {
+ enable = true;
+ masterhost = "localhost";
+ username = "testslave";
+ password = "krebspass";
+ packages = with pkgs;[ git nix ];
+ extraEnviron = { NIX_PATH="nixpkgs=${toString }"; };
+ };
+}
diff --git a/shared/2configs/cac-ci.nix b/shared/2configs/cac-ci.nix
deleted file mode 100644
index 06cce2746..000000000
--- a/shared/2configs/cac-ci.nix
+++ /dev/null
@@ -1,11 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-{
- environment.systemPackages = with pkgs;[
- get
- cac
- cacpanel
- jq
- ];
-}
diff --git a/shared/2configs/temp/dirs.nix b/shared/2configs/temp/dirs.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/dirs.nix
@@ -0,0 +1 @@
+_: { }
diff --git a/shared/2configs/temp/networking.nix b/shared/2configs/temp/networking.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/networking.nix
@@ -0,0 +1 @@
+_: { }
diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix
index cd3139754..a768e781b 100644
--- a/tv/1systems/wu.nix
+++ b/tv/1systems/wu.nix
@@ -234,7 +234,12 @@ with lib;
KERNEL=="hpet", GROUP="audio"
'';
- services.bitlbee.enable = true;
+ services.bitlbee = {
+ enable = true;
+ plugins = [
+ pkgs.bitlbee-facebook
+ ];
+ };
services.tor.client.enable = true;
services.tor.enable = true;
services.virtualboxHost.enable = true;
diff --git a/tv/2configs/git.nix b/tv/2configs/git.nix
index 4387be36a..800deff1b 100644
--- a/tv/2configs/git.nix
+++ b/tv/2configs/git.nix
@@ -33,6 +33,9 @@ let
nixpkgs = {};
push = {};
regfish = {};
+ soundcloud = {
+ desc = "SoundCloud command line interface";
+ };
stockholm = {
desc = "take all the computers hostage, they'll love you!";
};