stockholm/krebs/3modules/ci/default.nix

212 lines
6.6 KiB
Nix
Raw Normal View History

2018-09-09 18:01:51 +00:00
{ config, lib, pkgs, ... }:
with import <stockholm/lib>;
let
cfg = config.krebs.ci;
out = {
options.krebs.ci = api;
config = lib.mkIf cfg.enable imp;
};
api = {
enable = mkEnableOption "Enable krebs ci service";
repos = mkOption {
type = types.attrsOf (types.submodule ({ config, ...}: {
options = {
urls = mkOption {
type = types.listOf types.str;
default = [ "git@localhost:${config._module.args.name}" ];
};
};
}));
};
};
hostname = config.networking.hostName;
getJobs = pkgs.writeDash "get_jobs" ''
set -efu
2019-09-25 18:40:22 +00:00
${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2
2018-11-27 20:06:20 +00:00
json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)"
echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \
2018-11-26 23:58:07 +00:00
| while read -r host builder; do
gcroot=${shell.escape profileRoot}/$host-builder
${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder"
done
2018-11-27 20:06:20 +00:00
echo "$json"
2018-09-09 18:01:51 +00:00
'';
2018-11-13 20:38:28 +00:00
profileRoot = "/nix/var/nix/profiles/ci";
bcfg = config.services.buildbot-master;
2018-09-09 18:01:51 +00:00
imp = {
services.buildbot-master = {
workers = [ "worker.Worker('testworker', 'pass')" ];
changeSource = mapAttrsToList (name: repo:
concatMapStringsSep "," (url: ''
changes.GitPoller(
"${url}",
workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
project='${name}',
pollinterval=30
2018-09-09 18:01:51 +00:00
)
'') repo.urls
2018-09-09 18:01:51 +00:00
) cfg.repos;
schedulers = mapAttrsToList (name: repo: ''
schedulers.SingleBranchScheduler(
change_filter=util.ChangeFilter(
branch_re=".*",
project='${name}',
),
treeStableTimer=60,
name="${name}-all-branches",
builderNames=[
"${name}",
]
),
schedulers.ForceScheduler(
name="${name}",
builderNames=[
"${name}",
]
)
'') cfg.repos;
builders = [];
2018-09-09 18:01:51 +00:00
extraConfig = ''
# https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
from buildbot.plugins import util, steps
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
import json
import sys
2018-09-09 18:01:51 +00:00
class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
def __init__(self, **kwargs):
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.observer)
def extract_stages(self, stdout):
stages = json.loads(stdout)
return stages
@defer.inlineCallbacks
def run(self):
# run nix-instanstiate to generate the dict of stages
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
stages = self.extract_stages(self.observer.getStdout())
self.build.addStepsAfterCurrentStep([
2021-12-11 11:00:36 +00:00
steps.ShellCommand(
name=stage,
env=dict(
build_name = stage,
build_script = stages[stage],
),
2022-11-22 08:17:56 +00:00
timeout = 3600,
2021-12-11 11:00:36 +00:00
command="${pkgs.writeDash "build.sh" ''
set -xefu
profile=${shell.escape profileRoot}/$build_name
result=$("$build_script")
if [ -n "$result" ]; then
${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
fi
''}",
) for stage in stages
])
return result
${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
factory_${name} = util.BuildFactory()
factory_${name}.addStep(steps.Git(
2018-09-09 18:01:51 +00:00
repourl=util.Property('repository', '${head repo.urls}'),
2019-11-29 13:11:18 +00:00
method='clobber',
2018-09-09 18:01:51 +00:00
mode='full',
submodules=True,
))
factory_${name}.addStep(GenerateStagesCommand(
2018-09-09 18:01:51 +00:00
env={
"NIX_REMOTE": "daemon",
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
2018-09-09 18:01:51 +00:00
},
name="Generate build stages",
command=[
"${getJobs}"
],
haltOnFailure=True,
2018-09-09 18:01:51 +00:00
))
c['builders'].append(
2018-09-09 18:01:51 +00:00
util.BuilderConfig(
name="${name}",
workernames=['testworker'],
factory=factory_${name}
2018-09-09 18:01:51 +00:00
)
)
'') cfg.repos)}
# fancy irc notification by Mic92 https://github.com/Mic92/dotfiles/tree/master/nixos/eve/modules/buildbot
sys.path.append("${./modules}")
from irc_notify import NotifyFailedBuilds
c['services'].append(
NotifyFailedBuilds("irc://buildbot|test@irc.r:6667/#xxx")
)
'';
2018-09-09 18:01:51 +00:00
enable = true;
reporters = [
''
reporters.IRC(
host = "irc.r",
nick = "buildbot|${hostname}",
notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
channels = [{"channel": "#xxx"}],
showBlameList = True,
authz={'force': True},
)
''
];
buildbotUrl = "http://build.${hostname}.r/";
2018-09-09 18:01:51 +00:00
};
services.buildbot-worker = {
2018-09-09 18:01:51 +00:00
enable = true;
workerUser = "testworker";
workerPass = "pass";
packages = with pkgs; [ git gnutar gzip jq nix populate ];
2018-09-09 18:01:51 +00:00
};
2018-11-13 20:38:28 +00:00
system.activationScripts.buildbots-nix-profile = ''
${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot}
${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot}
${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot}
'';
users = {
groups.buildbots.gid = genid "buildbots";
users = {
buildbot.extraGroups = [ "buildbots" ];
bbworker.extraGroups = [ "buildbots" ];
2018-11-13 20:38:28 +00:00
};
};
2018-09-09 18:01:51 +00:00
};
in out