diff --git a/Makefile b/Makefile deleted file mode 100644 index 4258d9178..000000000 --- a/Makefile +++ /dev/null @@ -1,126 +0,0 @@ -stockholm ?= . - -export HOSTNAME ?= $(shell cat /proc/sys/kernel/hostname) - -export STOCKHOLM_VERSION ?= $(shell \ - version=git.$$(git describe --always --dirty); \ - case $$version in (*-dirty) version=$$version@$$HOSTNAME; esac; \ - date=$$(date +%y.%m); \ - printf '%s' "$$date.$$version"; \ -) - -system ?= $(HOSTNAME) -$(if $(system),,$(error unbound variable: system)) - -nixos-config ?= $(stockholm)/$(LOGNAME)/1systems/$(system)/config.nix -ifneq ($(words $(wildcard $(nixos-config))),1) -$(error bad nixos-config: $(nixos-config)) -endif - -# target = [target_user@]target_host[:target_port][/target_path] -ifdef target -_target_user != echo $(target) | sed -n 's/@.*//p' -_target_path != echo $(target) | sed -n 's/^[^/]*//p' -_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p' -_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p' -ifneq ($(_target_host),) -$(if $(target_host),$(error cannot define both, target_host and host in target)) -target_host ?= $(_target_host) -endif -ifneq ($(_target_user),) -$(if $(target_user),$(error cannot define both, target_user and user in target)) -target_user ?= $(_target_user) -endif -ifneq ($(_target_port),) -$(if $(target_port),$(error cannot define both, target_port and port in target)) -target_port ?= $(_target_port) -endif -ifneq ($(_target_path),) -$(if $(target_path),$(error cannot define both, target_path and path in target)) -target_path ?= $(_target_path) -endif -endif - -target_host ?= $(system) -target_user ?= root -target_port ?= 22 -target_path ?= /var/src - -$(if $(target_host),,$(error unbound variable: target_host)) -$(if $(target_user),,$(error unbound variable: target_user)) -$(if $(target_port),,$(error unbound variable: target_port)) -$(if $(target_path),,$(error unbound variable: target_path)) - -whatsupnix = \ - if type whatsupnix >/dev/null 2>&1; then \ - whatsupnix $(1); \ - else \ - cat; \ - fi - -build = \ - nix-build \ - -Q \ - --no-out-link \ - --show-trace \ - -I nixos-config=$(nixos-config) \ - -I stockholm=$(stockholm) \ - -E "with import ; $(1)" \ - $(2) \ - |& $(call whatsupnix) - -evaluate = \ - nix-instantiate \ - --eval \ - --readonly-mode \ - --show-trace \ - -I nixos-config=$(nixos-config) \ - -I stockholm=$(stockholm) \ - -E "let eval = import ; in with eval; $(1)" \ - $(2) - -ifeq ($(MAKECMDGOALS),) -$(error No goals specified) -endif - -# usage: make deploy system=foo [target=bar] -# usage: make test system=foo target=bar -deploy test: -ifdef target - nix-shell --run '$@ --system=$(system) --target=$(target)' -else - nix-shell --run '$@ --system=$(system)' -endif - -# usage: make populate system=foo -populate: populate-target = \ - $(target_user)@$(target_host):$(target_port)$(target_path) -ifeq ($(debug),true) -populate: populate-flags += --debug -endif -ifneq ($(ssh),) -populate: populate-flags += --ssh=$(ssh) -endif -populate: - nix-shell --run 'get-source $(LOGNAME)/1systems/$(system)/source.nix' \ - populate $(populate-target) $(populate-flags) - -# usage: make pkgs.populate -pkgs:;@$(error no package selected) -pkgs.%:;@$(call build,$@) - -# usage: make LOGNAME=krebs system=wolf eval.config.krebs.build.host.name -eval eval.:;@$(call evaluate,$${expr-eval}) -eval.%:;@$(call evaluate,$@) - -# usage: make install system=foo [target_host=bar] -install: ssh ?= ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -install: - $(ssh) $(target_user)@$(target_host) -p $(target_port) \ - env target_path=$(target_path) \ - sh -s prepare < krebs/4lib/infest/prepare.sh - $(MAKE) populate target_path=/mnt$(target_path) - $(ssh) $(target_user)@$(target_host) -p $(target_port) \ - env NIXOS_CONFIG=$(target_path)/nixos-config \ - STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ - nixos-install diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix index 18c8a86cd..26f392da8 100644 --- a/krebs/1systems/hotdog/config.nix +++ b/krebs/1systems/hotdog/config.nix @@ -8,6 +8,8 @@ imports = [ + + ]; krebs.build.host = config.krebs.hosts.hotdog; diff --git a/krebs/1systems/puyak/config.nix b/krebs/1systems/puyak/config.nix index 847f51161..6c950d414 100644 --- a/krebs/1systems/puyak/config.nix +++ b/krebs/1systems/puyak/config.nix @@ -7,7 +7,6 @@ - ]; diff --git a/krebs/1systems/wolf/config.nix b/krebs/1systems/wolf/config.nix index 0deb01f0a..e883a176d 100644 --- a/krebs/1systems/wolf/config.nix +++ b/krebs/1systems/wolf/config.nix @@ -12,7 +12,6 @@ in - diff --git a/krebs/2configs/repo-sync.nix b/krebs/2configs/repo-sync.nix index 157a30e69..b0b0b2f62 100644 --- a/krebs/2configs/repo-sync.nix +++ b/krebs/2configs/repo-sync.nix @@ -17,7 +17,7 @@ let verbose = false; channel = "#retiolum"; server = "ni.r"; - branches = [ "newest" ]; + branches = [ "master" ]; }; }); }; @@ -55,7 +55,7 @@ let }; latest = { url = "${mirror}${name}"; - ref = "heads/newest"; + ref = "heads/master"; }; }; krebs.git = defineRepo name true; diff --git a/krebs/2configs/shared-buildbot.nix b/krebs/2configs/shared-buildbot.nix index b534f0b62..7f243b506 100644 --- a/krebs/2configs/shared-buildbot.nix +++ b/krebs/2configs/shared-buildbot.nix @@ -1,183 +1,18 @@ { lib, config, pkgs, ... }: -# The buildbot config is self-contained and currently provides a way -# to test "krebs" configuration (infrastructure to be used by every krebsminister). +{ + imports = [ + + ]; -# You can add your own test, test steps as required. Deploy the config on a -# krebs host like wolf and everything should be fine. - -# TODO for all users schedule a build for fast tests -let - hostname = config.networking.hostName; -in { - # due to the fact that we actually build stuff on the box via the daemon, - # /nix/store should be cleaned up automatically as well - services.nginx = { - enable = true; - virtualHosts.build = { - serverAliases = [ "build.${hostname}.r" ]; - locations."/".extraConfig = '' - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port}; - ''; - }; - }; - - nix.gc.automatic = true; - nix.gc.dates = "05:23"; networking.firewall.allowedTCPPorts = [ 80 8010 9989 ]; - - krebs.buildbot.master = let - stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm" ; - in { - slaves = { - testslave = "krebspass"; - }; - change_source.stockholm = '' - stockholm_repo = '${stockholm-mirror-url}' - cs.append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branches=True, - project='stockholm', - pollinterval=60)) - ''; - scheduler = { - force-scheduler = '' - sched.append(schedulers.ForceScheduler( - name="force", - builderNames=[ - # "full-tests", - "fast-tests", - "build-local" - ])) - ''; - fast-tests-scheduler = '' - # test everything real quick - sched.append(schedulers.AnyBranchScheduler( - treeStableTimer=10, - name="fast-all-branches", - builderNames=["fast-tests"])) - ''; - test-cac-infest-master = '' - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^(krebs/.*|Makefile|default.nix|shell.nix)") - for file in change.files: - if r.match(file): - return True - return False - - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - treeStableTimer=60*60, # master was stable for the last hour - name="full-master", - builderNames=[ - # "full-tests", - "build-local" - ])) - ''; - }; - builder_pre = '' - # prepare grab_repo step for stockholm - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - - env = { - "LOGNAME": "krebs", - "NIX_REMOTE": "daemon", - "dummy_secrets": "true", - } - - # prepare nix-shell - # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "nix", - "(import ).pkgs.populate", - "(import ).pkgs.test.infest-cac-centos7" ] - # TODO: --pure , prepare ENV in nix-shell command: - # SSL_CERT_FILE,LOGNAME,NIX_REMOTE - nixshell = ["nix-shell", - "-I", "stockholm=.", - "-I", "nixpkgs=/var/src/nixpkgs", - "-p" ] + deps + [ "--run" ] - - # prepare addShell function - def addShell(factory,**kwargs): - factory.addStep(steps.ShellCommand(**kwargs)) - ''; - builder = { - fast-tests = '' - f = util.BuildFactory() - f.addStep(grab_repo) - - for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf", "test-centos7" ]: - addShell(f,name="build-{}".format(i),env=env, - command=nixshell + \ - ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \ - make \ - test \ - target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \ - method=eval \ - system={}".format(i)]) - - bu.append(util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - - ''; - # this build will try to build against local nixpkgs - # TODO change to do a 'local' populate and use the retrieved nixpkgs - build-local = '' - f = util.BuildFactory() - f.addStep(grab_repo) - - - bu.append(util.BuilderConfig(name="build-local", - slavenames=slavenames, - factory=f)) - ''; -# slow-tests = '' -# s = util.BuildFactory() -# s.addStep(grab_repo) -# -# # slave needs 2 files: -# # * cac.json -# # * retiolum -# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) -# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) -# addShell(s, name="infest-cac-centos7",env=env, -# sigtermTime=60, # SIGTERM 1 minute before SIGKILL -# timeout=10800, # 3h -# command=nixshell + ["infest-cac-centos7"]) -# -# bu.append(util.BuilderConfig(name="full-tests", -# slavenames=slavenames, -# factory=s)) -# ''; - }; - enable = true; - web = { - enable = true; - }; - irc = { - enable = true; - nick = "${hostname}bot"; - server = "ni.r"; - channels = [ "retiolum" ]; - allowForce = true; - }; - extraConfig = '' - c['buildbotURL'] = "http://build.${hostname}.r/" - ''; - }; - - krebs.buildbot.slave = { - enable = true; - masterhost = "localhost"; - username = "testslave"; - password = "krebspass"; - packages = with pkgs; [ gnumake jq nix populate ]; - # all nix commands will need a working nixpkgs installation - extraEnviron = { - NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./krebs/1systems/${hostname}/config.nix:stockholm=./"; }; + krebs.ci.enable = true; + krebs.ci.users.krebs ={ + all = true; + hosts = [ + "test-arch" + "test-centos6" + "test-centos7" + "test-all-krebs-modules" + ]; }; } diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 12c685b82..a7624c8f2 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -2,22 +2,6 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; - }) {}; - - buildbot = nixpkgs-fix.buildbot.overrideDerivation (old: { - postUnpack = "sourceRoot=\${sourceRoot}/master"; - patches = []; - src = pkgs.fetchFromGitHub { - owner = "krebscode"; - repo = "buildbot-classic"; - rev = "5b4f5f6f1"; - sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };}); - buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -364,7 +348,7 @@ let set -efux if [ ! -e ${workdir} ];then mkdir -p ${workdir} - ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + ${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir} fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg @@ -373,18 +357,18 @@ let ${ concatMapStringsSep "\n" (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets } # sanity - ${buildbot}/bin/buildbot checkconfig ${workdir} + ${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir} # TODO: maybe upgrade? not sure about this # normally we should write buildbot.tac by our own - # ${buildbot}/bin/buildbot upgrade-master ${workdir} + # ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir} chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start ${workdir}"; + ExecStop = "${pkgs.buildbot-classic}/bin/buildbot stop ${workdir}"; + ExecReload = "${pkgs.buildbot-classic}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 698bf3bcd..544f9c4e0 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,20 +2,6 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; - }) {}; - pkg = nixpkgs-fix.buildbot-slave.overrideDerivation (old: { - postUnpack = "sourceRoot=\${sourceRoot}/slave"; - patches = []; - src = pkgs.fetchFromGitHub { - owner = "krebscode"; - repo = "buildbot-classic"; - rev = "5b4f5f6f1"; - sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };}); buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' import os @@ -166,7 +152,6 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - buildbot = pkg; # TODO:make this in { PermissionsStartOnly = true; @@ -183,8 +168,8 @@ let chown buildbotSlave:buildbotSlave -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}"; + ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}"; PrivateTmp = "true"; User = "buildbotSlave"; Restart = "always"; diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix new file mode 100644 index 000000000..542a9252f --- /dev/null +++ b/krebs/3modules/ci.nix @@ -0,0 +1,175 @@ +{ config, pkgs, ... }: +with import ; +let + cfg = config.krebs.ci; + + hostname = config.networking.hostName; +in +{ + options.krebs.ci = { + enable = mkEnableOption "krebs continous integration"; + users = mkOption { + type = with types; attrsOf (submodule { + options = { + all = mkOption { + type = bool; + default = true; + }; + hosts = mkOption { + type = listOf str; + default = []; + }; + }; + }); + example = { + lass.all = true; + krebs = { + all = true; + hosts = [ + "test-all-krebs-modules" + "test-arch" + ]; + }; + }; + default = {}; + }; + }; + + config = mkIf cfg.enable { + services.nginx = { + enable = true; + virtualHosts.build = { + serverAliases = [ "build.${hostname}.r" ]; + locations."/".extraConfig = '' + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port}; + ''; + }; + }; + + nix.gc.automatic = true; + nix.gc.dates = "05:23"; + + krebs.buildbot.master = { + slaves = { + testslave = "lasspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.${hostname}.r/stockholm' + cs.append( + changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branches=True, + project='stockholm', + pollinterval=10 + ) + ) + ''; + scheduler = { + build-scheduler = '' + # build all hosts + sched.append( + schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch_re=".*"), + treeStableTimer=10, + name="build-all-branches", + builderNames=[ + "build-hosts" + ] + ) + ) + ''; + force-scheduler = '' + sched.append( + schedulers.ForceScheduler( + name="force", + builderNames=[ + "build-hosts" + ] + ) + ) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + grab_repo = steps.Git( + repourl=stockholm_repo, + mode='full' + ) + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + build-hosts = '' + f = util.BuildFactory() + f.addStep(grab_repo) + + def build_host(user, host): + addShell(f, + name="{}".format(host), + env={ + "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", + "NIX_REMOTE": "daemon", + "dummy_secrets": "true", + }, + command=[ + "nix-shell", "--run", + "test --user={} --system={} --target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user, host, user) + ] + ) + + ${let + user-hosts = mapAttrs (user: a: let + managed-hosts = attrNames (filterAttrs (_: h: (h.owner.name == user) && h.managed) config.krebs.hosts); + defined-hosts = a.hosts; + in + defined-hosts ++ (optionals a.all managed-hosts) + ) cfg.users; + + in + concatStringsSep "\n" ( + (mapAttrsToList (user: hosts: + concatMapStringsSep "\n" (host: + "build_host(\"${user}\", \"${host}\")" + ) hosts + ) user-hosts) + ) + } + + bu.append( + util.BuilderConfig( + name="build-hosts", + slavenames=slavenames, + factory=f + ) + ) + + ''; + }; + enable = true; + web.enable = true; + irc = { + enable = true; + nick = "build|${hostname}"; + server = "ni.r"; + channels = [ "retiolum" "noise" ]; + allowForce = true; + }; + extraConfig = '' + c['buildbotURL'] = "http://build.${hostname}.r/" + ''; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "lasspass"; + packages = with pkgs; [ gnumake jq nix populate ]; + }; + + }; +} diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 6123b6dd9..b0ad2baf5 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -12,6 +12,7 @@ let ./buildbot/master.nix ./buildbot/slave.nix ./build.nix + ./ci.nix ./current.nix ./exim.nix ./exim-retiolum.nix diff --git a/krebs/3modules/krebs/default.nix b/krebs/3modules/krebs/default.nix index 07543489a..27fbb7088 100644 --- a/krebs/3modules/krebs/default.nix +++ b/krebs/3modules/krebs/default.nix @@ -32,12 +32,15 @@ in { hosts = { hotdog = { owner = config.krebs.users.krebs; + managed = true; nets = { retiolum = { ip4.addr = "10.243.77.3"; ip6.addr = "42:0:0:0:0:0:77:3"; aliases = [ "hotdog.r" + "build.hotdog.r" + "cgit.hotdog.r" ]; tinc.pubkey = '' -----BEGIN RSA PUBLIC KEY----- @@ -56,6 +59,7 @@ in { }; puyak = { owner = config.krebs.users.krebs; + managed = true; nets = { retiolum = { ip4.addr = "10.243.77.2"; @@ -82,6 +86,7 @@ in { }; wolf = { owner = config.krebs.users.krebs; + managed = true; nets = { shack = { ip4.addr = "10.42.2.150" ; @@ -120,6 +125,11 @@ in { krebs = { pubkey = "lol"; # TODO krebs.users.krebs.pubkey should be unnecessary }; + hotdog-repo-sync = { + name = "hotdog-repo-sync"; + mail = "spam@krebsco.de"; + pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILzTvaR3QqOD3oEEGHQzg/sRnNbKJnZYcV9htDvXmu53"; + }; puyak-repo-sync = { name = "puyak-repo-sync"; mail = "spam@krebsco.de"; diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix index 139f02ddd..c554391f2 100644 --- a/krebs/3modules/lass/default.nix +++ b/krebs/3modules/lass/default.nix @@ -255,7 +255,7 @@ with import ; nets = rec { retiolum = { ip4.addr = "10.243.133.114"; - ip6.addr = "42:0000:0000:0000:0000:0000:d15f:1214"; + ip6.addr = "42:0:0:0:0:0:1ca0:1205"; aliases = [ "icarus.r" "cgit.icarus.r" @@ -276,6 +276,32 @@ with import ; ssh.privkey.path = ; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOPgQIMYiyD4/Co+nlOQWEzCKssemOEXAY/lbIZZaMhj"; }; + daedalus = { + cores = 2; + nets = rec { + retiolum = { + ip4.addr = "10.243.133.115"; + ip6.addr = "42:0:0:0:0:0:daed:a105"; + aliases = [ + "daedalus.r" + "cgit.daedalus.r" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAzlIJfYIoQGXishIQGFNOcaVoeelqy7a731FJ+VfrqeR8WURQ6D+8 + 5hz7go+l3Z7IhTc/HbpGFJ5QJJNFSuSpLfZVyi+cKAUVheTivIniHFIRw37JbJ4+ + qWTlVe3uvOiZ0cA9S6LrbzqAUTLbH0JlWj36mvGIPICDr9YSEkIUKbenxjJlIpX8 + ECEBm8RU1aq3PUo/cVjmpqircynVJBbRCXZiHoxyLXNmh23d0fCPCabEYWhJhgaR + arkYRls5A14HGMI52F3ehnhED3k0mU8/lb4OzYgk34FjuZGmyRWIfrEKnqL4Uu2w + 3pmEvswG1WYG/3+YE80C5OpCE4BUKAzYSwIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + secure = true; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAq5Ovdcsljr5dOl7+2sQNKpGpdX0SlOIuCZKEiWEp8g"; + }; iso = { cores = 1; managed = false; diff --git a/krebs/4lib/infest/prepare.sh b/krebs/4lib/infest/prepare.sh index d39aca348..ccfc4f49b 100644 --- a/krebs/4lib/infest/prepare.sh +++ b/krebs/4lib/infest/prepare.sh @@ -1,8 +1,8 @@ #! /bin/sh set -efu -nix_url=https://nixos.org/releases/nix/nix-1.10/nix-1.10-x86_64-linux.tar.bz2 -nix_sha256=504f7a3a85fceffb8766ae5e1005de9e02e489742f5a63cc3e7552120b138bf4 +nix_url=https://nixos.org/releases/nix/nix-1.11.13/nix-1.11.13-x86_64-linux.tar.bz2 +nix_sha256=c11411d52d8ad1ce3a68410015487282fd4651d3abefbbb13fa1f7803a2f60de prepare() {( if test -e /etc/os-release; then @@ -14,10 +14,6 @@ prepare() {( ;; centos) case $VERSION_ID in - 6) - prepare_centos "$@" - exit - ;; 7) prepare_centos "$@" exit @@ -51,13 +47,6 @@ prepare() {( esac ;; esac - elif test -e /etc/centos-release; then - case $(cat /etc/centos-release) in - 'CentOS release 6.5 (Final)') - prepare_centos "$@" - exit - ;; - esac fi echo "$0 prepare: unknown OS" >&2 exit -1 @@ -217,7 +206,7 @@ prepare_common() {( mkdir -p bin rm -f bin/nixos-install cp "$(type -p nixos-install)" bin/nixos-install - sed -i "s@^NIX_PATH=\"[^\"]*\"@NIX_PATH=$target_path@" bin/nixos-install + sed -i "s@NIX_PATH=\"[^\"]*\"@NIX_PATH=$target_path@" bin/nixos-install if ! grep -q '^PATH.*#krebs' .bashrc; then echo '. /root/.nix-profile/etc/profile.d/nix.sh' >> .bashrc diff --git a/krebs/5pkgs/simple/buildbot-classic-slave/default.nix b/krebs/5pkgs/simple/buildbot-classic-slave/default.nix new file mode 100644 index 000000000..c316889e4 --- /dev/null +++ b/krebs/5pkgs/simple/buildbot-classic-slave/default.nix @@ -0,0 +1,21 @@ +{ coreutils, fetchgit, fetchFromGitHub, buildbot-classic, python2Packages, ... }: + +python2Packages.buildPythonApplication { + name = "buildbot-classic-slave-0.8.12"; + namePrefix = ""; + + src = buildbot-classic.src; + postUnpack = "sourceRoot=\${sourceRoot}/slave"; + + patchPhase = '' + substituteInPlace buildslave/scripts/logwatcher.py --replace /usr/bin/tail ${coreutils}/bin/tail + ''; + + propagatedBuildInputs = [ python2Packages.twisted ]; + doCheck = false; + + postInstall = '' + mkdir -p "$out/share/man/man1" + cp docs/buildslave.1 "$out/share/man/man1" + ''; +} diff --git a/krebs/5pkgs/simple/buildbot-classic/default.nix b/krebs/5pkgs/simple/buildbot-classic/default.nix new file mode 100644 index 000000000..a3d924c4a --- /dev/null +++ b/krebs/5pkgs/simple/buildbot-classic/default.nix @@ -0,0 +1,47 @@ +{ fetchgit, fetchFromGitHub, python2Packages, ... }: +let + # https://github.com/NixOS/nixpkgs/issues/14026 + nixpkgs-fix = import (fetchgit { + url = https://github.com/nixos/nixpkgs; + rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; + sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; + }) {}; + +in nixpkgs-fix.buildPythonApplication { + name = "buildbot-classic-0.8.12"; + namePrefix = ""; + patches = []; + + src = fetchFromGitHub { + owner = "krebscode"; + repo = "buildbot-classic"; + rev = "5b4f5f6f1"; + sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; + }; + postUnpack = "sourceRoot=\${sourceRoot}/master"; + + patchPhase = + # The code insists on /usr/bin/tail, /usr/bin/make, etc. + '' echo "patching erroneous absolute path references..." + for i in $(find -name \*.py) + do + sed -i "$i" \ + -e "s|/usr/bin/python|$(type -P python)|g ; s|/usr/bin/||g" + done + + sed -i 's/==/>=/' setup.py + ''; + + propagatedBuildInputs = [ + python2Packages.jinja2 + python2Packages.twisted + nixpkgs-fix.pythonPackages.dateutil_1_5 + nixpkgs-fix.pythonPackages.sqlalchemy_migrate_0_7 + ]; + doCheck = false; + postInstall = '' + mkdir -p "$out/share/man/man1" + cp docs/buildbot.1 "$out/share/man/man1" + ''; +} + diff --git a/krebs/5pkgs/simple/populate/default.nix b/krebs/5pkgs/simple/populate/default.nix index 3ec432229..c2ca00590 100644 --- a/krebs/5pkgs/simple/populate/default.nix +++ b/krebs/5pkgs/simple/populate/default.nix @@ -13,12 +13,12 @@ in stdenv.mkDerivation rec { name = "populate"; - version = "1.2.0"; + version = "1.2.1"; src = fetchgit { url = http://cgit.ni.krebsco.de/populate; rev = "refs/tags/v${version}"; - sha256 = "0q3110hkkxn9bc3a63xbx1hyd1fpzz4wrck4lng3j5a9i1y1jm07"; + sha256 = "13viizpmjkqxc3d9mk2bzspgnal07bma5m9lc90mcdlk36na3pkq"; }; phases = [ diff --git a/krebs/5pkgs/writers.nix b/krebs/5pkgs/writers.nix index 49ca3557e..f1626078e 100644 --- a/krebs/5pkgs/writers.nix +++ b/krebs/5pkgs/writers.nix @@ -144,9 +144,14 @@ with import ; env = filevars // { passAsFile = attrNames filevars; }; in + # Use a subshell because 's genericBuild + # sources (or evaluates) the buildCommand and we don't want to modify its + # shell. In particular, exitHandler breaks in multiple ways with set -u. pkgs.runCommand name env /* sh */ '' - set -efu - ${concatMapStringsSep "\n" (getAttr "install") files} + ( + set -efu + ${concatMapStringsSep "\n" (getAttr "install") files} + ) ''; writeHaskell = diff --git a/lass/1systems/daedalus/config.nix b/lass/1systems/daedalus/config.nix new file mode 100644 index 000000000..290d8a780 --- /dev/null +++ b/lass/1systems/daedalus/config.nix @@ -0,0 +1,35 @@ +{ config, pkgs, ... }: + +{ + imports = [ + + + + + + + + + + + + + + + ]; + + krebs.build.host = config.krebs.hosts.daedalus; + + fileSystems = { + "/bku" = { + device = "/dev/mapper/pool-bku"; + fsType = "btrfs"; + options = ["defaults" "noatime" "ssd" "compress=lzo"]; + }; + }; + + services.udev.extraRules = '' + SUBSYSTEM=="net", ATTR{address}=="00:24:d7:f0:e8:c8", NAME="wl0" + SUBSYSTEM=="net", ATTR{address}=="f0:de:f1:8f:8a:78", NAME="et0" + ''; +} diff --git a/lass/1systems/daedalus/source.nix b/lass/1systems/daedalus/source.nix new file mode 100644 index 000000000..a15ac80c2 --- /dev/null +++ b/lass/1systems/daedalus/source.nix @@ -0,0 +1,4 @@ +import { + name = "daedalus"; + secure = true; +} diff --git a/lass/2configs/boot/stock-x220.nix b/lass/2configs/boot/stock-x220.nix new file mode 100644 index 000000000..54a382db7 --- /dev/null +++ b/lass/2configs/boot/stock-x220.nix @@ -0,0 +1,8 @@ +{ ... }: + +{ + boot = { + loader.systemd-boot.enable = true; + loader.efi.canTouchEfiVariables = true; + }; +} diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 7f0a3bff1..18fd9bb92 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -9,9 +9,11 @@ let ControlPersist 4h ''; + hostname = config.networking.hostName; + in { config.services.nginx.virtualHosts.build = { - serverAliases = [ "build.prism.r" ]; + serverAliases = [ "build.${hostname}.r" ]; locations."/".extraConfig = '' proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -20,7 +22,7 @@ in { }; config.krebs.buildbot.master = let - stockholm-mirror-url = http://cgit.prism.r/stockholm ; + stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm"; in { slaves = { testslave = "lasspass"; @@ -109,13 +111,13 @@ in { web.enable = true; irc = { enable = true; - nick = "buildbot-lass"; + nick = "build|${hostname}"; server = "ni.r"; channels = [ "retiolum" "noise" ]; allowForce = true; }; extraConfig = '' - c['buildbotURL'] = "http://build.prism.r/" + c['buildbotURL'] = "http://build.${hostname}.r/" ''; }; diff --git a/lass/2configs/exim-smarthost.nix b/lass/2configs/exim-smarthost.nix index fd2f1f765..942653bab 100644 --- a/lass/2configs/exim-smarthost.nix +++ b/lass/2configs/exim-smarthost.nix @@ -34,6 +34,7 @@ with import ; { from = "irgendwas@lassul.us"; to = lass.mail; } { from = "polo@lassul.us"; to = lass.mail; } { from = "shack@lassul.us"; to = lass.mail; } + { from = "nix@lassul.us"; to = lass.mail; } ]; system-aliases = [ { from = "mailer-daemon"; to = "postmaster"; } diff --git a/lib/types.nix b/lib/types.nix index f9ec7b1c3..236190ccd 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -36,6 +36,7 @@ rec { If true, then the host's configuration is defined in stockholm. ''; type = bool; + default = false; }; owner = mkOption { diff --git a/shell.nix b/shell.nix index 2973d4c51..57690d398 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,8 @@ let # high level commands # - # usage: deploy [--user=USER] --system=SYSTEM [--target=TARGET] + # usage: deploy [--force-populate] [--user=USER] + # --system=SYSTEM [--target=TARGET] cmds.deploy = pkgs.writeDash "cmds.deploy" '' set -efu @@ -15,11 +16,55 @@ let \test -n "''${target-}" || target=$system \test -n "''${user-}" || user=$LOGNAME . ${init.env} + . ${init.proxy} exec ${utils.deploy} ''; - # usage: test [--user=USER] --system=SYSTEM --target=TARGET + # usage: install [--force-populate] [--user=USER] + # --system=SYSTEM --target=TARGET + cmds.install = pkgs.writeBash "cmds.install" '' + set -efu + + command=install + . ${init.args} + \test -n "''${user-}" || user=$LOGNAME + . ${init.env} + + if \test "''${using_proxy-}" != true; then + ${pkgs.openssh}/bin/ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "$target_user@$target_host" -p "$target_port" \ + env target_path=$(quote "$target_path") \ + sh -s prepare < ${./krebs/4lib/infest/prepare.sh} + # TODO inline prepare.sh? + fi + + . ${init.proxy} + + # Reset PATH because we need access to nixos-install. + # TODO provide nixos-install instead of relying on prepare.sh + export PATH="$OLD_PATH" + + # these variables get defined by nix-shell (i.e. nix-build) from + # XDG_RUNTIME_DIR and reference the wrong directory (/run/user/0), + # which only exists on / and not at /mnt. + export NIX_BUILD_TOP=/tmp + export TEMPDIR=/tmp + export TEMP=/tmp + export TMPDIR=/tmp + export TMP=/tmp + export XDG_RUNTIME_DIR=/tmp + + export NIXOS_CONFIG="$target_path/nixos-config" + + cd + exec nixos-install + ''; + + # usage: test [--force-populate] [--user=USER] + # --system=SYSTEM --target=TARGET cmds.test = pkgs.writeDash "cmds.test" /* sh */ '' set -efu @@ -29,6 +74,7 @@ let . ${init.args} \test -n "''${user-}" || user=$LOGNAME . ${init.env} + . ${init.proxy} exec ${utils.build} config.system.build.toplevel ''; @@ -99,11 +145,13 @@ let init.args = pkgs.writeText "init.args" /* sh */ '' args=$(${pkgs.utillinux}/bin/getopt -n "$command" -s sh \ -o s:t:u: \ - -l system:,target:,user: \ + -l force-populate,system:,target:,user: \ -- "$@") if \test $? != 0; then exit 1; fi eval set -- "$args" + force_populate=false; while :; do case $1 in + --force-populate) force_populate=true; shift;; -s|--system) system=$2; shift 2;; -t|--target) target=$2; shift 2;; -u|--user) user=$2; shift 2;; @@ -114,9 +162,6 @@ let ''; init.env = pkgs.writeText "init.env" /* sh */ '' - source=''${source-$user/1systems/$system/source.nix} - - export source export system export target export user @@ -129,38 +174,35 @@ let export target_port="$(echo $target_object | ${pkgs.jq}/bin/jq -r .port)" export target_path="$(echo $target_object | ${pkgs.jq}/bin/jq -r .path)" export target_local="$(echo $target_object | ${pkgs.jq}/bin/jq -r .local)" + ''; + init.proxy = pkgs.writeText "init.proxy" /* sh */ '' if \test "''${using_proxy-}" != true; then - ${init.env.populate} + + source_file=$user/1systems/$system/source.nix + source=$(get-source "$source_file") + qualified_target=$target_user@$target_host:$target_port$target_path + if test "$force_populate" = true; then + echo "$source" | populate --force "$qualified_target" + else + echo "$source" | populate "$qualified_target" + fi + if \test "$target_local" != true; then - exec ${init.env.proxy} "$command" "$@" + exec ${pkgs.openssh}/bin/ssh \ + "$target_user@$target_host" -p "$target_port" \ + cd "$target_path/stockholm" \; \ + NIX_PATH=$(quote "$target_path") \ + STOCKHOLM_VERSION=$(quote "$STOCKHOLM_VERSION") \ + nix-shell --run "$(quote " + system=$(quote "$system") \ + target=$(quote "$target") \ + using_proxy=true \ + $(quote "$command" "$@") + ")" fi fi - '' // { - populate = pkgs.writeDash "init.env.populate" '' - set -efu - _source=$(get-source "$source") - echo $_source | - ${pkgs.populate}/bin/populate \ - "$target_user@$target_host:$target_port$target_path" \ - >&2 - unset _source - ''; - proxy = pkgs.writeDash "init.env.proxy" '' - set -efu - exec ${pkgs.openssh}/bin/ssh \ - "$target_user@$target_host" -p "$target_port" \ - cd "$target_path/stockholm" \; \ - NIX_PATH=$(quote "$target_path") \ - STOCKHOLM_VERSION=$(quote "$STOCKHOLM_VERSION") \ - nix-shell --run "$(quote " - system=$(quote "$system") \ - target=$(quote "$target") \ - using_proxy=true \ - $(quote "$@") - ")" - ''; - }; + ''; utils.build = pkgs.writeDash "utils.build" '' set -efu @@ -201,9 +243,13 @@ let in pkgs.stdenv.mkDerivation { name = "stockholm"; shellHook = /* sh */ '' + export OLD_PATH="$PATH" export NIX_PATH=stockholm=$PWD:nixpkgs=${toString } - export NIX_REMOTE=daemon + if test -e /nix/var/nix/daemon-socket/socket; then + export NIX_REMOTE=daemon + fi export PATH=${lib.makeBinPath [ + pkgs.populate shell.cmdspkg ]}