diff --git a/Makefile b/Makefile index aefd17147..5b898c54c 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ ifeq ($(filter),json) else filter() { cat; } endif - nix-instantiate \ + result=$$(nix-instantiate \ $${extraArgs-} \ --eval \ -A "$$get" \ @@ -45,8 +45,9 @@ endif --argstr current-host-name "$$HOSTNAME" \ --argstr current-user-name "$$LOGNAME" \ $${system+--argstr system "$$system"} \ - $${target+--argstr target "$$target"} \ - | filter + $${target+--argstr target "$$target"}) + echo "$$result" | filter + else $(error unbound variable: system[s]) endif diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 0fca52203..92400139c 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -9,6 +9,7 @@ let ${cfg.overrideConfig} '' else ""} ## Extra Config + ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)} ${cfg.extraConfig} ''; cfg = config.krebs.Reaktor; @@ -35,7 +36,6 @@ let ''; }; - overrideConfig = mkOption { default = null; type = types.nullOr types.str; @@ -44,6 +44,9 @@ let Reaktor default cfg can be retrieved via `reaktor get-config` ''; }; + plugins = mkOption { + default = [pkgs.ReaktorPlugins.nixos-version]; + }; extraConfig = mkOption { default = ""; type = types.string; @@ -51,6 +54,14 @@ let configuration appended to the default or overridden configuration ''; }; + + workdir = mkOption { + default = "/var/lib/Reaktor"; + type = types.str; + description = '' + Reaktor working directory + ''; + }; extraEnviron = mkOption { default = {}; type = types.attrsOf types.str; @@ -59,12 +70,17 @@ let REAKTOR_HOST REAKTOR_PORT REAKTOR_STATEDIR - REAKTOR_CHANNELS debug and nickname can be set separately via the Reaktor api ''; }; - + channels = mkOption { + default = [ "#krebs" ]; + type = types.listOf types.str; + description = '' + Channels the Reaktor should connect to at startup. + ''; + }; debug = mkOption { default = false; description = '' @@ -79,7 +95,7 @@ let name = "Reaktor"; uid = genid name; description = "Reaktor user"; - home = "/var/lib/Reaktor"; + home = cfg.workdir; createHome = true; }; @@ -101,6 +117,9 @@ let GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; REAKTOR_NICKNAME = cfg.nickname; REAKTOR_DEBUG = (if cfg.debug then "True" else "False"); + REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels; + state_dir = cfg.workdir; + } // cfg.extraEnviron; serviceConfig= { ExecStartPre = pkgs.writeScript "Reaktor-init" '' diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix new file mode 100644 index 000000000..01bb16a2b --- /dev/null +++ b/krebs/3modules/backup.nix @@ -0,0 +1,286 @@ +{ config, lib, pkgs, ... }: +with lib; +let + out = { + options.krebs.backup = api; + config = mkIf cfg.enable imp; + }; + + cfg = config.krebs.backup; + + api = { + enable = mkEnableOption "krebs.backup" // { default = true; }; + plans = mkOption { + default = {}; + type = types.attrsOf (types.submodule ({ + # TODO enable = mkEnableOption "TODO" // { default = true; }; + options = { + method = mkOption { + type = types.enum ["pull" "push"]; + }; + name = mkOption { + type = types.str; + }; + src = mkOption { + type = types.krebs.file-location; + }; + dst = mkOption { + type = types.krebs.file-location; + }; + startAt = mkOption { + type = types.str; + }; + snapshots = mkOption { + type = types.attrsOf (types.submodule { + options = { + format = mkOption { + type = types.str; # TODO date's +FORMAT + }; + retain = mkOption { + type = types.nullOr types.int; + default = null; # null = retain all snapshots + }; + }; + }); + }; + }; + })); + }; + }; + + imp = { + users.groups.backup.gid = genid "backup"; + users.users = {} + // { + root.openssh.authorizedKeys.keys = + map (plan: plan.dst.host.ssh.pubkey) + (filter isPullSrc (attrValues cfg.plans)) + ++ + map (plan: plan.src.host.ssh.pubkey) + (filter isPushDst (attrValues cfg.plans)) + ; + } + ; + systemd.services = + flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: { + name = "backup.${name}.pull"; + value = makePullService plan; + }) + // + flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: { + name = "backup.${name}.push"; + value = makePushService plan; + }) + ; + }; + + isPushSrc = plan: + plan.method == "push" && + plan.src.host.name == config.krebs.build.host.name; + + isPullSrc = plan: + plan.method == "pull" && + plan.src.host.name == config.krebs.build.host.name; + + isPushDst = plan: + plan.method == "push" && + plan.dst.host.name == config.krebs.build.host.name; + + isPullDst = plan: + plan.method == "pull" && + plan.dst.host.name == config.krebs.build.host.name; + + # TODO push destination needs this in the dst.user's PATH + service-path = [ + pkgs.coreutils + pkgs.gnused + pkgs.openssh + pkgs.rsync + pkgs.utillinux + ]; + + # TODO if there is plan.user, then use its privkey + makePushService = plan: assert isPushSrc plan; { + path = service-path; + serviceConfig = { + ExecStart = push plan; + Type = "oneshot"; + }; + startAt = plan.startAt; + }; + + makePullService = plan: assert isPullDst plan; { + path = service-path; + serviceConfig = { + ExecStart = pull plan; + Type = "oneshot"; + }; + startAt = plan.startAt; + }; + + push = plan: let + # We use writeDashBin and return the absolute path so systemd will produce + # nice names in the log, i.e. without the Nix store hash. + out = "${main}/bin/${main.name}"; + + main = writeDashBin "backup.${plan.name}.push" '' + set -efu + dst=${shell.escape plan.dst.path} + + mkdir -m 0700 -p "$dst" + exec flock -n "$dst" ${critical-section} + ''; + + critical-section = writeDash "backup.${plan.name}.push.critical-section" '' + # TODO check if there is a previous + set -efu + identity=${shell.escape plan.src.host.ssh.privkey.path} + src=${shell.escape plan.src.path} + dst_target=${shell.escape "root@${getFQDN plan.dst.host}"} + dst_path=${shell.escape plan.dst.path} + dst=$dst_target:$dst_path + + # Export NOW so runtime of rsync doesn't influence snapshot naming. + export NOW + NOW=$(date +%s) + + echo >&2 "update snapshot: current; $src -> $dst" + rsync >&2 \ + -aAXF --delete \ + -e "ssh -F /dev/null -i $identity" \ + --rsync-path ${shell.escape + "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \ + --link-dest="$dst_path/current" \ + "$src/" \ + "$dst/.partial" + + exec ssh -F /dev/null \ + -i "$identity" \ + "$dst_target" \ + -T \ + env NOW="$NOW" /bin/sh < ${remote-snapshot} + EOF + ''; + + remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" '' + set -efu + dst=${shell.escape plan.dst.path} + + if test -e "$dst/current"; then + mv "$dst/current" "$dst/.previous" + fi + mv "$dst/.partial" "$dst/current" + rm -fR "$dst/.previous" + echo >&2 + + (${(take-snapshots plan).text}) + ''; + + in out; + + # TODO admit plan.dst.user and its ssh identity + pull = plan: let + # We use writeDashBin and return the absolute path so systemd will produce + # nice names in the log, i.e. without the Nix store hash. + out = "${main}/bin/${main.name}"; + + main = writeDashBin "backup.${plan.name}.pull" '' + set -efu + dst=${shell.escape plan.dst.path} + + mkdir -m 0700 -p "$dst" + exec flock -n "$dst" ${critical-section} + ''; + + critical-section = writeDash "backup.${plan.name}.pull.critical-section" '' + # TODO check if there is a previous + set -efu + identity=${shell.escape plan.dst.host.ssh.privkey.path} + src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"} + dst=${shell.escape plan.dst.path} + + # Export NOW so runtime of rsync doesn't influence snapshot naming. + export NOW + NOW=$(date +%s) + + echo >&2 "update snapshot: current; $dst <- $src" + mkdir -m 0700 -p ${shell.escape plan.dst.path} + rsync >&2 \ + -aAXF --delete \ + -e "ssh -F /dev/null -i $identity" \ + --link-dest="$dst/current" \ + "$src/" \ + "$dst/.partial" + mv "$dst/current" "$dst/.previous" + mv "$dst/.partial" "$dst/current" + rm -fR "$dst/.previous" + echo >&2 + + exec ${take-snapshots plan} + ''; + in out; + + take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" '' + set -efu + NOW=''${NOW-$(date +%s)} + dst=${shell.escape plan.dst.path} + + snapshot() {( + : $ns $format $retain + name=$(date --date="@$NOW" +"$format") + if ! test -e "$dst/$ns/$name"; then + echo >&2 "create snapshot: $ns/$name" + mkdir -m 0700 -p "$dst/$ns" + rsync >&2 \ + -aAXF --delete \ + --link-dest="$dst/current" \ + "$dst/current/" \ + "$dst/$ns/.partial.$name" + mv "$dst/$ns/.partial.$name" "$dst/$ns/$name" + echo >&2 + fi + case $retain in + ([0-9]*) + delete_from=$(($retain + 1)) + ls -r "$dst/$ns" \ + | sed -n "$delete_from,\$p" \ + | while read old_name; do + echo >&2 "delete snapshot: $ns/$old_name" + rm -fR "$dst/$ns/$old_name" + done + ;; + (ALL) + : + ;; + esac + )} + + ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }: + toString (map shell.escape [ + "ns=${ns}" + "format=${format}" + "retain=${if retain == null then "ALL" else toString retain}" + "snapshot" + ])) + plan.snapshots)} + ''; + + # TODO getFQDN: admit hosts in other domains + getFQDN = host: "${host.name}.${config.krebs.search-domain}"; + + writeDash = name: text: pkgs.writeScript name '' + #! ${pkgs.dash}/bin/dash + ${text} + ''; + + writeDashBin = name: text: pkgs.writeTextFile { + executable = true; + destination = "/bin/${name}"; + name = name; + text = '' + #! ${pkgs.dash}/bin/dash + ${text} + ''; + }; + +in out diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix new file mode 100644 index 000000000..74385a433 --- /dev/null +++ b/krebs/3modules/buildbot/master.nix @@ -0,0 +1,385 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot = pkgs.buildbot; + buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' + # -*- python -*- + from buildbot.plugins import * + import re + import json + c = BuildmasterConfig = {} + + c['slaves'] = [] + slaves = json.loads('${builtins.toJSON cfg.slaves}') + slavenames = [ s for s in slaves ] + for k,v in slaves.items(): + c['slaves'].append(buildslave.BuildSlave(k, v)) + + # TODO: configure protocols? + c['protocols'] = {'pb': {'port': 9989}} + + ####### Build Inputs + c['change_source'] = cs = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Change_Source: Begin of ${n} + ${v} + #### Change_Source: End of ${n} + '') cfg.change_source )} + + ####### Build Scheduler + c['schedulers'] = sched = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Schedulers: Begin of ${n} + ${v} + #### Schedulers: End of ${n} + '') cfg.scheduler )} + + ###### Builder + c['builders'] = bu = [] + + # Builder Pre: Begin + ${cfg.builder_pre} + # Builder Pre: End + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Builder: Begin of ${n} + ${v} + #### Builder: End of ${n} + '') cfg.builder )} + + + ####### Status + c['status'] = st = [] + + # If you want to configure this url, override with extraConfig + c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" + + ${optionalString (cfg.web.enable) '' + from buildbot.status import html + from buildbot.status.web import authz, auth + authz_cfg=authz.Authz( + auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), + # TODO: configure harder + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = 'auth', + cancelPendingBuild = 'auth' + ) + # TODO: configure krebs.nginx + st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + ''} + + ${optionalString (cfg.irc.enable) '' + from buildbot.status import words + irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + channels=${builtins.toJSON cfg.irc.channels}, + notify_events={ + 'success': 1, + 'failure': 1, + 'exception': 1, + 'successToFailure': 1, + 'failureToSuccess': 1, + }${optionalString cfg.irc.allowForce ",allowForce=True"}) + c['status'].append(irc) + ''} + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Status: Begin of ${n} + ${v} + #### Status: End of ${n} + '') cfg.status )} + + ####### PROJECT IDENTITY + c['title'] = "${cfg.title}" + c['titleURL'] = "http://krebsco.de" + + + ####### DB URL + # TODO: configure + c['db'] = { + 'db_url' : "sqlite:///state.sqlite", + } + ${cfg.extraConfig} + ''; + + cfg = config.krebs.buildbot.master; + + api = { + enable = mkEnableOption "Buildbot Master"; + title = mkOption { + default = "Buildbot CI"; + type = types.str; + description = '' + Title of the Buildbot Installation + ''; + }; + workDir = mkOption { + default = "/var/lib/buildbot/master"; + type = types.str; + description = '' + Path to build bot master directory. + Will be created on startup. + ''; + }; + + secrets = mkOption { + default = []; + type = types.listOf types.str; + example = [ "cac.json" ]; + description = '' + List of all the secrets in which should be copied into the + buildbot master directory. + ''; + }; + + slaves = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of slavenames with their passwords + slavename = slavepassword + ''; + }; + + change_source = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + stockholm = '' + cs.append(changes.GitPoller( + 'http://cgit.gum/stockholm', + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + }; + description = '' + Attrset of all the change_sources which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to cs + ''; + }; + + scheduler = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + }; + description = '' + Attrset of all the schedulers which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to sched + ''; + }; + + builder_pre = mkOption { + default = ""; + type = types.lines; + example = '' + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + ''; + description = '' + some code before the builders are being assembled. + can be used to define functions used by multiple builders + ''; + }; + + builder = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + fast-test = '' + ''; + }; + description = '' + Attrset of all the builder which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to bu + ''; + }; + + status = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of all the extra status which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to st + + Right now IRC and Web status can be configured by setting + buildbot.master.irc.enable and + buildbot.master.web.enable + ''; + }; + + # Configurable Stati + web = mkOption { + default = {}; + type = types.submodule ({ config2, ... }: { + options = { + enable = mkEnableOption "Buildbot Master Web Status"; + username = mkOption { + default = "krebs"; + type = types.str; + description = '' + username for web authentication + ''; + }; + hostname = mkOption { + default = config.networking.hostName; + type = types.str; + description = '' + web interface Hostname + ''; + }; + password = mkOption { + default = "bob"; + type = types.str; + description = '' + password for web authentication + ''; + }; + port = mkOption { + default = 8010; + type = types.int; + description = '' + port for buildbot web status + ''; + }; + }; + }); + }; + + irc = mkOption { + default = {}; + type = types.submodule ({ config, ... }: { + options = { + enable = mkEnableOption "Buildbot Master IRC Status"; + channels = mkOption { + default = [ "nix-buildbot-meetup" ]; + type = with types; listOf str; + description = '' + irc channels the bot should connect to + ''; + }; + allowForce = mkOption { + default = false; + type = types.bool; + description = '' + Determines if builds can be forced via IRC + ''; + }; + nick = mkOption { + default = "nix-buildbot"; + type = types.str; + description = '' + nickname for IRC + ''; + }; + server = mkOption { + default = "irc.freenode.net"; + type = types.str; + description = '' + Buildbot Status IRC Server to connect to + ''; + }; + }; + }); + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + description = '' + extra config appended to the generated master.cfg + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotMaster = { + uid = genid "buildbotMaster"; + description = "Buildbot Master"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotMaster = { + gid = 672626386; + }; + + systemd.services.buildbotMaster = { + description = "Buildbot Master"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + # TODO: add extra dependencies to master like svn and cvs + path = [ pkgs.git ]; + environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; + serviceConfig = let + workdir="${lib.shell.escape cfg.workDir}"; + secretsdir="${lib.shell.escape (toString )}"; + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + if [ ! -e ${workdir} ];then + mkdir -p ${workdir} + ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + fi + # always override the master.cfg + cp ${buildbot-master-config} ${workdir}/master.cfg + + # copy secrets + ${ concatMapStringsSep "\n" + (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets } + # sanity + ${buildbot}/bin/buildbot checkconfig ${workdir} + + # TODO: maybe upgrade? not sure about this + # normally we should write buildbot.tac by our own + # ${buildbot}/bin/buildbot upgrade-master ${workdir} + + chmod 700 -R ${workdir} + chown buildbotMaster:buildbotMaster -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; + ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + PrivateTmp = "true"; + User = "buildbotMaster"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.krebs.buildbot.master = api; + config = mkIf cfg.enable imp; +} diff --git a/makefu/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix similarity index 97% rename from makefu/3modules/buildbot/slave.nix rename to krebs/3modules/buildbot/slave.nix index 7c9ea79c0..0e7796d8a 100644 --- a/makefu/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -39,7 +39,7 @@ let s.setServiceParent(application) ''; default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.makefu.buildbot.slave; + cfg = config.krebs.buildbot.slave; api = { enable = mkEnableOption "Buildbot Slave"; @@ -144,6 +144,7 @@ let path = default-packages ++ cfg.packages; environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; NIX_REMOTE="daemon"; } // cfg.extraEnviron; @@ -180,6 +181,6 @@ let }; in { - options.makefu.buildbot.slave = api; + options.krebs.buildbot.slave = api; config = mkIf cfg.enable imp; } diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index dddb2df50..65c1aa2ec 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -7,8 +7,11 @@ let out = { imports = [ ./apt-cacher-ng.nix + ./backup.nix ./bepasty-server.nix ./build.nix + ./buildbot/master.nix + ./buildbot/slave.nix ./current.nix ./exim-retiolum.nix ./exim-smarthost.nix diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix index e1315d8c8..ea012c38c 100644 --- a/krebs/3modules/exim-retiolum.nix +++ b/krebs/3modules/exim-retiolum.nix @@ -1,14 +1,12 @@ { config, pkgs, lib, ... }: -with builtins; with lib; let cfg = config.krebs.exim-retiolum; out = { options.krebs.exim-retiolum = api; - config = - mkIf cfg.enable imp; + config = mkIf cfg.enable imp; }; api = { @@ -16,13 +14,13 @@ let }; imp = { - services.exim = - # This configuration makes only sense for retiolum-enabled hosts. - # TODO modular configuration - assert config.krebs.retiolum.enable; - { - enable = true; - config = '' + services.exim = { + enable = true; + config = + # This configuration makes only sense for retiolum-enabled hosts. + # TODO modular configuration + assert config.krebs.retiolum.enable; + '' primary_hostname = ${retiolumHostname} domainlist local_domains = @ : localhost domainlist relay_to_domains = *.retiolum @@ -134,7 +132,7 @@ let begin authenticators ''; - }; + }; }; # TODO get the hostname from somewhere else. diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix index 26b0947bb..592ed475d 100644 --- a/krebs/3modules/lass/default.nix +++ b/krebs/3modules/lass/default.nix @@ -4,6 +4,38 @@ with lib; { hosts = addNames { + dishfire = { + cores = 4; + dc = "lass"; #dc = "cac"; + nets = rec { + internet = { + addrs4 = ["144.76.172.188"]; + aliases = [ + "dishfire.internet" + ]; + }; + retiolum = { + via = internet; + addrs4 = ["10.243.133.99"]; + addrs6 = ["42:0000:0000:0000:0000:0000:d15f:1233"]; + aliases = [ + "dishfire.retiolum" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs + Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7 + uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK + R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd + vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U + HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + #ssh.privkey.path = ; + #ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL21QDOEFdODFh6WAfNp6odrXo15pEsDQuGJfMu/cKzK"; + }; echelon = { cores = 2; dc = "lass"; #dc = "cac"; diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix index 1970a0777..31516d591 100644 --- a/krebs/3modules/makefu/default.nix +++ b/krebs/3modules/makefu/default.nix @@ -83,6 +83,9 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster"; + }; vbob = { @@ -108,6 +111,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos"; }; flap = rec { cores = 1; @@ -238,6 +243,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry"; }; filepimp = rec { cores = 1; @@ -287,6 +294,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch"; }; gum = rec { cores = 1; @@ -327,6 +336,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum"; }; }; users = addNames rec { diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix index b332676c6..518e46587 100644 --- a/krebs/3modules/shared/default.nix +++ b/krebs/3modules/shared/default.nix @@ -7,6 +7,7 @@ let "test-arch" "test-centos6" "test-centos7" + "test-all-krebs-modules" ] (name: { inherit name; cores = 1; diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index c52afa246..81ce659bd 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -177,4 +177,21 @@ types // rec { addr6 = str; hostname = str; label = str; + + krebs.file-location = types.submodule { + options = { + # TODO user + host = mkOption { + type = host; + }; + # TODO merge with ssl.privkey.path + path = mkOption { + type = types.either types.path types.str; + apply = x: { + path = toString x; + string = x; + }.${typeOf x}; + }; + }; + }; } diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix new file mode 100644 index 000000000..7490be4ca --- /dev/null +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -0,0 +1,120 @@ +{ stdenv, lib, pkgs, makeWrapper }: + +rec { + # Begin API + buildBaseReaktorPlugin = { name + , config # python extra configuration for plugin + , phases ? [] + , ... } @ attrs: + stdenv.mkDerivation (attrs // { + name = "Reaktor-plugin-" + name; + isReaktorPlugin = true; + }); + + buildSimpleReaktorPlugin = name: { script + , path ? [] + , env ? {} + , append_rule ? false # append the rule instead of insert + , pattern ? "" + , ... } @ attrs: + let + path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); }; + src_dir = pkgs.substituteAll ( { + inherit name; + dir = "bin"; + isExecutable = true; + src = script; + }); + src_file = "${src_dir}/bin/${name}"; + config = '' + public_commands.${if append_rule then "append(" else "insert(0," }{ + 'capname' : "${name}", + 'pattern' : ${if pattern == "" then + ''indirect_pattern.format("${name}")'' else + ''"${pattern}"'' }, + 'argv' : ["${src_file}"], + 'env' : ${builtins.toJSON (path_env // env)} }) + ''; + config_file = pkgs.writeText "plugin.py" config; + in buildBaseReaktorPlugin (attrs // rec { + inherit name config; + + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin $out/etc/Reaktor + ln -s ${src_file} $out/bin + wrapProgram $out/bin/${name} \ + --prefix PATH : ${path_env.PATH} + ln -s ${config_file} $out/etc/Reaktor/plugin.py + ''; + + }); + # End API + + # Begin Plugins + random-emoji = buildSimpleReaktorPlugin "emoji" { + path = with pkgs; [ gnused gnugrep xmlstarlet curl ]; + script = ./scripts/random-emoji.sh; + }; + + sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { + path = [ pkgs.gnused pkgs.python3 ]; + # only support s///gi the plugin needs to see every msg + # TODO: this will eat up the last regex, fix Reaktor to support fallthru + append_rule = true; + pattern = "^(?P.*)$$"; + script = ./scripts/sed-plugin.py; + }; + + shack-correct = buildSimpleReaktorPlugin "shack-correct" { + path = [ pkgs.gnused ]; + pattern = "^(?P.*Shack.*)$$"; + script = ./scripts/shack-correct.sh; + }; + + nixos-version = buildSimpleReaktorPlugin "nixos-version" { + script = pkgs.writeScript "nixos-version" '' + #! /bin/sh + . /etc/os-release + echo "$PRETTY_NAME" + ''; + }; + stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { + script = ./scripts/random-issue.sh; + path = with pkgs; [ git gnused lentil ]; + env = { "origin" = "http://cgit.gum/stockholm"; }; + }; + + titlebot = + let + pypkgs = pkgs.python3Packages; + titlebot_cmds = pypkgs.buildPythonPackage { + name = "titlebot_cmds"; + propagatedBuildInputs = with pypkgs; [ setuptools ]; + src = pkgs.fetchurl { + url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; + sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; + }; + }; + in buildBaseReaktorPlugin rec { + name = "titlebot"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out + ln -s ${titlebot_cmds}/* $out + ''; + config = '' + def titlebot_cmd(cmd): + from os import environ + return { 'capname': None, + 'env': { 'TITLEDB': + environ['state_dir']+'/suggestions.json' }, + 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', + 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } + for i in ['up','help','list','top','new']: + public_commands.insert(0,titlebot_cmd(i)) + commands.insert(0,titlebot_cmd('clear')) + ''; + }; +} diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh similarity index 100% rename from makefu/2configs/Reaktor/random-emoji.sh rename to krebs/5pkgs/Reaktor/scripts/random-emoji.sh diff --git a/makefu/2configs/Reaktor/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh similarity index 100% rename from makefu/2configs/Reaktor/random-issue.sh rename to krebs/5pkgs/Reaktor/scripts/random-issue.sh diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py similarity index 100% rename from makefu/2configs/Reaktor/sed-plugin.py rename to krebs/5pkgs/Reaktor/scripts/sed-plugin.py diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh similarity index 100% rename from makefu/2configs/Reaktor/shack-correct.sh rename to krebs/5pkgs/Reaktor/scripts/shack-correct.sh diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix index 3e3e2e1fc..3df4dffed 100644 --- a/krebs/5pkgs/cacpanel/default.nix +++ b/krebs/5pkgs/cacpanel/default.nix @@ -2,11 +2,11 @@ python3Packages.buildPythonPackage rec { name = "cacpanel-${version}"; - version = "0.2.1"; + version = "0.2.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz"; - sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3"; + sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8"; }; propagatedBuildInputs = with python3Packages; [ diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 7df7b7d3c..c4b1dafe4 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -26,6 +26,8 @@ subdirs // rec { inherit (subdirs) get jq; }; + ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {}; + execve = name: { filename, argv, envp ? {}, destination ? "" }: writeC name { inherit destination; } '' #include @@ -40,6 +42,10 @@ subdirs // rec { } ''; + test = { + infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {}; + }; + execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; }); writeC = name: { destination ? "" }: src: pkgs.runCommand name {} '' diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix new file mode 100644 index 000000000..7f2e3f231 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -0,0 +1,39 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "${shortname}-${version}"; + shortname = "infest-cac-centos7"; + version = "0.2.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes new file mode 100755 index 000000000..eee0bfc34 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -0,0 +1,142 @@ +#! /bin/sh + +# nix-shell -p gnumake jq openssh cac cacpanel +set -eufx + +# 2 secrets are required: + +krebs_cred=${krebs_cred-./cac.json} +retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} + +clear_defer(){ + echo "${trapstr:-exit}" + trap - INT TERM EXIT KILL +} +defer(){ + if test -z "${debug:-}"; then + trapstr="$1;${trapstr:-exit}" + trap "$trapstr" INT TERM EXIT KILL + fi +} + +# Sanity +if test ! -r "$krebs_cred";then + echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 +fi +if test ! -r "$retiolum_key";then + echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 +fi + +krebs_secrets=$(mktemp -d) +sec_file=$krebs_secrets/cac_config +krebs_ssh=$krebs_secrets/tempssh +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json +# we need to receive this key from buildmaster to speed up tinc bootstrap +defer "trap - INT TERM EXIT" +defer "rm -r $krebs_secrets" + +cat > $sec_file <&1\ + | jq -r .servername) + id=servername:$name + + clear_defer >/dev/null + defer "cac delete $id" + + # TODO: timeout? + + wait_login_cac(){ + # we wait for 15 minutes + for t in `seq 90`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 + } + # die on timeout + if ! wait_login_cac $id;then + echo "unable to boot a working system within time frame, retrying..." >&2 + echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + eval "$(clear_defer | sed 's/;exit//')" + else + echo "got a working system" >&2 + break + fi +done +clear_defer >/dev/null +defer "cac delete $id;$old_trapstr" + +mkdir -p shared/2configs/temp +cac generatenetworking $id > \ + shared/2configs/temp/networking.nix +# new temporary ssh key we will use to log in after infest +ssh-keygen -f $krebs_ssh -N "" +cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv +# we override the directories for secrets and stockholm +# additionally we set the ssh key we generated +ip=$(cac getserver $id | jq -r .ip) + +cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest +sh -x $krebs_secrets/infest + +# TODO: generate secrets directory $krebs_secrets for nix import +cac powerop $id reset + +wait_login(){ + # timeout + for t in `seq 90`;do + # now we have a working cac server + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i $krebs_ssh \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + root@$1 nixos-version ;then + return 0 + fi + sleep 10 + done + return 1 +} +wait_login $ip diff --git a/krebs/Zhosts/bobby b/krebs/Zhosts/bobby index ee111e695..aac6e377b 100644 --- a/krebs/Zhosts/bobby +++ b/krebs/Zhosts/bobby @@ -1,5 +1,6 @@ Subnet = 10.243.111.112/32 Subnet = 42:0:0:0:0:0:111:112/128 + -----BEGIN RSA PUBLIC KEY----- MIIBCgKCAQEA+AScnIqFdzGl+iRZTNZ7r91n/r1H4GzDsrAupUvJ4mi7nDN4eP8s uLvKtJp22RxfuF3Kf4KhHb8LHQ8bLLN/KDaNDXrCNBc69d7vvLsjoY+wfGLJNu4Y diff --git a/krebs/Zhosts/dishfire b/krebs/Zhosts/dishfire new file mode 100644 index 000000000..c4cf68b6b --- /dev/null +++ b/krebs/Zhosts/dishfire @@ -0,0 +1,12 @@ +Address = 144.76.172.188 +Subnet = 10.243.133.99 +Subnet = 42:0000:0000:0000:0000:0000:d15f:1233 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs +Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7 +uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK +R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd +vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U +HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB +-----END RSA PUBLIC KEY----- diff --git a/krebs/Zhosts/kebsco b/krebs/Zhosts/kebsco new file mode 100644 index 000000000..2fd1c5f42 --- /dev/null +++ b/krebs/Zhosts/kebsco @@ -0,0 +1,11 @@ +Subnet = 10.243.212.68 +Subnet = 42:9d30:3845:c822:988b:96c5:39ab:90b7 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEA0dEwTZh2uzJpP9GL7YRyiLuezJqYiJ8/4Bl4IPshJnuO9IGbEcto +0cFm9uM9gxxqggfaCi96DsIQNlyqff2vDfEj3mdIu9T3tkRROByQF8y1NWX29NyH +zZEX8Ri8u4U2KdYTEzPXEFxBEl0GQX9mMtlvwzCq7V4ueCcWB1xDA+DtJjpd894z +3FOw0rIxYmfYhLAL5B3rzF74bcHFGV30f4JWq11wLBkyR6/Q5gxgZzkKYGwdZ/SN +C6gg86abKdp65/Wq5P331IbwPBal1ZhGbaAo1y7JpjpLvZytI2jboXeQuPZ8P5hU +L3zKKceAibPKrw9+y8lb+IKoYLF7I1KYIwIDAQAB +-----END RSA PUBLIC KEY----- diff --git a/krebs/default.nix b/krebs/default.nix index ad0205426..81ddd3ea6 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -36,6 +36,7 @@ let out = { { system ? current-host-name , target ? system }@args: let + config = get-config system; in '' #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} @@ -47,6 +48,10 @@ let out = { ${builtins.readFile ./4lib/infest/install-nix.sh} ''} + # Prepare target source via bind-mounting + + (${populate (args // { infesting = true;}) }) + (${nixos-install args}) ${rootssh target '' @@ -98,7 +103,6 @@ let out = { #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} # krebs.nixos-install - (${populate args}) ${rootssh target '' export PATH; PATH=/root/.nix-profile/bin:$PATH @@ -205,6 +209,7 @@ let out = { populate = { system ? current-host-name , target ? system + , infesting ? false }@args: let out = '' #! /bin/sh @@ -217,6 +222,8 @@ let out = { ["dir" "git"])} ''; + + target_prefix=lib.optionalString infesting "/mnt"; config = get-config system; current-host = config.krebs.hosts.${current-host-name}; @@ -225,17 +232,18 @@ let out = { methods.dir = config: let can-push = config.host.name == current-host.name; + target-path = target_prefix + config.target-path; push-method = '' rsync \ --exclude .git \ --exclude .graveyard \ --exclude old \ --exclude tmp \ - --rsync-path='mkdir -p ${config.target-path} && rsync' \ + --rsync-path='mkdir -p ${target-path} && rsync' \ --delete-excluded \ -vrLptgoD \ ${config.path}/ \ - root@${target}:${config.target-path} + root@${target}:${target-path} ''; in if can-push then push-method else @@ -244,9 +252,10 @@ let out = { throw "No way to push ${dir} from ${current-host.name} to ${target}"; methods.git = config: - rootssh target '' - mkdir -p ${config.target-path} - cd ${config.target-path} + let target-path = target_prefix + config.target-path; + in rootssh target '' + mkdir -p ${target-path} + cd ${target-path} if ! test -e .git; then git init fi diff --git a/lass/1systems/dishfire.nix b/lass/1systems/dishfire.nix new file mode 100644 index 000000000..a1288d578 --- /dev/null +++ b/lass/1systems/dishfire.nix @@ -0,0 +1,43 @@ +{ config, lib, pkgs, ... }: + +{ + imports = [ + + ../2configs/base.nix + { + boot.loader.grub = { + device = "/dev/vda"; + splashImage = null; + }; + + boot.initrd.availableKernelModules = [ + "ata_piix" + "ehci_pci" + "uhci_hcd" + "virtio_pci" + "virtio_blk" + ]; + + fileSystems."/" = { + device = "/dev/mapper/pool-nix"; + fsType = "ext4"; + }; + + fileSystems."/boot" = { + device = "/dev/vda1"; + fsType = "ext4"; + }; + } + { + networking.dhcpcd.allowInterfaces = [ + "enp*" + "eth*" + ]; + } + { + sound.enable = false; + } + ]; + + krebs.build.host = config.krebs.hosts.dishfire; +} diff --git a/lass/1systems/mors.nix b/lass/1systems/mors.nix index 8af096f51..61f57f1f9 100644 --- a/lass/1systems/mors.nix +++ b/lass/1systems/mors.nix @@ -38,6 +38,10 @@ ../3modules/wordpress_nginx.nix ]; lass.wordpress."testserver.de" = { + multiSite = { + "1" = "testserver.de"; + "2" = "bla.testserver.de"; + }; }; services.mysql = { @@ -52,6 +56,27 @@ { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; } ]; } + { + #owncloud-test + #imports = singleton (sitesGenerators.createWordpress "testserver.de"); + imports = [ + ../3modules/owncloud_nginx.nix + ]; + lass.owncloud."owncloud-test.de" = { + }; + + #services.mysql = { + # enable = true; + # package = pkgs.mariadb; + # rootPassword = "/mysql_rootPassword"; + #}; + networking.extraHosts = '' + 10.243.0.2 owncloud-test.de + ''; + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; } + ]; + } ]; krebs.build.host = config.krebs.hosts.mors; @@ -59,11 +84,12 @@ networking.wireless.enable = true; networking.extraHosts = '' - 10.243.206.102 habsys.de - 10.243.206.102 pixelpocket.de - 10.243.206.102 karlaskop.de - 10.243.206.102 ubikmedia.de - 10.243.206.102 apanowicz.de + 213.239.205.240 wohnprojekt-rhh.de + 213.239.205.240 karlaskop.de + 213.239.205.240 makeup.apanowicz.de + 213.239.205.240 pixelpocket.de + 213.239.205.240 reich-gebaeudereinigung.de + 213.239.205.240 o.ubikmedia.de ''; hardware.enableAllFirmware = true; diff --git a/lass/2configs/base.nix b/lass/2configs/base.nix index 40f4e12c7..4685e1713 100644 --- a/lass/2configs/base.nix +++ b/lass/2configs/base.nix @@ -50,7 +50,8 @@ with lib; source = { git.nixpkgs = { url = https://github.com/Lassulus/nixpkgs; - rev = "363c8430f1efad8b03d5feae6b3a4f2fe7b29251"; + rev = "93d8671e2c6d1d25f126ed30e5e6f16764330119"; + target-path = "/var/src/nixpkgs"; }; dir.secrets = { host = config.krebs.hosts.mors; diff --git a/lass/2configs/baseX.nix b/lass/2configs/baseX.nix index 4e46c18d2..ede1c7b7b 100644 --- a/lass/2configs/baseX.nix +++ b/lass/2configs/baseX.nix @@ -31,10 +31,16 @@ in { environment.systemPackages = with pkgs; [ - powertop - sxiv + dmenu + gitAndTools.qgit + mpv much + pavucontrol + powertop push + slock + sxiv + xsel zathura #window manager stuff diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix new file mode 100644 index 000000000..4d02fb97a --- /dev/null +++ b/lass/2configs/buildbot-standalone.nix @@ -0,0 +1,78 @@ +{ lib, config, pkgs, ... }: +{ + #networking.firewall.allowedTCPPorts = [ 8010 9989 ]; + krebs.buildbot.master = { + slaves = { + testslave = "lasspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.mors/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["fast-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + addShell(f,name="mors-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=mors"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + }; + enable = true; + web.enable = true; + irc = { + enable = true; + nick = "lass-buildbot"; + server = "cd.retiolum"; + channels = [ "retiolum" ]; + allowForce = true; + }; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "lasspass"; + packages = with pkgs;[ git nix ]; + extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; + }; +} diff --git a/lass/2configs/xserver/default.nix b/lass/2configs/xserver/default.nix index da337f6a7..04d14c7ce 100644 --- a/lass/2configs/xserver/default.nix +++ b/lass/2configs/xserver/default.nix @@ -7,7 +7,6 @@ let user = config.users.users.mainUser; out = { - services.xserver = { display = 11; tty = 11; @@ -41,16 +40,6 @@ let }; }; - environment.systemPackages = [ - pkgs.gitAndTools.qgit - pkgs.mpv - pkgs.pavucontrol - pkgs.slock - pkgs.sxiv - pkgs.xsel - pkgs.zathura - ]; - security.setuidPrograms = [ "slock" ]; @@ -106,9 +95,6 @@ let set -efu export PATH; PATH=${makeSearchPath "bin" ([ pkgs.rxvt_unicode - pkgs.i3lock - pkgs.haskellPackages.yeganesh - pkgs.dmenu ] ++ config.environment.systemPackages)}:/var/setuid-wrappers settle() {( # Use PATH for a clean journal diff --git a/lass/5pkgs/newsbot-js/default.nix b/lass/5pkgs/newsbot-js/default.nix index ace2a976f..cabd7422c 100644 --- a/lass/5pkgs/newsbot-js/default.nix +++ b/lass/5pkgs/newsbot-js/default.nix @@ -26,8 +26,8 @@ in nodePackages.buildNodePackage { src = fetchgit { url = "http://cgit.echelon/newsbot-js/"; - rev = "b22729670236bfa6491207d57c5d7565137625ca"; - sha256 = "8ff00de56d85543399776c82d41d92ccc68000e5dce0f008d926748e188f3c69"; + rev = "802b172d0eed6c9625a9cb5db408f5cc8c01784e"; + sha256 = "794fc7845aca311f7cf7b6bdc109b5a25d0e2299322bc6612edadc477b2536e2"; }; phases = [ diff --git a/lass/5pkgs/xmonad-lass/Main.hs b/lass/5pkgs/xmonad-lass/Main.hs index ce5afe33a..faaa00aab 100644 --- a/lass/5pkgs/xmonad-lass/Main.hs +++ b/lass/5pkgs/xmonad-lass/Main.hs @@ -125,6 +125,8 @@ myKeyMap = --, ("M4-r", spawn "exe=$(yeganesh -x) && eval \"exec $exe\"") , ("", spawn "pactl -- set-sink-volume 0 +4%") , ("", spawn "pactl -- set-sink-volume 0 -4%") + , ("", spawn "pactl -- set-sink-mute 0 toggle") + , ("", spawn "pactl -- set-source-mute 1 toggle") , ("", gridselectWorkspace myWSConfig W.view) , ("M4-a", focusUrgent) diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix index 417a020fa..1907424ec 100644 --- a/makefu/1systems/gum.nix +++ b/makefu/1systems/gum.nix @@ -6,15 +6,19 @@ let internal-ip = head config.krebs.build.host.nets.retiolum.addrs4; in { imports = [ - # TODO: copy this config or move to krebs ../2configs/tinc-basic-retiolum.nix ../2configs/headless.nix ../2configs/fs/simple-swap.nix ../2configs/fs/single-partition-ext4.nix + ../2configs/smart-monitor.nix # ../2configs/iodined.nix ../2configs/git/cgit-retiolum.nix ../2configs/mattermost-docker.nix ../2configs/nginx/euer.test.nix + + ../2configs/exim-retiolum.nix + ../2configs/urlwatch.nix + ]; diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix index 161bfa3e9..a1b73c0c9 100644 --- a/makefu/1systems/pnp.nix +++ b/makefu/1systems/pnp.nix @@ -28,9 +28,6 @@ ../2configs/Reaktor/titlebot.nix ../2configs/Reaktor/shack-correct.nix - ../2configs/exim-retiolum.nix - ../2configs/urlwatch.nix - # ../2configs/graphite-standalone.nix ]; krebs.urlwatch.verbose = true; diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix index 28b77d330..690e26b36 100644 --- a/makefu/1systems/pornocauster.nix +++ b/makefu/1systems/pornocauster.nix @@ -26,6 +26,7 @@ # services ../2configs/git/brain-retiolum.nix ../2configs/tor.nix + # ../2configs/buildbot-standalone.nix # hardware specifics are in here ../2configs/hw/tp-x220.nix @@ -36,14 +37,14 @@ ]; nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; - buildbot = let - pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; - in pkgs1509.buildbot; }; - makefu.buildbot.master.enable = true; - #krebs.Reaktor.enable = true; - #krebs.Reaktor.nickname = "makefu|r"; + krebs.Reaktor = { + enable = true; + nickname = "makefu|r"; + plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ]; + }; + # nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ]; environment.systemPackages = with pkgs;[ diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix index cd2b3f657..f022311c9 100644 --- a/makefu/1systems/wry.nix +++ b/makefu/1systems/wry.nix @@ -18,8 +18,6 @@ in { ../2configs/iodined.nix - # Reaktor - ../2configs/Reaktor/simpleExtend.nix # other nginx ../2configs/nginx/euer.wiki.nix @@ -29,9 +27,21 @@ in { # collectd ../2configs/collectd/collectd-base.nix ]; + krebs.build.host = config.krebs.hosts.wry; - krebs.Reaktor.enable = true; + krebs.Reaktor = { + nickname = "Reaktor|bot"; + channels = [ "#krebs" "#shackspace" "#binaergewitter" ]; + enable = true; + plugins = with pkgs.ReaktorPlugins;[ + titlebot + # stockholm-issue + nixos-version + shack-correct + sed-plugin + random-emoji ]; + }; # bepasty to listen only on the correct interfaces krebs.bepasty.servers.internal.nginx.listen = [ "${internal-ip}:80" ]; @@ -59,11 +69,11 @@ in { }; networking = { - firewall = { + firewall = { allowPing = true; logRefusedConnections = false; allowedTCPPorts = [ 53 80 443 ]; - allowedUDPPorts = [ 655 ]; + allowedUDPPorts = [ 655 53 ]; }; interfaces.enp2s1.ip4 = [{ address = external-ip; diff --git a/makefu/2configs/Reaktor/full.nix b/makefu/2configs/Reaktor/full.nix deleted file mode 100644 index 50620890f..000000000 --- a/makefu/2configs/Reaktor/full.nix +++ /dev/null @@ -1,18 +0,0 @@ -_: -{ - # implementation of the complete Reaktor bot - imports = [ - #./stockholmLentil.nix - ./simpleExtend.nix - ./random-emoji.nix - ./titlebot.nix - ./shack-correct.nix - ./sed-plugin.nix - ]; - krebs.Reaktor.nickname = "Reaktor|bot"; - krebs.Reaktor.enable = true; - - krebs.Reaktor.extraEnviron = { - REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace"; - }; -} diff --git a/makefu/2configs/Reaktor/random-emoji.nix b/makefu/2configs/Reaktor/random-emoji.nix deleted file mode 100644 index 3113a826b..000000000 --- a/makefu/2configs/Reaktor/random-emoji.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - rpkg = pkgs.substituteAll( { - name="random-emoji"; - dir= "bin"; - isExecutable=true; - src= ./random-emoji.sh; - }); - rpkg-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["${rpkg}/bin/random-emoji"], - 'env' : { 'PATH':'${rpkg-path}' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/sed-plugin.nix b/makefu/2configs/Reaktor/sed-plugin.nix deleted file mode 100644 index a451e0d3e..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = ./sed-plugin.py; -in { - #TODO: this will eat up the last regex, fix Reaktor - krebs.Reaktor.extraConfig = '' - public_commands.append({ - 'capname' : "sed-plugin", - # only support s///gi - 'pattern' : '^(?P.*)$$', - 'argv' : ["${pkgs.python3}/bin/python3","${script}"], - 'env' : { 'state_dir' : workdir, - 'PATH':'${lib.makeSearchPath "bin" [pkgs.gnused]}' }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/shack-correct.nix b/makefu/2configs/Reaktor/shack-correct.nix deleted file mode 100644 index 8f30807f1..000000000 --- a/makefu/2configs/Reaktor/shack-correct.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = pkgs.substituteAll ( { - name="shack-correct"; - isExecutable=true; - dir = ""; - src = ./shack-correct.sh; - }); -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "shack-correct", - 'pattern' : '^(?P.*Shack.*)$$', - 'argv' : ["${script}"], - 'env' : { }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/simpleExtend.nix b/makefu/2configs/Reaktor/simpleExtend.nix deleted file mode 100644 index 95175a4e0..000000000 --- a/makefu/2configs/Reaktor/simpleExtend.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - nixos-version-script = pkgs.writeScript "nix-version" '' - #! /bin/sh - . /etc/os-release - echo "$PRETTY_NAME" - ''; -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "nixos-version", - 'pattern' : indirect_pattern.format("nixos-version"), - 'argv' : ["${nixos-version-script}"], - 'env' : { 'state_dir': workdir } }) - ''; -} - diff --git a/makefu/2configs/Reaktor/stockholmLentil.nix b/makefu/2configs/Reaktor/stockholmLentil.nix deleted file mode 100644 index 21f0305fb..000000000 --- a/makefu/2configs/Reaktor/stockholmLentil.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - random-issue = pkgs.substituteAll( { - name="random-issue"; - dir= "bin"; - isExecutable=true; - src= ./random-issue.sh; - }); - random-issue-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - git - gnused - lentil]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "stockholm-issue", - 'pattern' : indirect_pattern.format("stockholm-issue"), - 'argv' : ["${random-issue}/bin/random-issue"], - 'env' : { 'state_dir': workdir, - 'PATH':'${random-issue-path}', - 'origin':'http://cgit.pnp/stockholm' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/titlebot.nix b/makefu/2configs/Reaktor/titlebot.nix deleted file mode 100644 index 9ef02548b..000000000 --- a/makefu/2configs/Reaktor/titlebot.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ stdenv,config, lib, pkgs, ... }: - -with pkgs; -let - pypkgs = pkgs.python3Packages; - titlebot_cmds = pypkgs.buildPythonPackage { - name = "titlebot_cmds"; - propagatedBuildInputs = with pypkgs; [ setuptools ]; - src = fetchurl { - # https://github.com/makefu/reaktor-titlebot tag 2.1.0 - url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; - sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; - }; - }; - pub_cmds = ["up" "help" "list" "top" "highest" "undo" ]; - priv_cmds = [ "clear" ]; -in { - # TODO: write identify file in - # {config.users.extraUsers.Reaktor.home}/state/admin.lst - krebs.Reaktor.extraConfig = '' - def titlebot_cmd(cmd): - return { - 'capname': cmd, - 'env': { - 'TITLEDB': - '${config.users.extraUsers.Reaktor.home}/suggestions.json' - }, - 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', - 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } - # TODO: for each element in ${titlebot_cmds}/bin/* - public_commands.insert(0,titlebot_cmd('up')) - public_commands.insert(0,titlebot_cmd('help')) - public_commands.insert(0,titlebot_cmd('list')) - public_commands.insert(0,titlebot_cmd('top')) - public_commands.insert(0,titlebot_cmd('new')) - commands.insert(0,titlebot_cmd('clear')) - ''; -} diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index c0d7685e3..a0b49edaf 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -24,7 +24,7 @@ with lib; git.nixpkgs = { #url = https://github.com/NixOS/nixpkgs; url = mkDefault https://github.com/makefu/nixpkgs; - rev = mkDefault "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picking + rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking target-path = "/var/src/nixpkgs"; }; diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix new file mode 100644 index 000000000..7086f622b --- /dev/null +++ b/makefu/2configs/smart-monitor.nix @@ -0,0 +1,18 @@ +{ config, ... }: +{ + services.smartd = { + enable = true; + notifications = { + mail = { + enable = true; + recipient = config.krebs.users.makefu.mail; + }; + }; + # short daily, long weekly, check on boot + defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)"; + + devices = [{ + device = "/dev/sda"; + }]; + }; +} diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix index cd05f0114..a83279ba2 100644 --- a/makefu/2configs/urlwatch.nix +++ b/makefu/2configs/urlwatch.nix @@ -1,6 +1,22 @@ -{ config, ... }: +{ config, lib, ... }: { + nixpkgs.config.packageOverrides = pkgs: { + urlwatch = with pkgs.pythonPackages; buildPythonPackage rec { + name = "urlwatch-1.18"; + + propagatedBuildInputs = [ futures ]; + + src = pkgs.fetchurl { + url = "http://thp.io/2008/urlwatch/${name}.tar.gz"; + sha256 = "090qfgx249ks7103sap6w47f8302ix2k46wxhfssxwsqcqdl25vb"; + }; + + postFixup = '' + wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" + ''; + }; + }; krebs.urlwatch = { enable = true; mailto = config.krebs.users.makefu.mail; @@ -12,7 +28,7 @@ http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release https://pypi.python.org/simple/bepasty/ https://pypi.python.org/simple/xstatic/ - + http://guest:derpi@cvs2svn.tigris.org/svn/cvs2svn/tags/ ]; }; } diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix deleted file mode 100644 index 09edac94d..000000000 --- a/makefu/3modules/buildbot/master.nix +++ /dev/null @@ -1,263 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; -let - buildbot = pkgs.buildbot; - buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' - # -*- python -*- - from buildbot.plugins import * - import re - - c = BuildmasterConfig = {} - - c['slaves'] = [] - # TODO: template potential buildslaves - # TODO: set password? - slavenames= [ 'testslave' ] - for i in slavenames: - c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) - - c['protocols'] = {'pb': {'port': 9989}} - - ####### Build Inputs - stockholm_repo = 'http://cgit.gum/stockholm' - c['change_source'] = [] - c['change_source'].append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) - - ####### Build Scheduler - # TODO: configure scheduler - c['schedulers'] = [] - - # test the master real quick - fast = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - name="fast-master-test", - builderNames=["fast-tests"]) - - force = schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"]) - - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False - - full = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - name="full-master-test", - builderNames=["full-tests"]) - c['schedulers'] = [ fast, force, full ] - ###### The actual build - # couple of fast steps: - f = util.BuildFactory() - ## fetch repo - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - f.addStep(grab_repo) - - # the dependencies which are used by the test script - deps = [ "gnumake", "jq" ] - nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] - def addShell(f,**kwargs): - f.addStep(steps.ShellCommand(**kwargs)) - - addShell(f,name="centos7-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=test-centos7"]) - - addShell(f,name="wolf-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=wolf"]) - - c['builders'] = [] - c['builders'].append( - util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - - # TODO slow build - c['builders'].append( - util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=f)) - - ####### Status of Builds - c['status'] = [] - - from buildbot.status import html - from buildbot.status.web import authz, auth - # TODO: configure if http is wanted - authz_cfg=authz.Authz( - # TODO: configure user/pw - auth=auth.BasicAuth([("krebs","bob")]), - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = False, - stopAllBuilds = False, - cancelPendingBuild = False, - ) - # TODO: configure nginx - c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - - from buildbot.status import words - ${optionalString (cfg.irc.enable) '' - irc = words.IRC("${cfg.irc.server}", "krebsbuild", - # TODO: multiple channels - channels=["${cfg.irc.channel}"], - notify_events={ - #'success': 1, - #'failure': 1, - 'exception': 1, - 'successToFailure': 1, - 'failureToSuccess': 1, - }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['status'].append(irc) - ''} - - ####### PROJECT IDENTITY - c['title'] = "Stockholm" - c['titleURL'] = "http://krebsco.de" - - #c['buildbotURL'] = "http://buildbot.krebsco.de/" - # TODO: configure url - c['buildbotURL'] = "http://vbob:8010/" - - ####### DB URL - c['db'] = { - 'db_url' : "sqlite:///state.sqlite", - } - ${cfg.extraConfig} - ''; - - cfg = config.makefu.buildbot.master; - - api = { - enable = mkEnableOption "Buildbot Master"; - workDir = mkOption { - default = "/var/lib/buildbot/master"; - type = types.str; - description = '' - Path to build bot master directory. - Will be created on startup. - ''; - }; - irc = mkOption { - default = {}; - type = types.submodule ({ config, ... }: { - options = { - enable = mkEnableOption "Buildbot Master IRC Status"; - channel = mkOption { - default = "nix-buildbot-meetup"; - type = types.str; - description = '' - irc channel the bot should connect to - ''; - }; - allowForce = mkOption { - default = false; - type = types.bool; - description = '' - Determines if builds can be forced via IRC - ''; - }; - nick = mkOption { - default = "nix-buildbot"; - type = types.str; - description = '' - nickname for IRC - ''; - }; - server = mkOption { - default = "irc.freenode.net"; - type = types.str; - description = '' - Buildbot Status IRC Server to connect to - ''; - }; - }; - }); - }; - - extraConfig = mkOption { - default = ""; - type = types.lines; - description = '' - extra config appended to the generated master.cfg - ''; - }; - }; - - imp = { - - users.extraUsers.buildbotMaster = { - uid = genid "buildbotMaster"; - description = "Buildbot Master"; - home = cfg.workDir; - createHome = false; - }; - - users.extraGroups.buildbotMaster = { - gid = 672626386; - }; - - systemd.services.buildbotMaster = { - description = "Buildbot Master"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.git ]; - serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - # TODO: check if git is the only dep - in { - PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeScript "buildbot-master-init" '' - #!/bin/sh - set -efux - if [ ! -e ${workdir} ];then - mkdir -p ${workdir} - ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} - fi - # always override the master.cfg - cp ${buildbot-master-config} ${workdir}/master.cfg - # sanity - ${buildbot}/bin/buildbot checkconfig ${workdir} - - # TODO: maybe upgrade? not sure about this - # normally we should write buildbot.tac by our own - # ${buildbot}/bin/buildbot upgrade-master ${workdir} - - chmod 700 -R ${workdir} - chown buildbotMaster:buildbotMaster -R ${workdir} - ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; - PrivateTmp = "true"; - User = "buildbotMaster"; - Restart = "always"; - RestartSec = "10"; - }; - }; - }; -in -{ - options.makefu.buildbot.master = api; - config = mkIf cfg.enable imp; -} diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix index ffbf54cc0..a8a1f69d0 100644 --- a/makefu/3modules/default.nix +++ b/makefu/3modules/default.nix @@ -2,8 +2,6 @@ _: { imports = [ - ./buildbot/master.nix - ./buildbot/slave.nix ]; } diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix new file mode 100644 index 000000000..b98004dfe --- /dev/null +++ b/shared/1systems/test-all-krebs-modules.nix @@ -0,0 +1,45 @@ +{ config, pkgs, lib, ... }: +let + en = { enable = true;}; +in { + krebs = { + enable = true; + build.user = config.krebs.users.shared; + build.host = config.krebs.hosts.test-all-krebs-modules; + Reaktor.enable = true; + apt-cacher-ng.enable = true; + backup.enable = true; + bepasty.enable = true; + buildbot.master.enable = true; + buildbot.slave = { + enable = true; + username = "lol"; + password = "wut"; + }; + exim-retiolum.enable = true; + exim-smarthost = { + enable = true; + system-aliases = [ { from = "dick"; to = "butt"; } ]; + }; + go.enable = true; + iptables = { + enable = true; + tables = {}; + }; + nginx.enable = true; + realwallpaper.enable = true; + retiolum.enable = true; + retiolum-bootstrap.enable = true; + tinc_graphs.enable = true; + urlwatch.enable = true; + fetchWallpaper = { + enable = true; + url ="localhost"; + }; + }; + # just get the system running + boot.loader.grub.devices = ["/dev/sda"]; + fileSystems."/" = { + device = "/dev/lol"; + }; +} diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix index 077a5d61b..48cecc877 100644 --- a/shared/1systems/test-centos7.nix +++ b/shared/1systems/test-centos7.nix @@ -7,7 +7,8 @@ in { imports = [ ../2configs/base.nix ../2configs/os-templates/CAC-CentOS-7-64bit.nix - ../2configs/os-templates/temp-networking.nix + ../2configs/temp/networking.nix + ../2configs/temp/dirs.nix ]; sound.enable = false; diff --git a/shared/1systems/test-failing.nix b/shared/1systems/test-failing.nix new file mode 100644 index 000000000..81a9e48d6 --- /dev/null +++ b/shared/1systems/test-failing.nix @@ -0,0 +1,6 @@ +{ config, pkgs, ... }: + +{ + programs.ssh.startAgent = true; + programs.ssh.startAgent = false; +} diff --git a/shared/1systems/test-minimal-deploy.nix b/shared/1systems/test-minimal-deploy.nix new file mode 100644 index 000000000..ddd96f6b5 --- /dev/null +++ b/shared/1systems/test-minimal-deploy.nix @@ -0,0 +1,13 @@ +{ config, pkgs, lib, ... }: +{ + krebs = { + enable = true; + build.user = config.krebs.users.shared; + build.host = config.krebs.hosts.test-all-krebs-modules; + }; + # just get the system running + boot.loader.grub.devices = ["/dev/sda"]; + fileSystems."/" = { + device = "/dev/lol"; + }; +} diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index 2c51ac8fe..f05356f0f 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -11,7 +11,7 @@ in ../2configs/collectd-base.nix ../2configs/shack-nix-cacher.nix ../2configs/shack-drivedroid.nix - ../2configs/cac-ci.nix + ../2configs/buildbot-standalone.nix ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by @@ -33,8 +33,6 @@ in # uninteresting stuff ##################### krebs.build.host = config.krebs.hosts.wolf; - # TODO rename shared user to "krebs" - krebs.build.user = config.krebs.users.shared; krebs.build.target = "wolf"; boot.kernel.sysctl = { diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index df41eae1a..4d509d7a6 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -13,18 +13,22 @@ with lib; ]; }; + # TODO rename shared user to "krebs" + krebs.build.user = mkDefault config.krebs.users.shared; krebs.build.source = { git.nixpkgs = { url = https://github.com/NixOS/nixpkgs; rev = "6d31e9b81dcd4ab927bb3dc91b612dd5abfa2f80"; + target-path = "/var/src/nixpkgs"; }; dir.secrets = { host = config.krebs.current.host; - path = "${getEnv "HOME"}/secrets/krebs/wolf"; + path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; }; dir.stockholm = { host = config.krebs.current.host; - path = "${getEnv "HOME"}/stockholm"; + path = mkDefault "${getEnv "HOME"}/stockholm"; + target-path = "/var/src/stockholm"; }; }; @@ -65,7 +69,7 @@ with lib; config.krebs.users.lass.pubkey config.krebs.users.makefu.pubkey # TODO HARDER: - (readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub) + config.krebs.users.makefu-omo.pubkey config.krebs.users.tv.pubkey ]; diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix new file mode 100644 index 000000000..6ffd7fe8a --- /dev/null +++ b/shared/2configs/buildbot-standalone.nix @@ -0,0 +1,154 @@ +{ lib, config, pkgs, ... }: +let + pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; +in { + nixpkgs.config.packageOverrides = pkgs: { + buildbot = pkgs-unst.buildbot; + buildbot-slave = pkgs-unst.buildbot-slave; + }; + networking.firewall.allowedTCPPorts = [ 8010 9989 ]; + krebs.buildbot.master = { + secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; + slaves = { + testslave = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + test-cac-infest-master = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + treeStableTimer=60*60, # master was stable for the last hour + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + addShell(f,name="deploy-eval-centos7",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="deploy-eval-wolf",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) + + addShell(f,name="deploy-eval-cross-check",env=env, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + + addShell(f,name="instantiate-test-all-modules",env=env, + command=nixshell + \ + ["touch retiolum.rsa_key.priv; \ + nix-instantiate --eval -A \ + users.shared.test-all-krebs-modules.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + + addShell(f,name="instantiate-test-minimal-deploy",env=env, + command=nixshell + \ + ["nix-instantiate --eval -A \ + users.shared.test-minimal-deploy.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=7200, # 2h + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; + enable = true; + web = { + enable = true; + }; + irc = { + enable = true; + nick = "shared-buildbot"; + server = "cd.retiolum"; + channels = [ "retiolum" ]; + allowForce = true; + }; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "krebspass"; + packages = with pkgs;[ git nix ]; + # all nix commands will need a working nixpkgs installation + extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; + }; +} diff --git a/shared/2configs/cac-ci.nix b/shared/2configs/cac-ci.nix deleted file mode 100644 index 06cce2746..000000000 --- a/shared/2configs/cac-ci.nix +++ /dev/null @@ -1,11 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; -{ - environment.systemPackages = with pkgs;[ - get - cac - cacpanel - jq - ]; -} diff --git a/shared/2configs/temp/dirs.nix b/shared/2configs/temp/dirs.nix new file mode 100644 index 000000000..958608a54 --- /dev/null +++ b/shared/2configs/temp/dirs.nix @@ -0,0 +1 @@ +_: { } diff --git a/shared/2configs/temp/networking.nix b/shared/2configs/temp/networking.nix new file mode 100644 index 000000000..958608a54 --- /dev/null +++ b/shared/2configs/temp/networking.nix @@ -0,0 +1 @@ +_: { } diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix index 7cb903a44..8c2a9ae43 100644 --- a/tv/1systems/cd.nix +++ b/tv/1systems/cd.nix @@ -7,7 +7,6 @@ with lib; krebs.build.target = "root@cd.internet"; imports = [ - ../2configs/backup.nix ../2configs/hw/CAC-Developer-2.nix ../2configs/fs/CAC-CentOS-7-64bit.nix #../2configs/consul-server.nix diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index e1a9076dc..1f3e010a4 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -9,7 +9,6 @@ with lib; "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a"; imports = [ - ../2configs/backup.nix # TODO ../2configs/hw/x220.nix #../2configs/consul-client.nix ../2configs/git.nix diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix index 1cef0a6dc..51d3bb8a7 100644 --- a/tv/2configs/backup.nix +++ b/tv/2configs/backup.nix @@ -1,38 +1,22 @@ -{ config, lib, pkgs, ... }: +{ config, lib, ... }: with lib; -let - # Users that are allowed to connect to the backup user. - # Note: the user must own a push plan destination otherwise no rsync. - backup-users = [ - config.krebs.users.tv - ]; - - ## TODO parse.file-location admit user - ## loc has the form : - #parse.file-location = loc: let - # parts = splitString ":" loc; - # host-name = head parts; - # path = concatStringsSep ":" (tail parts); - #in { - # type = "types.krebs.file-location"; - # host = config.krebs.hosts.${host-name}; - # path = path; - #}; - - # TODO assert plan.dst.path & co - plans = with config.krebs.users; with config.krebs.hosts; addNames { +{ + krebs.backup.plans = addNames { xu-test-cd = { method = "push"; - #src = parse.file-location xu:/tmp/xu-test; - #dst = parse.file-location cd:/krebs/backup/xu-test; - src = { user = tv; host = xu; path = "/tmp/xu-test"; }; - dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; }; - startAt = "0,6,12,18:00"; - retain = { - hourly = 4; # sneakily depends on startAt - daily = 7; - weekly = 4; - monthly = 3; + + src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; }; + dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; }; + + #startAt = "0,6,12,18:00"; + startAt = "minutely"; + snapshots = { + minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; }; }; #xu-test-wu = { @@ -41,214 +25,18 @@ let #}; cd-test-xu = { method = "pull"; - #src = parse.file-location cd:/tmp/cd-test; - #dst = parse.file-location xu:/bku/cd-test; - src = { user = tv; host = cd; path = "/tmp/cd-test"; }; - dst = { user = tv; host = xu; path = "/bku/cd-test"; }; + src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; }; + dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; }; + startAt = "minutely"; + snapshots = { + minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; }; }; - - out = { - #options.krebs.backup = api; - config = imp; - }; - - imp = { - users.groups.backup.gid = genid "backup"; - users.users = map makeUser (filter isPushDst (attrValues plans)); - systemd.services = - flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: { - name = "backup.${name}"; - value = makePushService plan; - }); - }; - - - # TODO getFQDN: admit hosts in other domains - getFQDN = host: "${host.name}.${config.krebs.search-domain}"; - - isPushSrc = plan: - plan.method == "push" && - plan.src.host.name == config.krebs.build.host.name; - - makePushService = plan: assert isPushSrc plan; { - startAt = plan.startAt; - serviceConfig.ExecStart = writeSh plan "rsync" '' - exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [ - "-a" - "-e" - "${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}" - "${plan.src.path}" - "${plan.name}@${getFQDN plan.dst.host}::push" - ]} - ''; - }; - - isPushDst = plan: - plan.method == "push" && - plan.dst.host.name == config.krebs.build.host.name; - - makeUser = plan: assert isPushDst plan; rec { - name = plan.name; - uid = genid name; - group = config.users.groups.backup.name; - home = plan.dst.path; - createHome = true; - shell = "${writeSh plan "shell" '' - case $2 in - 'rsync --server --daemon .') - exec ${backup.rsync plan [ "--server" "--daemon" "." ]} - ;; - ''') - echo "ERROR: no command specified" >&2 - exit 23 - ;; - *) - echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2 - exit 23 - ;; - esac - ''}"; - openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ]; - }; - - rsync = plan: args: writeSh plan "rsync" '' - install -v -m 0700 -d ${plan.dst.path}/push >&2 - install -v -m 0700 -d ${plan.dst.path}/list >&2 - - ${pkgs.rsync}/bin/rsync \ - --config=${backup.rsyncd-conf plan { - post-xfer = writeSh plan "rsyncd.post-xfer" '' - case $RSYNC_EXIT_STATUS in 0) - exec ${backup.rsnapshot plan { - preexec = writeSh plan "rsnapshot.preexec" '' - touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL - ''; - postexec = writeSh plan "rsnapshot.postexec" '' - rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL - ''; - }} - esac - ''; - }} \ - ${toString (map shell.escape args)} - - fail=0 - for i in monthly weekly daily hourly; do - if test -e ${plan.dst.path}/rsnapshot.$i; then - rm ${plan.dst.path}/rsnapshot.$i - echo "ERROR: $i snapshot failed" >&2 - fail=1 - fi - done - if test $fail != 0; then - exit -1 - fi - ''; - - rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" '' - fake super = yes - use chroot = no - lock file = ${plan.dst.path}/rsyncd.lock - - [push] - max connections = 1 - path = ${plan.dst.path}/push - write only = yes - read only = no - post-xfer exec = ${conf.post-xfer} - - [list] - path = ${plan.dst.path}/list - read only = yes - write only = no - ''; - - rsnapshot = plan: conf: writeSh plan "rsnapshot" '' - rsnapshot() { - ${pkgs.proot}/bin/proot \ - -b /bin \ - -b /nix \ - -b /run/current-system \ - -b ${plan.dst.path} \ - -r ${plan.dst.path} \ - -w / \ - ${pkgs.rsnapshot}/bin/rsnapshot \ - -c ${pkgs.writeText "${plan.name}.rsnapshot.conf" '' - config_version 1.2 - snapshot_root ${plan.dst.path}/list - cmd_cp ${pkgs.coreutils}/bin/cp - cmd_du ${pkgs.coreutils}/bin/du - #cmd_rm ${pkgs.coreutils}/bin/rm - cmd_rsync ${pkgs.rsync}/bin/rsync - cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff - cmd_preexec ${conf.preexec} - cmd_postexec ${conf.postexec} - retain hourly 4 - retain daily 7 - retain weekly 4 - retain monthly 3 - lockfile ${plan.dst.path}/rsnapshot.pid - link_dest 1 - backup /push ./ - verbose 4 - ''} \ - "$@" - } - - cd ${plan.dst.path}/list/ - - now=$(date +%s) - is_older_than() { - test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \ - -ge $2 - } - - # TODO report stale snapshots - # i.e. there are $interval.$i > $interval.$max - - hour_s=3600 - day_s=86400 - week_s=604800 - month_s=2419200 # 4 weeks - - set -- - - if test -e weekly.3 && is_older_than monthly.0 $month_s; then - set -- "$@" monthly - fi - - if test -e daily.6 && is_older_than weekly.0 $week_s; then - set -- "$@" weekly - fi - - if test -e hourly.3 && is_older_than daily.0 $day_s; then - set -- "$@" daily - fi - - if is_older_than hourly.0 $hour_s; then - set -- "$@" hourly - fi - - - if test $# = 0; then - echo "taking no snapshots" >&2 - else - echo "taking snapshots: $@" >&2 - fi - - export RSNAPSHOT_INTERVAL - for RSNAPSHOT_INTERVAL; do - rsnapshot "$RSNAPSHOT_INTERVAL" - done - ''; - - writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" '' - #! ${pkgs.dash}/bin/dash - set -efu - export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])} - ${text} - ''; - -in out +} diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index 3400c13b6..c300633bb 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -28,6 +28,7 @@ with lib; imports = [ + ./backup.nix ./vim.nix { # stockholm dependencies