Merge remote-tracking branch 'prism/master'

krebs/3modules/buildbot/master.nix

@@ -3,15 +3,8 @@
 with config.krebs.lib;
 let
 
-  # https://github.com/NixOS/nixpkgs/issues/14026
-  nixpkgs-fix = import (pkgs.fetchgit {
-    url = https://github.com/nixos/nixpkgs;
-    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
-    sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632";
-  }) {};
-
-  buildbot = nixpkgs-fix.buildbot;
-  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writePython2 "buildbot-master.cfg" ''
     # -*- python -*-
     from buildbot.plugins import *
     import re

krebs/5pkgs/apt-cacher-ng/default.nix

@@ -2,11 +2,11 @@
 
 stdenv.mkDerivation rec {
   name = "apt-cacher-ng-${version}";
-  version = "0.9.3";
+  version = "0.9.3.2";
 
   src = fetchurl {
     url = "http://ftp.debian.org/debian/pool/main/a/apt-cacher-ng/apt-cacher-ng_${version}.orig.tar.xz";
-    sha256 = "1bd7l1wg0q1p9pg0v6lqflf2znydx8mrh2jxbvv6xsrp1473nfwg";
+    sha256 = "1bvng9mwrggvc93q2alj0x72i56wifnjs2dsycr17mapsv0f2gnc";
   };
 
   NIX_LDFLAGS = "-lpthread";

krebs/5pkgs/builders.nix

@@ -253,5 +253,27 @@ rec {
       ${pkgs.cabal2nix}/bin/cabal2nix ${path} > $out
     '');
 
+  writePython2 = name: src: pkgs.runCommand name {} /* sh */ ''
+    name=${assert types.filename.check name; name}
+    src=${shell.escape src}
+
+    # syntax check
+    printf '%s' "$src" > src.py
+    ${pkgs.python2}/bin/python -m py_compile src.py
+
+    cp src.py "$out"
+  '';
+
+  writePython3 = name: src: pkgs.runCommand name {} /* sh */ ''
+    name=${assert types.filename.check name; name}
+    src=${shell.escape src}
+
+    # syntax check
+    printf '%s' "$src" > src.py
+    ${pkgs.python3}/bin/python -m py_compile src.py
+
+    cp src.py "$out"
+  '';
+
   writeSed = makeScriptWriter "${pkgs.gnused}/bin/sed -f";
 }

lass/2configs/baseX.nix

@@ -16,7 +16,7 @@ in {
     systemWide = true;
   };
 
-  users.extraUsers.mainUser.extraGroups = [ "audio" ];
+  users.extraUsers.mainUser.extraGroups = [ "audio" "video" ];
 
   time.timeZone = "Europe/Berlin";
 

lass/2configs/buildbot-standalone.nix

@@ -8,9 +8,6 @@ let
     ControlPath /tmp/%u_sshmux_%r@%h:%p
     ControlPersist 4h
   '';
-  sshWrapper = pkgs.writeDash "ssh-wrapper" ''
-    ${pkgs.openssh}/bin/ssh -F ${sshHostConfig} -i ${shell.escape config.lass.build-ssh-privkey.path} "$@"
-  '';
 
 in {
   config.krebs.buildbot.master = let
@@ -66,10 +63,15 @@ in {
         "NIX_REMOTE": "daemon",
         "dummy_secrets": "true",
       }
+      env_shared = {
+        "LOGNAME": "shared",
+        "NIX_REMOTE": "daemon",
+        "dummy_secrets": "true",
+      }
 
       # prepare nix-shell
       # the dependencies which are used by the test script
-      deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
+      deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate", "openssh" ]
       # TODO: --pure , prepare ENV in nix-shell command:
       #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
       nixshell = ["nix-shell",
@@ -88,20 +90,20 @@ in {
         for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]:
           addShell(f,name="build-{}".format(i),env=env_lass,
                   command=nixshell + \
-                      ["make \
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
                             test \
-                            ssh=${sshWrapper} \
-                            target=build@localhost${config.users.users.build.home}/testbuild \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
                             method=build \
                             system={}".format(i)])
 
         for i in [ "x", "wry", "vbob", "wbob", "shoney" ]:
           addShell(f,name="build-{}".format(i),env=env_makefu,
                   command=nixshell + \
-                      ["make \
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
                             test \
-                            ssh=${sshWrapper} \
-                            target=build@localhost${config.users.users.build.home}/testbuild \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
                             method=build \
                             system={}".format(i)])
 
@@ -114,19 +116,35 @@ in {
       fast-tests = ''
         f = util.BuildFactory()
         f.addStep(grab_repo)
-        for i in [ "prism", "mors", "echelon" ]:
-          addShell(f,name="populate-{}".format(i),env=env_lass,
+        for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]:
+          addShell(f,name="build-{}".format(i),env=env_lass,
                   command=nixshell + \
-                            ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
+                            test \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
+                            method=eval \
+                            system={}".format(i)])
 
-        addShell(f,name="build-test-minimal",env=env_lass,
+        for i in [ "x", "wry", "vbob", "wbob", "shoney" ]:
+          addShell(f,name="build-{}".format(i),env=env_makefu,
                   command=nixshell + \
-                            ["nix-instantiate \
-                                  --show-trace --eval --strict --json \
-                                  -I nixos-config=./shared/1systems/test-minimal-deploy.nix  \
-                                  -I secrets=. \
-                                  -A config.system.build.toplevel"]
-                )
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
+                            test \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
+                            method=eval \
+                            system={}".format(i)])
+
+        for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf" ]:
+          addShell(f,name="build-{}".format(i),env=env_shared,
+                  command=nixshell + \
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
+                            test \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
+                            method=eval \
+                            system={}".format(i)])
 
         bu.append(util.BuilderConfig(name="fast-tests",
               slavenames=slavenames,

lass/2configs/default.nix

@@ -44,6 +44,11 @@ with config.krebs.lib;
         };
       };
     }
+    {
+      environment.variables = {
+        NIX_PATH = mkForce "secrets=/var/src/stockholm/null:/var/src";
+      };
+    }
   ];
 
   networking.hostName = config.krebs.build.host.name;

lass/2configs/nixpkgs.nix

@@ -3,6 +3,6 @@
 {
   krebs.build.source.nixpkgs.git = {
       url = https://github.com/lassulus/nixpkgs;
-      ref = "c6ca9c8c8b7eb8f8e68868e36fb90e162adf080f";
+      ref = "a75c0d9342ecb86dedd11f61a4e6f59ecc42d151";
     };
 }

makefu/2configs/default.nix

@@ -30,7 +30,7 @@ with config.krebs.lib;
           if getEnv "dummy_secrets" == "true"
             then toString <stockholm/makefu/6tests/data/secrets>
             else "/home/makefu/secrets/${host.name}";
-        stockholm.file = "/home/makefu/stockholm";
+        stockholm.file = getEnv "PWD";
 
         # Defaults for all stockholm users?
         nixos-config.symlink =

makefu/5pkgs/farpd/default.nix

@@ -17,7 +17,7 @@ stdenv.mkDerivation rec {
     libpcap
     libdnet
     pcre
-    libevent
+    libevent.out # requires .so and .h
     readline
     autoconf
     libtool
@@ -27,7 +27,10 @@ stdenv.mkDerivation rec {
     pythonPackages.sqlite3
   ];
   patches = [
-    ./farpd_0.2-11.diff
+    ( fetchurl {
+      url = https://launchpad.net/ubuntu/+archive/primary/+files/farpd_0.2-11.diff.gz;
+      sha256 = "2c246b37de8aab9c73f955fb77101adefd90637d03f582b9f8ffae2903af2f94";
+    })
   ];
   # removes user install script from Makefile before automake
   #patches = [ ./autoconf.patch ];
@@ -35,6 +38,7 @@ stdenv.mkDerivation rec {
     autoreconf -fi
 
     substituteInPlace configure \
+      --replace "dumbnet" "dnet" \
       --replace "libpcap.a" "libpcap.so" \
       --replace "libevent.a" "libevent.so" \
       --replace "net/bpf.h" "pcap/bpf.h"

shared/1systems/test-arch.nix

@@ -3,7 +3,6 @@
 {
   imports = [
     ../.
-    ../2configs/base.nix
     {
       boot.loader.grub = {
         device = "/dev/sda";

shared/1systems/test-centos6.nix

@@ -8,7 +8,6 @@ let
 in {
   imports = [
     ../.
-    ../2configs/base.nix
     ../2configs/os-templates/CAC-CentOS-6.5-64bit.nix
     {
       networking.interfaces.enp11s0.ip4 = [

shared/1systems/test-centos7.nix

@@ -6,7 +6,6 @@ let
 in {
   imports = [
     ../.
-    ../2configs/base.nix
     ../2configs/os-templates/CAC-CentOS-7-64bit.nix
     ../2configs/temp/networking.nix
     ../2configs/temp/dirs.nix

shared/1systems/wolf.nix

@@ -7,7 +7,6 @@ in
 {
   imports = [
     ../.
-    ../2configs/base.nix
     <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
     ../2configs/collectd-base.nix
     ../2configs/shack-nix-cacher.nix

shared/2configs/default.nix

@@ -11,10 +11,13 @@ with config.krebs.lib;
     nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
     nixpkgs.git = {
       url = https://github.com/NixOS/nixpkgs;
-      ref = "63b9785"; # stable @ 2016-06-01
+      ref = "9cb194cfa449c43f63185a25c8d10307aea3b358"; # nixos-16.03 @ 2016-08-05
     };
-    secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
-    stockholm.file = "${getEnv "HOME"}/stockholm";
+    secrets.file =
+      if getEnv "dummy_secrets" == "true"
+        then toString <stockholm/shared/6tests/data/secrets>
+        else "${getEnv "HOME"}/secrets/krebs/${host.name}";
+    stockholm.file = getEnv "PWD";
   };
 
   networking.hostName = config.krebs.build.host.name;

shared/2configs/shared-buildbot.nix

@@ -71,7 +71,11 @@
   # prepare grab_repo step for stockholm
   grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
 
-  env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+  env = {
+    "LOGNAME": "shared",
+    "NIX_REMOTE": "daemon",
+    "dummy_secrets": "true",
+  }
 
   # prepare nix-shell
   # the dependencies which are used by the test script
@@ -91,37 +95,24 @@
     '';
     builder = {
       fast-tests = ''
-  f = util.BuildFactory()
-  f.addStep(grab_repo)
-  for i in [ "test-centos7", "wolf", "test-failing" ]:
-    addShell(f,name="populate-{}".format(i),env=env,
-            command=nixshell + \
-                      ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
+        f = util.BuildFactory()
+        f.addStep(grab_repo)
 
-  # XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
-  addShell(f,name="instantiate-test-all-modules",env=env,
-            command=nixshell + \
-                      ["touch retiolum.rsa_key.priv; \
-                        nix-instantiate \
-                            --show-trace --eval --strict --json \
-                            -I nixos-config=./shared/1systems/test-all-krebs-modules.nix  \
-                            -I secrets=. \
-                            -A config.system.build.toplevel"]
-          )
+        for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf", "test-centos7" ]:
+          addShell(f,name="build-{}".format(i),env=env,
+                  command=nixshell + \
+                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
+                        make \
+                            test \
+                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
+                            method=eval \
+                            system={}".format(i)])
 
-  addShell(f,name="build-test-minimal",env=env,
-            command=nixshell + \
-                      ["nix-instantiate \
-                            --show-trace --eval --strict --json \
-                            -I nixos-config=./shared/1systems/test-minimal-deploy.nix  \
-                            -I secrets=. \
-                            -A config.system.build.toplevel"]
-          )
+        bu.append(util.BuilderConfig(name="fast-tests",
+              slavenames=slavenames,
+              factory=f))
 
-  bu.append(util.BuilderConfig(name="fast-tests",
-        slavenames=slavenames,
-        factory=f))
-      '';
+            '';
       # this build will try to build against local nixpkgs
       # TODO change to do a 'local' populate and use the retrieved nixpkgs
       build-local = ''

shared/6tests/data/secrets/grafana_security.nix

@@ -0,0 +1 @@
+{}

shared/default.nix

@@ -2,6 +2,7 @@ _:
 {
   imports = [
     ../krebs
+    ./2configs
     ./3modules
   ];
 }