Merge remote-tracking branch 'prism/master'

This commit is contained in:
tv 2016-08-07 09:54:10 +02:00
commit 3288d6848f
20 changed files with 106 additions and 1678 deletions

View File

@ -3,15 +3,8 @@
with config.krebs.lib;
let
# https://github.com/NixOS/nixpkgs/issues/14026
nixpkgs-fix = import (pkgs.fetchgit {
url = https://github.com/nixos/nixpkgs;
rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632";
}) {};
buildbot = nixpkgs-fix.buildbot;
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
buildbot = pkgs.buildbot;
buildbot-master-config = pkgs.writePython2 "buildbot-master.cfg" ''
# -*- python -*-
from buildbot.plugins import *
import re

View File

@ -2,11 +2,11 @@
stdenv.mkDerivation rec {
name = "apt-cacher-ng-${version}";
version = "0.9.3";
version = "0.9.3.2";
src = fetchurl {
url = "http://ftp.debian.org/debian/pool/main/a/apt-cacher-ng/apt-cacher-ng_${version}.orig.tar.xz";
sha256 = "1bd7l1wg0q1p9pg0v6lqflf2znydx8mrh2jxbvv6xsrp1473nfwg";
sha256 = "1bvng9mwrggvc93q2alj0x72i56wifnjs2dsycr17mapsv0f2gnc";
};
NIX_LDFLAGS = "-lpthread";

View File

@ -253,5 +253,27 @@ rec {
${pkgs.cabal2nix}/bin/cabal2nix ${path} > $out
'');
writePython2 = name: src: pkgs.runCommand name {} /* sh */ ''
name=${assert types.filename.check name; name}
src=${shell.escape src}
# syntax check
printf '%s' "$src" > src.py
${pkgs.python2}/bin/python -m py_compile src.py
cp src.py "$out"
'';
writePython3 = name: src: pkgs.runCommand name {} /* sh */ ''
name=${assert types.filename.check name; name}
src=${shell.escape src}
# syntax check
printf '%s' "$src" > src.py
${pkgs.python3}/bin/python -m py_compile src.py
cp src.py "$out"
'';
writeSed = makeScriptWriter "${pkgs.gnused}/bin/sed -f";
}

View File

@ -16,7 +16,7 @@ in {
systemWide = true;
};
users.extraUsers.mainUser.extraGroups = [ "audio" ];
users.extraUsers.mainUser.extraGroups = [ "audio" "video" ];
time.timeZone = "Europe/Berlin";

View File

@ -8,9 +8,6 @@ let
ControlPath /tmp/%u_sshmux_%r@%h:%p
ControlPersist 4h
'';
sshWrapper = pkgs.writeDash "ssh-wrapper" ''
${pkgs.openssh}/bin/ssh -F ${sshHostConfig} -i ${shell.escape config.lass.build-ssh-privkey.path} "$@"
'';
in {
config.krebs.buildbot.master = let
@ -66,10 +63,15 @@ in {
"NIX_REMOTE": "daemon",
"dummy_secrets": "true",
}
env_shared = {
"LOGNAME": "shared",
"NIX_REMOTE": "daemon",
"dummy_secrets": "true",
}
# prepare nix-shell
# the dependencies which are used by the test script
deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate", "openssh" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell",
@ -88,20 +90,20 @@ in {
for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]:
addShell(f,name="build-{}".format(i),env=env_lass,
command=nixshell + \
["make \
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
ssh=${sshWrapper} \
target=build@localhost${config.users.users.build.home}/testbuild \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=build \
system={}".format(i)])
for i in [ "x", "wry", "vbob", "wbob", "shoney" ]:
addShell(f,name="build-{}".format(i),env=env_makefu,
command=nixshell + \
["make \
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
ssh=${sshWrapper} \
target=build@localhost${config.users.users.build.home}/testbuild \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=build \
system={}".format(i)])
@ -114,19 +116,35 @@ in {
fast-tests = ''
f = util.BuildFactory()
f.addStep(grab_repo)
for i in [ "prism", "mors", "echelon" ]:
addShell(f,name="populate-{}".format(i),env=env_lass,
for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]:
addShell(f,name="build-{}".format(i),env=env_lass,
command=nixshell + \
["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=eval \
system={}".format(i)])
addShell(f,name="build-test-minimal",env=env_lass,
for i in [ "x", "wry", "vbob", "wbob", "shoney" ]:
addShell(f,name="build-{}".format(i),env=env_makefu,
command=nixshell + \
["nix-instantiate \
--show-trace --eval --strict --json \
-I nixos-config=./shared/1systems/test-minimal-deploy.nix \
-I secrets=. \
-A config.system.build.toplevel"]
)
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=eval \
system={}".format(i)])
for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf" ]:
addShell(f,name="build-{}".format(i),env=env_shared,
command=nixshell + \
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=eval \
system={}".format(i)])
bu.append(util.BuilderConfig(name="fast-tests",
slavenames=slavenames,

View File

@ -44,6 +44,11 @@ with config.krebs.lib;
};
};
}
{
environment.variables = {
NIX_PATH = mkForce "secrets=/var/src/stockholm/null:/var/src";
};
}
];
networking.hostName = config.krebs.build.host.name;

View File

@ -3,6 +3,6 @@
{
krebs.build.source.nixpkgs.git = {
url = https://github.com/lassulus/nixpkgs;
ref = "c6ca9c8c8b7eb8f8e68868e36fb90e162adf080f";
ref = "a75c0d9342ecb86dedd11f61a4e6f59ecc42d151";
};
}

View File

@ -30,7 +30,7 @@ with config.krebs.lib;
if getEnv "dummy_secrets" == "true"
then toString <stockholm/makefu/6tests/data/secrets>
else "/home/makefu/secrets/${host.name}";
stockholm.file = "/home/makefu/stockholm";
stockholm.file = getEnv "PWD";
# Defaults for all stockholm users?
nixos-config.symlink =

View File

@ -17,7 +17,7 @@ stdenv.mkDerivation rec {
libpcap
libdnet
pcre
libevent
libevent.out # requires .so and .h
readline
autoconf
libtool
@ -27,7 +27,10 @@ stdenv.mkDerivation rec {
pythonPackages.sqlite3
];
patches = [
./farpd_0.2-11.diff
( fetchurl {
url = https://launchpad.net/ubuntu/+archive/primary/+files/farpd_0.2-11.diff.gz;
sha256 = "2c246b37de8aab9c73f955fb77101adefd90637d03f582b9f8ffae2903af2f94";
})
];
# removes user install script from Makefile before automake
#patches = [ ./autoconf.patch ];
@ -35,6 +38,7 @@ stdenv.mkDerivation rec {
autoreconf -fi
substituteInPlace configure \
--replace "dumbnet" "dnet" \
--replace "libpcap.a" "libpcap.so" \
--replace "libevent.a" "libevent.so" \
--replace "net/bpf.h" "pcap/bpf.h"

File diff suppressed because it is too large Load Diff

View File

@ -3,7 +3,6 @@
{
imports = [
../.
../2configs/base.nix
{
boot.loader.grub = {
device = "/dev/sda";

View File

@ -8,7 +8,6 @@ let
in {
imports = [
../.
../2configs/base.nix
../2configs/os-templates/CAC-CentOS-6.5-64bit.nix
{
networking.interfaces.enp11s0.ip4 = [

View File

@ -6,7 +6,6 @@ let
in {
imports = [
../.
../2configs/base.nix
../2configs/os-templates/CAC-CentOS-7-64bit.nix
../2configs/temp/networking.nix
../2configs/temp/dirs.nix

View File

@ -7,7 +7,6 @@ in
{
imports = [
../.
../2configs/base.nix
<nixpkgs/nixos/modules/profiles/qemu-guest.nix>
../2configs/collectd-base.nix
../2configs/shack-nix-cacher.nix

View File

@ -11,10 +11,13 @@ with config.krebs.lib;
nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
nixpkgs.git = {
url = https://github.com/NixOS/nixpkgs;
ref = "63b9785"; # stable @ 2016-06-01
ref = "9cb194cfa449c43f63185a25c8d10307aea3b358"; # nixos-16.03 @ 2016-08-05
};
secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
stockholm.file = "${getEnv "HOME"}/stockholm";
secrets.file =
if getEnv "dummy_secrets" == "true"
then toString <stockholm/shared/6tests/data/secrets>
else "${getEnv "HOME"}/secrets/krebs/${host.name}";
stockholm.file = getEnv "PWD";
};
networking.hostName = config.krebs.build.host.name;

View File

@ -71,7 +71,11 @@
# prepare grab_repo step for stockholm
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
env = {
"LOGNAME": "shared",
"NIX_REMOTE": "daemon",
"dummy_secrets": "true",
}
# prepare nix-shell
# the dependencies which are used by the test script
@ -93,34 +97,21 @@
fast-tests = ''
f = util.BuildFactory()
f.addStep(grab_repo)
for i in [ "test-centos7", "wolf", "test-failing" ]:
addShell(f,name="populate-{}".format(i),env=env,
command=nixshell + \
["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
# XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
addShell(f,name="instantiate-test-all-modules",env=env,
for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf", "test-centos7" ]:
addShell(f,name="build-{}".format(i),env=env,
command=nixshell + \
["touch retiolum.rsa_key.priv; \
nix-instantiate \
--show-trace --eval --strict --json \
-I nixos-config=./shared/1systems/test-all-krebs-modules.nix \
-I secrets=. \
-A config.system.build.toplevel"]
)
addShell(f,name="build-test-minimal",env=env,
command=nixshell + \
["nix-instantiate \
--show-trace --eval --strict --json \
-I nixos-config=./shared/1systems/test-minimal-deploy.nix \
-I secrets=. \
-A config.system.build.toplevel"]
)
["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
make \
test \
target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
method=eval \
system={}".format(i)])
bu.append(util.BuilderConfig(name="fast-tests",
slavenames=slavenames,
factory=f))
'';
# this build will try to build against local nixpkgs
# TODO change to do a 'local' populate and use the retrieved nixpkgs

View File

@ -0,0 +1 @@
{}

View File

@ -2,6 +2,7 @@ _:
{
imports = [
../krebs
./2configs
./3modules
];
}