Merge remote-tracking branch 'gum/master'

krebs/3modules/bepasty-server.nix

@@ -96,9 +96,13 @@ let
         wantedBy = [ "multi-user.target" ];
         after = [ "network.target" ];
         restartIfChanged = true;
-        environment = {
+        environment = let
+          penv = python.buildEnv.override {
+            extraLibs = [ bepasty gevent ];
+          };
+        in {
           BEPASTY_CONFIG = "${server.workDir}/bepasty-${name}.conf";
-          PYTHONPATH= "${bepasty}/lib/${python.libPrefix}/site-packages:${gevent}/lib/${python.libPrefix}/site-packages";
+          PYTHONPATH= "${penv}/${python.sitePackages}/";
         };
 
         serviceConfig = {

krebs/3modules/makefu/default.nix

@@ -221,17 +221,17 @@ with config.krebs.lib;
           ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
-            MIICCgKCAgEAvmCBVNKT/Su4v9nl/Nm3STPo5QxWPg7xEkzIs3Oh39BS8+r6/7UQ
+            MIICCgKCAgEAs9bq++H4HF8EpZMfWGfoIsh/C+YNO2pg74UPBsP/tFFe71yzWwUn
-            rebib7mczb+ebZd+Rg2yFoGrWO8cmM0VcLy5bYRMK7in8XroLEjWecNNM4TRfNR4
+            U9LW0n3bBqCMQ/oDthbSMwCkS9JzcUi22QJEdjbQs/aay9gZR115b+UxWPocw0Ms
-            e53+LhcPdkxo0A3/D+yiut+A2Mkqe+4VXDm/JhAiAYkZTn7jUtj00Atrc7CWW1gN
+            ZoREKo3Oe0hETk7Ing8NdBDI0kCBh9QnvqQ3iKd0rBae3DYvcWlDsY93GLGMddgA
-            sP3jIgv4+CGftdSYOB4dm699B7OD9XDLci2kOaFqFl4cjDYUok03G0AduUlRx10v
+            7E9oa3EHVYH/MPZaeJtTknaJduanBSbiEb/xQOqxTadHoQASKU6DQD1czMH3hLG2
-            CKbKOTIdm8C36A902/3ms+Hyzkruu+VagGIZuPSwqXHJPCu7Ju+jarKQstMmpQi0
+            8Wn4MBj9fgKBAoIy092tIzPtE2QwAHO73yz4mSW/3r190hREgVbjuEPiw4w5mEyQ
-            PubweWDL0o/Dfz2qT3DuL4xDecIvGE6kv3m41hHJYiK+2/azTSehyPFbsVbL7w0V
+            j+NeN3f3heFKx+GCgdWH9xPw6m6qPdqUiGUPq91KXMOhNa8lLcTp95mHdCMesZCF
-            LgKN3usnZNcpTsBWxRGT7nMFSnX2FLDu7d9OfCuaXYxHVFLZaNrpccOq8NF/7Hbk
+            TFj7hf6y+SVt17Vo+YUL7UqnMtAm3eZZmwyDu0DfKFrdgz6MtDD+5dQp9g8VHpqw
-            DDW81W7CvLyJDlp0WLnAawSOGTUTPoYv/2wAapJ89i8QGCueGvEc6o2EcnBVMFEW
+            RfbaB1Srlr24EUYYoOBEF9CcIacFbsr+MKh+hQk5R0uEMSeAWARzxvvr69iMgdEC
-            ejWTQzyD816f4RsplnrRqLVlIMbr9Q/n5TvlgjjhX7IMEfMy4+7qLGRQkNbFzgwK
+            zDiu0rrRLN+CrfgkDir7pkRKxeA1lz8KpySyIZRziNg6mSHjKjih4++Bbu4N2ack
-            jxNG2fFSCjOEQitm0gAtx7QRIyvYr6c7/xiHz4AwxYzBmvQsL/OK57NO4+Krwgj5
+            86h84qBrA8lq2xsub4+HgKZGH2l5Y8tvlr+rx0mQKEJkT6XDKCXZFPfl2N0QrWGT
-            Vk8TQ2jGO7J4bB38zaxK+Lrtfl8i1AK1171JqFMhOc34JSJ7T4LWDMECAwEAAQ==
+            Dv7l2vn0QMj9E6+BdRhYaO/m3+cIZ9faM851nRj/gq2OOtzW3ekrne0CAwEAAQ==
             -----END RSA PUBLIC KEY-----
           '';
         };
@@ -272,6 +272,7 @@ with config.krebs.lib;
           addrs6 = ["42:f9f0::10"];
           aliases = [
             "omo.retiolum"
+            "omo.r"
           ];
           tinc.pubkey = ''
               -----BEGIN RSA PUBLIC KEY-----

krebs/4lib/infest/prepare.sh

@@ -98,6 +98,19 @@ prepare_nixos_iso() {
   sed -i "s@^NIX_PATH=\"[^\"]*\"@NIX_PATH=$target_path@" bin/nixos-install
 }
 
+get_nixos_install() {
+  echo "installing nixos-install" 2>&1
+  c=$(mktemp)
+
+  cat <<EOF > $c
+{ fileSystems."/" = {};
+    boot.loader.grub.enable = false;
+}
+EOF
+  export NIXOS_CONFIG=$c
+  nix-env -i -A config.system.build.nixos-install -f "<nixpkgs/nixos>"
+  rm -v $c
+}
 prepare_common() {(
 
   if ! getent group nixbld >/dev/null; then
@@ -191,6 +204,7 @@ prepare_common() {(
     mount --rbind /mnt/"$target_path" "$target_path"
   fi
 
+  get_nixos_install
   mkdir -p bin
   rm -f bin/nixos-install
   cp "$(type -p nixos-install)" bin/nixos-install

krebs/5pkgs/test/infest-cac-centos7/default.nix

@@ -1,9 +1,11 @@
-{ stdenv, coreutils,makeWrapper, cac-api, cac-cert, cac-panel, gnumake, gnused, jq, openssh, ... }:
+{ stdenv, coreutils, makeWrapper,
+  cac-api, cac-cert, cac-panel, gnumake, gnused, jq, openssh, sshpass, proot,
+  ... }:
 
 stdenv.mkDerivation rec {
   name = "${shortname}-${version}";
   shortname = "infest-cac-centos7";
-  version = "0.2.0";
+  version = "0.2.6";
 
   src = ./notes;
 
@@ -21,6 +23,8 @@ stdenv.mkDerivation rec {
     gnused
     jq
     openssh
+    sshpass
+    proot
   ];
 
   installPhase = ''
@@ -34,7 +38,7 @@ stdenv.mkDerivation rec {
   '';
   meta = with stdenv.lib; {
     homepage = http://krebsco.de;
-    description = "Krebs CI Scripts";
+    description = "infest a CaC box with stockholm";
     license = licenses.wtfpl;
     maintainers = [ maintainers.makefu ];
   };

krebs/5pkgs/test/infest-cac-centos7/notes

@@ -1,10 +1,26 @@
-# nix-shell -p gnumake jq openssh cac-api cac-panel sshpass
+#! /bin/sh
-set -eufx
+# usage: user=makefu target_system=wry debug=true \
+#         krebs_cred=~/secrets/cac.json \
+#         retiolum_key=~/secrets/wry/retiolum.rsa_key.priv \
+#           infest-cac-centos7
+
+# IMPORTANT: set debug to TRUE if you want to actually keep the system
+
+# must be run in <stockholm>
+set -euf
 
 # 2 secrets are required:
-
+#   login to panel
 krebs_cred=${krebs_cred-./cac.json}
+#   tinc retiolum key for host
 retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+# build this host
+user=${user:-shared}
+target=${target_system:-test-centos7}
+
+log(){
+  echo "[$(date +"%Y-%m-%d %T")] $@" 2>&1
+}
 
 clear_defer(){
   echo "${trapstr:-exit}"
@@ -14,9 +30,13 @@ defer(){
   if test -z "${debug:-}"; then
     trapstr="$1;${trapstr:-exit}"
     trap "$trapstr" INT TERM EXIT KILL
+  else
+    log "ignored defer: $1"
   fi
 }
 
+test -z "${debug:-}" && log "debug enabled, vm will not be deleted on error"
+
 # Sanity
 if test ! -r "$krebs_cred";then
   echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
@@ -25,6 +45,11 @@ if test ! -r "$retiolum_key";then
   echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
 fi
 
+if test ! -r "${user}/1systems/${target}.nix" ;then
+  echo "cannot find ${user}/1systems/${target}.nix , not started in stockholm directory?"
+  exit 1
+fi
+
 krebs_secrets=$(mktemp -d)
 sec_file=$krebs_secrets/cac_config
 krebs_ssh=$krebs_secrets/tempssh
@@ -32,7 +57,7 @@ export cac_resources_cache=$krebs_secrets/res_cache.json
 export cac_servers_cache=$krebs_secrets/servers_cache.json
 export cac_tasks_cache=$krebs_secrets/tasks_cache.json
 export cac_templates_cache=$krebs_secrets/templates_cache.json
-# we need to receive this key from buildmaster to speed up tinc bootstrap
+
 defer "trap - INT TERM EXIT"
 defer "rm -r $krebs_secrets"
 
@@ -42,10 +67,13 @@ cac_key="$(cac-panel --config $krebs_cred settings | jq -r .apicode)"
 EOF
 
 export cac_secrets=$sec_file
+log "adding own ip to allowed ips via cac-panel"
 cac-panel --config $krebs_cred add-api-ip
 
 # test login:
+log "updating cac-api state"
 cac-api update
+log "list of cac servers:"
 cac-api servers
 
 # preserve old trap
@@ -56,10 +84,10 @@ while true;do
   out=$(cac-api build cpu=1 ram=512 storage=10 os=26 2>&1)
   if name=$(echo "$out" | jq -r .servername);then
     id=servername:$name
-    echo "got a working machine, id=$id"
+    log "got a working machine, id=$id"
   else
-    echo "Unable to build a virtual machine, retrying in 15 seconds" >&2
+    elog "Unable to build a virtual machine, retrying in 15 seconds"
-    echo "Output of build program: $out" >&2
+    log "Output of build program: $out"
     sleep 15
     continue
   fi
@@ -74,22 +102,23 @@ while true;do
     for t in `seq 180`;do
       # now we have a working cac-api server
       if cac-api ssh $1 -o ConnectTimeout=10 \
-                    cat /etc/redhat-release | \
+                    cat /etc/redhat-release >/dev/null 2>&1 ;then
-                      grep CentOS ;then
         return 0
       fi
+      log "cac-api ssh $1 failed, retrying"
       sleep 10
     done
+    log "cac-api ssh failed for 30 minutes, assuming something else broke. bailing ou.t"
     return 1
   }
   # die on timeout
   if ! wait_login_cac $id;then
-    echo "unable to boot a working system within time frame, retrying..." >&2
+    log "unable to boot a working system within time frame, retrying..."
-    echo "Cleaning up old image,last status: $(cac-api update;cac-api getserver $id | jq -r .status)"
+    log "Cleaning up old image,last status: $(cac-api update;cac-api getserver $id | jq -r .status)"
     eval "$(clear_defer | sed 's/;exit//')"
     sleep 15
   else
-    echo "got a working system" >&2
+    log "got a working system: $id"
     break
   fi
 done
@@ -101,16 +130,16 @@ cac-api generatenetworking $id > \
   shared/2configs/temp/networking.nix
 # new temporary ssh key we will use to log in after install
 ssh-keygen -f $krebs_ssh -N ""
-cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
+cp "$retiolum_key" $krebs_secrets/retiolum.rsa_key.priv
 # we override the directories for secrets and stockholm
 # additionally we set the ssh key we generated
 ip=$(cac-api getserver $id | jq -r .ip)
 
 cat > shared/2configs/temp/dirs.nix <<EOF
 _: {
-  krebs.build.source.dir = {
+  krebs.build.source = {
-    secrets.path = "$krebs_secrets";
+    secrets = "$krebs_secrets";
-    stockholm.path = "$(pwd)";
+    stockholm = "$(pwd)";
   };
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     "$(cat ${krebs_ssh}.pub)"
@@ -118,14 +147,17 @@ _: {
 }
 EOF
 
+log "starting prepare and installation"
+# TODO: try harder
 make install \
-    LOGNAME=shared \
+    LOGNAME=${user} \
     SSHPASS="$(cac-api getserver $id | jq -r .rootpass)" \
     ssh='sshpass -e ssh -S none -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' \
-    system=test-centos7 \
+    system=${target} \
     target=$ip
-
+log "finalizing installation"
-# TODO: generate secrets directory $krebs_secrets for nix import
+cac-api ssh $id < krebs/4lib/infest/finalize.sh
+log "reset $id"
 cac-api powerop $id reset
 
 wait_login(){
@@ -137,11 +169,15 @@ wait_login(){
            -i $krebs_ssh \
            -o ConnectTimeout=10 \
            -o BatchMode=yes \
-           root@$1 nixos-version ;then
+           root@$1 nixos-version >/dev/null 2>&1;then
+      log "login to host $1 successful"
       return 0
     fi
+    log "unable to log into server, waiting"
     sleep 10
   done
+  log "unable to log in after 15 minutes, bailing out"
   return 1
 }
+log "waiting for system to come up"
 wait_login $ip

makefu/1systems/omo.nix

@@ -10,11 +10,25 @@ let
   homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3";
   # cryptsetup luksFormat $dev --cipher aes-xts-plain64 -s 512 -h sha512
   # cryptsetup luksAddKey $dev tmpkey
-  # cryptsetup luksOpen $dev crypt0
+  # cryptsetup luksOpen $dev crypt0 --key-file tmpkey --keyfile-size=4096
-  # mkfs.xfs /dev/mapper/crypt0 -L crypt0
+  # mkfs.ext4 /dev/mapper/crypt0 -L crypt0 -T largefile
+
+  # omo Chassis:
+  # __FRONT_
+  # |* d2   |
+  # |       |
+  # |* d3   |
+  # |       |
+  # |* d0   |
+  # |       |
+  # |* d1   |
+  # |*      |
+  # |  * r0 |
+  # |_______|
   cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6";
   cryptDisk1 = byid "ata-TP02000GB_TPW151006050068";
-  cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487";
+  cryptDisk2 = byid "ata-ST4000DM000-1F2168_Z303HVSG";
+  # cryptDisk3 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WMAZA1786907";
   # all physical disks
   allDisks = [ rootDisk cryptDisk0 cryptDisk1 cryptDisk2 ];
 in {
@@ -40,7 +54,6 @@ in {
   networking.firewall.allowedTCPPorts = [ 80 655 8080 ];
 
   # services.openssh.allowSFTP = false;
-  krebs.build.source.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce";
 
   # copy config from <secrets/sabnzbd.ini> to /var/lib/sabnzbd/
   services.sabnzbd.enable = true;

makefu/1systems/pornocauster.nix

@@ -8,6 +8,7 @@
     [ # Include the results of the hardware scan.
       ../.
       ../2configs/main-laptop.nix #< base-gui + zsh
+      ../2configs/laptop-utils.nix
 
       # Krebs
       ../2configs/tinc-basic-retiolum.nix
@@ -39,6 +40,9 @@
   nixpkgs.config.packageOverrides = pkgs: {
     tinc = pkgs.tinc_pre;
   };
+  # steam
+  hardware.opengl.driSupport32Bit = true;
+  hardware.pulseaudio.support32Bit = true;
 
   # configure pulseAudio to provide a HDMI sink as well
   networking.firewall.enable = true;

makefu/1systems/wry.nix

@@ -11,7 +11,6 @@ in {
       # TODO: copy this config or move to krebs
       ../../tv/2configs/hw/CAC.nix
       ../../tv/2configs/fs/CAC-CentOS-7-64bit.nix
-      ../2configs/unstable-sources.nix
       ../2configs/headless.nix
       ../2configs/tinc-basic-retiolum.nix
 
@@ -28,7 +27,8 @@ in {
       # collectd
       ../2configs/collectd/collectd-base.nix
   ];
-
+  services.nixosManual.enable = false;
+  programs.man.enable = false;
   krebs.build.host = config.krebs.hosts.wry;
 
   krebs.Reaktor = {

makefu/2configs/default.nix

@@ -4,13 +4,6 @@ with config.krebs.lib;
 {
   system.stateVersion = "15.09";
 
-  system.replaceRuntimeDependencies = with pkgs.lib;
-        [{original = pkgs.glibc; replacement = pkgs.stdenv.lib.overrideDerivation pkgs.glibc (oldAttr: { patches = oldAttr.patches ++
-          [(pkgs.fetchurl { url = "https://raw.githubusercontent.com/NixOS/nixpkgs/master/pkgs/development/libraries/glibc/cve-2015-7547.patch";
-                            sha256 = "0awpc4rp2x27rjpj83ps0rclmn73hsgfv2xxk18k82w4hdxqpp5r";})];
-        });}
-        ];
-
   imports = [
     {
       users.extraUsers =
@@ -29,7 +22,7 @@ with config.krebs.lib;
       source =  mapAttrs (_: mkDefault) {
         nixpkgs = {
           url = https://github.com/nixos/nixpkgs;
-          rev = "77f8f35d57618c1ba456d968524f2fb2c3448295"; # unstable @ 2015-01-27, tested on wry
+          rev = "40c586b7ce2c559374df435f46d673baf711c543"; # unstable @ 2016-02-27, tested on wry
         };
         secrets = "/home/makefu/secrets/${config.krebs.build.host.name}/";
         stockholm = "/home/makefu/stockholm";

makefu/2configs/deployment/mycube.connector.one.nix

@@ -16,7 +16,7 @@ in {
       vassals = {
         mycube-flask = {
           type = "normal";
-          python2Packages = self: with self; [ pkgs.mycube-flask flask redis werkzeug jinja2 markupsafe itsdangerous ];
+          pythonPackages = self: with self; [ pkgs.mycube-flask ];
           socket = wsgi-sock;
         };
       };

makefu/2configs/git/brain-retiolum.nix

@@ -14,6 +14,7 @@ let
 
   priv-repos = mapAttrs make-priv-repo {
     autosync = { };
+    pass     = { };
   };
 
   # TODO move users to separate module

makefu/2configs/git/cgit-retiolum.nix

@@ -15,7 +15,7 @@ let
     tinc_graphs = {
       desc = "Tinc Advanced Graph Generation";
     };
-    cac = { };
+    cac-api = { };
     init-stockholm = {
       desc = "Init stuff for stockholm";
     };

makefu/2configs/iodined.nix

@@ -1,4 +1,4 @@
-{ services,builtins,environment,pkgs, ... }:
+{ pkgs, config, ... }:
 
 let
   # TODO: make this a parameter
@@ -10,7 +10,7 @@ in {
     enable = true;
     domain = domain;
     ip = "172.16.10.1/24";
-    extraConfig = "-P ${pw}";
+    extraConfig = "-P ${pw} -l ${pkgs.lib.head config.krebs.build.host.nets.internet.addrs4}";
   };
 
 }

makefu/2configs/laptop-utils.nix

@@ -0,0 +1,65 @@
+{ pkgs, ... }:
+
+# tools i use when actually working with the host.
+# package version will now be maintained by nix-rebuild
+#
+# essentially `nix-env -q` of the main user
+# TODO: split gui and non-gui
+{
+  nixpkgs.config.firefox = {
+    enableAdobeFlash = true;
+  };
+
+  krebs.per-user.makefu.packages = with pkgs; [
+    # core
+    at_spi2_core
+    acpi
+    bc
+    exif
+    file
+    ntfs3g
+    pv
+    proot
+    sshpass
+    unzip
+    unrar
+    usbutils
+    zip
+
+    # dev
+    python35Packages.virtualenv
+
+
+    # gui
+    chromium
+    clipit
+    feh
+    firefox
+    keepassx
+    pcmanfm
+    skype
+    mirage
+    tightvnc
+    gnome3.dconf
+    vlc
+    virtmanager
+    wireshark
+    xdotool
+
+    # sectools
+    aria2
+    binwalk
+    dnsmasq
+    iodine
+    mtr
+    nmap
+
+
+    # stuff
+    cac-api
+    cac-panel
+    krebspaste
+    ledger
+    pass
+  ];
+}

makefu/2configs/mail-client.nix

@@ -3,12 +3,14 @@
 with config.krebs.lib;
 {
   environment.systemPackages = with pkgs; [
+    abook
+    gnupg
+    imapfilter
     msmtp
     mutt-kz
     notmuch
     offlineimap
-    imapfilter
+    openssl
-    gnupg
+    w3m
   ];
-
 }

makefu/2configs/main-laptop.nix

@@ -11,21 +11,11 @@ with config.krebs.lib;
     ./base-gui.nix
     ./fetchWallpaper.nix
     ./zsh-user.nix
+    ./laptop-utils.nix
   ];
 
   users.users.${config.krebs.build.user.name}.extraGroups = [ "dialout" ];
 
-  environment.systemPackages = with pkgs;[
-    vlc
-    firefox
-    chromium
-    keepassx
-    ntfs3g
-    at_spi2_core
-    gnome3.dconf
-    virtmanager
-    krebspaste
-  ];
 
   services.redshift = {
     enable = true;

makefu/3modules/snapraid.nix

@@ -35,7 +35,7 @@ let
     enable = mkEnableOption "snapraid";
 
     timerConfig = mkOption {
-      type = types.unspecified;
+      type = with types;attrsOf str;
       description = ''
         Start snapraid service
       '';

makefu/5pkgs/default.nix

@@ -9,7 +9,8 @@ in
     alsa-hdspconf = callPackage ./alsa-tools { alsaToolTarget="hdspconf";};
     alsa-hdsploader = callPackage ./alsa-tools { alsaToolTarget="hdsploader";};
     awesomecfg = callPackage ./awesomecfg {};
-    tw-upload-plugin = callPackage ./tw-upload-plugin {};
+    nodemcu-uploader = callPackage ./nodemcu-uploader {};
     mycube-flask = callPackage ./mycube-flask {};
+    tw-upload-plugin = callPackage ./tw-upload-plugin {};
   };
 }

makefu/5pkgs/mycube-flask/default.nix

@@ -2,7 +2,7 @@
 
 with pkgs.pythonPackages;buildPythonPackage rec {
   name = "mycube-flask-${version}";
-  version = "0.2.3";
+  version = "0.2.3.4";
   propagatedBuildInputs = [
     flask
     redis

makefu/5pkgs/nodemcu-uploader/default.nix

@@ -0,0 +1,22 @@
+{ lib, pkgs, pythonPackages, fetchurl, ... }:
+
+with pythonPackages; buildPythonPackage rec {
+  name = "nodemcu-uploader-${version}";
+  version = "0.2.2";
+  disabled = isPy3k || isPyPy;
+  propagatedBuildInputs = [
+    pyserial
+  ];
+  src = fetchurl {
+    url = "https://pypi.python.org/packages/source/n/nodemcu-uploader/nodemcu-uploader-${version}.tar.gz";
+    sha256 = "090giz84y9y3idgifp0yh80qqyv2czv6h3y55wyrlgf7qfbwbrvn";
+  };
+  # ImportError: No module named tests
+  # not sure what to do here
+  doCheck = false;
+  meta = {
+    homepage = https://github.com/kmpm/nodemcu-uploader;
+    description = "tool for uploading files to NodeMCU filesystem";
+    license = lib.licenses.mit;
+  };
+}

shared/2configs/base.nix

@@ -18,7 +18,7 @@ with config.krebs.lib;
   krebs.build.source = {
     nixpkgs = mkDefault {
       url = https://github.com/NixOS/nixpkgs;
-      rev = "77f8f35d57618c1ba456d968524f2fb2c3448295"; # for urlwatch-minidb
+      rev = "40c586b7ce2c559374df435f46d673baf711c543";
     };
     secrets =  mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
     stockholm = mkDefault "${getEnv "HOME"}/stockholm";