Merge remote-tracking branch 'gum/master'

krebs/3modules/makefu/default.nix

@@ -459,6 +459,7 @@ with import <stockholm/lib>;
           share.euer        IN A      ${nets.internet.ip4.addr}
           mattermost.euer   IN A      ${nets.internet.ip4.addr}
           gum               IN A      ${nets.internet.ip4.addr}
+          wikisearch        IN A      ${nets.internet.ip4.addr}
           pigstarter        IN A      ${nets.internet.ip4.addr}
           cgit.euer         IN A      ${nets.internet.ip4.addr}
           euer              IN A      ${nets.internet.ip4.addr}
@@ -490,6 +491,7 @@ with import <stockholm/lib>;
             "tracker.makefu.r"
 
             "graph.r"
+            "search.makefu.r"
             "wiki.makefu.r"
             "wiki.gum.r"
             "blog.makefu.r"

makefu/1systems/gum.nix

@@ -32,7 +32,7 @@ in {
       ../2configs/tools/sec.nix
 
       # services
-      ../2configs/gum-share.nix
+      ../2configs/share/gum.nix
       ../2configs/sabnzbd.nix
       ../2configs/torrent.nix
       ../2configs/iodined.nix
@@ -48,14 +48,25 @@ in {
       ../2configs/deployment/mycube.connector.one.nix
       ../2configs/deployment/graphs.nix
       ../2configs/deployment/owncloud.nix
-      ../2configs/deployment/wiki-irc.nix
+      ../2configs/deployment/wiki-irc-bot
       ../2configs/deployment/boot-euer.nix
+      ../2configs/deployment/hound
+      {
+        services.taskserver.enable = true;
+        services.taskserver.fqdn = config.krebs.build.host.name;
+        services.taskserver.listenHost = "::";
+        services.taskserver.organisations.home.users = [ "makefu" ];
+        networking.firewall.extraCommands = ''
+          iptables -A INPUT -i retiolum -p tcp --dport 53589 -j ACCEPT
+          ip6tables -A INPUT -i retiolum -p tcp --dport 53589 -j ACCEPT
+        '';
+      }
       # ../2configs/ipfs.nix
       ../2configs/syncthing.nix
 
       # ../2configs/opentracker.nix
-      ../2configs/logging/central-stats-client.nix
+      ../2configs/stats/client.nix
-      # ../2configs/logging/central-logging-client.nix
+      # ../2configs/logging/client.nix
 
   ];
   makefu.dl-dir = "/var/download";
@@ -78,7 +89,6 @@ in {
     ];
   };
 
-  makefu.taskserver.enable = true;
 
 
   # access
@@ -122,6 +132,8 @@ in {
           21031
           # taskserver
           53589
+          # temp vnc
+          18001
         ];
         allowedUDPPorts = [
           # tinc

makefu/1systems/omo.nix

@@ -50,11 +50,13 @@ in {
       # ../2configs/disable_v6.nix
       #../2configs/graphite-standalone.nix
       #../2configs/share-user-sftp.nix
-      ../2configs/omo-share.nix
+      ../2configs/share/omo.nix
       ../2configs/tinc/retiolum.nix
-      ../2configs/logging/central-stats-server.nix
+
-      # ../2configs/logging/central-logging-server.nix
+      # Logging
-      ../2configs/logging/central-stats-client.nix
+      ../2configs/stats/server.nix #influx + grafana
+      ../2configs/stats/client.nix
+      ../2configs/stats/external/aralast.nix # logs to influx
 
       # services
       ../2configs/syncthing.nix
@@ -180,7 +182,8 @@ in {
     uid = 9002;
     name = "misa";
   };
-  hardware.enableAllFirmware = true;
+  # hardware.enableAllFirmware = true;
+  hardware.enableRedistributableFirmware = true;
   hardware.cpu.intel.updateMicrocode = true;
 
   zramSwap.enable = true;

makefu/1systems/studio.nix

@@ -5,8 +5,10 @@
     ../2configs/vncserver.nix
     ../2configs/vim.nix
     ../2configs/disable_v6.nix
-    ../2configs/jack-on-pulse.nix
+    ../2configs/audio/jack-on-pulse.nix
+    ../2configs/audio/realtime-audio.nix
     ../2configs/gui/studio.nix
+    ../2configs/binary-cache/lass.nix
 
   ];
   makefu.gui.user = "user"; # we use an extra user

makefu/1systems/wbob.nix

@@ -18,6 +18,7 @@ in {
       ../2configs/mqtt.nix
       ../2configs/deployment/led-fader.nix
       # ../2configs/gui/wbob-kiosk.nix
+      ../2configs/stats/client.nix
 
       ../2configs/gui/studio.nix
       ../2configs/audio/jack-on-pulse.nix

makefu/2configs/audio/jack-on-pulse.nix

@@ -2,6 +2,7 @@
 let
   pulse = pkgs.pulseaudioFull;
   user = config.makefu.gui.user;
+  wait_time = 30;
 in
 {
   sound.enable = true;
@@ -13,16 +14,17 @@ in
   environment.systemPackages = with pkgs; [ jack2Full ];
   # from http://anderspapitto.com/posts/2015-11-26-overtone-on-nixos-with-jack-and-pulseaudio.html
 
-  systemd.services = {
+  systemd.user.services = {
     jackdbus = {
       description = "Runs jack, and points pulseaudio at it";
       serviceConfig = {
-        User = user;
         Type = "oneshot";
         ExecStart = pkgs.writeScript "start_jack.sh" ''
           #! ${pkgs.bash}/bin/bash
           . ${config.system.build.setEnvironment}
-          sleep 5 # wait for the gui to load
+
+          # TODO: correctly wait for pulseaudio, cannot use pulseaudio.service
+          sleep ${toString wait_time} # wait for the gui to load
 
           ${pkgs.jack2Full}/bin/jack_control start
           sleep 3 # give some time for sources/sinks to be created
@@ -37,9 +39,11 @@ in
           ${pkgs.jack2Full}/bin/jack_control stop
         '';
         RemainAfterExit = true;
+        Restart = "always";
+        RestartSec = "5";
       };
-      after = [ "display-manager.service" "sound.target" ];
+      # after = [ "display-manager.service" "sound.target" ];
-      wantedBy = [ "multi-user.target" ];
+      wantedBy = [ "default.target" ];
     };
   };
 }

makefu/2configs/deployment/hound/default.nix

@@ -0,0 +1,28 @@
+{ config, pkgs, ... }:
+{
+  services.nginx.virtualHosts."wikisearch.krebsco.de" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/".proxyPass = "http://localhost:6080";
+  };
+  services.hound = {
+    enable = true;
+    listen = "127.0.0.1:6080";
+    # package = pkgs.hound.overrideDerivation(oldAttrs: {
+    #   patches = [ ./keep-repo.patch ];
+    # });
+    config = ''{
+        "max-concurrent-indexers" : 2,
+        "dbpath" : "${config.services.hound.home}/data",
+        "repos" : {
+          "nixos-users-wiki": {
+            "url" : "https://github.com/nixos-users/wiki.wiki.git",
+            "url-pattern" : {
+              "base-url" : "{url}/{path}"
+            }
+          }
+        }
+      }'';
+  };
+
+}

makefu/2configs/deployment/led-fader.nix

@@ -29,7 +29,8 @@ in {
     environment = {
       NIX_PATH = "/var/src";
     };
-    wantedBy = [ "multi-user.target" ];
+    after = [ (lib.optional config.services.mosqitto.enable "mosquitto.service") ];
+    wantedBy = [ "multi-user.target"  ];
     serviceConfig = {
       # User = "nobody"; # need a user with permissions to run nix-shell
       ExecStart = "${pkg}/bin/ampel 4 ${pkg}/share/times.json";

makefu/2configs/deployment/wiki-irc-bot/default.nix

@@ -4,6 +4,10 @@ with lib;
 let
   port = 18872;
 in {
+  nixpkgs.config.packageOverrides = pkgs: with pkgs; {
+    logstash = pkgs.stdenv.lib.overrideDerivation  pkgs.logstash (old: {
+      patches = [ ./irc-out-notice.patch ]; });
+  };
   services.logstash = {
     enable = true;
     inputConfig = ''
@@ -40,10 +44,11 @@ in {
       file { path => "/tmp/logs.json" codec => "json_lines" }
       if [output] {
         irc {
-          channels => [ "#nixos" , "#krebs" ]
+          channels => [ "#krebs", "#nixos" ]
           host => "irc.freenode.net"
           nick => "nixos-users-wiki"
           format => "%{output}"
+          notice => true
         }
       }
     '';

makefu/2configs/deployment/wiki-irc-bot/irc-out-notice.patch

@@ -0,0 +1,26 @@
+index b63339d..8c8c747 100644
+--- a/vendor/bundle/jruby/1.9/gems/logstash-output-irc-2.0.4/lib/logstash/outputs/irc.rb
++++ b/vendor/bundle/jruby/1.9/gems/logstash-output-irc-2.0.4/lib/logstash/outputs/irc.rb
+@@ -48,6 +48,9 @@ class LogStash::Outputs::Irc < LogStash::Outputs::Base
+   # Static string after event
+   config :post_string, :validate => :string, :required => false
+ 
++  # Set this to true to send messages as notice
++  config :notice, :validate => :boolean, :default => false
++
+   public
+ 
+   def inject_bot(bot)
+@@ -90,9 +93,9 @@ class LogStash::Outputs::Irc < LogStash::Outputs::Base
+ 
+     @bot.channels.each do |channel|
+       @logger.debug("Sending to...", :channel => channel, :text => text)
+-      channel.msg(pre_string) if !@pre_string.nil?
+-      channel.msg(text)
+-      channel.msg(post_string) if !@post_string.nil?
++      channel.send(pre_string, :notice => @notice) if !@pre_string.nil?
++      channel.send(text, :notice => @notice)
++      channel.send(post_string, :notice => @notice) if !@post_string.nil?
+     end # channels.each
+   end # def receive
+ end # class LogStash::Outputs::Irc

makefu/2configs/gui/base.nix

@@ -24,7 +24,7 @@ in
     enable = true;
     layout = "us";
     xkbVariant = "altgr-intl";
-    xkbOptions = "ctrl:nocaps";
+    xkbOptions = "ctrl:nocaps, eurosign:e";
 
     windowManager = {
       awesome.enable = true;

makefu/2configs/stats/external/aralast.nix

@@ -0,0 +1,38 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+let
+  pkg = pkgs.stdenv.mkDerivation {
+    name = "aralast-master";
+    src = pkgs.fetchFromGitHub {
+      owner = "makefu";
+      repo = "aralast";
+      rev = "7121598";
+      sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m";
+    };
+    installPhase = ''
+      install -m755 -D aralast.sh $out/bin/aralast
+    '';
+  };
+in {
+  systemd.services.aralast  = {
+    description = "periodically fetch aramark";
+    path = [
+      pkgs.curl
+      pkgs.gnugrep
+      pkgs.gnused
+    ];
+    wantedBy = [ "multi-user.target" ];
+    environment = {
+      INFLUX_HOST = "localhost";
+      INFLUX_PORT = "8086";
+    };
+    # every 10 seconds when the cantina is open
+    startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0,15,30,45";
+    serviceConfig = {
+      User = "nobody";
+      ExecStart = "${pkg}/bin/aralast";
+      PrivateTmp = true;
+    };
+  };
+}

makefu/2configs/stats/server.nix

@@ -12,7 +12,9 @@ in {
   services.grafana.addr = "0.0.0.0";
 
   services.influxdb.enable = true;
-
+  # redirect grafana to stats.makefu.r
+  services.nginx.enable = true;
+  services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000";
   # forward these via nginx
   services.influxdb.extraConfig = {
     meta.hostname = config.krebs.build.host.name;

makefu/2configs/time-machine.nix

@@ -0,0 +1,31 @@
+let
+  time-machine-path = "/media/crypt2/backup/time-machine/misa";
+in {
+	networking.firewall.allowedTCPPorts = [
+    548 # netatalk
+  ];
+
+  services = {
+    netatalk = {
+      enable = true;
+
+      volumes = {
+        "misa-time-machine" = {
+          "time machine" = "yes";
+          path = time-machine-path;
+          "valid users" = "misa";
+        };
+      };
+    };
+
+    avahi = {
+      enable = true;
+      nssmdns = true;
+
+      publish = {
+        enable = true;
+        userServices = true;
+      };
+    };
+  };
+}

makefu/2configs/tools/games.nix

@@ -3,5 +3,6 @@
 {
   krebs.per-user.makefu.packages = with pkgs; [
     steam
+    games-user-env
   ];
 }

makefu/2configs/zsh-user.nix

@@ -44,4 +44,8 @@ in
       fi
       '';
   };
+
+  krebs.per-user.${mainUser}.packages = [
+    pkgs.nix-zsh-completions
+  ];
 }

makefu/3modules/default.nix

@@ -11,7 +11,6 @@ _:
     ./logging-config.nix
     ./server-config.nix
     ./snapraid.nix
-    ./taskserver.nix
     ./torrent.nix
     ./udpt.nix
     ./umts.nix

makefu/3modules/taskserver.nix

@@ -1,60 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with import <stockholm/lib>;
-let
-  cfg = config.makefu.taskserver;
-
-  out = {
-    options.makefu.taskserver = api;
-    config = lib.mkIf cfg.enable imp;
-  };
-
-  api = {
-    enable = mkEnableOption "taskserver";
-
-    workingDir = mkOption {
-      type = types.str;
-      default = "/var/lib/taskserver";
-    };
-
-    package = mkOption {
-      type = types.package;
-      default = pkgs.taskserver;
-    };
-
-
-  };
-
-  imp = {
-    environment.systemPackages = [ cfg.package ];
-    systemd.services.taskserver = {
-      description = "taskd server";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      restartIfChanged = true;
-      unitConfig = {
-        Documentation = "http://taskwarrior.org/docs/#taskd" ;
-        # https://taskwarrior.org/docs/taskserver/configure.html
-        ConditionPathExists = "${cfg.workingDir}/config";
-      };
-      serviceConfig = {
-        Type = "simple";
-        ExecStart = "${cfg.package}/bin/taskd server --data ${cfg.workingDir}";
-        WorkingDirectory = cfg.workingDir;
-        # PrivateTmp = true;
-        # InaccessibleDirectories = "/home /boot /opt /mnt /media";
-        User = "taskd";
-      };
-    };
-
-    users.users.taskd = {
-      uid = genid "taskd";
-      home = cfg.workingDir;
-      createHome = true;
-    };
-    users.groups.taskd.gid = genid "taskd";
-  };
-
-in
-out
-

shared/1systems/wolf.nix

@@ -6,19 +6,27 @@ in
   imports = [
     ../.
     <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
-    ../2configs/cgit-mirror.nix
     ../2configs/collectd-base.nix
+    ../2configs/shack/share.nix
+    ../2configs/central-stats-client.nix
+    ../2configs/save-diskspace.nix
+
+    ../2configs/cgit-mirror.nix
     ../2configs/graphite.nix
     ../2configs/repo-sync.nix
-    ../2configs/shack-drivedroid.nix
-    ../2configs/shack-nix-cacher.nix
     ../2configs/shared-buildbot.nix
-    ../2configs/share-shack.nix
+    ../2configs/shack/drivedroid.nix
-    ../2configs/central-stats-client.nix
+    ../2configs/shack/nix-cacher.nix
+
+    ../2configs/shack/mqtt_sub.nix
+    ../2configs/shack/muell_caller.nix
+
   ];
   # use your own binary cache, fallback use cache.nixos.org (which is used by
   # apt-cacher-ng in first place)
 
+  services.influxdb.enable = true;
+
   # local discovery in shackspace
   nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
   krebs.tinc.retiolum.extraConfig = "TCPOnly = yes";
@@ -46,6 +54,7 @@ in
 
   networking = {
     firewall.enable = false;
+    firewall.allowedTCPPorts = [ 8088 8086 8083 ];
     interfaces.enp0s3.ip4 = [{
       address = shack-ip;
       prefixLength = 20;
@@ -83,4 +92,5 @@ in
   ];
 
   time.timeZone = "Europe/Berlin";
+  sound.enable = false;
 }

shared/2configs/default.nix

@@ -11,7 +11,7 @@ with import <stockholm/lib>;
     nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
     nixpkgs.git = {
       url = https://github.com/NixOS/nixpkgs;
-      ref = "22da5d02466ffe465735986d705675982f3646a0"; # nixos-17.03 @ 2017-05-13
+      ref = "58e227052d40021d82d015f3f8da011ae54ea430"; # nixos-17.03 @ 2017-05-24
     };
     secrets.file =
       if getEnv "dummy_secrets" == "true"

shared/2configs/graphite.nix

@@ -22,12 +22,50 @@ with import <stockholm/lib>;
         MAX_CACHE_SIZE = inf
         MAX_UPDATES_PER_SECOND = 1
         MAX_CREATES_PER_MINUTE = 50
+        MAX_UPDATES_PER_SECOND_ONSHUTDOWN = 9001
         '';
       storageSchemas = ''
         [carbon]
         pattern = ^carbon\.
         retentions = 60:90d
 
+
+        [radiation_sensor]
+        pattern = ^sensors\.radiation\.
+        retentions = 1m:30d,5m:180d,10m:3y
+
+        [motion_sensors]
+        pattern = ^sensors\.motion\.
+        retentions = 1s:1h,60s:30d,300s:1y
+
+        [motion_sensors]
+        pattern = ^retiolum\.
+        retentions = 10s:1h,30s:30d,300s:1y
+
+        [homeassistant]
+        pattern = ^homeassistant\.
+        retentions = 10s:24h,30s:30d,300s:1y,3600s:5y
+
+        [ara]
+        pattern = ^ara\.
+        retentions = 60s:30d,300s:1y
+
+        [openweathermap]
+        pattern = ^weather\.openweathermap
+        retentions = 30m:30d,1h:5y
+
+        [stadtklima]
+        pattern = ^weather\.stadtklima-stuttgart
+        retentions = 15m:30d,30m:5y
+
+        [sensebox]
+        pattern = ^weather\.sensebox
+        retentions = 1m:90d,30m:5y
+
+        [elchos]
+        pattern = ^elchos\.
+        retentions = 10s:14d,1m:90d,10m:5y
+
         [default]
         pattern = .*
         retentions = 60s:30d,300s:1y

shared/2configs/save-diskspace.nix

@@ -0,0 +1,11 @@
+{lib, ... }:
+# TODO: do not check out nixpkgs master but fetch revision from github
+{
+  environment.noXlibs = true;
+  nix.gc.automatic = true;
+  nix.gc.dates = lib.mkDefault "03:10";
+  programs.info.enable = false;
+  programs.man.enable = false;
+  services.journald.extraConfig = "SystemMaxUse=50M";
+  services.nixosManual.enable = false;
+}

shared/2configs/shack/mqtt_sub.nix

@@ -0,0 +1,34 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+let
+  pkg = pkgs.stdenv.mkDerivation {
+    name = "mqtt2graphite-2017-05-29";
+    src = pkgs.fetchgit {
+      url = "https://github.com/shackspace/mqtt2graphite/";
+      rev = "8c060e6";
+      sha256 = "06x7a1j6sfyvvdxg0366fcslhn478anqh4m5hljyf0z29knvz7pg";
+    };
+    buildInputs = [
+      (pkgs.python35.withPackages (pythonPackages: with pythonPackages; [
+        docopt
+        paho-mqtt
+      ]))
+    ];
+    installPhase = ''
+      install -m755 -D sub.py  $out/bin/sub
+      install -m755 -D sub2.py  $out/bin/sub-new
+    '';
+  };
+in {
+  systemd.services.mqtt_sub  = {
+    description = "subscribe to mqtt, send to graphite";
+    # after = [ (lib.optional config.services.mosqitto.enable "mosquitto.service") ];
+    wantedBy = [ "multi-user.target"  ];
+    serviceConfig = {
+      User = "nobody";
+      ExecStart = "${pkg}/bin/sub-new";
+      PrivateTmp = true;
+    };
+  };
+}

shared/2configs/shack/muell_caller.nix

@@ -0,0 +1,41 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+let
+  pkg = pkgs.stdenv.mkDerivation {
+    name = "muell_caller-2017-06-01";
+    src = pkgs.fetchgit {
+      url = "https://github.com/shackspace/muell_caller/";
+      rev = "bbd4009";
+      sha256 = "1bfnfl2vdh0p5wzyz5p48qh04vvsg2445avg86fzhzragx25fqv0";
+    };
+    buildInputs = [
+      (pkgs.python3.withPackages (pythonPackages: with pythonPackages; [
+        docopt
+        requests2
+        paramiko
+        python
+      ]))
+    ];
+    installPhase = ''
+      install -m755 -D call.py  $out/bin/call-muell
+    '';
+  };
+  cfg = "${toString <secrets>}/tell.json";
+in {
+  systemd.services.call_muell = {
+    description = "call muell";
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      User = "nobody"; # TODO separate user
+      ExecStartPre = pkgs.writeDash "call-muell-pre" ''
+        cp ${cfg} /tmp/tell.json
+        chown nobody /tmp/tell.json
+      '';
+      ExecStart = "${pkg}/bin/call-muell --cfg /tmp/tell.json --mode mpd loop 60";
+      Restart = "always";
+      PrivateTmp = true;
+      PermissionsStartOnly = true;
+    };
+  };
+}