stockholm/lass/2configs/monitoring/server.nix

{pkgs, config, ...}:
with import <stockholm/lib>;
{
  services.influxdb.enable = true;

  services.influxdb.extraConfig = {
    meta.hostname = config.krebs.build.host.name;
    # meta.logging-enabled = true;
    http.bind-address = ":8086";
    admin.bind-address = ":8083";
    http.log-enabled = false;
    monitoring = {
      enabled = false;
      # write-interval = "24h";
    };
    collectd = [{
      enabled = true;
      typesdb = "${pkgs.collectd}/share/collectd/types.db";
      database = "collectd_db";
      port = 25826;
    }];
  };

  krebs.kapacitor =
    let
      db = "telegraf_db";
      echoToIrc = pkgs.writeDash "echo_irc" ''
        set -euf
        data="$(${pkgs.jq}/bin/jq -r .message)"
        export LOGNAME=prism-alarm
        ${pkgs.irc-announce}/bin/irc-announce \
          irc.r 6667 prism-alarm \#noise "$data" >/dev/null
      '';
    in {
      enable = true;
      alarms = {
        cpu = {
          database = db;
          text = ''
            var data = batch
              |query(${"'''"}
                SELECT mean("usage_user") AS mean
                FROM "${db}"."default"."cpu"
              ${"'''"})
              .period(10m)
              .every(1m)
              .groupBy('host')
              data |alert()
                .crit(lambda: "mean" > 90)
                .exec('${echoToIrc}')
              data |deadman(1.0,5m)
                .stateChangesOnly()
                .exec('${echoToIrc}')
          '';
        };
        ram = {
          database = db;
          text = ''
            var data = batch
              |query(${"'''"}
                SELECT mean("used_percent") AS mean
                FROM "${db}"."default"."mem"
              ${"'''"})
              .period(10m)
              .every(1m)
              .groupBy('host')
              data |alert()
                .crit(lambda: "mean" > 90)
                .exec('${echoToIrc}')
          '';
        };
      };
  };

  services.grafana = {
    enable = true;
    addr = "0.0.0.0";
    auth.anonymous.enable = true;
    security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
  };

  krebs.iptables.tables.filter.INPUT.rules = [
    { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; }
    { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; }
    { predicate = "-p udp -i retiolum --dport 25826"; target = "ACCEPT"; }
  ];
}