73 lines
2.0 KiB
Nix
73 lines
2.0 KiB
Nix
{pkgs, config, ...}:
|
|
|
|
with import <stockholm/lib>;
|
|
let
|
|
collectd-port = 25826;
|
|
influx-port = 8086;
|
|
grafana-port = 3000; # TODO nginx forward
|
|
in {
|
|
imports = [
|
|
../../lass/3modules/kapacitor.nix
|
|
];
|
|
services.grafana.enable = true;
|
|
services.grafana.addr = "0.0.0.0";
|
|
|
|
services.influxdb.enable = true;
|
|
|
|
# forward these via nginx
|
|
services.influxdb.extraConfig = {
|
|
meta.hostname = config.krebs.build.host.name;
|
|
# meta.logging-enabled = true;
|
|
http.bind-address = ":${toString influx-port}";
|
|
admin.bind-address = ":8083";
|
|
monitoring = {
|
|
enabled = false;
|
|
# write-interval = "24h";
|
|
};
|
|
collectd = [{
|
|
enabled = true;
|
|
typesdb = "${pkgs.collectd}/share/collectd/types.db";
|
|
database = "collectd_db";
|
|
port = collectd-port;
|
|
}];
|
|
};
|
|
lass.kapacitor =
|
|
let
|
|
echoToIrc = pkgs.writeDash "echo_irc" ''
|
|
set -euf
|
|
data="$(${pkgs.jq}/bin/jq -r .message)"
|
|
export LOGNAME=malarm
|
|
${pkgs.irc-announce}/bin/irc-announce \
|
|
irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null
|
|
'';
|
|
in {
|
|
enable = true;
|
|
alarms = {
|
|
cpu_deadman = ''
|
|
var data = batch
|
|
|query('''
|
|
SELECT mean("value") AS mean
|
|
FROM "collectd_db"."default"."cpu_value"
|
|
WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0)
|
|
''')
|
|
.period(10m)
|
|
.every(1m)
|
|
.groupBy('host')
|
|
data |alert()
|
|
.crit(lambda: "mean" < 50)
|
|
.stateChangesOnly()
|
|
.exec('${echoToIrc}')
|
|
data |deadman(1.0,5m)
|
|
.stateChangesOnly()
|
|
.exec('${echoToIrc}')
|
|
'';
|
|
};
|
|
|
|
};
|
|
networking.firewall.extraCommands = ''
|
|
iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
|
|
iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
|
|
iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
|
|
'';
|
|
}
|