From 2d79335e34d639657bd0045f9a6b777778a5ad82 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 17:07:16 +0100 Subject: [PATCH 01/14] tv.mail: wu -> nomic --- krebs/3modules/tv/default.nix | 2 +- tv/1systems/nomic.nix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index f8d3d8671..300fce017 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -352,7 +352,7 @@ with config.krebs.lib; pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGer9e2+Lew7vnisgBbsFNECEIkpNJgEaqQqgb9inWkQ mv@vod"; }; tv = { - mail = "tv@wu.retiolum"; + mail = "tv@nomic.retiolum"; pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDFR//RnCvEZAt0F6ExDsatKZ/DDdifanuSL360mqOhaFieKI34RoOwfQT9T+Ga52Vh5V2La6esvlph686EdgzeKLvDoxEwFM9ZYFBcMrNzu4bMTlgE7YUYw5JiORyXNfznBGnme6qpuvx9ibYhUyiZo99kM8ys5YrUHrP2JXQJMezDFZHxT4GFMOuSdh/1daGoKKD6hYL/jEHX8CI4E3BSmKK6ygYr1fVX0K0Tv77lIi5mLXucjR7CytWYWYnhM6DC3Hxpv2zRkPgf3k0x/Y1hrw3V/r0Me5h90pd2C8pFaWA2ZoUT/fmyVqvx1tZPYToU/O2dMItY0zgx2kR0yD+6g7Aahz3R+KlXkV8k5c8bbTbfGnZWDR1ZlbLRM9Yt5vosfwapUD90MmVkpmR3wUkO2sUKi80QfC7b4KvSDXQ+MImbGxMaU5Bnsq1PqLN95q+uat3nlAVBAELkcx51FlE9CaIS65y4J7FEDg8BE5JeuCNshh62VSYRXVSFt8bk3f/TFGgzC8OIo14BhVmiRQQ503Z1sROyf5xLX2a/EJavMm1i2Bs2TH6ROKY9z5Pz8hT5US0r381V8oG7TZyLF9HTtoy3wCYsgWA5EmLanjAsVU2YEeAA0rxzdtYP8Y2okFiJ6u+M4HQZ3Wg3peSodyp3vxdYce2vk4EKeqEFuuS82850DYb7Et7fmp+wQQUT8Q/bMO0DreWjHoMM5lE4LJ4ME6AxksmMiFtfo/4Fe2q9D+LAqZ+ANOcv9M+8Rn6ngiYmuRNd0l/a02q1PEvO6vTfXgcl4f7Z1IULHPEaDNZHCJS1K5RXYFqYQ6OHsTmOm7hnwaRAS97+VFMo1i5uvTx9nYaAcY7yzq3Ckfb67dMBKApGOpJpkvPgfrP7bgBO5rOZXM1opXqVPb09nljAhhAhyCTh1e/8+mJrBo0cLQ/LupQzVxGDgm3awSMPxsZAN45PSWz76zzxdDa1MMo51do+VJHfs7Wl0NcXAQrniOBYL9Wqt0qNkn1gY5smkkISGeQ/vxNap4MmzeZE7b5fpOy+2fpcRVQLpc4nooQzJvSVTFz+25lgZ6iHf45K87gQFMIAri1Pf/EDDpL87az+bRWvWi+BA2kMe1kf+Ay1LyMz8r+g51H0ma0bNFh6+fbWMfUiD9JCepIObclnUJ4NlWfcgHxTf17d/4tl6z4DTcLpCCk8Da77JouSHgvtcRbRlFV1OfhWZLXUsrlfpaQTiItv6TGIr3k7+7b66o3Qw/GQVs5GmYifaIZIz8n8my4XjkaMBd0SZfBzzvFjHMq6YUP9+SbjvReqofuoO+5tW1wTYZXitFFBfwuHlXm6w77K5QDBW6olT7pat41/F5eGxLcz tv@wu"; }; tv-nomic = { diff --git a/tv/1systems/nomic.nix b/tv/1systems/nomic.nix index 2c9775da7..c247bf796 100644 --- a/tv/1systems/nomic.nix +++ b/tv/1systems/nomic.nix @@ -10,6 +10,7 @@ with config.krebs.lib; ../2configs/hw/AO753.nix ../2configs/exim-retiolum.nix ../2configs/git.nix + ../2configs/mail-client.nix ../2configs/nginx-public_html.nix ../2configs/pulse.nix ../2configs/retiolum.nix From 468f294635523a39eaf1a6794571948066b63a1c Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 17:23:19 +0100 Subject: [PATCH 02/14] tv im: init --- tv/1systems/nomic.nix | 1 + tv/1systems/wu.nix | 20 +------------------- tv/2configs/im.nix | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 tv/2configs/im.nix diff --git a/tv/1systems/nomic.nix b/tv/1systems/nomic.nix index c247bf796..45320690b 100644 --- a/tv/1systems/nomic.nix +++ b/tv/1systems/nomic.nix @@ -10,6 +10,7 @@ with config.krebs.lib; ../2configs/hw/AO753.nix ../2configs/exim-retiolum.nix ../2configs/git.nix + ../2configs/im.nix ../2configs/mail-client.nix ../2configs/nginx-public_html.nix ../2configs/pulse.nix diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index 6154e4df9..7615c4e80 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -10,6 +10,7 @@ with config.krebs.lib; ../2configs/hw/w110er.nix ../2configs/exim-retiolum.nix ../2configs/git.nix + ../2configs/im.nix ../2configs/mail-client.nix ../2configs/nginx-public_html.nix ../2configs/pulse.nix @@ -23,19 +24,6 @@ with config.krebs.lib; hashPassword haskellPackages.lentil parallel - (pkgs.writeScriptBin "im" '' - #! ${pkgs.bash}/bin/bash - export PATH=${makeSearchPath "bin" (with pkgs; [ - tmux - gnugrep - weechat - ])} - if tmux list-sessions -F\#S | grep -q '^im''$'; then - exec tmux attach -t im - else - exec tmux new -s im weechat - fi - '') # root cryptsetup @@ -201,12 +189,6 @@ with config.krebs.lib; KERNEL=="hpet", GROUP="audio" ''; - services.bitlbee = { - enable = true; - plugins = [ - pkgs.bitlbee-facebook - ]; - }; services.tor.client.enable = true; services.tor.enable = true; services.virtualboxHost.enable = true; diff --git a/tv/2configs/im.nix b/tv/2configs/im.nix new file mode 100644 index 000000000..db1be7f0b --- /dev/null +++ b/tv/2configs/im.nix @@ -0,0 +1,24 @@ +{ config, lib, pkgs, ... }: +with config.krebs.lib; +{ + environment.systemPackages = with pkgs; [ + (pkgs.writeDashBin "im" '' + export PATH=${makeSearchPath "bin" (with pkgs; [ + tmux + gnugrep + weechat + ])} + if tmux list-sessions -F\#S | grep -q '^im''$'; then + exec tmux attach -t im + else + exec tmux new -s im weechat + fi + '') + ]; + services.bitlbee = { + enable = true; + plugins = [ + pkgs.bitlbee-facebook + ]; + }; +} From d71a8863ac10f34bfe30f950f32bbf57427e2ca7 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 17:30:39 +0100 Subject: [PATCH 03/14] tv krebs.backup.plans.nomic-home-xu: init --- tv/2configs/backup.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix index 641e2d586..decd8b286 100644 --- a/tv/2configs/backup.nix +++ b/tv/2configs/backup.nix @@ -2,6 +2,18 @@ with config.krebs.lib; { krebs.backup.plans = { + nomic-home-xu = { + method = "push"; + src = { host = config.krebs.hosts.nomic; path = "/home"; }; + dst = { host = config.krebs.hosts.xu; path = "/bku/nomic-home"; }; + startAt = "05:00"; + snapshots = { + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; + }; wu-home-xu = { method = "push"; src = { host = config.krebs.hosts.wu; path = "/home"; }; From a042769ee71e20176c5341bdcc029c3ac23646e1 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 17:45:58 +0100 Subject: [PATCH 04/14] xu,wu: rm pkgs that belong to xserver --- tv/1systems/wu.nix | 5 ----- tv/1systems/xu.nix | 4 ---- 2 files changed, 9 deletions(-) diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index 7615c4e80..2b6dca1c4 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -153,11 +153,7 @@ with config.krebs.lib; hardware.opengl.driSupport32Bit = true; environment.systemPackages = with pkgs; [ - xlibs.fontschumachermisc - slock ethtool - #firefoxWrapper # with plugins - #chromiumDevWrapper tinc iptables #jack2 @@ -165,7 +161,6 @@ with config.krebs.lib; security.setuidPrograms = [ "sendmail" # for cron - "slock" ]; services.printing.enable = true; diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index 5ec1fe52b..d4295d3b7 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -163,11 +163,7 @@ with config.krebs.lib; #hardware.opengl.driSupport32Bit = true; environment.systemPackages = with pkgs; [ - #xlibs.fontschumachermisc - #slock ethtool - #firefoxWrapper # with plugins - #chromiumDevWrapper tinc iptables #jack2 From 38cc636c8c1b8d95579ad9a980b29b494f383779 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 22:28:02 +0100 Subject: [PATCH 05/14] tv sudo: !lecture --- tv/2configs/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index c4a2d6baa..13699a3d5 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -50,6 +50,7 @@ with config.krebs.lib; { security.sudo.extraConfig = '' Defaults mailto="${config.krebs.users.tv.mail}" + Defaults !lecture ''; time.timeZone = "Europe/Berlin"; } From f5d0b2a4020b3bd5150f9861fb27a7faaddbd0b9 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 23:23:13 +0100 Subject: [PATCH 06/14] xu-qemu0: disable systemd-networkd-wait-online --- tv/2configs/xu-qemu0.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tv/2configs/xu-qemu0.nix b/tv/2configs/xu-qemu0.nix index 720a8acd8..2b67a8b84 100644 --- a/tv/2configs/xu-qemu0.nix +++ b/tv/2configs/xu-qemu0.nix @@ -27,6 +27,8 @@ with config.krebs.lib; networking.dhcpcd.denyInterfaces = [ "qemubr0" ]; systemd.network.enable = true; + systemd.services.systemd-networkd-wait-online.enable = false; + services.resolved.enable = mkForce false; boot.kernel.sysctl."net.ipv4.ip_forward" = 1; From 970eed274818cb49517fa5ef8b39a30d99302f1e Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 23:35:25 +0100 Subject: [PATCH 07/14] tv man: init --- tv/1systems/wu.nix | 3 +-- tv/1systems/xu.nix | 3 +-- tv/2configs/man.nix | 7 +++++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tv/2configs/man.nix diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index 2b6dca1c4..8c363d9fc 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -12,6 +12,7 @@ with config.krebs.lib; ../2configs/git.nix ../2configs/im.nix ../2configs/mail-client.nix + ../2configs/man.nix ../2configs/nginx-public_html.nix ../2configs/pulse.nix ../2configs/retiolum.nix @@ -40,14 +41,12 @@ with config.krebs.lib; haskellPackages.hledger htop jq - manpages mkpasswd netcat nix-repl nmap nq p7zip - posix_man_pages push qrencode texLive diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index d4295d3b7..c6a69a85a 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -11,6 +11,7 @@ with config.krebs.lib; ../2configs/exim-retiolum.nix ../2configs/git.nix ../2configs/mail-client.nix + ../2configs/man.nix ../2configs/nginx-public_html.nix ../2configs/pulse.nix ../2configs/retiolum.nix @@ -52,7 +53,6 @@ with config.krebs.lib; haskellPackages.hledger htop jq - manpages mkpasswd netcat nix-repl @@ -60,7 +60,6 @@ with config.krebs.lib; nq p7zip pass - posix_man_pages qrencode texLive tmux diff --git a/tv/2configs/man.nix b/tv/2configs/man.nix new file mode 100644 index 000000000..686e574fc --- /dev/null +++ b/tv/2configs/man.nix @@ -0,0 +1,7 @@ +{ config, lib, pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + manpages + posix_man_pages + ]; +} From f7d6e2043184401f7007b248fbe3af66b2752351 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 17 Feb 2016 23:35:43 +0100 Subject: [PATCH 08/14] tv man: inhibit warning break --- tv/2configs/man.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tv/2configs/man.nix b/tv/2configs/man.nix index 686e574fc..a84e60b73 100644 --- a/tv/2configs/man.nix +++ b/tv/2configs/man.nix @@ -1,5 +1,10 @@ { config, lib, pkgs, ... }: { + environment.etc."man.conf".source = pkgs.runCommand "man.conf" {} '' + ${pkgs.gnused}/bin/sed <${pkgs.man}/lib/man.conf >$out ' + s:^NROFF\t.*:& -Wbreak: + ' + ''; environment.systemPackages = with pkgs; [ manpages posix_man_pages From 7f1abe50ce0989d96c3d275a4d0481962848714f Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 00:50:10 +0100 Subject: [PATCH 09/14] xu-qemu0 host: setup iptables --- tv/2configs/xu-qemu0.nix | 18 ++++++++++++------ tv/3modules/iptables.nix | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/tv/2configs/xu-qemu0.nix b/tv/2configs/xu-qemu0.nix index 2b67a8b84..5be4899c8 100644 --- a/tv/2configs/xu-qemu0.nix +++ b/tv/2configs/xu-qemu0.nix @@ -15,17 +15,23 @@ in # # make [install] system=xu-qemu0 target_host=10.56.0.101 -# TODO iptables -A FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT -# TODO iptables -A FORWARD -i qemubr0 -s 10.56.0.1/24 -m conntrack --ctstate NEW -j ACCEPT -# TODO iptables -A POSTROUTING -t nat -j MASQUERADE -# TODO iptables -A INPUT -i qemubr0 -p udp -m udp --dport bootps -j ACCEPT -# TODO iptables -A INPUT -i qemubr0 -p udp -m udp --dport domain -j ACCEPT - with config.krebs.lib; { networking.dhcpcd.denyInterfaces = [ "qemubr0" ]; + tv.iptables.extra = { + nat.POSTROUTING = ["-j MASQUERADE"]; + filter.FORWARD = [ + "-m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT" + "-i qemubr0 -s 10.56.0.1/24 -m conntrack --ctstate NEW -j ACCEPT" + ]; + filter.INPUT = [ + "-i qemubr0 -p udp -m udp --dport bootps -j ACCEPT" + "-i qemubr0 -p udp -m udp --dport domain -j ACCEPT" + ]; + }; + systemd.network.enable = true; systemd.services.systemd-networkd-wait-online.enable = false; diff --git a/tv/3modules/iptables.nix b/tv/3modules/iptables.nix index c0fd7ec12..c0e71f24d 100644 --- a/tv/3modules/iptables.nix +++ b/tv/3modules/iptables.nix @@ -26,6 +26,21 @@ let type = with types; listOf (either int str); default = []; }; + + extra = { + nat.POSTROUTING = mkOption { + type = with types; listOf str; + default = []; + }; + filter.FORWARD = mkOption { + type = with types; listOf str; + default = []; + }; + filter.INPUT = mkOption { + type = with types; listOf str; + default = []; + }; + }; }; imp = { @@ -57,6 +72,11 @@ let }; }; + formatTable = table: + (concatStringsSep "\n" + (mapAttrsToList + (chain: concatMapStringsSep "\n" (rule: "-A ${chain} ${rule}")) + table)); rules = iptables-version: let accept-echo-request = { @@ -79,6 +99,7 @@ let ${concatMapStringsSep "\n" (rule: "-A OUTPUT ${rule}") [ "-o lo -p tcp -m tcp --dport 11423 -j REDIRECT --to-ports 22" ]} + ${formatTable cfg.extra.nat} COMMIT *filter :INPUT DROP [0:0] @@ -94,6 +115,7 @@ let ++ map accept-new-tcp (unique (map toString cfg.input-internet-accept-new-tcp)) ++ ["-i retiolum -j Retiolum"] )} + ${formatTable cfg.extra.filter} ${concatMapStringsSep "\n" (rule: "-A Retiolum ${rule}") ([] ++ optional (cfg.accept-echo-request == "retiolum") accept-echo-request ++ map accept-new-tcp (unique (map toString cfg.input-retiolum-accept-new-tcp)) From 8c264139b294cbbea27a042263a4b76991910f02 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 01:15:58 +0100 Subject: [PATCH 10/14] krebs.build: use $F5 to prefix verbose commands --- Makefile | 2 +- krebs/3modules/build.nix | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 60dfe8030..9dcd47548 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ evaluate = \ execute = \ result=$$($(call evaluate,-A config.krebs.build.$(1) --json)) && \ script=$$(echo "$$result" | jq -r .) && \ - echo "$$script" | sh + echo "$$script" | PS5=% sh # usage: make deploy system=foo [target_host=bar] deploy: ssh ?= ssh diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index c700fbc56..b8ea34ae2 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -42,12 +42,13 @@ let set -eu verbose() { - printf '+%s\n' "$(printf ' %q' "$@")" >&2 + printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 "$@" } - echo ${shell.escape git-script} \ - | ssh -p ${shell.escape target-port} \ + { printf 'PS5=%q%q\n' @ "$PS5" + echo ${shell.escape git-script} + } | verbose ssh -p ${shell.escape target-port} \ ${shell.escape "${target-user}@${target-host}"} -T unset tmpdir @@ -86,7 +87,7 @@ let set -efu verbose() { - printf '+%s\n' "$(printf ' %q' "$@")" >&2 + printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 "$@" } From 1de8d5c44d9a3fe15aaace950347986aeaa0a355 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 02:47:16 +0100 Subject: [PATCH 11/14] krebs.backup: talk about local and remote rsync --- krebs/3modules/backup.nix | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 0aa86dec9..97082f56a 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -117,6 +117,14 @@ let "$dst_user@$dst_host" \ -T "$with_dst_path_lock_script" } + rsh="ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" + local_rsync() { + rsync "$@" + } + remote_rsync=${shell.escape (concatStringsSep " && " [ + "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current" + "exec flock -n ${shell.escape plan.dst.path} rsync" + ])} ''; pull = '' identity=${shell.escape plan.dst.host.ssh.privkey.path} @@ -131,6 +139,12 @@ let dst_shell() { eval "$with_dst_path_lock_script" } + rsh="ssh -F /dev/null -i $identity ''${src_port:+-p $src_port}" + local_rsync() { + mkdir -m 0700 -p ${shell.escape plan.dst.path}/current + flock -n ${shell.escape plan.dst.path} rsync "$@" + } + remote_rsync=rsync ''; }} # Note that this only works because we trust date +%s to produce output @@ -140,13 +154,10 @@ let with_dst_path_lock_script="exec env start_date=$(date +%s) "${shell.escape "flock -n ${shell.escape plan.dst.path} /bin/sh" } - rsync >&2 \ + local_rsync >&2 \ -aAXF --delete \ - -e "ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" \ - --rsync-path ${shell.escape (concatStringsSep " && " [ - "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current" - "exec flock -n ${shell.escape plan.dst.path} rsync" - ])} \ + --rsh="$rsh" \ + --rsync-path="$remote_rsync" \ --link-dest="$dst_path/current" \ "$src/" \ "$dst/.partial" From db6342f58c534bd2ce631fd81d6956cdd8fe4637 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 02:55:46 +0100 Subject: [PATCH 12/14] tv: init backup plans xu-pull-cd-{ejabberd,home} --- tv/2configs/backup.nix | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix index decd8b286..b5512662f 100644 --- a/tv/2configs/backup.nix +++ b/tv/2configs/backup.nix @@ -2,41 +2,43 @@ with config.krebs.lib; { krebs.backup.plans = { + } // mapAttrs (_: recursiveUpdate { + snapshots = { + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; + }) { nomic-home-xu = { method = "push"; src = { host = config.krebs.hosts.nomic; path = "/home"; }; dst = { host = config.krebs.hosts.xu; path = "/bku/nomic-home"; }; startAt = "05:00"; - snapshots = { - daily = { format = "%Y-%m-%d"; retain = 7; }; - weekly = { format = "%YW%W"; retain = 4; }; - monthly = { format = "%Y-%m"; retain = 12; }; - yearly = { format = "%Y"; }; - }; }; wu-home-xu = { method = "push"; src = { host = config.krebs.hosts.wu; path = "/home"; }; dst = { host = config.krebs.hosts.xu; path = "/bku/wu-home"; }; startAt = "05:00"; - snapshots = { - daily = { format = "%Y-%m-%d"; retain = 7; }; - weekly = { format = "%YW%W"; retain = 4; }; - monthly = { format = "%Y-%m"; retain = 12; }; - yearly = { format = "%Y"; }; - }; }; xu-home-wu = { method = "push"; src = { host = config.krebs.hosts.xu; path = "/home"; }; dst = { host = config.krebs.hosts.wu; path = "/bku/xu-home"; }; startAt = "06:00"; - snapshots = { - daily = { format = "%Y-%m-%d"; retain = 7; }; - weekly = { format = "%YW%W"; retain = 4; }; - monthly = { format = "%Y-%m"; retain = 12; }; - yearly = { format = "%Y"; }; - }; + }; + xu-pull-cd-ejabberd = { + method = "pull"; + src = { host = config.krebs.hosts.cd; path = "/var/ejabberd"; }; + dst = { host = config.krebs.hosts.xu; path = "/bku/cd-ejabberd"; }; + startAt = "07:00"; + }; + xu-pull-cd-home = { + method = "pull"; + src = { host = config.krebs.hosts.cd; path = "/home"; }; + dst = { host = config.krebs.hosts.xu; path = "/bku/cd-home"; }; + startAt = "07:00"; }; } // mapAttrs (_: recursiveUpdate { snapshots = { From bca88ce7c65d424cdda61f4a588794e8d52ed394 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 16:42:11 +0100 Subject: [PATCH 13/14] krebs.urlwatch: add hooksFile and per-url filter --- krebs/3modules/urlwatch.nix | 69 ++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index cd4976a21..ed1a21260 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -3,7 +3,6 @@ # TODO multiple users # TODO inform about unused caches # cache = url: "${cfg.dataDir}/.urlwatch/cache/${hashString "sha1" url}" -# TODO hooks.py with config.krebs.lib; let @@ -32,6 +31,14 @@ let Content of the From: header of the generated mails. ''; }; + # TODO hooks :: attrsOf hook + hooksFile = mkOption { + type = with types; nullOr path; + default = null; + description = '' + File to use as hooks.py module. + ''; + }; mailto = mkOption { type = types.str; default = config.krebs.build.user.mail; @@ -48,7 +55,7 @@ let ''; }; urls = mkOption { - type = with types; listOf str; + type = with types; listOf (either str subtypes.job); default = []; description = "URL to watch."; example = [ @@ -56,7 +63,10 @@ let ]; apply = map (x: getAttr (typeOf x) { set = x; - string.url = x; + string = { + url = x; + filter = null; + }; }); }; verbose = mkOption { @@ -68,9 +78,12 @@ let }; }; - urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + urlsFile = pkgs.writeText "urls" + (concatMapStringsSep "\n---\n" toJSON cfg.urls); - configFile = toFile "urlwatch.yaml" (toJSON { + hooksFile = cfg.hooksFile; + + configFile = pkgs.writeText "urlwatch.yaml" (toJSON { display = { error = true; new = true; @@ -127,10 +140,10 @@ let User = user.name; PermissionsStartOnly = "true"; PrivateTmp = "true"; + SyslogIdentifier = "urlwatch"; Type = "oneshot"; ExecStartPre = - pkgs.writeScript "urlwatch-prestart" '' - #! /bin/sh + pkgs.writeDash "urlwatch-prestart" '' set -euf dataDir=$HOME @@ -140,31 +153,29 @@ let chown ${user.name}: "$dataDir" fi ''; - ExecStart = pkgs.writeScript "urlwatch" '' - #! /bin/sh + ExecStart = pkgs.writeDash "urlwatch" '' set -euf - from=${escapeShellArg cfg.from} - mailto=${escapeShellArg cfg.mailto} - urlsFile=${escapeShellArg urlsFile} - configFile=${escapeShellArg configFile} cd /tmp urlwatch \ ${optionalString cfg.verbose "-v"} \ - --urls="$urlsFile" \ - --config="$configFile" \ + --config=${shell.escape configFile} \ + ${optionalString (hooksFile != null) + "--hooks=${shell.escape hooksFile}" + } \ + --urls=${shell.escape urlsFile} \ > changes || : if test -s changes; then - date=$(date -R) - subject=$(sed -n 's/^\(CHANGED\|ERROR\|NEW\): //p' changes \ - | tr \\n \ ) { - echo "Date: $date" - echo "From: $from" - echo "Subject: $subject" - echo "To: $mailto" + echo Date: $(date -R) + echo From: ${shell.escape cfg.from} + echo Subject: $( + sed -n 's/^\(CHANGED\|ERROR\|NEW\): //p' changes \ + | tr '\n' ' ' + ) + echo To: ${shell.escape cfg.mailto} echo cat changes } | /var/setuid-wrappers/sendmail -t @@ -181,5 +192,15 @@ let name = "urlwatch"; uid = genid name; }; -in -out + + subtypes.job = types.submodule { + options = { + url = mkOption { + type = types.str; + }; + filter = mkOption { + type = with types; nullOr str; # TODO nullOr subtypes.filter + }; + }; + }; +in out From ffc47bf80d521635021b3f7a0122092708ebd2bf Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 18 Feb 2016 14:14:30 +0100 Subject: [PATCH 14/14] tv urlwatch: filter pypi/vncdotool/json through jq --- tv/2configs/urlwatch.nix | 41 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tv/2configs/urlwatch.nix b/tv/2configs/urlwatch.nix index 0106cddf7..51b53230b 100644 --- a/tv/2configs/urlwatch.nix +++ b/tv/2configs/urlwatch.nix @@ -1,5 +1,5 @@ -{ config, ... }: - +{ config, pkgs, ... }: +with config.krebs.lib; { krebs.urlwatch = { enable = true; @@ -52,8 +52,43 @@ # is derived from `configFile` in: https://raw.githubusercontent.com/NixOS/nixpkgs/master/nixos/modules/services/x11/xserver.nix - https://pypi.python.org/pypi/vncdotool + { + url = https://pypi.python.org/pypi/vncdotool/json; + filter = "system:${pkgs.jq}/bin/jq -r '.releases|keys[]'"; + } https://api.github.com/repos/kanaka/noVNC/tags ]; + hooksFile = toFile "hooks.py" '' + import subprocess + import urlwatch + + class CaseFilter(urlwatch.filters.FilterBase): + """Filter for piping data through an external process""" + + __kind__ = 'system' + + def filter(self, data, subfilter=None): + if subfilter is None: + raise ValueError('The system filter needs a command') + + proc = subprocess.Popen( + subfilter, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + (stdout, stderr) = proc.communicate(data.encode()) + + if proc.returncode != 0: + raise RuntimeError( + "system filter returned non-zero exit status %d; stderr:\n" + % proc.returncode + + stderr.decode() + ) + + return stdout.decode() + ''; }; }