{tv 2 => krebs 3} backup

krebs/3modules/backup.nix

@@ -0,0 +1,286 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  out = {
+    options.krebs.backup = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  cfg = config.krebs.backup;
+
+  api = {
+    enable = mkEnableOption "krebs.backup" // { default = true; };
+    plans = mkOption {
+      default = {};
+      type = types.attrsOf (types.submodule ({
+        # TODO enable = mkEnableOption "TODO" // { default = true; };
+        options = {
+          method = mkOption {
+            type = types.enum ["pull" "push"];
+          };
+          name = mkOption {
+            type = types.str;
+          };
+          src = mkOption {
+            type = types.krebs.file-location;
+          };
+          dst = mkOption {
+            type = types.krebs.file-location;
+          };
+          startAt = mkOption {
+            type = types.str;
+          };
+          snapshots = mkOption {
+            type = types.attrsOf (types.submodule {
+              options = {
+                format = mkOption {
+                  type = types.str; # TODO date's +FORMAT
+                };
+                retain = mkOption {
+                  type = types.nullOr types.int;
+                  default = null; # null = retain all snapshots
+                };
+              };
+            });
+          };
+        };
+      }));
+    };
+  };
+
+  imp = {
+    users.groups.backup.gid = genid "backup";
+    users.users = {}
+      // {
+        root.openssh.authorizedKeys.keys =
+          map (plan: plan.dst.host.ssh.pubkey)
+              (filter isPullSrc (attrValues cfg.plans))
+          ++
+          map (plan: plan.src.host.ssh.pubkey)
+              (filter isPushDst (attrValues cfg.plans))
+          ;
+      }
+      ;
+    systemd.services =
+      flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: {
+        name = "backup.${name}.pull";
+        value = makePullService plan;
+      })
+      //
+      flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: {
+        name = "backup.${name}.push";
+        value = makePushService plan;
+      })
+      ;
+  };
+
+  isPushSrc = plan:
+    plan.method == "push" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPullSrc = plan:
+    plan.method == "pull" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPushDst = plan:
+    plan.method == "push" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  isPullDst = plan:
+    plan.method == "pull" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  # TODO push destination needs this in the dst.user's PATH
+  service-path = [
+    pkgs.coreutils
+    pkgs.gnused
+    pkgs.openssh
+    pkgs.rsync
+    pkgs.utillinux
+  ];
+
+  # TODO if there is plan.user, then use its privkey
+  makePushService = plan: assert isPushSrc plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = push plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  makePullService = plan: assert isPullDst plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = pull plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  push = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.push" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.push.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.src.host.ssh.privkey.path}
+      src=${shell.escape plan.src.path}
+      dst_target=${shell.escape "root@${getFQDN plan.dst.host}"}
+      dst_path=${shell.escape plan.dst.path}
+      dst=$dst_target:$dst_path
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $src -> $dst"
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --rsync-path ${shell.escape
+            "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \
+          --link-dest="$dst_path/current" \
+          "$src/" \
+          "$dst/.partial"
+
+      exec ssh -F /dev/null \
+          -i "$identity" \
+          "$dst_target" \
+          -T \
+          env NOW="$NOW" /bin/sh < ${remote-snapshot}
+      EOF
+    '';
+
+    remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      if test -e "$dst/current"; then
+        mv "$dst/current" "$dst/.previous"
+      fi
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      (${(take-snapshots plan).text})
+    '';
+
+  in out;
+
+  # TODO admit plan.dst.user and its ssh identity
+  pull = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.pull" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.pull.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.dst.host.ssh.privkey.path}
+      src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"}
+      dst=${shell.escape plan.dst.path}
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $dst <- $src"
+      mkdir -m 0700 -p ${shell.escape plan.dst.path}
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --link-dest="$dst/current" \
+          "$src/" \
+          "$dst/.partial"
+      mv "$dst/current" "$dst/.previous"
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      exec ${take-snapshots plan}
+    '';
+  in out;
+
+  take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" ''
+    set -efu
+    NOW=''${NOW-$(date +%s)}
+    dst=${shell.escape plan.dst.path}
+
+    snapshot() {(
+      : $ns $format $retain
+      name=$(date --date="@$NOW" +"$format")
+      if ! test -e "$dst/$ns/$name"; then
+        echo >&2 "create snapshot: $ns/$name"
+        mkdir -m 0700 -p "$dst/$ns"
+        rsync >&2 \
+            -aAXF --delete \
+            --link-dest="$dst/current" \
+            "$dst/current/" \
+            "$dst/$ns/.partial.$name"
+        mv "$dst/$ns/.partial.$name" "$dst/$ns/$name"
+        echo >&2
+      fi
+      case $retain in
+        ([0-9]*)
+          delete_from=$(($retain + 1))
+          ls -r "$dst/$ns" \
+            | sed -n "$delete_from,\$p" \
+            | while read old_name; do
+                echo >&2 "delete snapshot: $ns/$old_name"
+                rm -fR "$dst/$ns/$old_name"
+              done
+          ;;
+        (ALL)
+          :
+          ;;
+      esac
+    )}
+
+    ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }:
+      toString (map shell.escape [
+        "ns=${ns}"
+        "format=${format}"
+        "retain=${if retain == null then "ALL" else toString retain}"
+        "snapshot"
+      ]))
+      plan.snapshots)}
+  '';
+
+  # TODO getFQDN: admit hosts in other domains
+  getFQDN = host: "${host.name}.${config.krebs.search-domain}";
+
+  writeDash = name: text: pkgs.writeScript name ''
+    #! ${pkgs.dash}/bin/dash
+    ${text}
+  '';
+
+  writeDashBin = name: text: pkgs.writeTextFile {
+    executable = true;
+    destination = "/bin/${name}";
+    name = name;
+    text = ''
+      #! ${pkgs.dash}/bin/dash
+      ${text}
+    '';
+  };
+
+in out

krebs/3modules/default.nix

@@ -7,6 +7,7 @@ let
   out = {
     imports = [
       ./apt-cacher-ng.nix
+      ./backup.nix
       ./bepasty-server.nix
       ./build.nix
       ./current.nix

krebs/4lib/types.nix

@@ -177,4 +177,21 @@ types // rec {
   addr6 = str;
   hostname = str;
   label = str;
+
+  krebs.file-location = types.submodule {
+    options = {
+      # TODO user
+      host = mkOption {
+        type = host;
+      };
+      # TODO merge with ssl.privkey.path
+      path = mkOption {
+        type = types.either types.path types.str;
+        apply = x: {
+          path = toString x;
+          string = x;
+        }.${typeOf x};
+      };
+    };
+  };
 }

tv/1systems/cd.nix

@@ -7,7 +7,6 @@ with lib;
   krebs.build.target = "root@cd.internet";
 
   imports = [
-    ../2configs/backup.nix
     ../2configs/hw/CAC-Developer-2.nix
     ../2configs/fs/CAC-CentOS-7-64bit.nix
     #../2configs/consul-server.nix

tv/1systems/xu.nix

@@ -9,7 +9,6 @@ with lib;
     "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a";
 
   imports = [
-    ../2configs/backup.nix # TODO
     ../2configs/hw/x220.nix
     #../2configs/consul-client.nix
     ../2configs/git.nix

tv/2configs/backup.nix

@@ -1,38 +1,22 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 with lib;
-let
+{
-  # Users that are allowed to connect to the backup user.
+  krebs.backup.plans = addNames {
-  # Note: the user must own a push plan destination otherwise no rsync.
-  backup-users = [
-    config.krebs.users.tv
-  ];
-
-  ## TODO parse.file-location admit user
-  ## loc has the form <host-name>:<abs-path>
-  #parse.file-location = loc: let
-  #  parts = splitString ":" loc;
-  #  host-name = head parts;
-  #  path = concatStringsSep ":" (tail parts);
-  #in {
-  #  type = "types.krebs.file-location";
-  #  host = config.krebs.hosts.${host-name};
-  #  path = path;
-  #};
-
-  # TODO assert plan.dst.path & co
-  plans = with config.krebs.users; with config.krebs.hosts; addNames {
     xu-test-cd = {
       method = "push";
-      #src = parse.file-location xu:/tmp/xu-test;
+
-      #dst = parse.file-location cd:/krebs/backup/xu-test;
+      src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; };
-      src = { user = tv; host = xu; path = "/tmp/xu-test"; };
+      dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; };
-      dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; };
+
-      startAt = "0,6,12,18:00";
+      #startAt = "0,6,12,18:00";
-      retain = {
+      startAt = "minutely";
-        hourly = 4; # sneakily depends on startAt
+      snapshots = {
-        daily = 7;
+        minutely = { format = "%Y-%m-%dT%H:%M"; retain =  5; };
-        weekly = 4;
+        hourly   = { format = "%Y-%m-%dT%H";    retain =  4; };
-        monthly = 3;
+        daily    = { format = "%Y-%m-%d";       retain =  7; };
+        weekly   = { format = "%YW%W";          retain =  4; };
+        monthly  = { format = "%Y-%m";          retain = 12; };
+        yearly   = { format = "%Y";                          };
       };
     };
     #xu-test-wu = {
@@ -41,214 +25,18 @@ let
     #};
     cd-test-xu = {
       method = "pull";
-      #src = parse.file-location cd:/tmp/cd-test;
+      src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; };
-      #dst = parse.file-location xu:/bku/cd-test;
+      dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; };
-      src = { user = tv; host = cd; path = "/tmp/cd-test"; };
+      startAt = "minutely";
-      dst = { user = tv; host = xu; path = "/bku/cd-test"; };
+      snapshots = {
+        minutely = { format = "%Y-%m-%dT%H:%M"; retain =  5; };
+        hourly   = { format = "%Y-%m-%dT%H";    retain =  4; };
+        daily    = { format = "%Y-%m-%d";       retain =  7; };
+        weekly   = { format = "%YW%W";          retain =  4; };
+        monthly  = { format = "%Y-%m";          retain = 12; };
+        yearly   = { format = "%Y";                          };
+      };
     };
 
   };
-
-  out = {
-    #options.krebs.backup = api;
-    config = imp;
-  };
-
-  imp = {
-    users.groups.backup.gid = genid "backup";
-    users.users = map makeUser (filter isPushDst (attrValues plans));
-    systemd.services =
-      flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: {
-        name = "backup.${name}";
-        value = makePushService plan;
-      });
-  };
-
-
-  # TODO getFQDN: admit hosts in other domains
-  getFQDN = host: "${host.name}.${config.krebs.search-domain}";
-
-  isPushSrc = plan:
-    plan.method == "push" &&
-    plan.src.host.name == config.krebs.build.host.name;
-
-  makePushService = plan: assert isPushSrc plan; {
-    startAt = plan.startAt;
-    serviceConfig.ExecStart = writeSh plan "rsync" ''
-      exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [
-        "-a"
-        "-e"
-        "${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}"
-        "${plan.src.path}"
-        "${plan.name}@${getFQDN plan.dst.host}::push"
-      ]}
-    '';
-  };
-
-  isPushDst = plan:
-    plan.method == "push" &&
-    plan.dst.host.name == config.krebs.build.host.name;
-
-  makeUser = plan: assert isPushDst plan; rec {
-    name = plan.name;
-    uid = genid name;
-    group = config.users.groups.backup.name;
-    home = plan.dst.path;
-    createHome = true;
-    shell = "${writeSh plan "shell" ''
-      case $2 in
-        'rsync --server --daemon .')
-          exec ${backup.rsync plan [ "--server" "--daemon" "." ]}
-          ;;
-        ''')
-          echo "ERROR: no command specified" >&2
-          exit 23
-          ;;
-        *)
-          echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2
-          exit 23
-          ;;
-      esac
-    ''}";
-    openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ];
-  };
-
-  rsync = plan: args: writeSh plan "rsync" ''
-    install -v -m 0700 -d ${plan.dst.path}/push >&2
-    install -v -m 0700 -d ${plan.dst.path}/list >&2
-
-    ${pkgs.rsync}/bin/rsync \
-        --config=${backup.rsyncd-conf plan {
-          post-xfer = writeSh plan "rsyncd.post-xfer" ''
-            case $RSYNC_EXIT_STATUS in 0)
-              exec ${backup.rsnapshot plan {
-                preexec = writeSh plan "rsnapshot.preexec" ''
-                  touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
-                '';
-                postexec = writeSh plan "rsnapshot.postexec" ''
-                  rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
-                '';
-              }}
-            esac
-          '';
-        }} \
-        ${toString (map shell.escape args)}
-
-    fail=0
-    for i in monthly weekly daily hourly; do
-      if test -e ${plan.dst.path}/rsnapshot.$i; then
-        rm ${plan.dst.path}/rsnapshot.$i
-        echo "ERROR: $i snapshot failed" >&2
-        fail=1
-      fi
-    done
-    if test $fail != 0; then
-      exit -1
-    fi
-  '';
-
-  rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" ''
-    fake super = yes
-    use chroot = no
-    lock file = ${plan.dst.path}/rsyncd.lock
-
-    [push]
-    max connections = 1
-    path = ${plan.dst.path}/push
-    write only = yes
-    read only = no
-    post-xfer exec = ${conf.post-xfer}
-
-    [list]
-    path = ${plan.dst.path}/list
-    read only = yes
-    write only = no
-  '';
-
-  rsnapshot = plan: conf: writeSh plan "rsnapshot" ''
-    rsnapshot() {
-      ${pkgs.proot}/bin/proot \
-          -b /bin \
-          -b /nix \
-          -b /run/current-system \
-          -b ${plan.dst.path} \
-          -r ${plan.dst.path} \
-          -w / \
-          ${pkgs.rsnapshot}/bin/rsnapshot \
-              -c ${pkgs.writeText "${plan.name}.rsnapshot.conf" ''
-                config_version	1.2
-                snapshot_root	${plan.dst.path}/list
-                cmd_cp	${pkgs.coreutils}/bin/cp
-                cmd_du	${pkgs.coreutils}/bin/du
-                #cmd_rm	${pkgs.coreutils}/bin/rm
-                cmd_rsync	${pkgs.rsync}/bin/rsync
-                cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-                cmd_preexec	${conf.preexec}
-                cmd_postexec	${conf.postexec}
-                retain	hourly	4
-                retain	daily	7
-                retain	weekly	4
-                retain	monthly	3
-                lockfile	${plan.dst.path}/rsnapshot.pid
-                link_dest	1
-                backup	/push	./
-                verbose	4
-              ''} \
-          "$@"
 }
-
-    cd ${plan.dst.path}/list/
-
-    now=$(date +%s)
-    is_older_than() {
-      test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \
-       -ge $2
-    }
-
-    # TODO report stale snapshots
-    # i.e. there are $interval.$i > $interval.$max
-
-    hour_s=3600
-    day_s=86400
-    week_s=604800
-    month_s=2419200 # 4 weeks
-
-    set --
-
-    if test -e weekly.3 && is_older_than monthly.0 $month_s; then
-      set -- "$@" monthly
-    fi
-
-    if test -e daily.6 && is_older_than weekly.0 $week_s; then
-      set -- "$@" weekly
-    fi
-
-    if test -e hourly.3 && is_older_than daily.0 $day_s; then
-      set -- "$@" daily
-    fi
-
-    if is_older_than hourly.0 $hour_s; then
-      set -- "$@" hourly
-    fi
-
-
-    if test $# = 0; then
-      echo "taking no snapshots" >&2
-    else
-      echo "taking snapshots: $@" >&2
-    fi
-
-    export RSNAPSHOT_INTERVAL
-    for RSNAPSHOT_INTERVAL; do
-      rsnapshot "$RSNAPSHOT_INTERVAL"
-    done
-  '';
-
-  writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" ''
-    #! ${pkgs.dash}/bin/dash
-    set -efu
-    export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])}
-    ${text}
-  '';
-
-in out

tv/2configs/default.nix

@@ -28,6 +28,7 @@ with lib;
 
   imports = [
     <secrets>
+    ./backup.nix
     ./vim.nix
     {
       # stockholm dependencies