{tv 2 => krebs 3} backup

This commit is contained in:
tv 2015-12-28 19:43:31 +01:00
parent b17e484d61
commit 5a9ccbef0a
7 changed files with 333 additions and 242 deletions

286
krebs/3modules/backup.nix Normal file
View File

@ -0,0 +1,286 @@
{ config, lib, pkgs, ... }:
with lib;
let
out = {
options.krebs.backup = api;
config = mkIf cfg.enable imp;
};
cfg = config.krebs.backup;
api = {
enable = mkEnableOption "krebs.backup" // { default = true; };
plans = mkOption {
default = {};
type = types.attrsOf (types.submodule ({
# TODO enable = mkEnableOption "TODO" // { default = true; };
options = {
method = mkOption {
type = types.enum ["pull" "push"];
};
name = mkOption {
type = types.str;
};
src = mkOption {
type = types.krebs.file-location;
};
dst = mkOption {
type = types.krebs.file-location;
};
startAt = mkOption {
type = types.str;
};
snapshots = mkOption {
type = types.attrsOf (types.submodule {
options = {
format = mkOption {
type = types.str; # TODO date's +FORMAT
};
retain = mkOption {
type = types.nullOr types.int;
default = null; # null = retain all snapshots
};
};
});
};
};
}));
};
};
imp = {
users.groups.backup.gid = genid "backup";
users.users = {}
// {
root.openssh.authorizedKeys.keys =
map (plan: plan.dst.host.ssh.pubkey)
(filter isPullSrc (attrValues cfg.plans))
++
map (plan: plan.src.host.ssh.pubkey)
(filter isPushDst (attrValues cfg.plans))
;
}
;
systemd.services =
flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: {
name = "backup.${name}.pull";
value = makePullService plan;
})
//
flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: {
name = "backup.${name}.push";
value = makePushService plan;
})
;
};
isPushSrc = plan:
plan.method == "push" &&
plan.src.host.name == config.krebs.build.host.name;
isPullSrc = plan:
plan.method == "pull" &&
plan.src.host.name == config.krebs.build.host.name;
isPushDst = plan:
plan.method == "push" &&
plan.dst.host.name == config.krebs.build.host.name;
isPullDst = plan:
plan.method == "pull" &&
plan.dst.host.name == config.krebs.build.host.name;
# TODO push destination needs this in the dst.user's PATH
service-path = [
pkgs.coreutils
pkgs.gnused
pkgs.openssh
pkgs.rsync
pkgs.utillinux
];
# TODO if there is plan.user, then use its privkey
makePushService = plan: assert isPushSrc plan; {
path = service-path;
serviceConfig = {
ExecStart = push plan;
Type = "oneshot";
};
startAt = plan.startAt;
};
makePullService = plan: assert isPullDst plan; {
path = service-path;
serviceConfig = {
ExecStart = pull plan;
Type = "oneshot";
};
startAt = plan.startAt;
};
push = plan: let
# We use writeDashBin and return the absolute path so systemd will produce
# nice names in the log, i.e. without the Nix store hash.
out = "${main}/bin/${main.name}";
main = writeDashBin "backup.${plan.name}.push" ''
set -efu
dst=${shell.escape plan.dst.path}
mkdir -m 0700 -p "$dst"
exec flock -n "$dst" ${critical-section}
'';
critical-section = writeDash "backup.${plan.name}.push.critical-section" ''
# TODO check if there is a previous
set -efu
identity=${shell.escape plan.src.host.ssh.privkey.path}
src=${shell.escape plan.src.path}
dst_target=${shell.escape "root@${getFQDN plan.dst.host}"}
dst_path=${shell.escape plan.dst.path}
dst=$dst_target:$dst_path
# Export NOW so runtime of rsync doesn't influence snapshot naming.
export NOW
NOW=$(date +%s)
echo >&2 "update snapshot: current; $src -> $dst"
rsync >&2 \
-aAXF --delete \
-e "ssh -F /dev/null -i $identity" \
--rsync-path ${shell.escape
"mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \
--link-dest="$dst_path/current" \
"$src/" \
"$dst/.partial"
exec ssh -F /dev/null \
-i "$identity" \
"$dst_target" \
-T \
env NOW="$NOW" /bin/sh < ${remote-snapshot}
EOF
'';
remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" ''
set -efu
dst=${shell.escape plan.dst.path}
if test -e "$dst/current"; then
mv "$dst/current" "$dst/.previous"
fi
mv "$dst/.partial" "$dst/current"
rm -fR "$dst/.previous"
echo >&2
(${(take-snapshots plan).text})
'';
in out;
# TODO admit plan.dst.user and its ssh identity
pull = plan: let
# We use writeDashBin and return the absolute path so systemd will produce
# nice names in the log, i.e. without the Nix store hash.
out = "${main}/bin/${main.name}";
main = writeDashBin "backup.${plan.name}.pull" ''
set -efu
dst=${shell.escape plan.dst.path}
mkdir -m 0700 -p "$dst"
exec flock -n "$dst" ${critical-section}
'';
critical-section = writeDash "backup.${plan.name}.pull.critical-section" ''
# TODO check if there is a previous
set -efu
identity=${shell.escape plan.dst.host.ssh.privkey.path}
src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"}
dst=${shell.escape plan.dst.path}
# Export NOW so runtime of rsync doesn't influence snapshot naming.
export NOW
NOW=$(date +%s)
echo >&2 "update snapshot: current; $dst <- $src"
mkdir -m 0700 -p ${shell.escape plan.dst.path}
rsync >&2 \
-aAXF --delete \
-e "ssh -F /dev/null -i $identity" \
--link-dest="$dst/current" \
"$src/" \
"$dst/.partial"
mv "$dst/current" "$dst/.previous"
mv "$dst/.partial" "$dst/current"
rm -fR "$dst/.previous"
echo >&2
exec ${take-snapshots plan}
'';
in out;
take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" ''
set -efu
NOW=''${NOW-$(date +%s)}
dst=${shell.escape plan.dst.path}
snapshot() {(
: $ns $format $retain
name=$(date --date="@$NOW" +"$format")
if ! test -e "$dst/$ns/$name"; then
echo >&2 "create snapshot: $ns/$name"
mkdir -m 0700 -p "$dst/$ns"
rsync >&2 \
-aAXF --delete \
--link-dest="$dst/current" \
"$dst/current/" \
"$dst/$ns/.partial.$name"
mv "$dst/$ns/.partial.$name" "$dst/$ns/$name"
echo >&2
fi
case $retain in
([0-9]*)
delete_from=$(($retain + 1))
ls -r "$dst/$ns" \
| sed -n "$delete_from,\$p" \
| while read old_name; do
echo >&2 "delete snapshot: $ns/$old_name"
rm -fR "$dst/$ns/$old_name"
done
;;
(ALL)
:
;;
esac
)}
${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }:
toString (map shell.escape [
"ns=${ns}"
"format=${format}"
"retain=${if retain == null then "ALL" else toString retain}"
"snapshot"
]))
plan.snapshots)}
'';
# TODO getFQDN: admit hosts in other domains
getFQDN = host: "${host.name}.${config.krebs.search-domain}";
writeDash = name: text: pkgs.writeScript name ''
#! ${pkgs.dash}/bin/dash
${text}
'';
writeDashBin = name: text: pkgs.writeTextFile {
executable = true;
destination = "/bin/${name}";
name = name;
text = ''
#! ${pkgs.dash}/bin/dash
${text}
'';
};
in out

View File

@ -7,6 +7,7 @@ let
out = { out = {
imports = [ imports = [
./apt-cacher-ng.nix ./apt-cacher-ng.nix
./backup.nix
./bepasty-server.nix ./bepasty-server.nix
./build.nix ./build.nix
./current.nix ./current.nix

View File

@ -177,4 +177,21 @@ types // rec {
addr6 = str; addr6 = str;
hostname = str; hostname = str;
label = str; label = str;
krebs.file-location = types.submodule {
options = {
# TODO user
host = mkOption {
type = host;
};
# TODO merge with ssl.privkey.path
path = mkOption {
type = types.either types.path types.str;
apply = x: {
path = toString x;
string = x;
}.${typeOf x};
};
};
};
} }

View File

@ -7,7 +7,6 @@ with lib;
krebs.build.target = "root@cd.internet"; krebs.build.target = "root@cd.internet";
imports = [ imports = [
../2configs/backup.nix
../2configs/hw/CAC-Developer-2.nix ../2configs/hw/CAC-Developer-2.nix
../2configs/fs/CAC-CentOS-7-64bit.nix ../2configs/fs/CAC-CentOS-7-64bit.nix
#../2configs/consul-server.nix #../2configs/consul-server.nix

View File

@ -9,7 +9,6 @@ with lib;
"7ae05edcdd14f6ace83ead9bf0d114e97c89a83a"; "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a";
imports = [ imports = [
../2configs/backup.nix # TODO
../2configs/hw/x220.nix ../2configs/hw/x220.nix
#../2configs/consul-client.nix #../2configs/consul-client.nix
../2configs/git.nix ../2configs/git.nix

View File

@ -1,38 +1,22 @@
{ config, lib, pkgs, ... }: { config, lib, ... }:
with lib; with lib;
let {
# Users that are allowed to connect to the backup user. krebs.backup.plans = addNames {
# Note: the user must own a push plan destination otherwise no rsync.
backup-users = [
config.krebs.users.tv
];
## TODO parse.file-location admit user
## loc has the form <host-name>:<abs-path>
#parse.file-location = loc: let
# parts = splitString ":" loc;
# host-name = head parts;
# path = concatStringsSep ":" (tail parts);
#in {
# type = "types.krebs.file-location";
# host = config.krebs.hosts.${host-name};
# path = path;
#};
# TODO assert plan.dst.path & co
plans = with config.krebs.users; with config.krebs.hosts; addNames {
xu-test-cd = { xu-test-cd = {
method = "push"; method = "push";
#src = parse.file-location xu:/tmp/xu-test;
#dst = parse.file-location cd:/krebs/backup/xu-test; src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; };
src = { user = tv; host = xu; path = "/tmp/xu-test"; }; dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; };
dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; };
startAt = "0,6,12,18:00"; #startAt = "0,6,12,18:00";
retain = { startAt = "minutely";
hourly = 4; # sneakily depends on startAt snapshots = {
daily = 7; minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; };
weekly = 4; hourly = { format = "%Y-%m-%dT%H"; retain = 4; };
monthly = 3; daily = { format = "%Y-%m-%d"; retain = 7; };
weekly = { format = "%YW%W"; retain = 4; };
monthly = { format = "%Y-%m"; retain = 12; };
yearly = { format = "%Y"; };
}; };
}; };
#xu-test-wu = { #xu-test-wu = {
@ -41,214 +25,18 @@ let
#}; #};
cd-test-xu = { cd-test-xu = {
method = "pull"; method = "pull";
#src = parse.file-location cd:/tmp/cd-test; src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; };
#dst = parse.file-location xu:/bku/cd-test; dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; };
src = { user = tv; host = cd; path = "/tmp/cd-test"; }; startAt = "minutely";
dst = { user = tv; host = xu; path = "/bku/cd-test"; }; snapshots = {
minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; };
hourly = { format = "%Y-%m-%dT%H"; retain = 4; };
daily = { format = "%Y-%m-%d"; retain = 7; };
weekly = { format = "%YW%W"; retain = 4; };
monthly = { format = "%Y-%m"; retain = 12; };
yearly = { format = "%Y"; };
};
}; };
}; };
out = {
#options.krebs.backup = api;
config = imp;
};
imp = {
users.groups.backup.gid = genid "backup";
users.users = map makeUser (filter isPushDst (attrValues plans));
systemd.services =
flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: {
name = "backup.${name}";
value = makePushService plan;
});
};
# TODO getFQDN: admit hosts in other domains
getFQDN = host: "${host.name}.${config.krebs.search-domain}";
isPushSrc = plan:
plan.method == "push" &&
plan.src.host.name == config.krebs.build.host.name;
makePushService = plan: assert isPushSrc plan; {
startAt = plan.startAt;
serviceConfig.ExecStart = writeSh plan "rsync" ''
exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [
"-a"
"-e"
"${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}"
"${plan.src.path}"
"${plan.name}@${getFQDN plan.dst.host}::push"
]}
'';
};
isPushDst = plan:
plan.method == "push" &&
plan.dst.host.name == config.krebs.build.host.name;
makeUser = plan: assert isPushDst plan; rec {
name = plan.name;
uid = genid name;
group = config.users.groups.backup.name;
home = plan.dst.path;
createHome = true;
shell = "${writeSh plan "shell" ''
case $2 in
'rsync --server --daemon .')
exec ${backup.rsync plan [ "--server" "--daemon" "." ]}
;;
''')
echo "ERROR: no command specified" >&2
exit 23
;;
*)
echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2
exit 23
;;
esac
''}";
openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ];
};
rsync = plan: args: writeSh plan "rsync" ''
install -v -m 0700 -d ${plan.dst.path}/push >&2
install -v -m 0700 -d ${plan.dst.path}/list >&2
${pkgs.rsync}/bin/rsync \
--config=${backup.rsyncd-conf plan {
post-xfer = writeSh plan "rsyncd.post-xfer" ''
case $RSYNC_EXIT_STATUS in 0)
exec ${backup.rsnapshot plan {
preexec = writeSh plan "rsnapshot.preexec" ''
touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
'';
postexec = writeSh plan "rsnapshot.postexec" ''
rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
'';
}}
esac
'';
}} \
${toString (map shell.escape args)}
fail=0
for i in monthly weekly daily hourly; do
if test -e ${plan.dst.path}/rsnapshot.$i; then
rm ${plan.dst.path}/rsnapshot.$i
echo "ERROR: $i snapshot failed" >&2
fail=1
fi
done
if test $fail != 0; then
exit -1
fi
'';
rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" ''
fake super = yes
use chroot = no
lock file = ${plan.dst.path}/rsyncd.lock
[push]
max connections = 1
path = ${plan.dst.path}/push
write only = yes
read only = no
post-xfer exec = ${conf.post-xfer}
[list]
path = ${plan.dst.path}/list
read only = yes
write only = no
'';
rsnapshot = plan: conf: writeSh plan "rsnapshot" ''
rsnapshot() {
${pkgs.proot}/bin/proot \
-b /bin \
-b /nix \
-b /run/current-system \
-b ${plan.dst.path} \
-r ${plan.dst.path} \
-w / \
${pkgs.rsnapshot}/bin/rsnapshot \
-c ${pkgs.writeText "${plan.name}.rsnapshot.conf" ''
config_version 1.2
snapshot_root ${plan.dst.path}/list
cmd_cp ${pkgs.coreutils}/bin/cp
cmd_du ${pkgs.coreutils}/bin/du
#cmd_rm ${pkgs.coreutils}/bin/rm
cmd_rsync ${pkgs.rsync}/bin/rsync
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
cmd_preexec ${conf.preexec}
cmd_postexec ${conf.postexec}
retain hourly 4
retain daily 7
retain weekly 4
retain monthly 3
lockfile ${plan.dst.path}/rsnapshot.pid
link_dest 1
backup /push ./
verbose 4
''} \
"$@"
} }
cd ${plan.dst.path}/list/
now=$(date +%s)
is_older_than() {
test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \
-ge $2
}
# TODO report stale snapshots
# i.e. there are $interval.$i > $interval.$max
hour_s=3600
day_s=86400
week_s=604800
month_s=2419200 # 4 weeks
set --
if test -e weekly.3 && is_older_than monthly.0 $month_s; then
set -- "$@" monthly
fi
if test -e daily.6 && is_older_than weekly.0 $week_s; then
set -- "$@" weekly
fi
if test -e hourly.3 && is_older_than daily.0 $day_s; then
set -- "$@" daily
fi
if is_older_than hourly.0 $hour_s; then
set -- "$@" hourly
fi
if test $# = 0; then
echo "taking no snapshots" >&2
else
echo "taking snapshots: $@" >&2
fi
export RSNAPSHOT_INTERVAL
for RSNAPSHOT_INTERVAL; do
rsnapshot "$RSNAPSHOT_INTERVAL"
done
'';
writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" ''
#! ${pkgs.dash}/bin/dash
set -efu
export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])}
${text}
'';
in out

View File

@ -28,6 +28,7 @@ with lib;
imports = [ imports = [
<secrets> <secrets>
./backup.nix
./vim.nix ./vim.nix
{ {
# stockholm dependencies # stockholm dependencies