stockholm/krebs/3modules/sync-containers.nix
2023-06-21 14:47:04 +02:00

244 lines
8.1 KiB
Nix

{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.krebs.sync-containers;
paths = cname: {
plain = "/var/lib/containers/${cname}/var/state";
ecryptfs = "${cfg.dataLocation}/${cname}/ecryptfs";
securefs = "${cfg.dataLocation}/${cname}/securefs";
luksfile = "${cfg.dataLocation}/${cname}/luksfile";
};
init = cname: {
plain = ''
echo 'no need for init'
'';
ecryptfs = ''
${pkgs.ecrypt}/bin/ecrypt init ${cfg.dataLocation}/${cname}/ecryptfs /var/lib/containers/${cname}/var/state
'';
securefs = ''
${pkgs.securefs}/bin/securefs create --format 3 ${cfg.dataLocation}/${cname}/securefs
'';
luksfile = ''
${pkgs.coreutils}/bin/truncate -s 10G '${(paths cname).luksfile}/fs.luks'
${pkgs.cryptsetup}/bin/cryptsetup luksFormat '${(paths cname).luksfile}/fs.luks'
${pkgs.cryptsetup}/bin/cryptsetup luksOpen '${(paths cname).luksfile}/fs.luks' 'luksfile-${cname}'
${pkgs.xfsprogs}/bin/mkfs.xfs '/dev/mapper/luksfile-${cname}'
'';
};
start = cname: {
plain = ''
:
'';
ecryptfs = ''
if [ -e ${cfg.dataLocation}/${cname}/ecryptfs/.cfg.json ]; then
if ! mount | grep -q '${cfg.dataLocation}/${cname}/ecryptfs on /var/lib/containers/${cname}/var/state type ecryptfs'; then
${pkgs.ecrypt}/bin/ecrypt mount ${cfg.dataLocation}/${cname}/ecryptfs /var/lib/containers/${cname}/var/state
fi
else
echo 'please run init-${cname} first'
exit 1
fi
'';
securefs = ''
## check if FS was initialized first
if ! ${pkgs.mount}/bin/mount | grep -q '^securefs on /var/lib/containers/${cname}/var/state type fuse.securefs'; then
${pkgs.securefs}/bin/securefs mount ${cfg.dataLocation}/${cname}/securefs /var/lib/containers/${cname}/var/state -b -o allow_other -o default_permissions
fi
'';
luksfile = ''
mkdir -p /var/lib/containers/${cname}/var/state
if ! test -e /dev/mapper/luksfile-${cname}; then
${pkgs.cryptsetup}/bin/cryptsetup luksOpen '${(paths cname).luksfile}/fs.luks' 'luksfile-${cname}'
fi
if ! ${pkgs.mount}/bin/mount | grep -q '^/dev/mapper/luksfile-${cname} on /var/lib/containers/${cname}/var/state'; then
mount '/dev/mapper/luksfile-${cname}' '/var/lib/containers/${cname}/var/state'
fi
'';
};
stop = cname: {
plain = ''
:
'';
ecryptfs = ''
${pkgs.ecrypt}/bin/ecrypt unmount ${cfg.dataLocation}/${cname}/ecryptfs /var/lib/containers/${cname}/var/state
'';
securefs = ''
umount /var/lib/containers/${cname}/var/state
'';
luksfile = ''
umount /var/lib/containers/${cname}/var/state
${pkgs.cryptsetup}/bin/cryptsetup luksClose luksfile-${cname}
'';
};
in {
options.krebs.sync-containers = {
dataLocation = mkOption {
description = ''
location where the encrypted sync-containers lie around
'';
default = "/var/lib/sync-containers";
type = types.absolute-pathname;
};
containers = mkOption {
type = types.attrsOf (types.submodule ({ config, ... }: {
options = {
name = mkOption {
description = ''
name of the container
'';
default = config._module.args.name;
type = types.str;
};
peers = mkOption {
description = ''
syncthing peers to share this container with
'';
default = [];
type = types.listOf types.str;
};
format = mkOption {
description = ''
file system encrption format of the container
'';
type = types.enum [ "plain" "ecryptfs" "securefs" "luksfile" ];
};
};
}));
default = {};
};
};
config = mkIf (cfg.containers != {}) {
programs.fuse.userAllowOther = true;
# allow syncthing to enter /var/lib/containers
system.activationScripts.containers-enter = mkDefault ''
${pkgs.coreutils}/bin/chmod a+x /var/lib/containers || :
'';
services.syncthing.folders = (mapAttrs' (_: ctr: nameValuePair "${(paths ctr.name).${ctr.format}}" ({
devices = ctr.peers;
ignorePerms = false;
})) cfg.containers);
krebs.acl = mapAttrs' (_: ctr: nameValuePair "${(paths ctr.name).${ctr.format}}" {
"u:syncthing:rX".parents = true;
"u:syncthing:rwX" = {};
}) cfg.containers;
systemd.services = mapAttrs' (n: ctr: nameValuePair "containers@${ctr.name}" ({
reloadIfChanged = mkForce false;
})) cfg.containers;
containers = mapAttrs' (n: ctr: nameValuePair ctr.name ({
config = { ... }: {
environment.systemPackages = [
pkgs.dhcpcd
pkgs.git
pkgs.jq
];
networking.useDHCP = mkForce true;
system.activationScripts.fuse = {
text = ''
${pkgs.coreutils}/bin/mknod /dev/fuse c 10 229
'';
deps = [];
};
};
allowedDevices = [
{ modifier = "rwm"; node = "/dev/fuse"; }
];
autoStart = false;
enableTun = true;
privateNetwork = true;
hostBridge = "ctr0";
})) cfg.containers;
networking.networkmanager.unmanaged = [ "ctr0" ];
networking.bridges.ctr0.interfaces = [];
networking.interfaces.ctr0.ipv4.addresses = [{
address = "10.233.0.1";
prefixLength = 24;
}];
# networking.nat = {
# enable = true;
# externalInterface = lib.mkDefault "et0";
# internalInterfaces = [ "ctr0" ];
# };
services.dhcpd4 = {
enable = true;
interfaces = [ "ctr0" ];
extraConfig = ''
option subnet-mask 255.255.255.0;
option routers 10.233.0.1;
# option domain-name-servers 8.8.8.8; # TODO configure dns server
subnet 10.233.0.0 netmask 255.255.255.0 {
range 10.233.0.10 10.233.0.250;
}
'';
};
users.users.root.packages = flatten (mapAttrsToList (n: ctr: [
(pkgs.writeDashBin "init-${ctr.name}" ''
set -euf
set -x
mkdir -p /var/lib/containers/${ctr.name}/var/state
STATE=$(/run/current-system/sw/bin/nixos-container status ${ctr.name})
if [ "$STATE" = 'up' ]; then
/run/current-system/sw/bin/nixos-container stop ${ctr.name}
fi
${(init ctr.name).${ctr.format}}
${(start ctr.name).${ctr.format}}
/run/current-system/sw/bin/nixos-container start ${ctr.name}
/run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "deploy-${ctr.name}" ''
set -x
mkdir -p /var/state/var_src
ln -sfTr /var/state/var_src /var/src
touch /etc/NIXOS
''}
target_ip=$(/run/current-system/sw/bin/nixos-container run ${ctr.name} -- ip -j a s eth0 | jq -r '.[].addr_info[] | select(.family=="inet") | .local')
echo "deploy to $target_ip"
'')
(pkgs.writeDashBin "start-${ctr.name}" ''
set -euf
set -x
mkdir -p /var/lib/containers/${ctr.name}/var/state
${(start ctr.name).${ctr.format}}
STATE=$(/run/current-system/sw/bin/nixos-container status ${ctr.name})
if [ "$STATE" = 'down' ]; then
/run/current-system/sw/bin/nixos-container start ${ctr.name}
fi
/run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "deploy-${ctr.name}" ''
set -x
mkdir -p /var/state/var_src
ln -sfTr /var/state/var_src /var/src
touch /etc/NIXOS
''}
if [ -h /var/lib/containers/${ctr.name}/var/src/nixos-config ] && (! ping -c1 -q -w5 ${ctr.name}.r); then
/run/current-system/sw/bin/nixos-container run ${ctr.name} -- nixos-rebuild -I /var/src switch
else
echo 'no nixos config, or target already online, bailing out'
${(stop ctr.name).${ctr.format}}
/run/current-system/sw/bin/nixos-container stop ${ctr.name}
fi
'')
(pkgs.writeDashBin "stop-${ctr.name}" ''
set -euf
/run/current-system/sw/bin/nixos-container stop ${ctr.name}
${(stop ctr.name).${ctr.format}}
'')
]) cfg.containers);
};
}