227 lines
8.0 KiB
Nix
227 lines
8.0 KiB
Nix
|
{ config, lib, pkgs, ... }: let
|
||
|
|
cfg = config.lass.sync-containers3;
|
||
|
|
slib = pkgs.stockholm.lib;
|
||
|
|
in {
|
||
|
|
options.lass.sync-containers3 = {
|
||
|
|
inContainer = {
|
||
|
|
enable = lib.mkEnableOption "container config for syncing";
|
||
|
|
pubkey = lib.mkOption {
|
||
|
|
type = lib.types.str; # TODO ssh key
|
||
|
|
};
|
||
|
|
};
|
||
|
|
containers = lib.mkOption {
|
||
|
|
default = {};
|
||
|
|
type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
|
||
|
|
options = {
|
||
|
|
name = lib.mkOption {
|
||
|
|
type = lib.types.str;
|
||
|
|
default = config._module.args.name;
|
||
|
|
};
|
||
|
|
sshKey = lib.mkOption {
|
||
|
|
type = slib.types.absolute-pathname;
|
||
|
|
};
|
||
|
|
luksKey = lib.mkOption {
|
||
|
|
type = slib.types.absolute-pathname;
|
||
|
|
default = config.sshKey;
|
||
|
|
};
|
||
|
|
ephemeral = lib.mkOption {
|
||
|
|
type = lib.types.bool;
|
||
|
|
default = false;
|
||
|
|
};
|
||
|
|
};
|
||
|
|
}));
|
||
|
|
};
|
||
|
|
};
|
||
|
|
config = lib.mkMerge [
|
||
|
|
(lib.mkIf (cfg.containers != {}) {
|
||
|
|
|
||
|
|
containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name {
|
||
|
|
config = {
|
||
|
|
environment.systemPackages = [
|
||
|
|
pkgs.dhcpcd
|
||
|
|
pkgs.git
|
||
|
|
pkgs.jq
|
||
|
|
];
|
||
|
|
networking.useDHCP = lib.mkForce true;
|
||
|
|
systemd.services.autoswitch = {
|
||
|
|
environment = {
|
||
|
|
NIX_REMOTE = "daemon";
|
||
|
|
};
|
||
|
|
wantedBy = [ "multi-user.target" ];
|
||
|
|
serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ''
|
||
|
|
set -efu
|
||
|
|
ln -frs /var/state/var_src /var/src
|
||
|
|
if test -e /var/src/nixos-config; then
|
||
|
|
/run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
|
||
|
|
fi
|
||
|
|
'';
|
||
|
|
unitConfig.X-StopOnRemoval = false;
|
||
|
|
};
|
||
|
|
};
|
||
|
|
autoStart = false;
|
||
|
|
enableTun = true;
|
||
|
|
ephemeral = ctr.ephemeral;
|
||
|
|
privateNetwork = true;
|
||
|
|
hostBridge = "ctr0";
|
||
|
|
bindMounts = {
|
||
|
|
"/etc/resolv.conf".hostPath = "/etc/resolv.conf";
|
||
|
|
"/var/lib/self/disk" = {
|
||
|
|
hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
|
||
|
|
isReadOnly = false;
|
||
|
|
};
|
||
|
|
"/var/state" = {
|
||
|
|
hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
|
||
|
|
isReadOnly = false;
|
||
|
|
};
|
||
|
|
};
|
||
|
|
}) cfg.containers;
|
||
|
|
|
||
|
|
systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [
|
||
|
|
{ "${ctr.name}_syncer" = {
|
||
|
|
path = with pkgs; [
|
||
|
|
coreutils
|
||
|
|
consul
|
||
|
|
rsync
|
||
|
|
openssh
|
||
|
|
systemd
|
||
|
|
];
|
||
|
|
startAt = "*:0/1";
|
||
|
|
serviceConfig = {
|
||
|
|
User = "${ctr.name}_container";
|
||
|
|
LoadCredential = [
|
||
|
|
"ssh_key:${ctr.sshKey}"
|
||
|
|
];
|
||
|
|
ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
|
||
|
|
set -efu
|
||
|
|
! systemctl is-active --quiet container@${ctr.name}.service
|
||
|
|
'';
|
||
|
|
ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
|
||
|
|
set -efux
|
||
|
|
consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" ''
|
||
|
|
set -efux
|
||
|
|
if ping -c 1 ${ctr.name}.r; then
|
||
|
|
touch "$HOME"/incomplete
|
||
|
|
rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk
|
||
|
|
rm "$HOME"/incomplete
|
||
|
|
fi
|
||
|
|
''}
|
||
|
|
'';
|
||
|
|
};
|
||
|
|
}; }
|
||
|
|
{ "${ctr.name}_scheduler" = {
|
||
|
|
wantedBy = [ "multi-user.target" ];
|
||
|
|
path = with pkgs; [
|
||
|
|
coreutils
|
||
|
|
consul
|
||
|
|
cryptsetup
|
||
|
|
mount
|
||
|
|
util-linux
|
||
|
|
systemd
|
||
|
|
retry
|
||
|
|
];
|
||
|
|
serviceConfig = let
|
||
|
|
containerDirectory = lib.removeSuffix "/%i" config.systemd.services."container@${ctr.name}".environment.root;
|
||
|
|
in {
|
||
|
|
Restart = "always";
|
||
|
|
RestartSec = "5s";
|
||
|
|
ExecStart = "${pkgs.consul}/bin/consul lock -verbose -monitor-retry 100 container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
|
||
|
|
set -efux
|
||
|
|
if test -e /var/lib/sync-containers3/${ctr.name}/incomplete; then
|
||
|
|
echo 'data is inconistent, start aborted'
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
trap ${pkgs.writers.writeDash "stop-${ctr.name}" ''
|
||
|
|
set -efux
|
||
|
|
/run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
|
||
|
|
umount /var/lib/sync-containers3/${ctr.name}/state || :
|
||
|
|
cryptsetup luksClose ${ctr.name} || :
|
||
|
|
''} INT TERM EXIT
|
||
|
|
consul kv put containers/${ctr.name}/host ${config.networking.hostName}
|
||
|
|
cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
|
||
|
|
mkdir -p /var/lib/sync-containers3/${ctr.name}/state
|
||
|
|
mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
|
||
|
|
/run/current-system/sw/bin/nixos-container start ${ctr.name}
|
||
|
|
set +x
|
||
|
|
until /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null; do sleep 5; done
|
||
|
|
while retry -t 5 -d 60 -- /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r > /dev/null; do sleep 5; done
|
||
|
|
echo "lost tinc connection to container, shutting down"
|
||
|
|
''}";
|
||
|
|
};
|
||
|
|
}; }
|
||
|
|
]) (lib.attrValues cfg.containers)));
|
||
|
|
|
||
|
|
systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" {
|
||
|
|
timerConfig = {
|
||
|
|
RandomizedDelaySec = 100;
|
||
|
|
};
|
||
|
|
}) cfg.containers;
|
||
|
|
|
||
|
|
users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" {
|
||
|
|
}) cfg.containers;
|
||
|
|
users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({
|
||
|
|
group = "container_${ctr.name}";
|
||
|
|
isNormalUser = true;
|
||
|
|
uid = slib.genid_uint31 "container_${ctr.name}";
|
||
|
|
home = "/var/lib/sync-containers3/${ctr.name}";
|
||
|
|
createHome = true;
|
||
|
|
homeMode = "705";
|
||
|
|
})) cfg.containers;
|
||
|
|
|
||
|
|
})
|
||
|
|
(lib.mkIf (cfg.containers != {}) {
|
||
|
|
# networking
|
||
|
|
networking.networkmanager.unmanaged = [ "ctr0" ];
|
||
|
|
networking.interfaces.dummy0.virtual = true;
|
||
|
|
networking.bridges.ctr0.interfaces = [ "dummy0" ];
|
||
|
|
networking.interfaces.ctr0.ipv4.addresses = [{
|
||
|
|
address = "10.233.0.1";
|
||
|
|
prefixLength = 24;
|
||
|
|
}];
|
||
|
|
systemd.services."dhcpd-ctr0" = {
|
||
|
|
wantedBy = [ "multi-user.target" ];
|
||
|
|
after = [ "network.target" ];
|
||
|
|
serviceConfig = {
|
||
|
|
Type = "forking";
|
||
|
|
Restart = "always";
|
||
|
|
DynamicUser = true;
|
||
|
|
StateDirectory = "dhcpd-ctr0";
|
||
|
|
User = "dhcpd-ctr0";
|
||
|
|
Group = "dhcpd-ctr0";
|
||
|
|
AmbientCapabilities = [
|
||
|
|
"CAP_NET_RAW" # to send ICMP messages
|
||
|
|
"CAP_NET_BIND_SERVICE" # to bind on DHCP port (67)
|
||
|
|
];
|
||
|
|
ExecStartPre = "${pkgs.coreutils}/bin/touch /var/lib/dhcpd-ctr0/dhcpd.leases";
|
||
|
|
ExecStart = "${pkgs.dhcp}/bin/dhcpd -4 -lf /var/lib/dhcpd-ctr0/dhcpd.leases -cf ${pkgs.writeText "dhpd.conf" ''
|
||
|
|
default-lease-time 600;
|
||
|
|
max-lease-time 7200;
|
||
|
|
authoritative;
|
||
|
|
ddns-update-style interim;
|
||
|
|
log-facility local1; # see dhcpd.nix
|
||
|
|
|
||
|
|
option subnet-mask 255.255.255.0;
|
||
|
|
option routers 10.233.0.1;
|
||
|
|
# option domain-name-servers 8.8.8.8; # TODO configure dns server
|
||
|
|
subnet 10.233.0.0 netmask 255.255.255.0 {
|
||
|
|
range 10.233.0.10 10.233.0.250;
|
||
|
|
}
|
||
|
|
''} ctr0";
|
||
|
|
};
|
||
|
|
};
|
||
|
|
})
|
||
|
|
(lib.mkIf cfg.inContainer.enable {
|
||
|
|
users.groups.container_sync = {};
|
||
|
|
users.users.container_sync = {
|
||
|
|
group = "container_sync";
|
||
|
|
uid = slib.genid_uint31 "container_sync";
|
||
|
|
isNormalUser = true;
|
||
|
|
home = "/var/lib/self";
|
||
|
|
createHome = true;
|
||
|
|
openssh.authorizedKeys.keys = [
|
||
|
|
cfg.inContainer.pubkey
|
||
|
|
];
|
||
|
|
};
|
||
|
|
})
|
||
|
|
];
|
||
|
|
}
|