krebs.build.populate: init and drop support for v1

This commit is contained in:
tv 2016-02-04 05:40:26 +01:00
parent 89d6f319d0
commit b857a48632
11 changed files with 204 additions and 442 deletions

108
Makefile
View File

@ -1,87 +1,51 @@
#
# usage:
# make infest system=foo [target=bar]
# make [deploy] system=foo [target=bar]
# make [deploy] systems='foo bar'
# make eval get=users.tv.wu.config.time.timeZone [filter=json]
#
.ONESHELL:
.SHELLFLAGS := -eufc
ifdef systems
$(systems):
@
unset target
parallel \
--line-buffer \
-j0 \
--no-notice \
--tagstring {} \
-q make -s systems= system={} ::: $(systems)
else ifdef system
.PHONY: deploy infest
deploy infest:;@
export get=krebs.$@
export filter=json
script=$$(make -s eval)
echo "$$script" | sh
.PHONY: eval
eval:
@
ifeq ($(filter),json)
extraArgs='--json --strict'
filter() { jq -r .; }
else
filter() { cat; }
ifndef system
$(error unbound variable: system)
endif
result=$$(nix-instantiate \
$${extraArgs-} \
--eval \
-A "$$get" \
-I stockholm="$$PWD" \
'<stockholm>' \
--argstr current-host-name "$$HOSTNAME" \
--argstr current-user-name "$$LOGNAME" \
$${system+--argstr system "$$system"} \
$${target+--argstr target "$$target"})
echo "$$result" | filter
export target_host ?= $(system)
export target_user ?= root
export target_path ?= /var/src
# usage: make deploy system=foo [target_host=bar]
.PHONY: deploy
deploy: populate ;@set -x
ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path"
# usage: make populate system=foo [target_host=bar]
.PHONY: populate
populate: export lib = \
let nlib = import <nixpkgs/lib>; in \
nlib // import krebs/4lib { lib = nlib; } // builtins
populate: export source = \
with builtins; \
with (import ./. {}).users.$${getEnv "LOGNAME"}.$${getEnv "system"}; \
assert config.krebs.build.source-version == 2; \
config.krebs.build.source
populate:;@
result=$$(nix-instantiate \
--eval \
--json \
--arg lib "$$lib" \
--arg source "$$source" \
--argstr target-user "$$target_user" \
--argstr target-host "$$target_host" \
--argstr target-path "$$target_path" \
-A populate \
krebs/v2)
script=$$(echo "$$result" | jq -r .)
echo "$$script" | sh
# usage: make rebuild system=foo [target_host=bar] [operation=switch]
.PHONY: rebuild
rebuild: populate ;@set -x
ssh "$$target_user@$$target_host" \
nixos-rebuild "$${operation-switch}" -I "$$target_path"
result=$$(make -s eval get=config.krebs.build.populate filter=json)
echo "$$result" | sh
# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json]
.PHONY: eval
eval:;@
ifeq ($(filter),json)
extraArgs='--json --strict'
filter() { echo "$$1" | jq -r .; }
else
$(error unbound variable: system[s])
filter() { echo "$$1"; }
endif
result=$$(nix-instantiate \
$${extraArgs-} \
--show-trace \
--readonly-mode \
--eval \
-A "$$get" \
--arg configuration "<stockholm/$$LOGNAME/1systems/$$system.nix>")
filter "$$result"
## usage: make install system=foo target=
#.PHONY: install
#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
#install:;@set -x
# $(ssh) "$$target_user@$$target_host" \
# env target_path=/var/src \
# sh -s prepare < krebs/4lib/infest/prepare.sh
# make -s populate target_path=/mnt"$$target_path"
# $(ssh) "$$target_user@$$target_host" \
# env NIXOS_CONFIG=/var/src/nixos-config \
# nixos-install

View File

@ -1,9 +1,15 @@
{ current-host-name ?
{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config>
, system ? builtins.currentSystem
, current-host-name ?
let v = builtins.getEnv "HOSTNAME"; in
if v != "" then v else builtins.readFile /proc/sys/kernel/hostname
, current-user-name ?
let v = builtins.getEnv "LOGNAME"; in
if v != "" then v else abort "undefined variable: LOGNAME"
, nixpkgs-path ?
if (builtins.tryEval <nixpkgs/krebs>).success
then <upstream-nixpkgs>
else <nixpkgs>
, StrictHostKeyChecking ? "yes"
}@args:
@ -11,7 +17,8 @@ let stockholm = {
inherit krebs;
inherit users;
inherit lib;
inherit pkgs;
inherit config options pkgs;
system = config.system.build.toplevel;
};
krebs = import ./krebs (args // { inherit lib stockholm; });
@ -20,7 +27,7 @@ let stockholm = {
nlib = import (slib.npath "lib");
klib = import (slib.kpath "4lib") { lib = nlib; };
slib = rec {
npath = p: <nixpkgs> + "/${p}";
npath = p: nixpkgs-path + "/${p}";
kpath = p: ./. + "/krebs/${p}";
upath = p: ./. + "/${current-user-name}/${p}";
};
@ -29,7 +36,7 @@ let stockholm = {
(import p { lib = nlib // klib; });
in nlib // klib // slib // ulib // builtins;
inherit (eval {}) pkgs;
inherit (eval configuration) config options pkgs;
base-module = { config, ... }: {
imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [
@ -48,6 +55,7 @@ let stockholm = {
};
eval = config: import (lib.npath "nixos/lib/eval-config.nix") {
inherit system;
specialArgs = {
inherit lib;
};

View File

@ -28,81 +28,157 @@ let
type = types.user;
};
options.krebs.build.source-version = mkOption {
type = types.enum [ 1 2 ];
default = 1;
};
options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" {
v1 = {
dir = mkOption {
type = let
default-host = config.krebs.current.host;
in types.attrsOf (types.submodule ({ config, ... }: {
options = {
host = mkOption {
type = types.host;
default = default-host;
};
path = mkOption {
type = types.str;
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
url = mkOption {
type = types.str;
default = "file://${config.host.name}${config.path}";
};
};
}));
default = {};
};
git = mkOption {
type = with types; attrsOf (submodule ({ config, ... }: {
options = {
url = mkOption {
type = types.str; # TODO must be shell safe
};
rev = mkOption {
type = types.str;
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
};
}));
default = {};
};
};
v2 = let
raw = types.either types.str types.path;
url = types.submodule {
options = {
url = mkOption {
type = types.str;
};
rev = mkOption {
type = types.str;
};
dev = mkOption {
type = types.str;
};
options.krebs.build.source = let
raw = types.either types.str types.path;
url = types.submodule {
options = {
url = mkOption {
type = types.str;
};
rev = mkOption {
type = types.str;
};
dev = mkOption {
type = types.str;
};
};
in mkOption {
type = types.attrsOf (types.either types.str url);
apply = let f = mapAttrs (_: value: {
string = value;
path = toString value;
set = f value;
}.${typeOf value}); in f;
default = {};
};
in mkOption {
type = types.attrsOf (types.either types.str url);
apply = let f = mapAttrs (_: value: {
string = value;
path = toString value;
set = f value;
}.${typeOf value}); in f;
default = {};
};
options.krebs.build.populate = mkOption {
type = types.str;
default = let
source = config.krebs.build.source;
target-user = maybeEnv "target_user" "root";
target-host = maybeEnv "target_host" config.krebs.build.host.name;
target-path = maybeEnv "target_path" "/var/src";
out = ''
#! /bin/sh
set -eu
verbose() {
printf '+%s\n' "$(printf ' %q' "$@")" >&2
"$@"
}
echo ${shell.escape git-script} \
| ssh ${shell.escape "${target-user}@${target-host}"} -T
unset tmpdir
trap '
rm "$tmpdir"/*
rmdir "$tmpdir"
trap - EXIT INT QUIT
' EXIT INT QUIT
tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir"
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: let dst = removePrefix "symlink:" (get-url spec); in
"verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
symlink-specs)}
verbose proot \
-b $tmpdir:${shell.escape target-path} \
${concatStringsSep " \\\n "
(mapAttrsToList
(name: spec:
"-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
file-specs)} \
rsync \
-f ${shell.escape "P /*"} \
${concatMapStringsSep " \\\n "
(name: "-f ${shell.escape "R /${name}"}")
(attrNames file-specs)} \
--delete \
-vFrlptD \
${shell.escape target-path}/ \
${shell.escape "${target-user}@${target-host}:${target-path}"}
'';
get-schema = uri:
if substring 0 1 uri == "/"
then "file"
else head (splitString ":" uri);
has-schema = schema: uri: get-schema uri == schema;
get-url = spec: {
string = spec;
path = toString spec;
set = get-url spec.url;
}.${typeOf spec};
git-specs =
filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
file-specs =
filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
symlink-specs =
filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
git-script = ''
#! /bin/sh
set -efu
verbose() {
printf '+%s\n' "$(printf ' %q' "$@")" >&2
"$@"
}
fetch_git() {(
dst_dir=$1
src_url=$2
src_ref=$3
if ! test -e "$dst_dir"; then
git clone "$src_url" "$dst_dir"
fi
cd "$dst_dir"
if ! url=$(git config remote.origin.url); then
git remote add origin "$src_url"
elif test "$url" != "$src_url"; then
git remote set-url origin "$src_url"
fi
# TODO resolve src_ref to commit hash
hash=$src_ref
if ! test "$(git log --format=%H -1)" = "$hash"; then
git fetch origin
git checkout "$hash" -- "$dst_dir"
git checkout "$hash"
fi
git clean -dxf
)}
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: toString (map shell.escape [
"verbose"
"fetch_git"
"${target-path}/${name}"
spec.url
spec.rev
]))
git-specs)}
'';
in out;
};
};

View File

@ -1,3 +1,5 @@
assert false;
{ current-host-name
, current-user-name
, lib
@ -6,30 +8,11 @@
}:
let out = {
inherit deploy;
inherit infest;
inherit init;
inherit nixos-install;
inherit populate;
};
deploy =
{ system ? current-host-name
, target ? system
}@args: let
config = get-config system;
in ''
#! /bin/sh
# krebs.deploy
set -efu
(${populate args})
${rootssh target ''
${nix-install args}
${config.krebs.build.profile}/bin/switch-to-configuration switch
''}
echo OK
'';
infest =
{ system ? current-host-name
, target ? system
@ -45,9 +28,6 @@ let out = {
${builtins.readFile ./4lib/infest/install-nix.sh}
''}
# Prepare target source via bind-mounting
(${nixos-install args})
${rootssh target ''
@ -169,9 +149,7 @@ let out = {
get-config = system: let
config = stockholm.users.${current-user-name}.${system}.config
or (abort "unknown system: ${system}, user: ${current-user-name}");
in
assert config.krebs.build.source-version == 1;
config;
in config;
nix-install =
{ system ? current-host-name
@ -203,73 +181,6 @@ let out = {
])}
'';
populate =
{ system ? current-host-name
, target ? system
, root ? ""
}@args:
let out = ''
#! /bin/sh
set -efu
${lib.concatStringsSep "\n"
(lib.concatMap
(type: lib.mapAttrsToList (_: methods.${type})
config.krebs.build.source.${type})
["dir" "git"])}
'';
config = get-config system;
current-host = config.krebs.hosts.${current-host-name};
current-user = config.krebs.users.${current-user-name};
methods.dir = config:
let
can-push = config.host.name == current-host.name;
target-path = root + config.target-path;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
--rsync-path='mkdir -p ${target-path} && rsync' \
--delete-excluded \
-vrlptD \
${config.path}/ \
root@${target}:${target-path}
'';
in
if can-push then push-method else
let dir = "file://${config.host.name}${config.path}"; in
# /!\ revise this message when using more than just push-method
throw "No way to push ${dir} from ${current-host.name} to ${target}";
methods.git = config:
let target-path = root + config.target-path;
in rootssh target ''
mkdir -p ${target-path}
cd ${target-path}
if ! test -e .git; then
git init
fi
if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
git remote add origin ${config.url}
elif test "$cur_url" != ${config.url}; then
git remote set-url origin ${config.url}
fi
if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
git fetch origin
git checkout ${config.rev} -- .
git checkout -q ${config.rev}
git submodule init
git submodule update
fi
git clean -dxf
'';
in out;
rootssh = target: script:
let
flags = "-o StrictHostKeyChecking=${StrictHostKeyChecking}";

View File

@ -1,132 +0,0 @@
{ lib
, source
, target-user ? "root"
, target-host
, target-path ? "/var/src"
}:
with lib;
let
out = {
inherit populate;
};
populate = ''
#! /bin/sh
set -eu
verbose() {
printf '+%s\n' "$(printf ' %q' "$@")" >&2
"$@"
}
echo ${shell.escape git-script} \
| ssh ${shell.escape "${target-user}@${target-host}"} -T
unset tmpdir
trap '
rm "$tmpdir"/*
rmdir "$tmpdir"
trap - EXIT INT QUIT
' EXIT INT QUIT
tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir"
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: let dst = removePrefix "symlink:" (get-url spec); in
"verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
symlink-specs)}
verbose proot \
-b $tmpdir:${shell.escape target-path} \
${concatStringsSep " \\\n "
(mapAttrsToList
(name: spec:
"-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
file-specs)} \
rsync \
-f ${shell.escape "P /*"} \
${concatMapStringsSep " \\\n "
(name: "-f ${shell.escape "R /${name}"}")
(attrNames file-specs)} \
--delete \
-vFrlptD \
${shell.escape target-path}/ \
${shell.escape "${target-user}@${target-host}:${target-path}"}
'';
get-schema = uri:
if substring 0 1 uri == "/"
then "file"
else head (splitString ":" uri);
has-schema = schema: uri: get-schema uri == schema;
get-url = spec: {
string = spec;
path = toString spec;
set = get-url spec.url;
}.${typeOf spec};
git-specs =
filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
file-specs =
filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
symlink-specs =
filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
git-script = ''
#! /bin/sh
set -efu
verbose() {
printf '+%s\n' "$(printf ' %q' "$@")" >&2
"$@"
}
fetch_git() {(
dst_dir=$1
src_url=$2
src_ref=$3
if ! test -e "$dst_dir"; then
git clone "$src_url" "$dst_dir"
fi
cd "$dst_dir"
if ! url=$(git config remote.origin.url); then
git remote add origin "$src_url"
elif test "$url" != "$src_url"; then
git remote set-url origin "$src_url"
fi
# TODO resolve src_ref to commit hash
hash=$src_ref
if ! test "$(git log --format=%H -1)" = "$hash"; then
git fetch origin
git checkout "$hash" -- "$dst_dir"
git checkout "$hash"
fi
git clean -dxf
)}
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: toString (map shell.escape [
"verbose"
"fetch_git"
"${target-path}/${name}"
spec.url
spec.rev
]))
git-specs)}
'';
in out

0
nixpkgs/krebs Normal file
View File

View File

@ -1,65 +1 @@
{ configuration ? import <upstream-nixpkgs/nixos/lib/from-env.nix> "NIXOS_CONFIG" <nixos-config>
, system ? builtins.currentSystem
}:
let
eval-config = modules: import <upstream-nixpkgs/nixos/lib/eval-config.nix> {
inherit system;
modules = modules ++ [({ config, lib, ... }: with lib; {
imports = filter dir.has-default-nix (concatLists [
(map (p: p + "/2configs") [ <stockholm-private> ])
(map (p: p + "/3modules") [ <stockholm-krebs> <stockholm-private> ])
]);
krebs.current = {
enable = true;
host = config.krebs.hosts.${readFile /proc/sys/kernel/hostname};
user = config.krebs.users.${getEnv "LOGNAME"};
};
nixpkgs.config.packageOverrides = pkgs: let
kpkgs = import <stockholm-krebs/5pkgs> { inherit lib pkgs; };
upkgs = import <stockholm-private/5pkgs> { inherit lib; pkgs = pkgs // kpkgs; };
in kpkgs // upkgs;
})];
specialArgs = {
lib = let
nlib = import <upstream-nixpkgs/lib> // builtins;
klib = nlib // import <stockholm-krebs/4lib> { lib = nlib; };
ulib = klib // (with klib; let p = <stockholm-private> + "/4lib"; in
optionalAttrs (dir.has-default-nix p)
(import p { lib = klib; }));
in ulib;
};
};
eval = eval-config [
configuration
];
# This is for `nixos-rebuild build-vm'.
vm = eval-config [
configuration
<upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
];
# This is for `nixos-rebuild build-vm-with-bootloader'.
vm-with-bootloader = eval-config [
configuration
<upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
{ virtualisation.useBootLoader = true; }
];
in
{
inherit (eval) config options;
system = eval.config.system.build.toplevel;
vm = vm.config.system.build.vm;
vmWithBootLoader = vm-with-bootloader.config.system.build.vm;
# The following are used by nixos-rebuild.
nixFallback = eval.pkgs.nixUnstable;
}
import <stockholm>

View File

@ -1 +1 @@
../../upstream-nixpkgs/nixos/lib
../../../upstream-nixpkgs/nixos/lib

View File

@ -1 +1 @@
../../upstream-nixpkgs/nixos/modules
../../../upstream-nixpkgs/nixos/modules

1
root Symbolic link
View File

@ -0,0 +1 @@
../stockholm-user

View File

@ -8,11 +8,9 @@ with lib;
krebs.build = {
user = config.krebs.users.tv;
target = mkDefault "root@${config.krebs.build.host.name}";
source-version = 2;
source = mapAttrs (_: mkDefault) ({
nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix";
nixpkgs = symlink:stockholm/nixpkgs;
null = "symlink:stockholm/null";
secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
secrets-common = "/home/tv/secrets/common";
stockholm = "/home/tv/stockholm";
@ -104,7 +102,7 @@ with lib;
};
environment.variables = {
NIX_PATH = mkForce "secrets=/var/src/null:/var/src";
NIX_PATH = mkForce "secrets=/var/src/stockholm/null:/var/src";
};
programs.bash = {