Merge branch 'tv'

This commit is contained in:
lassulus 2015-10-01 14:48:51 +02:00
commit 4bacf702b0
13 changed files with 387 additions and 357 deletions

View File

@ -1,7 +1,8 @@
#
# usage:
# make system=foo
# make systems='foo bar'
# make infest system=foo [target=bar]
# make [deploy] system=foo [target=bar]
# make [deploy] systems='foo bar'
# make eval get=tv.wu.config.time.timeZone [filter=json]
#
@ -11,6 +12,7 @@
ifdef systems
$(systems):
@
unset target
parallel \
--line-buffer \
-j0 \
@ -20,7 +22,7 @@ $(systems):
else ifdef system
.PHONY: deploy infest
deploy infest:;@
export get=$$LOGNAME.${system}.config.krebs.build.scripts.$@
export get=krebs.$@
export filter=json
make -s eval | sh
@ -39,8 +41,11 @@ endif
--eval \
-A "$$get" \
'<stockholm>' \
--argstr user-name "$$LOGNAME" \
--argstr host-name "$$HOSTNAME" \
--argstr current-date "$$(date -Is)" \
--argstr current-host-name "$$HOSTNAME" \
--argstr current-user-name "$$LOGNAME" \
$${system+--argstr system "$$system"} \
$${target+--argstr target "$$target"} \
| filter
else
$(error unbound variable: system[s])

View File

@ -1,36 +1,34 @@
{ user-name, host-name }:
{ current-date
, current-host-name
, current-user-name
}:
let
lib = import <nixpkgs/lib>;
krebs-modules-path = ./krebs/3modules;
krebs-pkgs-path = ./krebs/5pkgs;
user-modules-path = ./. + "/${user-name}/3modules";
user-pkgs-path = ./. + "/${user-name}/5pkgs";
user-modules-path = ./. + "/${current-user-name}/3modules";
user-pkgs-path = ./. + "/${current-user-name}/5pkgs";
out =
(lib.mapAttrs (k: v: mk-namespace (./. + "/${k}"))
(lib.filterAttrs
(k: v: !lib.hasPrefix "." k && v == "directory" &&
builtins.pathExists (./. + "/${k}/1systems"))
(builtins.readDir ./.)));
lib.mapAttrs (_: builtins.getAttr "main")
(lib.filterAttrs (_: builtins.hasAttr "main")
(lib.mapAttrs
(k: v:
if lib.hasPrefix "." k || v != "directory" then
{}
else if builtins.pathExists (./. + "/${k}/default.nix") then
{ main = import (./. + "/${k}"); }
else if builtins.pathExists (./. + "/${k}/1systems") then
{ main = mk-namespace (./. + "/${k}"); }
else
{})
(builtins.readDir ./.)));
eval = path: import <nixpkgs/nixos/lib/eval-config.nix> {
system = builtins.currentSystem;
modules = [
({ config, ... }:
with import ./krebs/4lib { inherit lib; };
{
options.krebs.exec.host = mkOption {
type = types.host;
default = config.krebs.hosts.${host-name};
};
options.krebs.exec.user = mkOption {
type = types.user;
default = config.krebs.users.${user-name};
};
}
)
path
krebs-modules-path
user-modules-path

72
krebs/3modules/build.nix Normal file
View File

@ -0,0 +1,72 @@
{ config, lib, ... }:
with import ../4lib { inherit lib; };
let
target = config.krebs.build // { user.name = "root"; };
out = {
# TODO deprecate krebs.build.host
options.krebs.build.host = mkOption {
type = types.host;
};
# TODO make krebs.build.profile shell safe
options.krebs.build.profile = mkOption {
type = types.str;
default = "/nix/var/nix/profiles/system";
};
# TODO make krebs.build.target.host :: host
options.krebs.build.target = mkOption {
type = with types; nullOr str;
default = null;
};
# TODO deprecate krebs.build.user
options.krebs.build.user = mkOption {
type = types.user;
};
options.krebs.build.source.dir = mkOption {
type = types.attrsOf (types.submodule ({ config, ... }: {
options = {
host = mkOption {
type = types.host;
};
path = mkOption {
type = types.str;
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
url = mkOption {
type = types.str;
default = "file://${config.host.name}${config.path}";
};
};
}));
default = {};
};
options.krebs.build.source.git = mkOption {
type = with types; attrsOf (submodule ({ config, ... }: {
options = {
url = mkOption {
type = types.str; # TODO must be shell safe
};
rev = mkOption {
type = types.str;
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
};
}));
default = {};
};
};
in out

View File

@ -1,309 +0,0 @@
{ config, lib, ... }:
with import ../../4lib { inherit lib; };
let
target = config.krebs.build // { user.name = "root"; };
out = {
# TODO deprecate krebs.build.host
options.krebs.build.host = mkOption {
type = types.host;
};
# TODO make krebs.build.profile shell safe
options.krebs.build.profile = mkOption {
type = types.str;
default = "/nix/var/nix/profiles/system";
};
# TODO make krebs.build.target.host :: host
options.krebs.build.target = mkOption {
type = with types; nullOr str;
default = null;
};
# TODO deprecate krebs.build.user
options.krebs.build.user = mkOption {
type = types.user;
};
options.krebs.build.scripts.init = lib.mkOption {
type = lib.types.str;
default =
let
inherit (config.krebs.build) host;
in
''
#! /bin/sh
set -efu
hostname=${host.name}
secrets_dir=${config.krebs.build.source.dir.secrets.path}
key_type=ed25519
key_file=$secrets_dir/ssh.id_$key_type
key_comment=$hostname
if test -e "$key_file"; then
echo "Warning: privkey already exists: $key_file" >&2
else
ssh-keygen \
-C "$key_comment" \
-t "$key_type" \
-f "$key_file" \
-N ""
rm "$key_file.pub"
fi
pubkey=$(ssh-keygen -y -f "$key_file")
cat<<EOF
# put following into config.krebs.hosts.$hostname:
ssh.pubkey = $(echo $pubkey | jq -R .);
EOF
'';
};
options.krebs.build.scripts.deploy = lib.mkOption {
type = lib.types.str;
default = ''
set -efu
(${config.krebs.build.scripts._source})
${ssh-target ''
${config.krebs.build.scripts._nix-env}
${config.krebs.build.profile}/bin/switch-to-configuration switch
''}
echo OK
'';
};
options.krebs.build.scripts.infest = lib.mkOption {
type = lib.types.str;
default = ''
set -efu
export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \
-o 'HostName ${target.host.infest.addr}' \
-o 'Port ${toString target.host.infest.port}' \
"
ssh() {
eval "$RSYNC_RSH \"\$@\""
}
${ssh-target ''
${readFile ./infest/prepare.sh}
${readFile ./infest/install-nix.sh}
''}
(${config.krebs.build.scripts._source})
${ssh-target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
src=$(type -p nixos-install)
cat_src() {
sed < "$src" "$(
{ sed < "$src" -n '
/^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
/^nixpkgs=/=
/^NIX_PATH=/,/^$/{/./=}
# Disable: Copy the NixOS/Nixpkgs sources to the target as
# the initial contents of the NixOS channel.
/^srcs=/,/^ln -sfn /=
'
} | sed 's:$:s/^/#krebs#/:'
)"
}
# Location to insert config.krebs.build.scripts._nix-env
i=$(sed -n '/^echo "building the system configuration/=' "$src")
{
cat_src | sed -n "1,$i{p}"
cat ${doc config.krebs.build.scripts._nix-env}
cat_src | sed -n "$i,\''${$i!p}"
} > nixos-install
chmod +x nixos-install
# Wrap inserted config.krebs.build.scripts._nix-env into chroot.
nix_env=$(cat_src | sed -n '
s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q
')
echo nix-env is $nix_env
sed -i '
s:^nix-env:chroot $mountPoint '"$nix_env"':
' nixos-install
./nixos-install
${readFile ./infest/finalize.sh}
''}
'';
};
options.krebs.build.scripts._nix-env = lib.mkOption {
type = lib.types.str;
default = ''
set -efu
NIX_PATH=${config.krebs.build.source.NIX_PATH} \
nix-env \
-f '<stockholm>' \
-Q \
--argstr user-name ${config.krebs.exec.user.name} \
--argstr host-name ${target.host.name} \
--profile ${config.krebs.build.profile} \
--set \
-A ${lib.escapeShellArg (lib.concatStringsSep "." [
config.krebs.build.user.name
config.krebs.build.host.name
"system"
])}
'';
};
options.krebs.build.scripts._source = lib.mkOption {
type = lib.types.str;
default = ''
set -efu
${
lib.concatStringsSep "\n"
(lib.mapAttrsToList
(name: { scripts, url, ... }: "(${scripts._source})")
(config.krebs.build.source.dir //
config.krebs.build.source.git))
}
'';
};
options.krebs.build.source.NIX_PATH = mkOption {
type = types.str;
default =
lib.concatStringsSep ":"
(lib.mapAttrsToList (name: _: "${name}=/root/${name}")
(config.krebs.build.source.dir //
config.krebs.build.source.git));
};
options.krebs.build.source.dir = mkOption {
type =
let
exec = config.krebs.exec;
in
types.attrsOf (types.submodule ({ config, ... }:
let
url = "file://${config.host.name}${config.path}";
can-link = config.host.name == target.host.name;
can-push = config.host.name == exec.host.name;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
--rsync-path='mkdir -p ${config.target-path} && rsync' \
--delete-excluded \
-vrLptgoD \
${config.path}/ \
${target.user.name}@${target.host.name}:${config.target-path}
'';
in
{
options = {
host = mkOption {
type = types.host;
description = ''
define the host where the directory is stored on.
XXX: currently it is just used to check if rsync is working,
becomes part of url
'';
};
path = mkOption {
type = types.str;
};
scripts._source = mkOption {
type = types.str;
default =
#if can-link then link-method else
if can-push then push-method else
throw "cannot source ${url}";
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
url = mkOption {
type = types.str;
default = "file://${config.host.name}${config.path}";
};
};
}
));
default = {};
};
options.krebs.build.source.git = mkOption {
type =
let
target = config.krebs.build // { user.name = "root"; };
in
with types; attrsOf (submodule ({ config, ... }:
{
options = {
url = mkOption {
type = types.str; # TODO must be shell safe
};
rev = mkOption {
type = types.str;
};
scripts._source = mkOption {
type = types.str;
default = ssh-target ''
mkdir -p ${config.target-path}
cd ${config.target-path}
if ! test -e .git; then
git init
fi
if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
git remote add origin ${config.url}
elif test "$cur_url" != ${config.url}; then
git remote set-url origin ${config.url}
fi
if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
git fetch origin
git checkout ${config.rev} -- .
git checkout -q ${config.rev}
git submodule init
git submodule update
fi
git clean -dxf
'';
};
target-path = mkOption {
type = types.str;
default = "/root/${config._module.args.name}";
};
};
}
));
default = {};
};
};
doc = s:
let b = "EOF${hashString "sha256" s}"; in
''
<<\${b}
${s}
${b}
'';
ssh-target = script:
"ssh root@${target.host.name} -T ${doc ''
set -efu
${script}
''}";
in out

View File

@ -6,7 +6,7 @@ let
out = {
imports = [
./build
./build.nix
./exim-retiolum.nix
./exim-smarthost.nix
./github-hosts-sync.nix
@ -600,7 +600,7 @@ let
infest.addr = head nets.internet.addrs4;
nets = rec {
internet = {
addrs4 = ["104.233.84.173"];
addrs4 = ["104.233.84.215"];
aliases = [
"mkdir.internet"
];

View File

@ -6,7 +6,7 @@ with lib;
rec {
escape =
let
isSafeChar = c: match "[-./0-9_a-zA-Z]" c != null;
isSafeChar = c: match "[-+./0-9:=A-Z_a-z]" c != null;
in
stringAsChars (c:
if isSafeChar c then c

View File

@ -1,12 +1,12 @@
{ coreutils, gnugrep, gnused, fetchgit, jq, nix, stdenv, ... }:
stdenv.mkDerivation {
name = "get-1.1.1";
name = "get-1.3.0";
src = fetchgit {
url = http://cgit.cd.retiolum/get;
rev = "e64826a4f5f74cbaa895e538b97d0e523e9709f9";
sha256 = "4d1aa07bba52f697cf7aa7ad1b02b9ff41598dfea83c578e77b8d81e3e8830d2";
rev = "fbe8f8d12ede9762fceb15b9944b69a4ee6331eb";
sha256 = "bcdf036f8b5d1467285d0998aeac7e48280adfb9e1278f9f424c9c8b5e6ed8fa";
};
phases = [

263
krebs/default.nix Normal file
View File

@ -0,0 +1,263 @@
{ current-date
, current-host-name
, current-user-name
}@current: rec {
deploy =
{ system ? current-host-name
, target ? system
}@args: let
config = lib.get-config system;
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.deploy
set -efu
(${lib.populate args})
${lib.rootssh target ''
${lib.install args}
${config.krebs.build.profile}/bin/switch-to-configuration switch
''}
echo OK
'';
infest =
{ system ? current-host-name
, target ? system
}@args: let
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.infest
set -efu
# XXX type -p is non-standard
#export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \
# -o 'HostName $ {target.host.infest.addr}' \
# -o 'Port $ {toString target.host.infest.port}' \
#"
#ssh() {
# eval "$RSYNC_RSH \"\$@\""
#}
${lib.rootssh target ''
${builtins.readFile ./4lib/infest/prepare.sh}
${builtins.readFile ./4lib/infest/install-nix.sh}
''}
(${lib.populate args})
${lib.rootssh target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
src=$(type -p nixos-install)
cat_src() {
sed < "$src" "$(
{ sed < "$src" -n '
/^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
/^nixpkgs=/=
/^NIX_PATH=/,/^$/{/./=}
# Disable: Copy the NixOS/Nixpkgs sources to the target as
# the initial contents of the NixOS channel.
/^srcs=/,/^ln -sfn /=
'
} | sed 's:$:s/^/#krebs#/:'
)"
}
# Location to insert lib.install
i=$(sed -n '/^echo "building the system configuration/=' "$src")
{
cat_src | sed -n "1,$i{p}"
cat ${lib.doc (lib.install args)}
cat_src | sed -n "$i,\''${$i!p}"
} > nixos-install
chmod +x nixos-install
## Wrap inserted lib.install into chroot.
#nix_env=$(cat_src | sed -n '
# s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q
#')
#echo nix-env is $nix_env
#sed -i '
# s:^nix-env:chroot $mountPoint '"$nix_env"':
#' nixos-install
./nixos-install
${builtins.readFile ./4lib/infest/finalize.sh}
''}
'';
init =
{ system ? current-host-name
}@args: let
config = lib.get-config system;
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.init
set -efu
system=${lib.shell.escape system}
secrets_dir=${config.krebs.build.source.dir.secrets.path}
key_type=ed25519
key_file=$secrets_dir/ssh.id_$key_type
key_comment=$system
if test -e "$key_file"; then
echo "Warning: privkey already exists: $key_file" >&2
else
ssh-keygen \
-C "$key_comment" \
-t "$key_type" \
-f "$key_file" \
-N ""
rm "$key_file.pub"
fi
pubkey=$(ssh-keygen -y -f "$key_file")
cat<<EOF
# put following into config.krebs.hosts.$system:
ssh.pubkey = $(echo $pubkey | jq -R .);
EOF
'';
lib = import ./4lib { lib = import <nixpkgs/lib>; } // rec {
stockholm = import ../. current;
get-config = system:
stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
doc = s:
let b = "EOF${builtins.hashString "sha256" s}"; in
''
<<\${b}
${s}
${b}
'';
rootssh = target: script:
"ssh root@${target} -T ${lib.doc ''
set -efu
${script}
''}";
install =
{ system ? current-host-name
, target ? system
}:
let
stockholm = import ../. {
inherit current-date;
inherit current-host-name;
inherit current-user-name;
};
config = stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
nix-path =
lib.concatStringsSep ":"
(lib.mapAttrsToList (name: _: "${name}=/root/${name}")
(config.krebs.build.source.dir //
config.krebs.build.source.git));
in ''
set -efu
NIX_PATH=${lib.shell.escape nix-path} \
nix-env \
--show-trace \
-f '<stockholm>' \
-Q \
--argstr current-date ${lib.shell.escape current-date} \
--argstr current-host-name ${lib.shell.escape current-host-name} \
--argstr current-user-name ${lib.shell.escape current-user-name} \
--profile ${lib.shell.escape config.krebs.build.profile} \
--set \
-A ${lib.escapeShellArg (lib.concatStringsSep "." [
config.krebs.build.user.name
config.krebs.build.host.name
"system"
])}
'';
populate =
{ system ? current-host-name
, target ? system
}@args:
let out = ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
set -efu
${lib.concatStringsSep "\n"
(lib.concatMap
(type: lib.mapAttrsToList (_: methods.${type})
config.krebs.build.source.${type})
["dir" "git"])}
'';
stockholm = import ../. {
inherit current-date;
inherit current-host-name;
inherit current-user-name;
};
config = stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
current-host = config.krebs.hosts.${current-host-name};
current-user = config.krebs.users.${current-user-name};
target-host = config.krebs.hosts.${system};
methods.dir = config:
let
can-link = config.host.name == target-host.name;
can-push = config.host.name == current-host.name;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
--rsync-path='mkdir -p ${config.target-path} && rsync' \
--delete-excluded \
-vrLptgoD \
${config.path}/ \
root@${target}:${config.target-path}
'';
url = "file://${config.host.name}${config.path}";
in
#if can-link then link-method else
if can-push then push-method else
throw "cannot source ${url}";
methods.git = config:
lib.rootssh target ''
mkdir -p ${config.target-path}
cd ${config.target-path}
if ! test -e .git; then
git init
fi
if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
git remote add origin ${config.url}
elif test "$cur_url" != ${config.url}; then
git remote set-url origin ${config.url}
fi
if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
git fetch origin
git checkout ${config.rev} -- .
git checkout -q ${config.rev}
git submodule init
git submodule update
fi
git clean -dxf
'';
in out;
};
}

View File

@ -11,7 +11,7 @@ with lib;
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
rev = "bd84ebaa1e0359f41350e053ed24592b169b5714";
rev = "e916273209560b302ab231606babf5ce1c481f08";
};
dir.secrets = {
host = config.krebs.hosts.wu;

View File

@ -114,6 +114,18 @@ let
gnused
])}
green() { printf '\x0303,99%s\x0F' "$1"; }
red() { printf '\x0304,99%s\x0F' "$1"; }
orange() { printf '\x0307,99%s\x0F' "$1"; }
pink() { printf '\x0313,99%s\x0F' "$1"; }
gray() { printf '\x0314,99%s\x0F' "$1"; }
unset message
add_message() {
message="''${message+$message
}$*"
}
nick=${escapeShellArg nick}
channel=${escapeShellArg channel}
server=${escapeShellArg server}
@ -124,7 +136,6 @@ let
empty=0000000000000000000000000000000000000000
unset message
while read oldrev newrev ref; do
if [ $oldrev = $empty ]; then
@ -162,27 +173,17 @@ let
esac
#$host $GIT_SSH_REPO $ref $link
message="''${message+$message
}$GIT_SSH_USER $receive_mode $link"
message=''${message+$message
}$(
green() { printf '\x0303,99%s\x0F' "$1"; }
red() { printf '\x0304,99%s\x0F' "$1"; }
orange() { printf '\x0307,99%s\x0F' "$1"; }
gray() { printf '\x0314,99%s\x0F' "$1"; }
add_message $(pink push) $link $(gray "($receive_mode)")
add_message "$(
git log \
--format="$(orange %h) %s $(gray '(%ar)')" \
--reverse \
$id2..$id
git diff --stat $id2..$id \
| sed '
$!s/+/'$(green '&')'/g
$!s/-/'$(red '&')'/g
'
)
| sed '$!s/\(+*\)\(-*\)$/'$(green '\1')$(red '\2')'/'
)"
done