Merge remote-tracking branch 'cd/master'

This commit is contained in:
lassulus 2015-10-20 23:47:29 +02:00
commit f6187b3796
8 changed files with 284 additions and 262 deletions

View File

@ -3,7 +3,7 @@
# make infest system=foo [target=bar]
# make [deploy] system=foo [target=bar]
# make [deploy] systems='foo bar'
# make eval get=tv.wu.config.time.timeZone [filter=json]
# make eval get=users.tv.wu.config.time.timeZone [filter=json]
#
.ONESHELL:

View File

@ -1,72 +1,112 @@
# Welcome to the top-level default.nix of stockholm.
#
# You can discover the whole thing easily using the `get` utility,
# which can be found at http://cgit.cd.krebsco.de/get/tree/get
# To install `get` on any Nix-enabled system, use:
#
# nix-env -f /path/to/stockholm -iA pkgs.get
#
# The "current" arguments are used to provide information about the user who's
# evaluating this file. This information is used to determine which user
# namespace is to be used. Of course there's nothing trying to prevent you
# from forging this information. E.g. you could try to generate the deployment
# script for some random user's system, targeting some random host:
#
# LOGNAME=tv get krebs.deploy system=nomic target=8.8.8.8
#
{ current-date ? abort "current-date not defined"
, current-host-name ? abort "current-host-name not defined"
, current-user-name ? builtins.getEnv "LOGNAME"
}:
}@current:
assert current-user-name != "";
let stockholm = {
# The generated scripts to deploy (or infest) systems can be found in the
# `krebs` attribute. There's also an init script, but it's in its early
# stages, not well integrated and mostly useless at the moment. :)
#
# You'll also find lib here, which is nixpkgs/lib + krebs lib, but nobody
# is really accessing this directly, as this lib gets reexported below.
inherit krebs;
let
lib = import <nixpkgs/lib>;
klib = import ./krebs/4lib { inherit lib; };
in with klib; let
# All systems of all users can be found here.
#
# /!\ Please note that `get users.${user-name}.${host-name}.system` is a
# bad idea because it will produce vast amounts of output. These are the
# actual and complete system derivations that can be installed on the
# respective host.
#
# Another thing to notice here is that other user's systems might not be
# evaluable because of missing secrets. If you _are_ able to evaluate
# another user's system, then you probably share a similar naming scheme
# for your secret files! :)
inherit users;
nspath = ns: p: ./. + "/${ns}/${p}";
kpath = nspath "krebs";
upath = nspath current-user-name;
# Additionally, output lib and pkgs for easy access from the shell.
# Notice how we're evaluating just the base module to obtain pkgs.
inherit lib;
inherit (eval {}) pkgs;
};
stockholm = {
krebs = import ./krebs (current // { inherit stockholm; });
inherit (krebs) lib;
# Path resolvers for common and individual files.
# Example: `upath "3modules"` produces the current user's 3modules directory
kpath = lib.nspath "krebs";
upath = lib.nspath current-user-name;
# This is the base module. Its purpose is to provide modules and
# packages, both common ones, found in krebs/ as well as the current user's,
# found in the user's namespace.
base-module = {
imports = map (f: f "3modules") [ kpath upath ];
nixpkgs.config.packageOverrides = pkgs:
let
# Notice the ordering. Krebs packages can only depend on Nixpkgs,
# whereas user packages additionally can depend on krebs packages.
kpkgs = import (kpath "5pkgs") { inherit pkgs; };
upkgs = import (upath "5pkgs") { pkgs = pkgs // kpkgs; };
in
kpkgs // upkgs;
};
out =
{ inherit (eval {}) config options pkgs; } //
lib.mapAttrs
(name: _:
if builtins.pathExists (nspath name "default.nix")
then import (nspath name "default.nix")
else import-1systems (nspath name "1systems"))
(lib.filterAttrs
(n: t: !lib.hasPrefix "." n && t == "directory")
(builtins.readDir ./.));
eval = path: import <nixpkgs/nixos/lib/eval-config.nix> {
# The above base module is used together with a NixOS configuration to
# produce a system. Notice how stockholm really just provides additional
# packages and modules on top of NixOS. Some of this stuff might become
# useful to a broader audience, at which point it should probably be merged
# and pull-requested for inclusion into NixOS/nixpkgs.
# TODO provide krebs lib, so modules don't have to import it awkwardly
eval = config: import <nixpkgs/nixos/lib/eval-config.nix> {
modules = [
stockholm
path
base-module
config
];
};
import-1systems = path: lib.mapAttrs (_: mk-system) (nixDir path);
# Any top-level directory other than krebs/ is considered to be a user
# namespace, configuring a bunch of systems.
# Have a look at the definition of install in krebs/default.nix to see how
# nix-env is using this attribute set to obtain the system to be installed.
# TODO move user namespaces' to users/, so no exception for krebs/ is needed
users =
lib.mapAttrs
(name: _: eval-all-systems (lib.nspath name "1systems"))
(lib.filterAttrs
(n: t: !lib.hasPrefix "." n && t == "directory" && n != "krebs")
(builtins.readDir ./.));
mk-system = path: rec {
# Given a path to a user namespace, provide an attribute of evaluated
# system configurations, keyed by system names (AKA host names).
eval-all-systems = path:
lib.mapAttrs'
(n: _: (lib.nameValuePair (lib.removeSuffix ".nix" n)
(eval-system (path + "/${n}"))))
(builtins.readDir path);
eval-system = path: rec {
inherit (eval path) config options;
system = config.system.build.toplevel;
fetch = import ./krebs/0tools/fetch.nix { inherit config lib; };
};
nixDir = path:
builtins.listToAttrs
(catMaybes
(lib.mapAttrsToList
(k: v: {
directory =
let p = path + "/${k}/default.nix"; in
if builtins.pathExists p
then Just (lib.nameValuePair k p)
else Nothing;
regular =
let p = path + "/${k}"; in
if lib.hasSuffix ".nix" p
then Just (lib.nameValuePair (lib.removeSuffix ".nix" k) p)
else Nothing;
}.${v} or Nothing)
(builtins.readDir path)));
in out
in stockholm

View File

@ -24,9 +24,21 @@ let
"${config.networking.hostName}.retiolum"
];
};
listen = mkOption {
type = with types; either str (listOf str);
default = "80";
apply = x:
if typeOf x != "list"
then [x]
else x;
};
locations = mkOption {
type = with types; listOf (attrsOf str);
};
extraConfig = mkOption {
type = with types; str;
default = "";
};
};
default = {};
};
@ -60,10 +72,11 @@ let
}
'';
to-server = { server-names, locations, ... }: ''
to-server = { server-names, listen, locations, extraConfig, ... }: ''
server {
listen 80;
${concatMapStringsSep "\n" (x: "listen ${x};") listen}
server_name ${toString server-names};
${extraConfig}
${indent (concatStrings (map to-location locations))}
}
'';

View File

@ -3,13 +3,8 @@
with builtins;
with lib;
let
maybe = import ./maybe.nix { inherit lib; };
in
builtins //
lib //
maybe //
rec {
eq = x: y: x == y;

View File

@ -1,10 +0,0 @@
{ ... }:
rec {
Just = x: { type = "maybe"; value = x; };
Nothing = { type = "maybe"; };
isMaybe = x: builtins.typeOf x == "set" && x.type or false == "maybe";
isJust = x: isMaybe x && builtins.hasAttr "value" x;
fromJust = x: assert isJust x; x.value;
catMaybes = xs: map fromJust (builtins.filter isJust xs);
}

View File

@ -9,12 +9,12 @@
, ... }:
stdenv.mkDerivation {
name = "push-1.0.0";
name = "push-1.1.0";
src = fetchgit {
url = http://cgit.cd.retiolum/push;
rev = "513da89fe50b3bad3d758855f5622c4508977e4a";
sha256 = "6124e1d4d4ef57455e2f06891e06fb01d3786846efaf9b79e3176d89988e1b4e";
rev = "c5f4bda5bd00bad7778bbd5a9af8d476de0de920";
sha256 = "d335b644b791214263cee5c6659538c8e45326531b0588e5e7eb3bd9ef969800";
};
phases = [

View File

@ -1,21 +1,30 @@
{ current-date
, current-host-name
, current-user-name
}@current: rec {
, stockholm
}:
let out = {
inherit deploy;
inherit infest;
inherit init;
inherit lib;
inherit nixos-install;
};
deploy =
{ system ? current-host-name
, target ? system
}@args: let
config = lib.get-config system;
config = get-config system;
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.deploy
set -efu
(${lib.populate args})
${lib.rootssh target ''
${lib.install args}
(${populate args})
${rootssh target ''
${nix-install args}
${config.krebs.build.profile}/bin/switch-to-configuration switch
''}
echo OK
@ -31,63 +40,14 @@
# krebs.infest
set -efu
# XXX type -p is non-standard
#export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \
# -o 'HostName $ {target.host.infest.addr}' \
# -o 'Port $ {toString target.host.infest.port}' \
#"
#ssh() {
# eval "$RSYNC_RSH \"\$@\""
#}
${lib.rootssh target ''
${rootssh target ''
${builtins.readFile ./4lib/infest/prepare.sh}
${builtins.readFile ./4lib/infest/install-nix.sh}
''}
(${lib.populate args})
${lib.rootssh target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
src=$(type -p nixos-install)
cat_src() {
sed < "$src" "$(
{ sed < "$src" -n '
/^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
/^nixpkgs=/=
/^NIX_PATH=/,/^$/{/./=}
# Disable: Copy the NixOS/Nixpkgs sources to the target as
# the initial contents of the NixOS channel.
/^srcs=/,/^ln -sfn /=
'
} | sed 's:$:s/^/#krebs#/:'
)"
}
# Location to insert lib.install
i=$(sed -n '/^echo "building the system configuration/=' "$src")
{
cat_src | sed -n "1,$i{p}"
cat ${lib.doc (lib.install args)}
cat_src | sed -n "$i,\''${$i!p}"
} > nixos-install
chmod +x nixos-install
## Wrap inserted lib.install into chroot.
#nix_env=$(cat_src | sed -n '
# s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q
#')
#echo nix-env is $nix_env
#sed -i '
# s:^nix-env:chroot $mountPoint '"$nix_env"':
#' nixos-install
unset SSL_CERT_FILE
./nixos-install
(${nixos-install args})
${rootssh target ''
${builtins.readFile ./4lib/infest/finalize.sh}
''}
'';
@ -95,7 +55,7 @@
init =
{ system ? current-host-name
}@args: let
config = lib.get-config system;
config = get-config system;
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
@ -127,138 +87,166 @@
EOF
'';
nixos-install =
{ system ? current-host-name
, target ? system
}@args: let
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.nixos-install
(${populate args})
${rootssh target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
src=$(type -p nixos-install)
cat_src() {
sed < "$src" "$(
{ sed < "$src" -n '
/^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
/^nixpkgs=/=
/^NIX_PATH=/,/^$/{/./=}
# Disable: Copy the NixOS/Nixpkgs sources to the target as
# the initial contents of the NixOS channel.
/^srcs=/,/^ln -sfn /=
'
} | sed 's:$:s/^/#krebs#/:'
)"
}
# Location to insert `nix-install`
i=$(sed -n '/^echo "building the system configuration/=' "$src")
{
cat_src | sed -n "1,$i{p}"
cat ${doc (nix-install args)}
cat_src | sed -n "$i,\''${$i!p}"
} > nixos-install
chmod +x nixos-install
unset SSL_CERT_FILE
./nixos-install
''}
'';
lib = import ./4lib { lib = import <nixpkgs/lib>; } // rec {
stockholm = import ../. current;
get-config = system:
stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
doc = s:
let b = "EOF${builtins.hashString "sha256" s}"; in
''
<<\${b}
${s}
${b}
'';
rootssh = target: script:
"ssh root@${target} -T ${lib.doc ''
set -efu
${script}
''}";
install =
{ system ? current-host-name
, target ? system
}:
let
stockholm = import ../. {
inherit current-date;
inherit current-host-name;
inherit current-user-name;
};
config = stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
nix-path =
lib.concatStringsSep ":"
(lib.mapAttrsToList (name: _: "${name}=/root/${name}")
(config.krebs.build.source.dir //
config.krebs.build.source.git));
in ''
set -efu
NIX_PATH=${lib.shell.escape nix-path} \
nix-env \
--show-trace \
-f '<stockholm>' \
-Q \
--argstr current-date ${lib.shell.escape current-date} \
--argstr current-host-name ${lib.shell.escape current-host-name} \
--argstr current-user-name ${lib.shell.escape current-user-name} \
--profile ${lib.shell.escape config.krebs.build.profile} \
--set \
-A ${lib.escapeShellArg (lib.concatStringsSep "." [
config.krebs.build.user.name
config.krebs.build.host.name
"system"
])}
'';
populate =
{ system ? current-host-name
, target ? system
}@args:
let out = ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
set -efu
${lib.concatStringsSep "\n"
(lib.concatMap
(type: lib.mapAttrsToList (_: methods.${type})
config.krebs.build.source.${type})
["dir" "git"])}
'';
stockholm = import ../. {
inherit current-date;
inherit current-host-name;
inherit current-user-name;
};
config = stockholm.${current-user-name}.${system}.config
or (abort "unknown system: ${system}");
current-host = config.krebs.hosts.${current-host-name};
current-user = config.krebs.users.${current-user-name};
target-host = config.krebs.hosts.${system};
methods.dir = config:
let
can-link = config.host.name == target-host.name;
can-push = config.host.name == current-host.name;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
--rsync-path='mkdir -p ${config.target-path} && rsync' \
--delete-excluded \
-vrLptgoD \
${config.path}/ \
root@${target}:${config.target-path}
'';
url = "file://${config.host.name}${config.path}";
in
#if can-link then link-method else
if can-push then push-method else
throw "cannot source ${url}";
methods.git = config:
lib.rootssh target ''
mkdir -p ${config.target-path}
cd ${config.target-path}
if ! test -e .git; then
git init
fi
if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
git remote add origin ${config.url}
elif test "$cur_url" != ${config.url}; then
git remote set-url origin ${config.url}
fi
if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
git fetch origin
git checkout ${config.rev} -- .
git checkout -q ${config.rev}
git submodule init
git submodule update
fi
git clean -dxf
'';
in out;
stockholm-path = ../.;
nspath = ns: p: stockholm-path + "/${ns}/${p}";
};
}
doc = s:
let b = "EOF${builtins.hashString "sha256" s}"; in
''
<<\${b}
${s}
${b}
'';
get-config = system:
stockholm.users.${current-user-name}.${system}.config
or (abort "unknown system: ${system}, user: ${current-user-name}");
nix-install =
{ system ? current-host-name
, target ? system
}:
let
config = get-config system;
nix-path =
lib.concatStringsSep ":"
(lib.mapAttrsToList (name: _: "${name}=/root/${name}")
(config.krebs.build.source.dir //
config.krebs.build.source.git));
in ''
set -efu
NIX_PATH=${lib.shell.escape nix-path} \
nix-env \
--show-trace \
-f '<stockholm>' \
--argstr current-date ${lib.shell.escape current-date} \
--argstr current-host-name ${lib.shell.escape current-host-name} \
--argstr current-user-name ${lib.shell.escape current-user-name} \
--profile ${lib.shell.escape config.krebs.build.profile} \
--set \
-A ${lib.escapeShellArg (lib.concatStringsSep "." [
"users"
config.krebs.build.user.name
config.krebs.build.host.name
"system"
])}
'';
populate =
{ system ? current-host-name
, target ? system
}@args:
let out = ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
set -efu
${lib.concatStringsSep "\n"
(lib.concatMap
(type: lib.mapAttrsToList (_: methods.${type})
config.krebs.build.source.${type})
["dir" "git"])}
'';
config = get-config system;
current-host = config.krebs.hosts.${current-host-name};
current-user = config.krebs.users.${current-user-name};
methods.dir = config:
let
can-push = config.host.name == current-host.name;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
--rsync-path='mkdir -p ${config.target-path} && rsync' \
--delete-excluded \
-vrLptgoD \
${config.path}/ \
root@${target}:${config.target-path}
'';
in
if can-push then push-method else
let dir = "file://${config.host.name}${config.path}"; in
# /!\ revise this message when using more than just push-method
throw "No way to push ${dir} from ${current-host.name} to ${target}";
methods.git = config:
rootssh target ''
mkdir -p ${config.target-path}
cd ${config.target-path}
if ! test -e .git; then
git init
fi
if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
git remote add origin ${config.url}
elif test "$cur_url" != ${config.url}; then
git remote set-url origin ${config.url}
fi
if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
git fetch origin
git checkout ${config.rev} -- .
git checkout -q ${config.rev}
git submodule init
git submodule update
fi
git clean -dxf
'';
in out;
rootssh = target: script:
"ssh root@${target} -T ${doc ''
set -efu
${script}
''}";
in out

View File

@ -29,10 +29,6 @@
# 2014-09-24 ref https://github.com/4z3/xintmap
http://www.mathstat.dal.ca/~selinger/quipper/
# 2014-12-12 remove nixopsUnstable when nixops get's bumped to 1.3
# ref https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/package-management/nixops/unstable.nix
http://nixos.org/releases/nixops/
## other
https://nixos.org/channels/nixos-unstable/git-revision