diff --git a/Makefile b/Makefile index 8834cb7c8..aefd17147 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # make infest system=foo [target=bar] # make [deploy] system=foo [target=bar] # make [deploy] systems='foo bar' -# make eval get=tv.wu.config.time.timeZone [filter=json] +# make eval get=users.tv.wu.config.time.timeZone [filter=json] # .ONESHELL: diff --git a/default.nix b/default.nix index 841882f54..11bae7d98 100644 --- a/default.nix +++ b/default.nix @@ -1,72 +1,112 @@ +# Welcome to the top-level default.nix of stockholm. +# +# You can discover the whole thing easily using the `get` utility, +# which can be found at http://cgit.cd.krebsco.de/get/tree/get +# To install `get` on any Nix-enabled system, use: +# +# nix-env -f /path/to/stockholm -iA pkgs.get +# +# The "current" arguments are used to provide information about the user who's +# evaluating this file. This information is used to determine which user +# namespace is to be used. Of course there's nothing trying to prevent you +# from forging this information. E.g. you could try to generate the deployment +# script for some random user's system, targeting some random host: +# +# LOGNAME=tv get krebs.deploy system=nomic target=8.8.8.8 +# { current-date ? abort "current-date not defined" , current-host-name ? abort "current-host-name not defined" , current-user-name ? builtins.getEnv "LOGNAME" -}: +}@current: -assert current-user-name != ""; +let stockholm = { + # The generated scripts to deploy (or infest) systems can be found in the + # `krebs` attribute. There's also an init script, but it's in its early + # stages, not well integrated and mostly useless at the moment. :) + # + # You'll also find lib here, which is nixpkgs/lib + krebs lib, but nobody + # is really accessing this directly, as this lib gets reexported below. + inherit krebs; -let - lib = import ; - klib = import ./krebs/4lib { inherit lib; }; -in with klib; let + # All systems of all users can be found here. + # + # /!\ Please note that `get users.${user-name}.${host-name}.system` is a + # bad idea because it will produce vast amounts of output. These are the + # actual and complete system derivations that can be installed on the + # respective host. + # + # Another thing to notice here is that other user's systems might not be + # evaluable because of missing secrets. If you _are_ able to evaluate + # another user's system, then you probably share a similar naming scheme + # for your secret files! :) + inherit users; - nspath = ns: p: ./. + "/${ns}/${p}"; - kpath = nspath "krebs"; - upath = nspath current-user-name; + # Additionally, output lib and pkgs for easy access from the shell. + # Notice how we're evaluating just the base module to obtain pkgs. + inherit lib; + inherit (eval {}) pkgs; + }; - stockholm = { + krebs = import ./krebs (current // { inherit stockholm; }); + inherit (krebs) lib; + + # Path resolvers for common and individual files. + # Example: `upath "3modules"` produces the current user's 3modules directory + kpath = lib.nspath "krebs"; + upath = lib.nspath current-user-name; + + # This is the base module. Its purpose is to provide modules and + # packages, both common ones, found in krebs/ as well as the current user's, + # found in the user's namespace. + base-module = { imports = map (f: f "3modules") [ kpath upath ]; nixpkgs.config.packageOverrides = pkgs: let + # Notice the ordering. Krebs packages can only depend on Nixpkgs, + # whereas user packages additionally can depend on krebs packages. kpkgs = import (kpath "5pkgs") { inherit pkgs; }; upkgs = import (upath "5pkgs") { pkgs = pkgs // kpkgs; }; in kpkgs // upkgs; }; - out = - { inherit (eval {}) config options pkgs; } // - lib.mapAttrs - (name: _: - if builtins.pathExists (nspath name "default.nix") - then import (nspath name "default.nix") - else import-1systems (nspath name "1systems")) - (lib.filterAttrs - (n: t: !lib.hasPrefix "." n && t == "directory") - (builtins.readDir ./.)); - - eval = path: import { + # The above base module is used together with a NixOS configuration to + # produce a system. Notice how stockholm really just provides additional + # packages and modules on top of NixOS. Some of this stuff might become + # useful to a broader audience, at which point it should probably be merged + # and pull-requested for inclusion into NixOS/nixpkgs. + # TODO provide krebs lib, so modules don't have to import it awkwardly + eval = config: import { modules = [ - stockholm - path + base-module + config ]; }; - import-1systems = path: lib.mapAttrs (_: mk-system) (nixDir path); + # Any top-level directory other than krebs/ is considered to be a user + # namespace, configuring a bunch of systems. + # Have a look at the definition of install in krebs/default.nix to see how + # nix-env is using this attribute set to obtain the system to be installed. + # TODO move user namespaces' to users/, so no exception for krebs/ is needed + users = + lib.mapAttrs + (name: _: eval-all-systems (lib.nspath name "1systems")) + (lib.filterAttrs + (n: t: !lib.hasPrefix "." n && t == "directory" && n != "krebs") + (builtins.readDir ./.)); - mk-system = path: rec { + # Given a path to a user namespace, provide an attribute of evaluated + # system configurations, keyed by system names (AKA host names). + eval-all-systems = path: + lib.mapAttrs' + (n: _: (lib.nameValuePair (lib.removeSuffix ".nix" n) + (eval-system (path + "/${n}")))) + (builtins.readDir path); + + eval-system = path: rec { inherit (eval path) config options; system = config.system.build.toplevel; - fetch = import ./krebs/0tools/fetch.nix { inherit config lib; }; }; - nixDir = path: - builtins.listToAttrs - (catMaybes - (lib.mapAttrsToList - (k: v: { - directory = - let p = path + "/${k}/default.nix"; in - if builtins.pathExists p - then Just (lib.nameValuePair k p) - else Nothing; - regular = - let p = path + "/${k}"; in - if lib.hasSuffix ".nix" p - then Just (lib.nameValuePair (lib.removeSuffix ".nix" k) p) - else Nothing; - }.${v} or Nothing) - (builtins.readDir path))); - -in out +in stockholm diff --git a/krebs/3modules/nginx.nix b/krebs/3modules/nginx.nix index 702e8a7f6..0530921a0 100644 --- a/krebs/3modules/nginx.nix +++ b/krebs/3modules/nginx.nix @@ -24,9 +24,21 @@ let "${config.networking.hostName}.retiolum" ]; }; + listen = mkOption { + type = with types; either str (listOf str); + default = "80"; + apply = x: + if typeOf x != "list" + then [x] + else x; + }; locations = mkOption { type = with types; listOf (attrsOf str); }; + extraConfig = mkOption { + type = with types; str; + default = ""; + }; }; default = {}; }; @@ -60,10 +72,11 @@ let } ''; - to-server = { server-names, locations, ... }: '' + to-server = { server-names, listen, locations, extraConfig, ... }: '' server { - listen 80; + ${concatMapStringsSep "\n" (x: "listen ${x};") listen} server_name ${toString server-names}; + ${extraConfig} ${indent (concatStrings (map to-location locations))} } ''; diff --git a/krebs/4lib/default.nix b/krebs/4lib/default.nix index f15514fe3..0a6101314 100644 --- a/krebs/4lib/default.nix +++ b/krebs/4lib/default.nix @@ -3,13 +3,8 @@ with builtins; with lib; -let - maybe = import ./maybe.nix { inherit lib; }; -in - builtins // lib // -maybe // rec { eq = x: y: x == y; diff --git a/krebs/4lib/maybe.nix b/krebs/4lib/maybe.nix deleted file mode 100644 index 5189c99a1..000000000 --- a/krebs/4lib/maybe.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ ... }: - -rec { - Just = x: { type = "maybe"; value = x; }; - Nothing = { type = "maybe"; }; - isMaybe = x: builtins.typeOf x == "set" && x.type or false == "maybe"; - isJust = x: isMaybe x && builtins.hasAttr "value" x; - fromJust = x: assert isJust x; x.value; - catMaybes = xs: map fromJust (builtins.filter isJust xs); -} diff --git a/krebs/5pkgs/push/default.nix b/krebs/5pkgs/push/default.nix index ce496d9c3..410b43465 100644 --- a/krebs/5pkgs/push/default.nix +++ b/krebs/5pkgs/push/default.nix @@ -9,12 +9,12 @@ , ... }: stdenv.mkDerivation { - name = "push-1.0.0"; + name = "push-1.1.0"; src = fetchgit { url = http://cgit.cd.retiolum/push; - rev = "513da89fe50b3bad3d758855f5622c4508977e4a"; - sha256 = "6124e1d4d4ef57455e2f06891e06fb01d3786846efaf9b79e3176d89988e1b4e"; + rev = "c5f4bda5bd00bad7778bbd5a9af8d476de0de920"; + sha256 = "d335b644b791214263cee5c6659538c8e45326531b0588e5e7eb3bd9ef969800"; }; phases = [ diff --git a/krebs/default.nix b/krebs/default.nix index b98fad558..de805a89c 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -1,21 +1,30 @@ { current-date , current-host-name , current-user-name -}@current: rec { +, stockholm +}: + +let out = { + inherit deploy; + inherit infest; + inherit init; + inherit lib; + inherit nixos-install; + }; deploy = { system ? current-host-name , target ? system }@args: let - config = lib.get-config system; + config = get-config system; in '' #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} # krebs.deploy set -efu - (${lib.populate args}) - ${lib.rootssh target '' - ${lib.install args} + (${populate args}) + ${rootssh target '' + ${nix-install args} ${config.krebs.build.profile}/bin/switch-to-configuration switch ''} echo OK @@ -31,63 +40,14 @@ # krebs.infest set -efu - # XXX type -p is non-standard - #export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \ - # -o 'HostName $ {target.host.infest.addr}' \ - # -o 'Port $ {toString target.host.infest.port}' \ - #" - #ssh() { - # eval "$RSYNC_RSH \"\$@\"" - #} - - ${lib.rootssh target '' + ${rootssh target '' ${builtins.readFile ./4lib/infest/prepare.sh} ${builtins.readFile ./4lib/infest/install-nix.sh} ''} - (${lib.populate args}) - - ${lib.rootssh target '' - export PATH; PATH=/root/.nix-profile/bin:$PATH - - src=$(type -p nixos-install) - cat_src() { - sed < "$src" "$( - { sed < "$src" -n ' - /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/= - /^nixpkgs=/= - /^NIX_PATH=/,/^$/{/./=} - - # Disable: Copy the NixOS/Nixpkgs sources to the target as - # the initial contents of the NixOS channel. - /^srcs=/,/^ln -sfn /= - ' - } | sed 's:$:s/^/#krebs#/:' - )" - } - - # Location to insert lib.install - i=$(sed -n '/^echo "building the system configuration/=' "$src") - - { - cat_src | sed -n "1,$i{p}" - cat ${lib.doc (lib.install args)} - cat_src | sed -n "$i,\''${$i!p}" - } > nixos-install - chmod +x nixos-install - - ## Wrap inserted lib.install into chroot. - #nix_env=$(cat_src | sed -n ' - # s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q - #') - #echo nix-env is $nix_env - #sed -i ' - # s:^nix-env:chroot $mountPoint '"$nix_env"': - #' nixos-install - - unset SSL_CERT_FILE - ./nixos-install + (${nixos-install args}) + ${rootssh target '' ${builtins.readFile ./4lib/infest/finalize.sh} ''} ''; @@ -95,7 +55,7 @@ init = { system ? current-host-name }@args: let - config = lib.get-config system; + config = get-config system; in '' #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} @@ -127,138 +87,166 @@ EOF ''; + nixos-install = + { system ? current-host-name + , target ? system + }@args: let + in '' + #! /bin/sh + # ${current-date} ${current-user-name}@${current-host-name} + # krebs.nixos-install + (${populate args}) + + ${rootssh target '' + export PATH; PATH=/root/.nix-profile/bin:$PATH + + src=$(type -p nixos-install) + cat_src() { + sed < "$src" "$( + { sed < "$src" -n ' + /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/= + /^nixpkgs=/= + /^NIX_PATH=/,/^$/{/./=} + + # Disable: Copy the NixOS/Nixpkgs sources to the target as + # the initial contents of the NixOS channel. + /^srcs=/,/^ln -sfn /= + ' + } | sed 's:$:s/^/#krebs#/:' + )" + } + + # Location to insert `nix-install` + i=$(sed -n '/^echo "building the system configuration/=' "$src") + + { + cat_src | sed -n "1,$i{p}" + cat ${doc (nix-install args)} + cat_src | sed -n "$i,\''${$i!p}" + } > nixos-install + chmod +x nixos-install + + unset SSL_CERT_FILE + ./nixos-install + ''} + ''; + lib = import ./4lib { lib = import ; } // rec { - - stockholm = import ../. current; - - get-config = system: - stockholm.${current-user-name}.${system}.config - or (abort "unknown system: ${system}"); - - doc = s: - let b = "EOF${builtins.hashString "sha256" s}"; in - '' - <<\${b} - ${s} - ${b} - ''; - - rootssh = target: script: - "ssh root@${target} -T ${lib.doc '' - set -efu - ${script} - ''}"; - - install = - { system ? current-host-name - , target ? system - }: - let - stockholm = import ../. { - inherit current-date; - inherit current-host-name; - inherit current-user-name; - }; - - config = stockholm.${current-user-name}.${system}.config - or (abort "unknown system: ${system}"); - - nix-path = - lib.concatStringsSep ":" - (lib.mapAttrsToList (name: _: "${name}=/root/${name}") - (config.krebs.build.source.dir // - config.krebs.build.source.git)); - in '' - set -efu - NIX_PATH=${lib.shell.escape nix-path} \ - nix-env \ - --show-trace \ - -f '' \ - -Q \ - --argstr current-date ${lib.shell.escape current-date} \ - --argstr current-host-name ${lib.shell.escape current-host-name} \ - --argstr current-user-name ${lib.shell.escape current-user-name} \ - --profile ${lib.shell.escape config.krebs.build.profile} \ - --set \ - -A ${lib.escapeShellArg (lib.concatStringsSep "." [ - config.krebs.build.user.name - config.krebs.build.host.name - "system" - ])} - ''; - - populate = - { system ? current-host-name - , target ? system - }@args: - let out = '' - #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} - set -efu - ${lib.concatStringsSep "\n" - (lib.concatMap - (type: lib.mapAttrsToList (_: methods.${type}) - config.krebs.build.source.${type}) - ["dir" "git"])} - ''; - - stockholm = import ../. { - inherit current-date; - inherit current-host-name; - inherit current-user-name; - }; - - config = stockholm.${current-user-name}.${system}.config - or (abort "unknown system: ${system}"); - - current-host = config.krebs.hosts.${current-host-name}; - current-user = config.krebs.users.${current-user-name}; - - target-host = config.krebs.hosts.${system}; - - methods.dir = config: - let - can-link = config.host.name == target-host.name; - can-push = config.host.name == current-host.name; - push-method = '' - rsync \ - --exclude .git \ - --exclude .graveyard \ - --exclude old \ - --exclude tmp \ - --rsync-path='mkdir -p ${config.target-path} && rsync' \ - --delete-excluded \ - -vrLptgoD \ - ${config.path}/ \ - root@${target}:${config.target-path} - ''; - url = "file://${config.host.name}${config.path}"; - in - #if can-link then link-method else - if can-push then push-method else - throw "cannot source ${url}"; - - methods.git = config: - lib.rootssh target '' - mkdir -p ${config.target-path} - cd ${config.target-path} - if ! test -e .git; then - git init - fi - if ! cur_url=$(git config remote.origin.url 2>/dev/null); then - git remote add origin ${config.url} - elif test "$cur_url" != ${config.url}; then - git remote set-url origin ${config.url} - fi - if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then - git fetch origin - git checkout ${config.rev} -- . - git checkout -q ${config.rev} - git submodule init - git submodule update - fi - git clean -dxf - ''; - in out; + stockholm-path = ../.; + nspath = ns: p: stockholm-path + "/${ns}/${p}"; }; -} + + doc = s: + let b = "EOF${builtins.hashString "sha256" s}"; in + '' + <<\${b} + ${s} + ${b} + ''; + + get-config = system: + stockholm.users.${current-user-name}.${system}.config + or (abort "unknown system: ${system}, user: ${current-user-name}"); + + nix-install = + { system ? current-host-name + , target ? system + }: + let + config = get-config system; + + nix-path = + lib.concatStringsSep ":" + (lib.mapAttrsToList (name: _: "${name}=/root/${name}") + (config.krebs.build.source.dir // + config.krebs.build.source.git)); + in '' + set -efu + NIX_PATH=${lib.shell.escape nix-path} \ + nix-env \ + --show-trace \ + -f '' \ + --argstr current-date ${lib.shell.escape current-date} \ + --argstr current-host-name ${lib.shell.escape current-host-name} \ + --argstr current-user-name ${lib.shell.escape current-user-name} \ + --profile ${lib.shell.escape config.krebs.build.profile} \ + --set \ + -A ${lib.escapeShellArg (lib.concatStringsSep "." [ + "users" + config.krebs.build.user.name + config.krebs.build.host.name + "system" + ])} + ''; + + populate = + { system ? current-host-name + , target ? system + }@args: + let out = '' + #! /bin/sh + # ${current-date} ${current-user-name}@${current-host-name} + set -efu + ${lib.concatStringsSep "\n" + (lib.concatMap + (type: lib.mapAttrsToList (_: methods.${type}) + config.krebs.build.source.${type}) + ["dir" "git"])} + ''; + + config = get-config system; + + current-host = config.krebs.hosts.${current-host-name}; + current-user = config.krebs.users.${current-user-name}; + + methods.dir = config: + let + can-push = config.host.name == current-host.name; + push-method = '' + rsync \ + --exclude .git \ + --exclude .graveyard \ + --exclude old \ + --exclude tmp \ + --rsync-path='mkdir -p ${config.target-path} && rsync' \ + --delete-excluded \ + -vrLptgoD \ + ${config.path}/ \ + root@${target}:${config.target-path} + ''; + in + if can-push then push-method else + let dir = "file://${config.host.name}${config.path}"; in + # /!\ revise this message when using more than just push-method + throw "No way to push ${dir} from ${current-host.name} to ${target}"; + + methods.git = config: + rootssh target '' + mkdir -p ${config.target-path} + cd ${config.target-path} + if ! test -e .git; then + git init + fi + if ! cur_url=$(git config remote.origin.url 2>/dev/null); then + git remote add origin ${config.url} + elif test "$cur_url" != ${config.url}; then + git remote set-url origin ${config.url} + fi + if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then + git fetch origin + git checkout ${config.rev} -- . + git checkout -q ${config.rev} + git submodule init + git submodule update + fi + git clean -dxf + ''; + in out; + + rootssh = target: script: + "ssh root@${target} -T ${doc '' + set -efu + ${script} + ''}"; + +in out diff --git a/tv/2configs/urlwatch.nix b/tv/2configs/urlwatch.nix index c1c5d19d9..373803cc6 100644 --- a/tv/2configs/urlwatch.nix +++ b/tv/2configs/urlwatch.nix @@ -29,10 +29,6 @@ # 2014-09-24 ref https://github.com/4z3/xintmap http://www.mathstat.dal.ca/~selinger/quipper/ - # 2014-12-12 remove nixopsUnstable when nixops get's bumped to 1.3 - # ref https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/package-management/nixops/unstable.nix - http://nixos.org/releases/nixops/ - ## other https://nixos.org/channels/nixos-unstable/git-revision