Merge remote-tracking branch 'cd/master'

Makefile

@@ -3,7 +3,7 @@
 #		make infest system=foo [target=bar]
 #		make [deploy] system=foo [target=bar]
 #		make [deploy] systems='foo bar'
-#		make eval get=tv.wu.config.time.timeZone [filter=json]
+#		make eval get=users.tv.wu.config.time.timeZone [filter=json]
 #
 
 .ONESHELL:

default.nix

@@ -1,72 +1,112 @@
+# Welcome to the top-level default.nix of stockholm.
+#
+# You can discover the whole thing easily using the `get` utility,
+# which can be found at http://cgit.cd.krebsco.de/get/tree/get
+# To install `get` on any Nix-enabled system, use:
+#
+#     nix-env -f /path/to/stockholm -iA pkgs.get
+#
+# The "current" arguments are used to provide information about the user who's
+# evaluating this file.  This information is used to determine which user
+# namespace is to be used.  Of course there's nothing trying to prevent you
+# from forging this information.  E.g. you could try to generate the deployment
+# script for some random user's system, targeting some random host:
+#
+#     LOGNAME=tv get krebs.deploy system=nomic target=8.8.8.8
+#
 { current-date ? abort "current-date not defined"
 , current-host-name ? abort "current-host-name not defined"
 , current-user-name ? builtins.getEnv "LOGNAME"
-}:
+}@current:
 
-assert current-user-name != "";
+let stockholm = {
+    # The generated scripts to deploy (or infest) systems can be found in the
+    # `krebs` attribute.  There's also an init script, but it's in its early
+    # stages, not well integrated and mostly useless at the moment. :)
+    #
+    # You'll also find lib here, which is nixpkgs/lib + krebs lib, but nobody
+    # is really accessing this directly, as this lib gets reexported below.
+    inherit krebs;
 
-let
+    # All systems of all users can be found here.
-  lib = import <nixpkgs/lib>;
+    #
-  klib = import ./krebs/4lib { inherit lib; };
+    # /!\ Please note that `get users.${user-name}.${host-name}.system` is a
-in with klib; let
+    # bad idea because it will produce vast amounts of output.  These are the
+    # actual and complete system derivations that can be installed on the
+    # respective host.
+    #
+    # Another thing to notice here is that other user's systems might not be
+    # evaluable because of missing secrets.  If you _are_ able to evaluate
+    # another user's system, then you probably share a similar naming scheme
+    # for your secret files! :)
+    inherit users;
 
-  nspath = ns: p: ./. + "/${ns}/${p}";
+    # Additionally, output lib and pkgs for easy access from the shell.
-  kpath = nspath "krebs";
+    # Notice how we're evaluating just the base module to obtain pkgs.
-  upath = nspath current-user-name;
+    inherit lib;
+    inherit (eval {}) pkgs;
+  };
 
-  stockholm = {
+  krebs = import ./krebs (current // { inherit stockholm; });
+  inherit (krebs) lib;
+
+  # Path resolvers for common and individual files.
+  # Example: `upath "3modules"` produces the current user's 3modules directory
+  kpath = lib.nspath "krebs";
+  upath = lib.nspath current-user-name;
+
+  # This is the base module.  Its purpose is to provide modules and
+  # packages, both common ones, found in krebs/ as well as the current user's,
+  # found in the user's namespace.
+  base-module = {
     imports = map (f: f "3modules") [ kpath upath ];
 
     nixpkgs.config.packageOverrides = pkgs:
       let
+        # Notice the ordering.  Krebs packages can only depend on Nixpkgs,
+        # whereas user packages additionally can depend on krebs packages.
         kpkgs = import (kpath "5pkgs") { inherit pkgs; };
         upkgs = import (upath "5pkgs") { pkgs = pkgs // kpkgs; };
       in
       kpkgs // upkgs;
   };
 
-  out =
+  # The above base module is used together with a NixOS configuration to
-    { inherit (eval {}) config options pkgs; } //
+  # produce a system.  Notice how stockholm really just provides additional
-    lib.mapAttrs
+  # packages and modules on top of NixOS.  Some of this stuff might become
-      (name: _:
+  # useful to a broader audience, at which point it should probably be merged
-        if builtins.pathExists (nspath name "default.nix")
+  # and pull-requested for inclusion into NixOS/nixpkgs.
-          then import (nspath name "default.nix")
+  # TODO provide krebs lib, so modules don't have to import it awkwardly
-          else import-1systems (nspath name "1systems"))
+  eval = config: import <nixpkgs/nixos/lib/eval-config.nix> {
-      (lib.filterAttrs
-        (n: t: !lib.hasPrefix "." n && t == "directory")
-        (builtins.readDir ./.));
-
-  eval = path: import <nixpkgs/nixos/lib/eval-config.nix> {
     modules = [
-      stockholm
+      base-module
-      path
+      config
     ];
   };
 
-  import-1systems = path: lib.mapAttrs (_: mk-system) (nixDir path);
+  # Any top-level directory other than krebs/ is considered to be a user
+  # namespace, configuring a bunch of systems.
+  # Have a look at the definition of install in krebs/default.nix to see how
+  # nix-env is using this attribute set to obtain the system to be installed.
+  # TODO move user namespaces' to users/, so no exception for krebs/ is needed
+  users =
+    lib.mapAttrs
+      (name: _: eval-all-systems (lib.nspath name "1systems"))
+      (lib.filterAttrs
+        (n: t: !lib.hasPrefix "." n && t == "directory" && n != "krebs")
+        (builtins.readDir ./.));
 
-  mk-system = path: rec {
+  # Given a path to a user namespace, provide an attribute of evaluated
+  # system configurations, keyed by system names (AKA host names).
+  eval-all-systems = path:
+    lib.mapAttrs'
+      (n: _: (lib.nameValuePair (lib.removeSuffix ".nix" n)
+                                (eval-system (path + "/${n}"))))
+      (builtins.readDir path);
+
+  eval-system = path: rec {
     inherit (eval path) config options;
     system = config.system.build.toplevel;
-    fetch = import ./krebs/0tools/fetch.nix { inherit config lib; };
   };
 
-  nixDir = path:
+in stockholm
-    builtins.listToAttrs
-      (catMaybes
-        (lib.mapAttrsToList
-          (k: v: {
-            directory =
-              let p = path + "/${k}/default.nix"; in
-              if builtins.pathExists p
-                then Just (lib.nameValuePair k p)
-                else Nothing;
-            regular =
-              let p = path + "/${k}"; in
-              if lib.hasSuffix ".nix" p
-                then Just (lib.nameValuePair (lib.removeSuffix ".nix" k) p)
-                else Nothing;
-          }.${v} or Nothing)
-          (builtins.readDir path)));
-
-in out

krebs/3modules/nginx.nix

@@ -24,9 +24,21 @@ let
             "${config.networking.hostName}.retiolum"
           ];
         };
+        listen = mkOption {
+          type = with types; either str (listOf str);
+          default = "80";
+          apply = x:
+            if typeOf x != "list"
+              then [x]
+              else x;
+        };
         locations = mkOption {
           type = with types; listOf (attrsOf str);
         };
+        extraConfig = mkOption {
+          type = with types; str;
+          default = "";
+        };
       };
       default = {};
     };
@@ -60,10 +72,11 @@ let
     }
   '';
 
-  to-server = { server-names, locations, ... }: ''
+  to-server = { server-names, listen, locations, extraConfig, ... }: ''
     server {
-      listen 80;
+      ${concatMapStringsSep "\n" (x: "listen ${x};") listen}
       server_name ${toString server-names};
+      ${extraConfig}
       ${indent (concatStrings (map to-location locations))}
     }
   '';

krebs/4lib/default.nix

@@ -3,13 +3,8 @@
 with builtins;
 with lib;
 
-let
-  maybe = import ./maybe.nix { inherit lib; };
-in
-
 builtins //
 lib //
-maybe //
 rec {
 
   eq = x: y: x == y;

krebs/4lib/maybe.nix

@@ -1,10 +0,0 @@
-{ ... }:
-
-rec {
-  Just = x: { type = "maybe"; value = x; };
-  Nothing = { type = "maybe"; };
-  isMaybe = x: builtins.typeOf x == "set" && x.type or false == "maybe";
-  isJust = x: isMaybe x && builtins.hasAttr "value" x;
-  fromJust = x: assert isJust x; x.value;
-  catMaybes = xs: map fromJust (builtins.filter isJust xs);
-}

krebs/5pkgs/push/default.nix

@@ -9,12 +9,12 @@
 , ... }:
 
 stdenv.mkDerivation {
-  name = "push-1.0.0";
+  name = "push-1.1.0";
 
   src = fetchgit {
     url = http://cgit.cd.retiolum/push;
-    rev = "513da89fe50b3bad3d758855f5622c4508977e4a";
+    rev = "c5f4bda5bd00bad7778bbd5a9af8d476de0de920";
-    sha256 = "6124e1d4d4ef57455e2f06891e06fb01d3786846efaf9b79e3176d89988e1b4e";
+    sha256 = "d335b644b791214263cee5c6659538c8e45326531b0588e5e7eb3bd9ef969800";
   };
 
   phases = [

krebs/default.nix

@@ -1,21 +1,30 @@
 { current-date
 , current-host-name
 , current-user-name
-}@current: rec {
+, stockholm
+}:
+
+let out = {
+    inherit deploy;
+    inherit infest;
+    inherit init;
+    inherit lib;
+    inherit nixos-install;
+  };
 
   deploy =
     { system ? current-host-name
     , target ? system
     }@args: let
-      config = lib.get-config system;
+      config = get-config system;
     in ''
       #! /bin/sh
       # ${current-date} ${current-user-name}@${current-host-name}
       # krebs.deploy
       set -efu
-      (${lib.populate args})
+      (${populate args})
-      ${lib.rootssh target ''
+      ${rootssh target ''
-        ${lib.install args}
+        ${nix-install args}
         ${config.krebs.build.profile}/bin/switch-to-configuration switch
       ''}
       echo OK
@@ -31,63 +40,14 @@
       # krebs.infest
       set -efu
 
-      # XXX type -p is non-standard
+      ${rootssh target ''
-      #export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \
-      #  -o 'HostName $ {target.host.infest.addr}' \
-      #  -o 'Port $ {toString target.host.infest.port}' \
-      #"
-      #ssh() {
-      #  eval "$RSYNC_RSH \"\$@\""
-      #}
-
-      ${lib.rootssh target ''
         ${builtins.readFile ./4lib/infest/prepare.sh}
         ${builtins.readFile ./4lib/infest/install-nix.sh}
       ''}
 
-      (${lib.populate args})
+      (${nixos-install args})
-
-      ${lib.rootssh target ''
-        export PATH; PATH=/root/.nix-profile/bin:$PATH
-
-        src=$(type -p nixos-install)
-        cat_src() {
-          sed < "$src" "$(
-            { sed < "$src" -n '
-                  /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
-                  /^nixpkgs=/=
-                  /^NIX_PATH=/,/^$/{/./=}
-
-                  # Disable: Copy the NixOS/Nixpkgs sources to the target as
-                  # the initial contents of the NixOS channel.
-                  /^srcs=/,/^ln -sfn /=
-                '
-            } | sed 's:$:s/^/#krebs#/:'
-          )"
-        }
-
-        # Location to insert lib.install
-        i=$(sed -n '/^echo "building the system configuration/=' "$src")
-
-        {
-          cat_src | sed -n "1,$i{p}"
-          cat ${lib.doc (lib.install args)}
-          cat_src | sed -n "$i,\''${$i!p}"
-        } > nixos-install
-        chmod +x nixos-install
-
-        ## Wrap inserted lib.install into chroot.
-        #nix_env=$(cat_src | sed -n '
-        #  s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q
-        #')
-        #echo nix-env is $nix_env
-        #sed -i '
-        #  s:^nix-env:chroot $mountPoint '"$nix_env"':
-        #' nixos-install
-
-        unset SSL_CERT_FILE
-        ./nixos-install
 
+      ${rootssh target ''
         ${builtins.readFile ./4lib/infest/finalize.sh}
       ''}
     '';
@@ -95,7 +55,7 @@
   init =
     { system ? current-host-name
     }@args: let
-      config = lib.get-config system;
+      config = get-config system;
     in ''
       #! /bin/sh
       # ${current-date} ${current-user-name}@${current-host-name}
@@ -127,138 +87,166 @@
       EOF
     '';
 
+  nixos-install =
+    { system ? current-host-name
+    , target ? system
+    }@args: let
+    in ''
+      #! /bin/sh
+      # ${current-date} ${current-user-name}@${current-host-name}
+      # krebs.nixos-install
+      (${populate args})
+
+      ${rootssh target ''
+        export PATH; PATH=/root/.nix-profile/bin:$PATH
+
+        src=$(type -p nixos-install)
+        cat_src() {
+          sed < "$src" "$(
+            { sed < "$src" -n '
+                  /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/=
+                  /^nixpkgs=/=
+                  /^NIX_PATH=/,/^$/{/./=}
+
+                  # Disable: Copy the NixOS/Nixpkgs sources to the target as
+                  # the initial contents of the NixOS channel.
+                  /^srcs=/,/^ln -sfn /=
+                '
+            } | sed 's:$:s/^/#krebs#/:'
+          )"
+        }
+
+        # Location to insert `nix-install`
+        i=$(sed -n '/^echo "building the system configuration/=' "$src")
+
+        {
+          cat_src | sed -n "1,$i{p}"
+          cat ${doc (nix-install args)}
+          cat_src | sed -n "$i,\''${$i!p}"
+        } > nixos-install
+        chmod +x nixos-install
+
+        unset SSL_CERT_FILE
+        ./nixos-install
+      ''}
+    '';
+
   lib = import ./4lib { lib = import <nixpkgs/lib>; } // rec {
-
+    stockholm-path = ../.;
-    stockholm = import ../. current;
+    nspath = ns: p: stockholm-path + "/${ns}/${p}";
-
-    get-config = system:
-      stockholm.${current-user-name}.${system}.config
-        or (abort "unknown system: ${system}");
-
-    doc = s:
-      let b = "EOF${builtins.hashString "sha256" s}"; in
-      ''
-      <<\${b}
-      ${s}
-      ${b}
-      '';
-
-    rootssh = target: script:
-      "ssh root@${target} -T ${lib.doc ''
-        set -efu
-        ${script}
-      ''}";
-
-    install =
-      { system ? current-host-name
-      , target ? system
-      }:
-      let
-        stockholm = import ../. {
-          inherit current-date;
-          inherit current-host-name;
-          inherit current-user-name;
-        };
-
-        config = stockholm.${current-user-name}.${system}.config
-          or (abort "unknown system: ${system}");
-
-        nix-path =
-          lib.concatStringsSep ":"
-            (lib.mapAttrsToList (name: _: "${name}=/root/${name}")
-              (config.krebs.build.source.dir //
-               config.krebs.build.source.git));
-      in ''
-        set -efu
-        NIX_PATH=${lib.shell.escape nix-path} \
-        nix-env \
-          --show-trace \
-          -f '<stockholm>' \
-          -Q \
-          --argstr current-date ${lib.shell.escape current-date} \
-          --argstr current-host-name ${lib.shell.escape current-host-name} \
-          --argstr current-user-name ${lib.shell.escape current-user-name} \
-          --profile ${lib.shell.escape config.krebs.build.profile} \
-          --set \
-          -A ${lib.escapeShellArg (lib.concatStringsSep "." [
-                config.krebs.build.user.name
-                config.krebs.build.host.name
-                "system"
-              ])}
-      '';
-
-    populate =
-      { system ? current-host-name
-      , target ? system
-      }@args:
-      let out = ''
-          #! /bin/sh
-          # ${current-date} ${current-user-name}@${current-host-name}
-          set -efu
-          ${lib.concatStringsSep "\n"
-            (lib.concatMap
-              (type: lib.mapAttrsToList (_: methods.${type})
-                                        config.krebs.build.source.${type})
-              ["dir" "git"])}
-        '';
-
-        stockholm = import ../. {
-          inherit current-date;
-          inherit current-host-name;
-          inherit current-user-name;
-        };
-
-        config = stockholm.${current-user-name}.${system}.config
-            or (abort "unknown system: ${system}");
-
-        current-host = config.krebs.hosts.${current-host-name};
-        current-user = config.krebs.users.${current-user-name};
-
-        target-host = config.krebs.hosts.${system};
-
-        methods.dir = config:
-          let
-            can-link = config.host.name == target-host.name;
-            can-push = config.host.name == current-host.name;
-            push-method = ''
-              rsync \
-                --exclude .git \
-                --exclude .graveyard \
-                --exclude old \
-                --exclude tmp \
-                --rsync-path='mkdir -p ${config.target-path} && rsync' \
-                --delete-excluded \
-                -vrLptgoD \
-                ${config.path}/ \
-                root@${target}:${config.target-path}
-            '';
-            url = "file://${config.host.name}${config.path}";
-          in
-          #if can-link then link-method else
-          if can-push then push-method else
-          throw "cannot source ${url}";
-
-        methods.git = config:
-          lib.rootssh target ''
-            mkdir -p ${config.target-path}
-            cd ${config.target-path}
-            if ! test -e .git; then
-              git init
-            fi
-            if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
-              git remote add origin ${config.url}
-            elif test "$cur_url" != ${config.url}; then
-              git remote set-url origin ${config.url}
-            fi
-            if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
-              git fetch origin
-              git checkout ${config.rev} -- .
-              git checkout -q ${config.rev}
-              git submodule init
-              git submodule update
-            fi
-            git clean -dxf
-          '';
-      in out;
   };
-}
+
+  doc = s:
+    let b = "EOF${builtins.hashString "sha256" s}"; in
+    ''
+    <<\${b}
+    ${s}
+    ${b}
+    '';
+
+  get-config = system:
+    stockholm.users.${current-user-name}.${system}.config
+      or (abort "unknown system: ${system}, user: ${current-user-name}");
+
+  nix-install =
+    { system ? current-host-name
+    , target ? system
+    }:
+    let
+      config = get-config system;
+
+      nix-path =
+        lib.concatStringsSep ":"
+          (lib.mapAttrsToList (name: _: "${name}=/root/${name}")
+            (config.krebs.build.source.dir //
+             config.krebs.build.source.git));
+    in ''
+      set -efu
+      NIX_PATH=${lib.shell.escape nix-path} \
+      nix-env \
+        --show-trace \
+        -f '<stockholm>' \
+        --argstr current-date ${lib.shell.escape current-date} \
+        --argstr current-host-name ${lib.shell.escape current-host-name} \
+        --argstr current-user-name ${lib.shell.escape current-user-name} \
+        --profile ${lib.shell.escape config.krebs.build.profile} \
+        --set \
+        -A ${lib.escapeShellArg (lib.concatStringsSep "." [
+              "users"
+              config.krebs.build.user.name
+              config.krebs.build.host.name
+              "system"
+            ])}
+    '';
+
+  populate =
+    { system ? current-host-name
+    , target ? system
+    }@args:
+    let out = ''
+        #! /bin/sh
+        # ${current-date} ${current-user-name}@${current-host-name}
+        set -efu
+        ${lib.concatStringsSep "\n"
+          (lib.concatMap
+            (type: lib.mapAttrsToList (_: methods.${type})
+                                      config.krebs.build.source.${type})
+            ["dir" "git"])}
+      '';
+
+      config = get-config system;
+
+      current-host = config.krebs.hosts.${current-host-name};
+      current-user = config.krebs.users.${current-user-name};
+
+      methods.dir = config:
+        let
+          can-push = config.host.name == current-host.name;
+          push-method = ''
+            rsync \
+              --exclude .git \
+              --exclude .graveyard \
+              --exclude old \
+              --exclude tmp \
+              --rsync-path='mkdir -p ${config.target-path} && rsync' \
+              --delete-excluded \
+              -vrLptgoD \
+              ${config.path}/ \
+              root@${target}:${config.target-path}
+          '';
+        in
+        if can-push then push-method else
+        let dir = "file://${config.host.name}${config.path}"; in
+        # /!\ revise this message when using more than just push-method
+        throw "No way to push ${dir} from ${current-host.name} to ${target}";
+
+      methods.git = config:
+        rootssh target ''
+          mkdir -p ${config.target-path}
+          cd ${config.target-path}
+          if ! test -e .git; then
+            git init
+          fi
+          if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
+            git remote add origin ${config.url}
+          elif test "$cur_url" != ${config.url}; then
+            git remote set-url origin ${config.url}
+          fi
+          if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
+            git fetch origin
+            git checkout ${config.rev} -- .
+            git checkout -q ${config.rev}
+            git submodule init
+            git submodule update
+          fi
+          git clean -dxf
+        '';
+    in out;
+
+  rootssh = target: script:
+    "ssh root@${target} -T ${doc ''
+      set -efu
+      ${script}
+    ''}";
+
+in out

tv/2configs/urlwatch.nix

@@ -29,10 +29,6 @@
       # 2014-09-24 ref https://github.com/4z3/xintmap
       http://www.mathstat.dal.ca/~selinger/quipper/
 
-      # 2014-12-12 remove nixopsUnstable when nixops get's bumped to 1.3
-      # ref https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/package-management/nixops/unstable.nix
-      http://nixos.org/releases/nixops/
-
       ## other
 
       https://nixos.org/channels/nixos-unstable/git-revision