krebs.build.populate: init and drop support for v1

Makefile

@@ -1,87 +1,51 @@
-#
-# usage:
-#		make infest system=foo [target=bar]
-#		make [deploy] system=foo [target=bar]
-#		make [deploy] systems='foo bar'
-#		make eval get=users.tv.wu.config.time.timeZone [filter=json]
-#
-
 .ONESHELL:
 .SHELLFLAGS := -eufc
 
-ifdef systems
+ifndef system
-$(systems):
+$(error unbound variable: system)
-	@
-	unset target
-	parallel \
-		--line-buffer \
-		-j0 \
-		--no-notice \
-		--tagstring {} \
-		-q make -s systems= system={} ::: $(systems)
-else ifdef system
-.PHONY: deploy infest
-deploy infest:;@
-	export get=krebs.$@
-	export filter=json
-	script=$$(make -s eval)
-	echo "$$script" | sh
-
-.PHONY: eval
-eval:
-	@
-ifeq ($(filter),json)
-	extraArgs='--json --strict'
-	filter() { jq -r .; }
-else
-	filter() { cat; }
 endif
-	result=$$(nix-instantiate \
-		$${extraArgs-} \
-		--eval \
-		-A "$$get" \
-		-I stockholm="$$PWD" \
-		'<stockholm>' \
-		--argstr current-host-name "$$HOSTNAME" \
-		--argstr current-user-name "$$LOGNAME" \
-		$${system+--argstr system "$$system"} \
-		$${target+--argstr target "$$target"})
-	echo "$$result" | filter
 
 export target_host ?= $(system)
 export target_user ?= root
 export target_path ?= /var/src
 
+# usage: make deploy system=foo [target_host=bar]
+.PHONY: deploy
+deploy: populate ;@set -x
+	ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path"
+
 # usage: make populate system=foo [target_host=bar]
 .PHONY: populate
-populate: export lib = \
-	let nlib = import <nixpkgs/lib>; in \
-	nlib // import krebs/4lib { lib = nlib; } // builtins
-populate: export source = \
-	with builtins; \
-	with (import ./. {}).users.$${getEnv "LOGNAME"}.$${getEnv "system"}; \
-	assert config.krebs.build.source-version == 2; \
-	config.krebs.build.source
 populate:;@
-	result=$$(nix-instantiate \
+	result=$$(make -s eval get=config.krebs.build.populate filter=json)
-			--eval \
+	echo "$$result" | sh
-			--json \
-			--arg lib "$$lib" \
-			--arg source "$$source" \
-			--argstr target-user "$$target_user" \
-			--argstr target-host "$$target_host" \
-			--argstr target-path "$$target_path" \
-			-A populate \
-			krebs/v2)
-	script=$$(echo "$$result" | jq -r .)
-	echo "$$script" | sh
-
-# usage: make rebuild system=foo [target_host=bar] [operation=switch]
-.PHONY: rebuild
-rebuild: populate ;@set -x
-	ssh "$$target_user@$$target_host" \
-		nixos-rebuild "$${operation-switch}" -I "$$target_path"
 
+# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json]
+.PHONY: eval
+eval:;@
+ifeq ($(filter),json)
+	extraArgs='--json --strict'
+	filter() { echo "$$1" | jq -r .; }
 else
-$(error unbound variable: system[s])
+	filter() { echo "$$1"; }
 endif
+	result=$$(nix-instantiate \
+		$${extraArgs-} \
+		--show-trace \
+		--readonly-mode \
+		--eval \
+		-A "$$get" \
+		--arg configuration "<stockholm/$$LOGNAME/1systems/$$system.nix>")
+	filter "$$result"
+
+## usage: make install system=foo target=
+#.PHONY: install
+#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+#install:;@set -x
+#	$(ssh) "$$target_user@$$target_host" \
+#		env target_path=/var/src \
+#			sh -s prepare < krebs/4lib/infest/prepare.sh
+#	make -s populate target_path=/mnt"$$target_path"
+#	$(ssh) "$$target_user@$$target_host" \
+#		env NIXOS_CONFIG=/var/src/nixos-config \
+#			nixos-install

default.nix

@@ -1,9 +1,15 @@
-{ current-host-name ?
+{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config>
+, system ? builtins.currentSystem
+, current-host-name ?
     let v = builtins.getEnv "HOSTNAME"; in
     if v != "" then v else builtins.readFile /proc/sys/kernel/hostname
 , current-user-name ?
     let v = builtins.getEnv "LOGNAME"; in
     if v != "" then v else abort "undefined variable: LOGNAME"
+, nixpkgs-path ?
+    if (builtins.tryEval <nixpkgs/krebs>).success
+      then <upstream-nixpkgs>
+      else <nixpkgs>
 , StrictHostKeyChecking ? "yes"
 }@args:
 
@@ -11,7 +17,8 @@ let stockholm = {
     inherit krebs;
     inherit users;
     inherit lib;
-    inherit pkgs;
+    inherit config options pkgs;
+    system = config.system.build.toplevel;
   };
 
   krebs = import ./krebs (args // { inherit lib stockholm; });
@@ -20,7 +27,7 @@ let stockholm = {
     nlib = import (slib.npath "lib");
     klib = import (slib.kpath "4lib") { lib = nlib; };
     slib = rec {
-      npath = p: <nixpkgs> + "/${p}";
+      npath = p: nixpkgs-path + "/${p}";
       kpath = p: ./. + "/krebs/${p}";
       upath = p: ./. + "/${current-user-name}/${p}";
     };
@@ -29,7 +36,7 @@ let stockholm = {
                          (import p { lib = nlib // klib; });
   in nlib // klib // slib // ulib // builtins;
 
-  inherit (eval {}) pkgs;
+  inherit (eval configuration) config options pkgs;
 
   base-module = { config, ... }: {
     imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [
@@ -48,6 +55,7 @@ let stockholm = {
   };
 
   eval = config: import (lib.npath "nixos/lib/eval-config.nix") {
+    inherit system;
     specialArgs = {
       inherit lib;
     };

krebs/3modules/build.nix

@@ -28,81 +28,157 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source-version = mkOption {
+    options.krebs.build.source = let
-      type = types.enum [ 1 2 ];
+      raw = types.either types.str types.path;
-      default = 1;
+      url = types.submodule {
-    };
+        options = {
-
+          url = mkOption {
-    options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" {
+            type = types.str;
-      v1 = {
+          };
-        dir = mkOption {
+          rev = mkOption {
-          type = let
+            type = types.str;
-            default-host = config.krebs.current.host;
+          };
-          in types.attrsOf (types.submodule ({ config, ... }: {
+          dev = mkOption {
-            options = {
+            type = types.str;
-              host = mkOption {
-                type = types.host;
-                default = default-host;
-              };
-              path = mkOption {
-                type = types.str;
-              };
-              target-path = mkOption {
-                type = types.str;
-                default = "/root/${config._module.args.name}";
-              };
-              url = mkOption {
-                type = types.str;
-                default = "file://${config.host.name}${config.path}";
-              };
-            };
-          }));
-          default = {};
-        };
-
-        git = mkOption {
-          type = with types; attrsOf (submodule ({ config, ... }: {
-            options = {
-              url = mkOption {
-                type = types.str; # TODO must be shell safe
-              };
-              rev = mkOption {
-                type = types.str;
-              };
-              target-path = mkOption {
-                type = types.str;
-                default = "/root/${config._module.args.name}";
-              };
-            };
-          }));
-          default = {};
-        };
-      };
-
-      v2 = let
-        raw = types.either types.str types.path;
-        url = types.submodule {
-          options = {
-            url = mkOption {
-              type = types.str;
-            };
-            rev = mkOption {
-              type = types.str;
-            };
-            dev = mkOption {
-              type = types.str;
-            };
           };
         };
-      in mkOption {
-        type = types.attrsOf (types.either types.str url);
-        apply = let f = mapAttrs (_: value: {
-          string = value;
-          path = toString value;
-          set = f value;
-        }.${typeOf value}); in f;
-        default = {};
       };
+    in mkOption {
+      type = types.attrsOf (types.either types.str url);
+      apply = let f = mapAttrs (_: value: {
+        string = value;
+        path = toString value;
+        set = f value;
+      }.${typeOf value}); in f;
+      default = {};
+    };
+
+    options.krebs.build.populate = mkOption {
+      type = types.str;
+      default = let
+        source = config.krebs.build.source;
+        target-user = maybeEnv "target_user" "root";
+        target-host = maybeEnv "target_host" config.krebs.build.host.name;
+        target-path = maybeEnv "target_path" "/var/src";
+        out = ''
+          #! /bin/sh
+          set -eu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          echo ${shell.escape git-script} \
+            | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+          unset tmpdir
+          trap '
+            rm "$tmpdir"/*
+            rmdir "$tmpdir"
+            trap - EXIT INT QUIT
+          '        EXIT INT QUIT
+          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+          chmod 0755 "$tmpdir"
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+              symlink-specs)}
+
+          verbose proot \
+              -b $tmpdir:${shell.escape target-path} \
+              ${concatStringsSep " \\\n    "
+                (mapAttrsToList
+                  (name: spec:
+                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+                  file-specs)} \
+              rsync \
+                  -f ${shell.escape "P /*"} \
+                  ${concatMapStringsSep " \\\n        "
+                    (name: "-f ${shell.escape "R /${name}"}")
+                    (attrNames file-specs)} \
+                  --delete \
+                  -vFrlptD \
+                  ${shell.escape target-path}/ \
+                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
+        '';
+
+        get-schema = uri:
+          if substring 0 1 uri == "/"
+            then "file"
+            else head (splitString ":" uri);
+
+        has-schema = schema: uri: get-schema uri == schema;
+
+        get-url = spec: {
+          string = spec;
+          path = toString spec;
+          set = get-url spec.url;
+        }.${typeOf spec};
+
+        git-specs =
+          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+        file-specs =
+          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+        symlink-specs =
+          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+        git-script = ''
+          #! /bin/sh
+          set -efu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          fetch_git() {(
+            dst_dir=$1
+            src_url=$2
+            src_ref=$3
+
+            if ! test -e "$dst_dir"; then
+              git clone "$src_url" "$dst_dir"
+            fi
+
+            cd "$dst_dir"
+
+            if ! url=$(git config remote.origin.url); then
+              git remote add origin "$src_url"
+            elif test "$url" != "$src_url"; then
+              git remote set-url origin "$src_url"
+            fi
+
+            # TODO resolve src_ref to commit hash
+            hash=$src_ref
+
+            if ! test "$(git log --format=%H -1)" = "$hash"; then
+              git fetch origin
+              git checkout "$hash" -- "$dst_dir"
+              git checkout "$hash"
+            fi
+
+            git clean -dxf
+          )}
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: toString (map shell.escape [
+                "verbose"
+                "fetch_git"
+                "${target-path}/${name}"
+                spec.url
+                spec.rev
+              ]))
+              git-specs)}
+        '';
+      in out;
     };
 
   };

krebs/default.nix

@@ -1,3 +1,5 @@
+assert false;
+
 { current-host-name
 , current-user-name
 , lib
@@ -6,30 +8,11 @@
 }:
 
 let out = {
-    inherit deploy;
     inherit infest;
     inherit init;
     inherit nixos-install;
-    inherit populate;
   };
 
-  deploy =
-    { system ? current-host-name
-    , target ? system
-    }@args: let
-      config = get-config system;
-    in ''
-      #! /bin/sh
-      # krebs.deploy
-      set -efu
-      (${populate args})
-      ${rootssh target ''
-        ${nix-install args}
-        ${config.krebs.build.profile}/bin/switch-to-configuration switch
-      ''}
-      echo OK
-    '';
-
   infest =
     { system ? current-host-name
     , target ? system
@@ -45,9 +28,6 @@ let out = {
         ${builtins.readFile ./4lib/infest/install-nix.sh}
       ''}
 
-      # Prepare target source via bind-mounting
-
-
       (${nixos-install args})
 
       ${rootssh target ''
@@ -169,9 +149,7 @@ let out = {
   get-config = system: let
     config = stockholm.users.${current-user-name}.${system}.config
       or (abort "unknown system: ${system}, user: ${current-user-name}");
-  in
+  in config;
-  assert config.krebs.build.source-version == 1;
-  config;
 
   nix-install =
     { system ? current-host-name
@@ -203,73 +181,6 @@ let out = {
             ])}
     '';
 
-  populate =
-    { system ? current-host-name
-    , target ? system
-    , root ? ""
-    }@args:
-    let out = ''
-        #! /bin/sh
-        set -efu
-        ${lib.concatStringsSep "\n"
-          (lib.concatMap
-            (type: lib.mapAttrsToList (_: methods.${type})
-                                      config.krebs.build.source.${type})
-            ["dir" "git"])}
-      '';
-
-
-      config = get-config system;
-
-      current-host = config.krebs.hosts.${current-host-name};
-      current-user = config.krebs.users.${current-user-name};
-
-      methods.dir = config:
-        let
-          can-push = config.host.name == current-host.name;
-          target-path = root + config.target-path;
-          push-method = ''
-            rsync \
-              --exclude .git \
-              --exclude .graveyard \
-              --exclude old \
-              --exclude tmp \
-              --rsync-path='mkdir -p ${target-path} && rsync' \
-              --delete-excluded \
-              -vrlptD \
-              ${config.path}/ \
-              root@${target}:${target-path}
-          '';
-        in
-        if can-push then push-method else
-        let dir = "file://${config.host.name}${config.path}"; in
-        # /!\ revise this message when using more than just push-method
-        throw "No way to push ${dir} from ${current-host.name} to ${target}";
-
-      methods.git = config:
-        let target-path = root + config.target-path;
-        in rootssh target ''
-          mkdir -p ${target-path}
-          cd ${target-path}
-          if ! test -e .git; then
-            git init
-          fi
-          if ! cur_url=$(git config remote.origin.url 2>/dev/null); then
-            git remote add origin ${config.url}
-          elif test "$cur_url" != ${config.url}; then
-            git remote set-url origin ${config.url}
-          fi
-          if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then
-            git fetch origin
-            git checkout ${config.rev} -- .
-            git checkout -q ${config.rev}
-            git submodule init
-            git submodule update
-          fi
-          git clean -dxf
-        '';
-    in out;
-
   rootssh = target: script:
     let
       flags = "-o StrictHostKeyChecking=${StrictHostKeyChecking}";

krebs/v2/default.nix

@@ -1,132 +0,0 @@
-{ lib
-, source
-, target-user ? "root"
-, target-host
-, target-path ? "/var/src"
-}:
-with lib;
-let
-  out = {
-    inherit populate;
-  };
-
-  populate = ''
-    #! /bin/sh
-    set -eu
-
-    verbose() {
-      printf '+%s\n' "$(printf ' %q' "$@")" >&2
-      "$@"
-    }
-
-    echo ${shell.escape git-script} \
-      | ssh ${shell.escape "${target-user}@${target-host}"} -T
-
-    unset tmpdir
-    trap '
-      rm "$tmpdir"/*
-      rmdir "$tmpdir"
-      trap - EXIT INT QUIT
-    '        EXIT INT QUIT
-    tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
-    chmod 0755 "$tmpdir"
-
-    ${concatStringsSep "\n"
-      (mapAttrsToList
-        (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
-          "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
-        symlink-specs)}
-
-    verbose proot \
-        -b $tmpdir:${shell.escape target-path} \
-        ${concatStringsSep " \\\n    "
-          (mapAttrsToList
-            (name: spec:
-              "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
-            file-specs)} \
-        rsync \
-            -f ${shell.escape "P /*"} \
-            ${concatMapStringsSep " \\\n        "
-              (name: "-f ${shell.escape "R /${name}"}")
-              (attrNames file-specs)} \
-            --delete \
-            -vFrlptD \
-            ${shell.escape target-path}/ \
-            ${shell.escape "${target-user}@${target-host}:${target-path}"}
-  '';
-
-  get-schema = uri:
-    if substring 0 1 uri == "/"
-      then "file"
-      else head (splitString ":" uri);
-
-  has-schema = schema: uri: get-schema uri == schema;
-
-  get-url = spec: {
-    string = spec;
-    path = toString spec;
-    set = get-url spec.url;
-  }.${typeOf spec};
-
-  git-specs =
-    filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
-    filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
-    filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
-
-  file-specs =
-    filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
-
-  symlink-specs =
-    filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
-
-  git-script = ''
-    #! /bin/sh
-    set -efu
-
-    verbose() {
-      printf '+%s\n' "$(printf ' %q' "$@")" >&2
-      "$@"
-    }
-
-    fetch_git() {(
-      dst_dir=$1
-      src_url=$2
-      src_ref=$3
-
-      if ! test -e "$dst_dir"; then
-        git clone "$src_url" "$dst_dir"
-      fi
-
-      cd "$dst_dir"
-
-      if ! url=$(git config remote.origin.url); then
-        git remote add origin "$src_url"
-      elif test "$url" != "$src_url"; then
-        git remote set-url origin "$src_url"
-      fi
-
-      # TODO resolve src_ref to commit hash
-      hash=$src_ref
-
-      if ! test "$(git log --format=%H -1)" = "$hash"; then
-        git fetch origin
-        git checkout "$hash" -- "$dst_dir"
-        git checkout "$hash"
-      fi
-
-      git clean -dxf
-    )}
-
-    ${concatStringsSep "\n"
-      (mapAttrsToList
-        (name: spec: toString (map shell.escape [
-          "verbose"
-          "fetch_git"
-          "${target-path}/${name}"
-          spec.url
-          spec.rev
-        ]))
-        git-specs)}
-  '';
-
-in out

nixpkgs/nixos/default.nix

@@ -1,65 +1 @@
-{ configuration ? import <upstream-nixpkgs/nixos/lib/from-env.nix> "NIXOS_CONFIG" <nixos-config>
+import <stockholm>
-, system ? builtins.currentSystem
-}:
-
-let
-  eval-config = modules: import <upstream-nixpkgs/nixos/lib/eval-config.nix> {
-    inherit system;
-    modules = modules ++ [({ config, lib, ... }: with lib; {
-      imports = filter dir.has-default-nix (concatLists [
-        (map (p: p + "/2configs") [ <stockholm-private> ])
-        (map (p: p + "/3modules") [ <stockholm-krebs> <stockholm-private> ])
-      ]);
-
-      krebs.current = {
-        enable = true;
-        host = config.krebs.hosts.${readFile /proc/sys/kernel/hostname};
-        user = config.krebs.users.${getEnv "LOGNAME"};
-      };
-
-      nixpkgs.config.packageOverrides = pkgs: let
-        kpkgs = import <stockholm-krebs/5pkgs> { inherit lib pkgs; };
-        upkgs = import <stockholm-private/5pkgs> { inherit lib; pkgs = pkgs // kpkgs; };
-      in kpkgs // upkgs;
-    })];
-    specialArgs = {
-      lib = let
-        nlib = import <upstream-nixpkgs/lib> // builtins;
-        klib = nlib // import <stockholm-krebs/4lib> { lib = nlib; };
-        ulib = klib // (with klib; let p = <stockholm-private> + "/4lib"; in
-          optionalAttrs (dir.has-default-nix p)
-                        (import p { lib = klib; }));
-      in ulib;
-    };
-  };
-
-  eval = eval-config [
-    configuration
-  ];
-
-  # This is for `nixos-rebuild build-vm'.
-  vm = eval-config [
-    configuration
-    <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
-  ];
-
-  # This is for `nixos-rebuild build-vm-with-bootloader'.
-  vm-with-bootloader = eval-config [
-    configuration
-    <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
-    { virtualisation.useBootLoader = true; }
-  ];
-in
-
-{
-  inherit (eval) config options;
-
-  system = eval.config.system.build.toplevel;
-
-  vm = vm.config.system.build.vm;
-
-  vmWithBootLoader = vm-with-bootloader.config.system.build.vm;
-
-  # The following are used by nixos-rebuild.
-  nixFallback = eval.pkgs.nixUnstable;
-}

nixpkgs/nixos/lib

@@ -1 +1 @@
-../../upstream-nixpkgs/nixos/lib
+../../../upstream-nixpkgs/nixos/lib

nixpkgs/nixos/modules

@@ -1 +1 @@
-../../upstream-nixpkgs/nixos/modules
+../../../upstream-nixpkgs/nixos/modules

root

@@ -0,0 +1 @@
+../stockholm-user

tv/2configs/default.nix

@@ -8,11 +8,9 @@ with lib;
   krebs.build = {
     user = config.krebs.users.tv;
     target = mkDefault "root@${config.krebs.build.host.name}";
-    source-version = 2;
     source = mapAttrs (_: mkDefault) ({
       nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix";
       nixpkgs = symlink:stockholm/nixpkgs;
-      null = "symlink:stockholm/null";
       secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
       secrets-common = "/home/tv/secrets/common";
       stockholm = "/home/tv/stockholm";
@@ -104,7 +102,7 @@ with lib;
       };
 
       environment.variables = {
-        NIX_PATH = mkForce "secrets=/var/src/null:/var/src";
+        NIX_PATH = mkForce "secrets=/var/src/stockholm/null:/var/src";
       };
 
       programs.bash = {