diff --git a/Makefile b/Makefile index 384c872ab..60dfe8030 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,32 @@ -ifndef system -$(error unbound variable: system) +stockholm ?= . + +ifndef nixos-config +$(if $(system),,$(error unbound variable: system)) +nixos-config = ./$(LOGNAME)/1systems/$(system).nix +endif + +# target = [target_user@]target_host[:target_port][/target_path] +ifdef target +_target_user != echo $(target) | sed -n 's/@.*//p' +_target_path != echo $(target) | sed -n 's/^[^/]*//p' +_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p' +_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p' +ifneq ($(_target_host),) +$(if $(target_host),$(error cannot define both, target_host and host in target)) +target_host ?= $(_target_host) +endif +ifneq ($(_target_user),) +$(if $(target_user),$(error cannot define both, target_user and user in target)) +target_user ?= $(_target_user) +endif +ifneq ($(_target_port),) +$(if $(target_port),$(error cannot define both, target_port and port in target)) +target_port ?= $(_target_port) +endif +ifneq ($(_target_path),) +$(if $(target_path),$(error cannot define both, target_path and path in target)) +target_path ?= $(_target_path) +endif endif export target_host ?= $(system) @@ -7,13 +34,18 @@ export target_user ?= root export target_port ?= 22 export target_path ?= /var/src +$(if $(target_host),,$(error unbound variable: target_host)) +$(if $(target_user),,$(error unbound variable: target_user)) +$(if $(target_port),,$(error unbound variable: target_port)) +$(if $(target_path),,$(error unbound variable: target_path)) + evaluate = \ nix-instantiate \ --eval \ --readonly-mode \ --show-trace \ - -I nixos-config=./$(LOGNAME)/1systems/$(system).nix \ - -I stockholm=. \ + -I nixos-config=$(nixos-config) \ + -I stockholm=$(stockholm) \ $(1) execute = \ @@ -22,9 +54,10 @@ execute = \ echo "$$script" | sh # usage: make deploy system=foo [target_host=bar] +deploy: ssh ?= ssh deploy: $(call execute,populate) - ssh $(target_user)@$(target_host) -p $(target_port) \ + $(ssh) $(target_user)@$(target_host) -p $(target_port) \ nixos-rebuild switch --show-trace -I $(target_path) # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name @@ -41,3 +74,21 @@ install: $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env NIXOS_CONFIG=$(target_path)/nixos-config \ nixos-install + +# usage: make test system=foo [target=bar] [method={eval,build}] +method ?= eval +ifeq ($(method),build) +test: command = nix-build --no-out-link +else +ifeq ($(method),eval) +test: command ?= nix-instantiate --eval --json --readonly-mode --strict +else +$(error bad method: $(method)) +endif +endif +test: ssh ?= ssh +test: + $(call execute,populate) + $(ssh) $(target_user)@$(target_host) -p $(target_port) \ + $(command) --show-trace -I $(target_path) \ + -A config.system.build.toplevel $(target_path)/stockholm diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index a1f446188..c700fbc56 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -20,35 +20,19 @@ let type = types.user; }; - options.krebs.build.source = let - raw = types.either types.str types.path; - url = types.submodule { + options.krebs.build.source = mkOption { + type = with types; attrsOf (either str (submodule { options = { - url = mkOption { - type = types.str; - }; - rev = mkOption { - type = types.str; - }; - dev = mkOption { - type = types.str; - }; + url = str; + rev = str; }; - }; - in mkOption { - type = types.attrsOf (types.either types.str url); - apply = let f = mapAttrs (_: value: { - string = value; - path = toString value; - set = f value; - }.${typeOf value}); in f; + })); default = {}; }; options.krebs.build.populate = mkOption { type = types.str; default = let - source = config.krebs.build.source; target-user = maybeEnv "target_user" "root"; target-host = maybeEnv "target_host" config.krebs.build.host.name; target-port = maybeEnv "target_port" "22"; @@ -75,24 +59,21 @@ let tmpdir=$(mktemp -dt stockholm.XXXXXXXX) chmod 0755 "$tmpdir" - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: let dst = removePrefix "symlink:" (get-url spec); in - "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") - symlink-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: symlink: '' + verbose ln -s ${shell.escape symlink.target} \ + "$tmpdir"/${shell.escape name} + '') source-by-method.symlink)} verbose proot \ - -b $tmpdir:${shell.escape target-path} \ - ${concatStringsSep " \\\n " - (mapAttrsToList - (name: spec: - "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") - file-specs)} \ + -b "$tmpdir":${shell.escape target-path} \ + ${concatStringsSep " \\\n " (mapAttrsToList (name: file: + "-b ${shell.escape "${file.path}:${target-path}/${name}"}" + ) source-by-method.file)} \ rsync \ -f ${shell.escape "P /*"} \ - ${concatMapStringsSep " \\\n " - (name: "-f ${shell.escape "R /${name}"}") - (attrNames file-specs)} \ + ${concatMapStringsSep " \\\n " (name: + "-f ${shell.escape "R /${name}"}" + ) (attrNames source-by-method.file)} \ --delete \ -vFrlptD \ -e ${shell.escape "ssh -p ${target-port}"} \ @@ -100,30 +81,6 @@ let ${shell.escape "${target-user}@${target-host}:${target-path}"} ''; - get-schema = uri: - if substring 0 1 uri == "/" - then "file" - else head (splitString ":" uri); - - has-schema = schema: uri: get-schema uri == schema; - - get-url = spec: { - string = spec; - path = toString spec; - set = get-url spec.url; - }.${typeOf spec}; - - git-specs = - filterAttrs (_: spec: has-schema "https" (get-url spec)) source // - filterAttrs (_: spec: has-schema "http" (get-url spec)) source // - filterAttrs (_: spec: has-schema "git" (get-url spec)) source; - - file-specs = - filterAttrs (_: spec: has-schema "file" (get-url spec)) source; - - symlink-specs = - filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; - git-script = '' #! /bin/sh set -efu @@ -162,20 +119,42 @@ let git clean -dxf )} - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: toString (map shell.escape [ - "verbose" - "fetch_git" - "${target-path}/${name}" - spec.url - spec.rev - ])) - git-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: git: '' + verbose fetch_git ${concatMapStringsSep " " shell.escape [ + "${target-path}/${name}" + git.url + git.rev + ]} + '') source-by-method.git)} ''; in out; }; }; + source-by-method = let + known-methods = ["git" "file" "symlink"]; + in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme { + git = source-by-scheme.http or {} // + source-by-scheme.https or {}; + }; + + source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out { + ${v.scheme}.${k} = v; + }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source); + + normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) { + path = f (toString x); + string = f { + url = if substring 0 1 x == "/" then "file://${x}" else x; + }; + set = let scheme = head (splitString ":" x.url); in recursiveUpdate x { + inherit scheme; + } // { + symlink.target = removePrefix "symlink:" x.url; + file.path = # TODO file://host/... + assert hasPrefix "file:///" x.url; + removePrefix "file://" x.url; + }.${scheme} or {}; + }; in f) config.krebs.build.source; in out diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 825cb3413..2a1dbe31a 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -338,8 +338,8 @@ let SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; }; serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - secretsdir="${lib.shell.escape (toString )}"; + workdir = shell.escape cfg.workDir; + secretsdir = shell.escape (toString ); in { PermissionsStartOnly = true; Type = "forking"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 7705ac31c..248b46132 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -149,9 +149,9 @@ let } // cfg.extraEnviron; serviceConfig = let - workdir = "${lib.shell.escape cfg.workDir}"; - contact = "${lib.shell.escape cfg.contact}"; - description = "${lib.shell.escape cfg.description}"; + workdir = shell.escape cfg.workDir; + contact = shell.escape cfg.contact; + description = shell.escape cfg.description; buildbot = pkgs.buildbot-slave; # TODO:make this in { diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 16a74e7c1..c06f3754e 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -31,6 +31,7 @@ let ./setuid.nix ./tinc_graphs.nix ./urlwatch.nix + ./repo-sync.nix ]; options.krebs = api; config = lib.mkIf cfg.enable imp; diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix new file mode 100644 index 000000000..7a7c80a75 --- /dev/null +++ b/krebs/3modules/repo-sync.nix @@ -0,0 +1,109 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.krebs.repo-sync; + + out = { + options.krebs.repo-sync = api; + config = mkIf cfg.enable imp; + }; + + api = { + enable = mkEnableOption "repo-sync"; + config = mkOption { + type = with types;attrsOf (attrsOf (attrsOf str)); + example = literalExample '' + # see `repo-sync --help` + # `ref` provides sane defaults and can be omitted + + # attrset will be converted to json and be used as config + { + makefu = { + origin = { + url = http://github.com/makefu/repo ; + ref = "heads/dev" ; + }; + mirror = { + url = "git@internal:mirror" ; + ref = "heads/github-mirror-dev" ; + }; + }; + lass = { + origin = { + url = http://github.com/lass/repo ; + }; + mirror = { + url = "git@internal:mirror" ; + }; + }; + "@latest" = { + mirror = { + url = "git@internal:mirror"; + ref = "heads/master"; + }; + }; + }; + ''; + }; + timerConfig = mkOption { + type = types.attrsOf types.str; + default = { + OnCalendar = "*:00,15,30,45"; + }; + }; + stateDir = mkOption { + type = types.str; + default = "/var/lib/repo-sync"; + }; + privateKeyFile = mkOption { + type = types.str; + description = '' + used by repo-sync to identify with ssh service + ''; + default = toString ; + }; + }; + repo-sync-config = pkgs.writeText "repo-sync-config.json" + (builtins.toJSON cfg.config); + + imp = { + users.users.repo-sync = { + name = "repo-sync"; + uid = config.krebs.lib.genid "repo-sync"; + description = "repo-sync user"; + home = cfg.stateDir; + createHome = true; + }; + + systemd.timers.repo-sync = { + description = "repo-sync timer"; + wantedBy = [ "timers.target" ]; + + timerConfig = cfg.timerConfig; + }; + systemd.services.repo-sync = { + description = "repo-sync"; + after = [ "network.target" ]; + + path = with pkgs; [ ]; + + environment = { + GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv"; + }; + + serviceConfig = { + Type = "simple"; + PermissionsStartOnly = true; + ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" '' + #! /bin/sh + cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv + chown repo-sync ${cfg.stateDir}/ssh.priv + ''; + ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}"; + WorkingDirectory = cfg.stateDir; + User = "repo-sync"; + }; + }; + }; +in out diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix index 90f838de9..789c03f36 100644 --- a/krebs/5pkgs/repo-sync/default.nix +++ b/krebs/5pkgs/repo-sync/default.nix @@ -1,15 +1,17 @@ { lib, pkgs, python3Packages, fetchurl, ... }: + with python3Packages; buildPythonPackage rec { name = "repo-sync-${version}"; - version = "0.1.1"; + version = "0.2.5"; disabled = isPy26 || isPy27; propagatedBuildInputs = [ docopt GitPython + pkgs.git ]; src = fetchurl { url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz"; - sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv"; + sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84"; }; meta = { homepage = http://github.com/makefu/repo-sync; diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index 317591433..96691aed8 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -14,6 +14,7 @@ in ../2configs/shack-drivedroid.nix ../2configs/shared-buildbot.nix ../2configs/cgit-mirror.nix + ../2configs/repo-sync.nix # ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix index d30f1444f..b984535c9 100644 --- a/shared/2configs/cgit-mirror.nix +++ b/shared/2configs/cgit-mirror.nix @@ -3,7 +3,7 @@ with config.krebs.lib; let rules = with git; singleton { - user = [ git-sync ]; + user = [ wolf-repo-sync ]; repo = [ stockholm-mirror ]; perm = push ''refs/*'' [ non-fast-forward create delete merge ]; }; @@ -22,14 +22,15 @@ let }; }; - git-sync = { - name = "git-sync"; + wolf-repo-sync = { + name = "wolf-repo-sync"; mail = "spam@krebsco.de"; # TODO put git-sync pubkey somewhere more appropriate - pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync''; + pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf''; }; in { + krebs.users.wolf-repo-sync = wolf-repo-sync; krebs.git = { enable = true; root-title = "Shared Repos"; diff --git a/shared/2configs/repo-sync.nix b/shared/2configs/repo-sync.nix new file mode 100644 index 000000000..b23cb1675 --- /dev/null +++ b/shared/2configs/repo-sync.nix @@ -0,0 +1,28 @@ +{ config, lib, pkgs, ... }: + +with lib; +{ + krebs.repo-sync = let + # TODO addMirrorURL function + mirror = "git@wolf:stockholm-mirror"; + in { + enable = true; + config = { + makefu = { + origin.url = http://cgit.gum/stockholm ; + mirror.url = mirror; + }; + tv = { + origin.url = http://cgit.cd/stockholm ; + mirror.url = mirror; + }; + lassulus = { + origin.url = http://cgit.cloudkrebs/stockholm ; + mirror.url = mirror; + }; + "@latest" = { + mirror.url = mirror; + }; + }; + }; +} diff --git a/shared/2configs/shack-drivedroid.nix b/shared/2configs/shack-drivedroid.nix index 08a6b0697..6133ccc99 100644 --- a/shared/2configs/shack-drivedroid.nix +++ b/shared/2configs/shack-drivedroid.nix @@ -1,7 +1,8 @@ { pkgs, lib, config, ... }: +with config.krebs.lib; let repodir = "/var/srv/drivedroid"; - srepodir = lib.shell.escape repodir; + srepodir = shell.escape repodir; in { environment.systemPackages = [ pkgs.drivedroid-gen-repo ]; @@ -40,5 +41,4 @@ in }; }; }; - } diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index af877f5d8..ebf5f4a1e 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -1,18 +1,22 @@ { lib, config, pkgs, ... }: -# The buildbot config is seilf-contained and provides a way to test "shared" -# configuration (infrastructure to be used by every krebsminister). +# The buildbot config is self-contained and currently provides a way +# to test "shared" configuration (infrastructure to be used by every krebsminister). # You can add your own test, test steps as required. Deploy the config on a # shared host like wolf and everything should be fine. + +# TODO for all users schedule a build for fast tests { networking.firewall.allowedTCPPorts = [ 8010 9989 ]; - krebs.buildbot.master = { + krebs.buildbot.master = let + stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ; + in { secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; slaves = { testslave = "krebspass"; }; change_source.stockholm = '' - stockholm_repo = 'http://cgit.wolf/stockholm-mirror' + stockholm_repo = '${stockholm-mirror-url}' cs.append(changes.GitPoller( stockholm_repo, workdir='stockholm-poller', branches=True, @@ -23,16 +27,15 @@ force-scheduler = '' sched.append(schedulers.ForceScheduler( name="force", - builderNames=["full-tests"])) + builderNames=["full-tests","fast-tests"])) ''; fast-tests-scheduler = '' - # test the master real quick + # test everything real quick sched.append(schedulers.SingleBranchScheduler( ## all branches change_filter=util.ChangeFilter(branch_re=".*"), - # change_filter=util.ChangeFilter(branch="master"), - treeStableTimer=10, #only test the latest push - name="fast-master-test", + # treeStableTimer=10, + name="fast-test-all-branches", builderNames=["fast-tests"])) ''; test-cac-infest-master = '' @@ -61,7 +64,7 @@ # prepare nix-shell # the dependencies which are used by the test script deps = [ "gnumake", "jq","nix","rsync", - "(import {}).pkgs.test.infest-cac-centos7" ] + "(import ).pkgs.test.infest-cac-centos7" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", @@ -133,7 +136,7 @@ }; irc = { enable = true; - nick = "shared-buildbot"; + nick = "wolfbot"; server = "cd.retiolum"; channels = [ "retiolum" ]; allowForce = true; @@ -147,6 +150,7 @@ password = "krebspass"; packages = with pkgs;[ git nix ]; # all nix commands will need a working nixpkgs installation - extraEnviron = { NIX_PATH="/var/src"; }; + extraEnviron = { + NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; }; }; }