Merge remote-tracking branch 'cd/master'

This commit is contained in:
lassulus 2016-02-17 15:02:45 +01:00
commit f48e90915f
12 changed files with 275 additions and 99 deletions

View File

@ -1,5 +1,32 @@
ifndef system stockholm ?= .
$(error unbound variable: system)
ifndef nixos-config
$(if $(system),,$(error unbound variable: system))
nixos-config = ./$(LOGNAME)/1systems/$(system).nix
endif
# target = [target_user@]target_host[:target_port][/target_path]
ifdef target
_target_user != echo $(target) | sed -n 's/@.*//p'
_target_path != echo $(target) | sed -n 's/^[^/]*//p'
_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
ifneq ($(_target_host),)
$(if $(target_host),$(error cannot define both, target_host and host in target))
target_host ?= $(_target_host)
endif
ifneq ($(_target_user),)
$(if $(target_user),$(error cannot define both, target_user and user in target))
target_user ?= $(_target_user)
endif
ifneq ($(_target_port),)
$(if $(target_port),$(error cannot define both, target_port and port in target))
target_port ?= $(_target_port)
endif
ifneq ($(_target_path),)
$(if $(target_path),$(error cannot define both, target_path and path in target))
target_path ?= $(_target_path)
endif
endif endif
export target_host ?= $(system) export target_host ?= $(system)
@ -7,13 +34,18 @@ export target_user ?= root
export target_port ?= 22 export target_port ?= 22
export target_path ?= /var/src export target_path ?= /var/src
$(if $(target_host),,$(error unbound variable: target_host))
$(if $(target_user),,$(error unbound variable: target_user))
$(if $(target_port),,$(error unbound variable: target_port))
$(if $(target_path),,$(error unbound variable: target_path))
evaluate = \ evaluate = \
nix-instantiate \ nix-instantiate \
--eval \ --eval \
--readonly-mode \ --readonly-mode \
--show-trace \ --show-trace \
-I nixos-config=./$(LOGNAME)/1systems/$(system).nix \ -I nixos-config=$(nixos-config) \
-I stockholm=. \ -I stockholm=$(stockholm) \
$(1) $(1)
execute = \ execute = \
@ -22,9 +54,10 @@ execute = \
echo "$$script" | sh echo "$$script" | sh
# usage: make deploy system=foo [target_host=bar] # usage: make deploy system=foo [target_host=bar]
deploy: ssh ?= ssh
deploy: deploy:
$(call execute,populate) $(call execute,populate)
ssh $(target_user)@$(target_host) -p $(target_port) \ $(ssh) $(target_user)@$(target_host) -p $(target_port) \
nixos-rebuild switch --show-trace -I $(target_path) nixos-rebuild switch --show-trace -I $(target_path)
# usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
@ -41,3 +74,21 @@ install:
$(ssh) $(target_user)@$(target_host) -p $(target_port) \ $(ssh) $(target_user)@$(target_host) -p $(target_port) \
env NIXOS_CONFIG=$(target_path)/nixos-config \ env NIXOS_CONFIG=$(target_path)/nixos-config \
nixos-install nixos-install
# usage: make test system=foo [target=bar] [method={eval,build}]
method ?= eval
ifeq ($(method),build)
test: command = nix-build --no-out-link
else
ifeq ($(method),eval)
test: command ?= nix-instantiate --eval --json --readonly-mode --strict
else
$(error bad method: $(method))
endif
endif
test: ssh ?= ssh
test:
$(call execute,populate)
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
$(command) --show-trace -I $(target_path) \
-A config.system.build.toplevel $(target_path)/stockholm

View File

@ -20,35 +20,19 @@ let
type = types.user; type = types.user;
}; };
options.krebs.build.source = let options.krebs.build.source = mkOption {
raw = types.either types.str types.path; type = with types; attrsOf (either str (submodule {
url = types.submodule {
options = { options = {
url = mkOption { url = str;
type = types.str; rev = str;
}; };
rev = mkOption { }));
type = types.str;
};
dev = mkOption {
type = types.str;
};
};
};
in mkOption {
type = types.attrsOf (types.either types.str url);
apply = let f = mapAttrs (_: value: {
string = value;
path = toString value;
set = f value;
}.${typeOf value}); in f;
default = {}; default = {};
}; };
options.krebs.build.populate = mkOption { options.krebs.build.populate = mkOption {
type = types.str; type = types.str;
default = let default = let
source = config.krebs.build.source;
target-user = maybeEnv "target_user" "root"; target-user = maybeEnv "target_user" "root";
target-host = maybeEnv "target_host" config.krebs.build.host.name; target-host = maybeEnv "target_host" config.krebs.build.host.name;
target-port = maybeEnv "target_port" "22"; target-port = maybeEnv "target_port" "22";
@ -75,24 +59,21 @@ let
tmpdir=$(mktemp -dt stockholm.XXXXXXXX) tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir" chmod 0755 "$tmpdir"
${concatStringsSep "\n" ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
(mapAttrsToList verbose ln -s ${shell.escape symlink.target} \
(name: spec: let dst = removePrefix "symlink:" (get-url spec); in "$tmpdir"/${shell.escape name}
"verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") '') source-by-method.symlink)}
symlink-specs)}
verbose proot \ verbose proot \
-b $tmpdir:${shell.escape target-path} \ -b "$tmpdir":${shell.escape target-path} \
${concatStringsSep " \\\n " ${concatStringsSep " \\\n " (mapAttrsToList (name: file:
(mapAttrsToList "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
(name: spec: ) source-by-method.file)} \
"-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
file-specs)} \
rsync \ rsync \
-f ${shell.escape "P /*"} \ -f ${shell.escape "P /*"} \
${concatMapStringsSep " \\\n " ${concatMapStringsSep " \\\n " (name:
(name: "-f ${shell.escape "R /${name}"}") "-f ${shell.escape "R /${name}"}"
(attrNames file-specs)} \ ) (attrNames source-by-method.file)} \
--delete \ --delete \
-vFrlptD \ -vFrlptD \
-e ${shell.escape "ssh -p ${target-port}"} \ -e ${shell.escape "ssh -p ${target-port}"} \
@ -100,30 +81,6 @@ let
${shell.escape "${target-user}@${target-host}:${target-path}"} ${shell.escape "${target-user}@${target-host}:${target-path}"}
''; '';
get-schema = uri:
if substring 0 1 uri == "/"
then "file"
else head (splitString ":" uri);
has-schema = schema: uri: get-schema uri == schema;
get-url = spec: {
string = spec;
path = toString spec;
set = get-url spec.url;
}.${typeOf spec};
git-specs =
filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
file-specs =
filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
symlink-specs =
filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
git-script = '' git-script = ''
#! /bin/sh #! /bin/sh
set -efu set -efu
@ -162,20 +119,42 @@ let
git clean -dxf git clean -dxf
)} )}
${concatStringsSep "\n" ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
(mapAttrsToList verbose fetch_git ${concatMapStringsSep " " shell.escape [
(name: spec: toString (map shell.escape [
"verbose"
"fetch_git"
"${target-path}/${name}" "${target-path}/${name}"
spec.url git.url
spec.rev git.rev
])) ]}
git-specs)} '') source-by-method.git)}
''; '';
in out; in out;
}; };
}; };
source-by-method = let
known-methods = ["git" "file" "symlink"];
in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
git = source-by-scheme.http or {} //
source-by-scheme.https or {};
};
source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
${v.scheme}.${k} = v;
}) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
path = f (toString x);
string = f {
url = if substring 0 1 x == "/" then "file://${x}" else x;
};
set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
inherit scheme;
} // {
symlink.target = removePrefix "symlink:" x.url;
file.path = # TODO file://host/...
assert hasPrefix "file:///" x.url;
removePrefix "file://" x.url;
}.${scheme} or {};
}; in f) config.krebs.build.source;
in out in out

View File

@ -338,8 +338,8 @@ let
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
}; };
serviceConfig = let serviceConfig = let
workdir="${lib.shell.escape cfg.workDir}"; workdir = shell.escape cfg.workDir;
secretsdir="${lib.shell.escape (toString <secrets>)}"; secretsdir = shell.escape (toString <secrets>);
in { in {
PermissionsStartOnly = true; PermissionsStartOnly = true;
Type = "forking"; Type = "forking";

View File

@ -149,9 +149,9 @@ let
} // cfg.extraEnviron; } // cfg.extraEnviron;
serviceConfig = let serviceConfig = let
workdir = "${lib.shell.escape cfg.workDir}"; workdir = shell.escape cfg.workDir;
contact = "${lib.shell.escape cfg.contact}"; contact = shell.escape cfg.contact;
description = "${lib.shell.escape cfg.description}"; description = shell.escape cfg.description;
buildbot = pkgs.buildbot-slave; buildbot = pkgs.buildbot-slave;
# TODO:make this # TODO:make this
in { in {

View File

@ -31,6 +31,7 @@ let
./setuid.nix ./setuid.nix
./tinc_graphs.nix ./tinc_graphs.nix
./urlwatch.nix ./urlwatch.nix
./repo-sync.nix
]; ];
options.krebs = api; options.krebs = api;
config = lib.mkIf cfg.enable imp; config = lib.mkIf cfg.enable imp;

View File

@ -0,0 +1,109 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.krebs.repo-sync;
out = {
options.krebs.repo-sync = api;
config = mkIf cfg.enable imp;
};
api = {
enable = mkEnableOption "repo-sync";
config = mkOption {
type = with types;attrsOf (attrsOf (attrsOf str));
example = literalExample ''
# see `repo-sync --help`
# `ref` provides sane defaults and can be omitted
# attrset will be converted to json and be used as config
{
makefu = {
origin = {
url = http://github.com/makefu/repo ;
ref = "heads/dev" ;
};
mirror = {
url = "git@internal:mirror" ;
ref = "heads/github-mirror-dev" ;
};
};
lass = {
origin = {
url = http://github.com/lass/repo ;
};
mirror = {
url = "git@internal:mirror" ;
};
};
"@latest" = {
mirror = {
url = "git@internal:mirror";
ref = "heads/master";
};
};
};
'';
};
timerConfig = mkOption {
type = types.attrsOf types.str;
default = {
OnCalendar = "*:00,15,30,45";
};
};
stateDir = mkOption {
type = types.str;
default = "/var/lib/repo-sync";
};
privateKeyFile = mkOption {
type = types.str;
description = ''
used by repo-sync to identify with ssh service
'';
default = toString <secrets/wolf-repo-sync.rsa_key.priv>;
};
};
repo-sync-config = pkgs.writeText "repo-sync-config.json"
(builtins.toJSON cfg.config);
imp = {
users.users.repo-sync = {
name = "repo-sync";
uid = config.krebs.lib.genid "repo-sync";
description = "repo-sync user";
home = cfg.stateDir;
createHome = true;
};
systemd.timers.repo-sync = {
description = "repo-sync timer";
wantedBy = [ "timers.target" ];
timerConfig = cfg.timerConfig;
};
systemd.services.repo-sync = {
description = "repo-sync";
after = [ "network.target" ];
path = with pkgs; [ ];
environment = {
GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv";
};
serviceConfig = {
Type = "simple";
PermissionsStartOnly = true;
ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" ''
#! /bin/sh
cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv
chown repo-sync ${cfg.stateDir}/ssh.priv
'';
ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}";
WorkingDirectory = cfg.stateDir;
User = "repo-sync";
};
};
};
in out

View File

@ -1,15 +1,17 @@
{ lib, pkgs, python3Packages, fetchurl, ... }: { lib, pkgs, python3Packages, fetchurl, ... }:
with python3Packages; buildPythonPackage rec { with python3Packages; buildPythonPackage rec {
name = "repo-sync-${version}"; name = "repo-sync-${version}";
version = "0.1.1"; version = "0.2.5";
disabled = isPy26 || isPy27; disabled = isPy26 || isPy27;
propagatedBuildInputs = [ propagatedBuildInputs = [
docopt docopt
GitPython GitPython
pkgs.git
]; ];
src = fetchurl { src = fetchurl {
url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz"; url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz";
sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv"; sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84";
}; };
meta = { meta = {
homepage = http://github.com/makefu/repo-sync; homepage = http://github.com/makefu/repo-sync;

View File

@ -14,6 +14,7 @@ in
../2configs/shack-drivedroid.nix ../2configs/shack-drivedroid.nix
../2configs/shared-buildbot.nix ../2configs/shared-buildbot.nix
../2configs/cgit-mirror.nix ../2configs/cgit-mirror.nix
../2configs/repo-sync.nix
# ../2configs/graphite.nix # ../2configs/graphite.nix
]; ];
# use your own binary cache, fallback use cache.nixos.org (which is used by # use your own binary cache, fallback use cache.nixos.org (which is used by

View File

@ -3,7 +3,7 @@
with config.krebs.lib; with config.krebs.lib;
let let
rules = with git; singleton { rules = with git; singleton {
user = [ git-sync ]; user = [ wolf-repo-sync ];
repo = [ stockholm-mirror ]; repo = [ stockholm-mirror ];
perm = push ''refs/*'' [ non-fast-forward create delete merge ]; perm = push ''refs/*'' [ non-fast-forward create delete merge ];
}; };
@ -22,14 +22,15 @@ let
}; };
}; };
git-sync = { wolf-repo-sync = {
name = "git-sync"; name = "wolf-repo-sync";
mail = "spam@krebsco.de"; mail = "spam@krebsco.de";
# TODO put git-sync pubkey somewhere more appropriate # TODO put git-sync pubkey somewhere more appropriate
pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync''; pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf'';
}; };
in { in {
krebs.users.wolf-repo-sync = wolf-repo-sync;
krebs.git = { krebs.git = {
enable = true; enable = true;
root-title = "Shared Repos"; root-title = "Shared Repos";

View File

@ -0,0 +1,28 @@
{ config, lib, pkgs, ... }:
with lib;
{
krebs.repo-sync = let
# TODO addMirrorURL function
mirror = "git@wolf:stockholm-mirror";
in {
enable = true;
config = {
makefu = {
origin.url = http://cgit.gum/stockholm ;
mirror.url = mirror;
};
tv = {
origin.url = http://cgit.cd/stockholm ;
mirror.url = mirror;
};
lassulus = {
origin.url = http://cgit.cloudkrebs/stockholm ;
mirror.url = mirror;
};
"@latest" = {
mirror.url = mirror;
};
};
};
}

View File

@ -1,7 +1,8 @@
{ pkgs, lib, config, ... }: { pkgs, lib, config, ... }:
with config.krebs.lib;
let let
repodir = "/var/srv/drivedroid"; repodir = "/var/srv/drivedroid";
srepodir = lib.shell.escape repodir; srepodir = shell.escape repodir;
in in
{ {
environment.systemPackages = [ pkgs.drivedroid-gen-repo ]; environment.systemPackages = [ pkgs.drivedroid-gen-repo ];
@ -40,5 +41,4 @@ in
}; };
}; };
}; };
} }

View File

@ -1,18 +1,22 @@
{ lib, config, pkgs, ... }: { lib, config, pkgs, ... }:
# The buildbot config is seilf-contained and provides a way to test "shared" # The buildbot config is self-contained and currently provides a way
# configuration (infrastructure to be used by every krebsminister). # to test "shared" configuration (infrastructure to be used by every krebsminister).
# You can add your own test, test steps as required. Deploy the config on a # You can add your own test, test steps as required. Deploy the config on a
# shared host like wolf and everything should be fine. # shared host like wolf and everything should be fine.
# TODO for all users schedule a build for fast tests
{ {
networking.firewall.allowedTCPPorts = [ 8010 9989 ]; networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = { krebs.buildbot.master = let
stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ;
in {
secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
slaves = { slaves = {
testslave = "krebspass"; testslave = "krebspass";
}; };
change_source.stockholm = '' change_source.stockholm = ''
stockholm_repo = 'http://cgit.wolf/stockholm-mirror' stockholm_repo = '${stockholm-mirror-url}'
cs.append(changes.GitPoller( cs.append(changes.GitPoller(
stockholm_repo, stockholm_repo,
workdir='stockholm-poller', branches=True, workdir='stockholm-poller', branches=True,
@ -23,16 +27,15 @@
force-scheduler = '' force-scheduler = ''
sched.append(schedulers.ForceScheduler( sched.append(schedulers.ForceScheduler(
name="force", name="force",
builderNames=["full-tests"])) builderNames=["full-tests","fast-tests"]))
''; '';
fast-tests-scheduler = '' fast-tests-scheduler = ''
# test the master real quick # test everything real quick
sched.append(schedulers.SingleBranchScheduler( sched.append(schedulers.SingleBranchScheduler(
## all branches ## all branches
change_filter=util.ChangeFilter(branch_re=".*"), change_filter=util.ChangeFilter(branch_re=".*"),
# change_filter=util.ChangeFilter(branch="master"), # treeStableTimer=10,
treeStableTimer=10, #only test the latest push name="fast-test-all-branches",
name="fast-master-test",
builderNames=["fast-tests"])) builderNames=["fast-tests"]))
''; '';
test-cac-infest-master = '' test-cac-infest-master = ''
@ -61,7 +64,7 @@
# prepare nix-shell # prepare nix-shell
# the dependencies which are used by the test script # the dependencies which are used by the test script
deps = [ "gnumake", "jq","nix","rsync", deps = [ "gnumake", "jq","nix","rsync",
"(import <stockholm> {}).pkgs.test.infest-cac-centos7" ] "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
# TODO: --pure , prepare ENV in nix-shell command: # TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE # SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell", nixshell = ["nix-shell",
@ -133,7 +136,7 @@
}; };
irc = { irc = {
enable = true; enable = true;
nick = "shared-buildbot"; nick = "wolfbot";
server = "cd.retiolum"; server = "cd.retiolum";
channels = [ "retiolum" ]; channels = [ "retiolum" ];
allowForce = true; allowForce = true;
@ -147,6 +150,7 @@
password = "krebspass"; password = "krebspass";
packages = with pkgs;[ git nix ]; packages = with pkgs;[ git nix ];
# all nix commands will need a working nixpkgs installation # all nix commands will need a working nixpkgs installation
extraEnviron = { NIX_PATH="/var/src"; }; extraEnviron = {
NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
}; };
} }