Merge remote-tracking branch 'cd/master'

This commit is contained in:
lassulus 2016-02-17 15:02:45 +01:00
commit f48e90915f
12 changed files with 275 additions and 99 deletions

View File

@ -1,5 +1,32 @@
ifndef system
$(error unbound variable: system)
stockholm ?= .
ifndef nixos-config
$(if $(system),,$(error unbound variable: system))
nixos-config = ./$(LOGNAME)/1systems/$(system).nix
endif
# target = [target_user@]target_host[:target_port][/target_path]
ifdef target
_target_user != echo $(target) | sed -n 's/@.*//p'
_target_path != echo $(target) | sed -n 's/^[^/]*//p'
_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
ifneq ($(_target_host),)
$(if $(target_host),$(error cannot define both, target_host and host in target))
target_host ?= $(_target_host)
endif
ifneq ($(_target_user),)
$(if $(target_user),$(error cannot define both, target_user and user in target))
target_user ?= $(_target_user)
endif
ifneq ($(_target_port),)
$(if $(target_port),$(error cannot define both, target_port and port in target))
target_port ?= $(_target_port)
endif
ifneq ($(_target_path),)
$(if $(target_path),$(error cannot define both, target_path and path in target))
target_path ?= $(_target_path)
endif
endif
export target_host ?= $(system)
@ -7,13 +34,18 @@ export target_user ?= root
export target_port ?= 22
export target_path ?= /var/src
$(if $(target_host),,$(error unbound variable: target_host))
$(if $(target_user),,$(error unbound variable: target_user))
$(if $(target_port),,$(error unbound variable: target_port))
$(if $(target_path),,$(error unbound variable: target_path))
evaluate = \
nix-instantiate \
--eval \
--readonly-mode \
--show-trace \
-I nixos-config=./$(LOGNAME)/1systems/$(system).nix \
-I stockholm=. \
-I nixos-config=$(nixos-config) \
-I stockholm=$(stockholm) \
$(1)
execute = \
@ -22,9 +54,10 @@ execute = \
echo "$$script" | sh
# usage: make deploy system=foo [target_host=bar]
deploy: ssh ?= ssh
deploy:
$(call execute,populate)
ssh $(target_user)@$(target_host) -p $(target_port) \
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
nixos-rebuild switch --show-trace -I $(target_path)
# usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
@ -41,3 +74,21 @@ install:
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env NIXOS_CONFIG=$(target_path)/nixos-config \
nixos-install
# usage: make test system=foo [target=bar] [method={eval,build}]
method ?= eval
ifeq ($(method),build)
test: command = nix-build --no-out-link
else
ifeq ($(method),eval)
test: command ?= nix-instantiate --eval --json --readonly-mode --strict
else
$(error bad method: $(method))
endif
endif
test: ssh ?= ssh
test:
$(call execute,populate)
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
$(command) --show-trace -I $(target_path) \
-A config.system.build.toplevel $(target_path)/stockholm

View File

@ -20,35 +20,19 @@ let
type = types.user;
};
options.krebs.build.source = let
raw = types.either types.str types.path;
url = types.submodule {
options.krebs.build.source = mkOption {
type = with types; attrsOf (either str (submodule {
options = {
url = mkOption {
type = types.str;
url = str;
rev = str;
};
rev = mkOption {
type = types.str;
};
dev = mkOption {
type = types.str;
};
};
};
in mkOption {
type = types.attrsOf (types.either types.str url);
apply = let f = mapAttrs (_: value: {
string = value;
path = toString value;
set = f value;
}.${typeOf value}); in f;
}));
default = {};
};
options.krebs.build.populate = mkOption {
type = types.str;
default = let
source = config.krebs.build.source;
target-user = maybeEnv "target_user" "root";
target-host = maybeEnv "target_host" config.krebs.build.host.name;
target-port = maybeEnv "target_port" "22";
@ -75,24 +59,21 @@ let
tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir"
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: let dst = removePrefix "symlink:" (get-url spec); in
"verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
symlink-specs)}
${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
verbose ln -s ${shell.escape symlink.target} \
"$tmpdir"/${shell.escape name}
'') source-by-method.symlink)}
verbose proot \
-b $tmpdir:${shell.escape target-path} \
${concatStringsSep " \\\n "
(mapAttrsToList
(name: spec:
"-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
file-specs)} \
-b "$tmpdir":${shell.escape target-path} \
${concatStringsSep " \\\n " (mapAttrsToList (name: file:
"-b ${shell.escape "${file.path}:${target-path}/${name}"}"
) source-by-method.file)} \
rsync \
-f ${shell.escape "P /*"} \
${concatMapStringsSep " \\\n "
(name: "-f ${shell.escape "R /${name}"}")
(attrNames file-specs)} \
${concatMapStringsSep " \\\n " (name:
"-f ${shell.escape "R /${name}"}"
) (attrNames source-by-method.file)} \
--delete \
-vFrlptD \
-e ${shell.escape "ssh -p ${target-port}"} \
@ -100,30 +81,6 @@ let
${shell.escape "${target-user}@${target-host}:${target-path}"}
'';
get-schema = uri:
if substring 0 1 uri == "/"
then "file"
else head (splitString ":" uri);
has-schema = schema: uri: get-schema uri == schema;
get-url = spec: {
string = spec;
path = toString spec;
set = get-url spec.url;
}.${typeOf spec};
git-specs =
filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
file-specs =
filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
symlink-specs =
filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
git-script = ''
#! /bin/sh
set -efu
@ -162,20 +119,42 @@ let
git clean -dxf
)}
${concatStringsSep "\n"
(mapAttrsToList
(name: spec: toString (map shell.escape [
"verbose"
"fetch_git"
${concatStringsSep "\n" (mapAttrsToList (name: git: ''
verbose fetch_git ${concatMapStringsSep " " shell.escape [
"${target-path}/${name}"
spec.url
spec.rev
]))
git-specs)}
git.url
git.rev
]}
'') source-by-method.git)}
'';
in out;
};
};
source-by-method = let
known-methods = ["git" "file" "symlink"];
in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
git = source-by-scheme.http or {} //
source-by-scheme.https or {};
};
source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
${v.scheme}.${k} = v;
}) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
path = f (toString x);
string = f {
url = if substring 0 1 x == "/" then "file://${x}" else x;
};
set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
inherit scheme;
} // {
symlink.target = removePrefix "symlink:" x.url;
file.path = # TODO file://host/...
assert hasPrefix "file:///" x.url;
removePrefix "file://" x.url;
}.${scheme} or {};
}; in f) config.krebs.build.source;
in out

View File

@ -338,8 +338,8 @@ let
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
};
serviceConfig = let
workdir="${lib.shell.escape cfg.workDir}";
secretsdir="${lib.shell.escape (toString <secrets>)}";
workdir = shell.escape cfg.workDir;
secretsdir = shell.escape (toString <secrets>);
in {
PermissionsStartOnly = true;
Type = "forking";

View File

@ -149,9 +149,9 @@ let
} // cfg.extraEnviron;
serviceConfig = let
workdir = "${lib.shell.escape cfg.workDir}";
contact = "${lib.shell.escape cfg.contact}";
description = "${lib.shell.escape cfg.description}";
workdir = shell.escape cfg.workDir;
contact = shell.escape cfg.contact;
description = shell.escape cfg.description;
buildbot = pkgs.buildbot-slave;
# TODO:make this
in {

View File

@ -31,6 +31,7 @@ let
./setuid.nix
./tinc_graphs.nix
./urlwatch.nix
./repo-sync.nix
];
options.krebs = api;
config = lib.mkIf cfg.enable imp;

View File

@ -0,0 +1,109 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.krebs.repo-sync;
out = {
options.krebs.repo-sync = api;
config = mkIf cfg.enable imp;
};
api = {
enable = mkEnableOption "repo-sync";
config = mkOption {
type = with types;attrsOf (attrsOf (attrsOf str));
example = literalExample ''
# see `repo-sync --help`
# `ref` provides sane defaults and can be omitted
# attrset will be converted to json and be used as config
{
makefu = {
origin = {
url = http://github.com/makefu/repo ;
ref = "heads/dev" ;
};
mirror = {
url = "git@internal:mirror" ;
ref = "heads/github-mirror-dev" ;
};
};
lass = {
origin = {
url = http://github.com/lass/repo ;
};
mirror = {
url = "git@internal:mirror" ;
};
};
"@latest" = {
mirror = {
url = "git@internal:mirror";
ref = "heads/master";
};
};
};
'';
};
timerConfig = mkOption {
type = types.attrsOf types.str;
default = {
OnCalendar = "*:00,15,30,45";
};
};
stateDir = mkOption {
type = types.str;
default = "/var/lib/repo-sync";
};
privateKeyFile = mkOption {
type = types.str;
description = ''
used by repo-sync to identify with ssh service
'';
default = toString <secrets/wolf-repo-sync.rsa_key.priv>;
};
};
repo-sync-config = pkgs.writeText "repo-sync-config.json"
(builtins.toJSON cfg.config);
imp = {
users.users.repo-sync = {
name = "repo-sync";
uid = config.krebs.lib.genid "repo-sync";
description = "repo-sync user";
home = cfg.stateDir;
createHome = true;
};
systemd.timers.repo-sync = {
description = "repo-sync timer";
wantedBy = [ "timers.target" ];
timerConfig = cfg.timerConfig;
};
systemd.services.repo-sync = {
description = "repo-sync";
after = [ "network.target" ];
path = with pkgs; [ ];
environment = {
GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv";
};
serviceConfig = {
Type = "simple";
PermissionsStartOnly = true;
ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" ''
#! /bin/sh
cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv
chown repo-sync ${cfg.stateDir}/ssh.priv
'';
ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}";
WorkingDirectory = cfg.stateDir;
User = "repo-sync";
};
};
};
in out

View File

@ -1,15 +1,17 @@
{ lib, pkgs, python3Packages, fetchurl, ... }:
with python3Packages; buildPythonPackage rec {
name = "repo-sync-${version}";
version = "0.1.1";
version = "0.2.5";
disabled = isPy26 || isPy27;
propagatedBuildInputs = [
docopt
GitPython
pkgs.git
];
src = fetchurl {
url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz";
sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv";
sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84";
};
meta = {
homepage = http://github.com/makefu/repo-sync;

View File

@ -14,6 +14,7 @@ in
../2configs/shack-drivedroid.nix
../2configs/shared-buildbot.nix
../2configs/cgit-mirror.nix
../2configs/repo-sync.nix
# ../2configs/graphite.nix
];
# use your own binary cache, fallback use cache.nixos.org (which is used by

View File

@ -3,7 +3,7 @@
with config.krebs.lib;
let
rules = with git; singleton {
user = [ git-sync ];
user = [ wolf-repo-sync ];
repo = [ stockholm-mirror ];
perm = push ''refs/*'' [ non-fast-forward create delete merge ];
};
@ -22,14 +22,15 @@ let
};
};
git-sync = {
name = "git-sync";
wolf-repo-sync = {
name = "wolf-repo-sync";
mail = "spam@krebsco.de";
# TODO put git-sync pubkey somewhere more appropriate
pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync'';
pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf'';
};
in {
krebs.users.wolf-repo-sync = wolf-repo-sync;
krebs.git = {
enable = true;
root-title = "Shared Repos";

View File

@ -0,0 +1,28 @@
{ config, lib, pkgs, ... }:
with lib;
{
krebs.repo-sync = let
# TODO addMirrorURL function
mirror = "git@wolf:stockholm-mirror";
in {
enable = true;
config = {
makefu = {
origin.url = http://cgit.gum/stockholm ;
mirror.url = mirror;
};
tv = {
origin.url = http://cgit.cd/stockholm ;
mirror.url = mirror;
};
lassulus = {
origin.url = http://cgit.cloudkrebs/stockholm ;
mirror.url = mirror;
};
"@latest" = {
mirror.url = mirror;
};
};
};
}

View File

@ -1,7 +1,8 @@
{ pkgs, lib, config, ... }:
with config.krebs.lib;
let
repodir = "/var/srv/drivedroid";
srepodir = lib.shell.escape repodir;
srepodir = shell.escape repodir;
in
{
environment.systemPackages = [ pkgs.drivedroid-gen-repo ];
@ -40,5 +41,4 @@ in
};
};
};
}

View File

@ -1,18 +1,22 @@
{ lib, config, pkgs, ... }:
# The buildbot config is seilf-contained and provides a way to test "shared"
# configuration (infrastructure to be used by every krebsminister).
# The buildbot config is self-contained and currently provides a way
# to test "shared" configuration (infrastructure to be used by every krebsminister).
# You can add your own test, test steps as required. Deploy the config on a
# shared host like wolf and everything should be fine.
# TODO for all users schedule a build for fast tests
{
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = {
krebs.buildbot.master = let
stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ;
in {
secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
slaves = {
testslave = "krebspass";
};
change_source.stockholm = ''
stockholm_repo = 'http://cgit.wolf/stockholm-mirror'
stockholm_repo = '${stockholm-mirror-url}'
cs.append(changes.GitPoller(
stockholm_repo,
workdir='stockholm-poller', branches=True,
@ -23,16 +27,15 @@
force-scheduler = ''
sched.append(schedulers.ForceScheduler(
name="force",
builderNames=["full-tests"]))
builderNames=["full-tests","fast-tests"]))
'';
fast-tests-scheduler = ''
# test the master real quick
# test everything real quick
sched.append(schedulers.SingleBranchScheduler(
## all branches
change_filter=util.ChangeFilter(branch_re=".*"),
# change_filter=util.ChangeFilter(branch="master"),
treeStableTimer=10, #only test the latest push
name="fast-master-test",
# treeStableTimer=10,
name="fast-test-all-branches",
builderNames=["fast-tests"]))
'';
test-cac-infest-master = ''
@ -61,7 +64,7 @@
# prepare nix-shell
# the dependencies which are used by the test script
deps = [ "gnumake", "jq","nix","rsync",
"(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
"(import <stockholm>).pkgs.test.infest-cac-centos7" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell",
@ -133,7 +136,7 @@
};
irc = {
enable = true;
nick = "shared-buildbot";
nick = "wolfbot";
server = "cd.retiolum";
channels = [ "retiolum" ];
allowForce = true;
@ -147,6 +150,7 @@
password = "krebspass";
packages = with pkgs;[ git nix ];
# all nix commands will need a working nixpkgs installation
extraEnviron = { NIX_PATH="/var/src"; };
extraEnviron = {
NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
};
}