Mic92/stockholm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'pnp/master'

Browse Source
This commit is contained in:
tv 2015-10-22 20:14:01 +02:00
commit 128e5feae9
20 changed files with 485 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
krebs/3modules/bepasty-server.nix Normal file
View File

@ -0,0 +1,164 @@
{ config, lib, pkgs, ... }:
with lib;
let
gunicorn = pkgs.pythonPackages.gunicorn;
bepasty = pkgs.pythonPackages.bepasty-server;
gevent = pkgs.pythonPackages.gevent;
python = pkgs.pythonPackages.python;
cfg = config.krebs.bepasty;
out = {
options.krebs.bepasty = api;
config = mkIf cfg.enable (mkMerge [
(mkIf cfg.serveNginx nginx-imp)
imp
]);
};
api = {
enable = mkEnableOption "Bepasty Servers";
serveNginx = mkEnableOption "Serve Bepasty Servers with Nginx";
servers = mkOption {
type = with types; attrsOf optionSet;
options = singleton {
nginx = mkOption {
# TODO use the correct type
type = with types; attrsOf unspecified;
description = ''
additional nginx configuration. see krebs.nginx for all options
'';
};
secretKey = mkOption {
type = types.str;
description = ''
server secret for safe session cookies, must be set.
'';
default = "";
};
# we create a wsgi socket in $workDir/gunicorn-${name}.wsgi
workDir = mkOption {
type = types.str;
description = ''
Path to the working directory (used for sockets and pidfile).
Defaults to the users home directory. Must be accessible to nginx,
permissions will be set to 755
'';
default = config.users.extraUsers.bepasty.home;
};
dataDir = mkOption {
type = types.str;
description = ''
Defaults to the new users home dir which defaults to
/var/lib/bepasty-server/data
'';
default = "${config.users.extraUsers.bepasty.home}/data";
};
extraConfig = mkOption {
type = types.str;
default = "";
# TODO configure permissions in separate
example = ''
PERMISSIONS = {
'myadminsecret': 'admin,list,create,read,delete',
}
MAX_ALLOWED_FILE_SIZE = 5 * 1000 * 1000
'';
};
defaultPermissions = mkOption {
# TODO: listOf str
type = types.str;
description = ''
default permissions for all unauthenticated users.
'';
example = "read,create,delete";
default = "read";
};
};
default = {};
};
};
imp = {
# Configures systemd services for each configured server
# environment.systemPackages = [ bepasty gunicorn gevent ];
systemd.services = mapAttrs' (name: server:
nameValuePair "bepasty-server-${name}" {
description = "Bepasty Server ${name}";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
restartIfChanged = true;
environment = {
BEPASTY_CONFIG = "${server.workDir}/bepasty-${name}.conf";
PYTHONPATH= "${bepasty}/lib/${python.libPrefix}/site-packages:${gevent}/lib/${python.libPrefix}/site-packages";
};
serviceConfig = {
Type = "simple";
PrivateTmp = true;
ExecStartPre = assert server.secretKey != ""; pkgs.writeScript "bepasty-server.${name}-init" ''
#!/bin/sh
mkdir -p "${server.dataDir}" "${server.workDir}"
chown bepasty:bepasty "${server.workDir}" "${server.dataDir}"
cat > "${server.workDir}/bepasty-${name}.conf" <<EOF
SITENAME="${name}"
STORAGE_FILESYSTEM_DIRECTORY="${server.dataDir}"
SECRET_KEY="${server.secretKey}"
DEFAULT_PERMISSIONS="${server.defaultPermissions}"
${server.extraConfig}
EOF
'';
ExecStart = ''${gunicorn}/bin/gunicorn bepasty.wsgi --name ${name} \
-u bepasty \
-g bepasty \
--workers 3 --log-level=info \
--bind=unix:${server.workDir}/gunicorn-${name}.sock \
--pid ${server.workDir}/gunicorn-${name}.pid \
-k gevent
'';
};
}
) cfg.servers;
users.extraUsers.bepasty = {
uid = 2796546855; #genid bepasty
group = "bepasty";
home = "/var/lib/bepasty-server";
};
users.extraGroups.bepasty = {
gid = 2796546855; #genid bepasty
};
};
nginx-imp = {
assertions = [{ assertion = config.krebs.nginx.enable;
message = "krebs.nginx.enable must be true"; }];
krebs.nginx.servers = mapAttrs' (name: server:
nameValuePair("bepasty-server-${name}")
(mkMerge [ server.nginx {
extraConfig = ''
client_max_body_size 32M;
'';
locations = [
(nameValuePair "/" ''
proxy_set_header Host $http_host;
proxy_pass http://unix:${server.workDir}/gunicorn-${name}.sock;
'')
(nameValuePair "/static/" ''
alias ${bepasty}/lib/${python.libPrefix}/site-packages/bepasty/static/;
'')
];
}])) cfg.servers ;
};
in
out

11
krebs/3modules/default.nix
View File

@ -6,6 +6,7 @@ let
out = {
imports = [
./bepasty-server.nix
./build.nix
./exim-retiolum.nix
./exim-smarthost.nix
@ -14,8 +15,10 @@ let
./iptables.nix
./nginx.nix
./Reaktor.nix
./retiolum-bootstrap.nix
./realwallpaper.nix
./retiolum.nix
./tinc_graphs.nix
./urlwatch.nix
];
options.krebs = api;
@ -102,10 +105,14 @@ let
# Implements environment.etc."zones/<zone-name>"
environment.etc = let
stripEmptyLines = s: concatStringsSep "\n"
(remove "\n" (remove "" (splitString "\n" s)));
all-zones = foldAttrs (sum: current: sum + "\n" +current ) ""
([cfg.zone-head-config] ++ combined-hosts) ;
([cfg.zone-head-config] ++ combined-hosts);
combined-hosts = (mapAttrsToList (name: value: value.extraZones) cfg.hosts );
in lib.mapAttrs' (name: value: nameValuePair (("zones/" + name)) ({ text=value; })) all-zones;
in lib.mapAttrs' (name: value: nameValuePair
("zones/" + name)
{ text=(stripEmptyLines value); }) all-zones;
krebs.exim-smarthost.internet-aliases = let
format = from: to:

15
krebs/3modules/makefu/default.nix
View File

@ -127,10 +127,8 @@ with lib;
"krebsco.de" = ''
IN MX 10 mx42
euer IN MX 1 aspmx.l.google.com.
io IN NS pigstarter.krebsco.de.
pigstarter IN A ${head nets.internet.addrs4}
gold IN A ${head nets.internet.addrs4}
tinc IN A ${head nets.internet.addrs4}
boot IN A ${head nets.internet.addrs4}'';
};
nets = {
@ -165,8 +163,11 @@ with lib;
dc = "makefu"; #dc = "cac";
extraZones = {
"krebsco.de" = ''
wry IN A ${head nets.internet.addrs4}
graphs IN A ${head nets.internet.addrs4}
wry IN A ${head nets.internet.addrs4}
io IN NS wry.krebsco.de.
graphs IN A ${head nets.internet.addrs4}
paste 60 IN A ${head nets.internet.addrs4}
tinc IN A ${head nets.internet.addrs4}
'';
};
nets = rec {
@ -174,6 +175,7 @@ with lib;
addrs4 = ["104.233.87.86"];
aliases = [
"wry.internet"
"paste.internet"
];
};
retiolum = {
@ -182,6 +184,8 @@ with lib;
addrs6 = ["42:6e1e:cc8a:7cef:827:f938:8c64:baad"];
aliases = [
"graphs.wry.retiolum"
"paste.wry.retiolum"
"paste.retiolum"
"wry.retiolum"
];
tinc.pubkey = ''
@ -210,8 +214,7 @@ with lib;
"krebsco.de" = ''
omo IN A ${head nets.internet.addrs4}
euer IN A ${head nets.internet.addrs4}
gum IN A ${head nets.internet.addrs4}
paste IN A ${head nets.internet.addrs4}'';
gum IN A ${head nets.internet.addrs4} '';
};
nets = {
internet = {

2
krebs/3modules/nginx.nix
View File

@ -36,7 +36,7 @@ let
type = with types; listOf (attrsOf str);
};
extraConfig = mkOption {
type = with types; str;
type = with types; string;
default = "";
};
};

64
krebs/3modules/retiolum-bootstrap.nix Normal file
View File

@ -0,0 +1,64 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.krebs.retiolum-bootstrap;
out = {
options.krebs.retiolum-bootstrap = api;
config = mkIf cfg.enable imp ;
};
api = {
enable = mkEnableOption "retiolum boot strap for tinc.krebsco.de";
hostname = mkOption {
type = types.str;
description = "hostname which serves tinc boot";
default = "tinc.krebsco.de" ;
};
listen = mkOption {
type = with types; listOf str;
description = ''Addresses to listen on (nginx-syntax).
ssl will be configured, http will be redirected to ssl.
Make sure to have at least 1 ssl port configured.
'';
default = [ "80" "443 ssl" ] ;
};
ssl_certificate_key = mkOption {
type = types.str;
description = "Certificate key to use for ssl";
default = "/root/secrets/tinc.krebsco.de.key";
};
ssl_certificate = mkOption {
type = types.str;
description = "Certificate file to use for ssl";
default = "/root/secrets/tinc.krebsco.de.crt" ;
};
# in use:
# <secrets/tinc.krebsco.de.crt>
# <secrets/tinc.krebsco.de.key>
};
imp = {
krebs.nginx.servers = assert config.krebs.nginx.enable; {
retiolum-boot-ssl = {
server-names = singleton cfg.hostname;
listen = cfg.listen;
extraConfig = ''
ssl_certificate ${cfg.ssl_certificate};
ssl_certificate_key ${cfg.ssl_certificate_key};
if ($scheme = http){
return 301 https://$server_name$request_uri;
}
root ${pkgs.retiolum-bootstrap};
try_files $uri $uri/retiolum.sh;
'';
locations = [];
};
};
};
in
out

66
makefu/3modules/tinc_graphs.nix → krebs/3modules/tinc_graphs.nix
View File

@ -2,12 +2,12 @@
with lib;
let
cfg = config.makefu.tinc_graphs;
cfg = config.krebs.tinc_graphs;
internal_dir = "${cfg.workingDir}/internal";
external_dir = "${cfg.workingDir}/external";
out = {
options.makefu.tinc_graphs = api;
options.krebs.tinc_graphs = api;
config = mkIf cfg.enable imp ;
};
@ -20,26 +20,38 @@ let
default = "${pkgs.geolite-legacy}/share/GeoIP/GeoIPCity.dat";
};
krebsNginx = {
# configure krebs nginx to serve the new graphs
enable = mkEnableOption "tinc_graphs nginx";
nginx = {
enable = mkEnableOption "enable tinc_graphs to be served with nginx";
hostnames_complete = mkOption {
#TODO: this is not a secure way to serve these graphs,better listen to
# the correct interface, krebs.nginx does not support this yet
anonymous = {
server-names = mkOption {
type = with types; listOf str;
description = "hostnames which serve anonymous graphs";
default = [ "graphs.${config.krebs.build.host.name}" ];
};
listen = mkOption {
# use the type of the nginx listen option
type = with types; listOf str;
description = "listen address for anonymous graphs";
default = [ "80" ];
};
type = with types; listOf str;
description = "hostname which serves complete graphs";
default = [ "graphs.${config.krebs.build.host.name}" ];
};
hostnames_anonymous = mkOption {
type = with types; listOf str;
description = ''
hostname which serves anonymous graphs
must be different from hostname_complete
'';
default = [ "anongraphs.${config.krebs.build.host.name}" ];
complete = {
server-names = mkOption {
type = with types; listOf str;
description = "hostname which serves complete graphs";
default = [ "graphs.${config.krebs.build.host.name}" ];
};
listen = mkOption {
type = with types; listOf str;
description = "listen address for complete graphs";
default = [ "127.0.0.1:80" ];
};
};
};
@ -83,7 +95,9 @@ let
ExecStartPre = pkgs.writeScript "tinc_graphs-init" ''
#!/bin/sh
mkdir -p "${external_dir}" "${internal_dir}"
if ! test -e "${cfg.workingDir}/internal/index.html"; then
cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/internal/" "${internal_dir}"
fi
'';
ExecStart = "${pkgs.tinc_graphs}/bin/all-the-graphs";
@ -94,10 +108,10 @@ let
# this is needed because homedir is created with 700
chmod 755 "${cfg.workingDir}"
'';
PrivateTmp = "yes";
User = "root"; # tinc cannot be queried as user,
# seems to be a tinc-pre issue
privateTmp = true;
};
};
@ -107,25 +121,23 @@ let
createHome = true;
};
krebs.nginx.servers = mkIf cfg.krebsNginx.enable {
tinc_graphs_complete = {
server-names = cfg.krebsNginx.hostnames_complete;
krebs.nginx.servers = mkIf cfg.nginx.enable {
tinc_graphs_complete = mkMerge [ cfg.nginx.complete {
locations = [
(nameValuePair "/" ''
autoindex on;
root ${internal_dir};
'')
];
};
tinc_graphs_anonymous = {
server-names = cfg.krebsNginx.hostnames_anonymous;
}] ;
tinc_graphs_anonymous = mkMerge [ cfg.nginx.anonymous {
locations = [
(nameValuePair "/" ''
autoindex on;
root ${external_dir};
'')
];
};
}];
};
};

29
krebs/5pkgs/retiolum-bootstrap/default.nix Normal file
View File

@ -0,0 +1,29 @@
{ stdenv,lib,fetchurl, ... }:
with lib;
stdenv.mkDerivation rec {
name = "retiolum-bootstrap";
version = "4.2.3";
src = fetchurl {
url = https://raw.githubusercontent.com/krebscode/painload/master/retiolum/scripts/tinc_setup/new_install.sh;
sha256 = "03kmil8q2xm3rdm2jxyah7vww84pw6w01d0c3siid9zpn2j7la9s";
};
phases = [
"installPhase"
];
installPhase = ''
mkdir -p "$out"
cp -a ${src} $out/retiolum.sh
'';
meta = {
description = "Retiolum boostrap scripts";
url = https://github.com/krebscode/painload;
license = licenses.wtfpl;
platforms = platforms.unix;
maintainers = with maintainers; [ makefu ];
};
}

1
makefu/1systems/pornocauster.nix
View File

@ -21,6 +21,7 @@
# applications
../2configs/exim-retiolum.nix
../2configs/mail-client.nix
#../2configs/virtualization.nix
../2configs/virtualization-virtualbox.nix
../2configs/wwan.nix

91
makefu/1systems/wry.nix
View File

@ -1,47 +1,72 @@
{ config, lib, pkgs, ... }:
with lib;
let
ip = (lib.head config.krebs.build.host.nets.internet.addrs4);
external-ip = head config.krebs.build.host.nets.internet.addrs4;
internal-ip = head config.krebs.build.host.nets.retiolum.addrs4;
in {
imports = [
../../tv/2configs/CAC-CentOS-7-64bit.nix
../2configs/base.nix
../2configs/base-sources.nix
../2configs/tinc-basic-retiolum.nix
# TODO: copy this config or move to krebs
../../tv/2configs/CAC-CentOS-7-64bit.nix
../2configs/base.nix
../2configs/unstable-sources.nix
../2configs/tinc-basic-retiolum.nix
../2configs/bepasty-dual.nix
../2configs/iodined.nix
# Reaktor
../2configs/Reaktor/simpleExtend.nix
];
networking.firewall.allowPing = true;
networking.interfaces.enp2s1.ip4 = [
{
address = ip;
prefixLength = 24;
}
];
networking.defaultGateway = "104.233.87.1";
networking.nameservers = [
"8.8.8.8"
];
# based on ../../tv/2configs/CAC-Developer-2.nix
sound.enable = false;
# prepare graphs
nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
krebs.nginx.enable = true;
makefu.tinc_graphs.enable = true;
makefu.tinc_graphs.krebsNginx = {
enable = true;
# TODO: remove hard-coded hostname
hostnames_complete = [ "graphs.wry" ];
hostnames_anonymous = [ "graphs.krebsco.de" ];
};
networking.firewall.allowedTCPPorts = [80];
krebs.build = {
user = config.krebs.users.makefu;
target = "root@${ip}";
target = "root@wry";
host = config.krebs.hosts.wry;
};
krebs.Reaktor.enable = true;
# bepasty to listen only on the correct interfaces
krebs.bepasty.servers.internal.nginx.listen = [ "${internal-ip}:80" ];
krebs.bepasty.servers.external.nginx.listen = [ "${external-ip}:80" "${external-ip}:443 ssl" ];
# prepare graphs
krebs.nginx.enable = true;
krebs.retiolum-bootstrap.enable = true;
nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
krebs.tinc_graphs = {
enable = true;
nginx = {
enable = true;
# TODO: remove hard-coded hostname
complete = {
listen = [ "${internal-ip}:80" ];
server-names = [ "graphs.wry" ];
};
anonymous = {
listen = [ "${external-ip}:80" ] ;
server-names = [ "graphs.krebsco.de" ];
};
};
};
networking = {
firewall.allowPing = true;
firewall.allowedTCPPorts = [ 53 80 443 ];
interfaces.enp2s1.ip4 = [{
address = external-ip;
prefixLength = 24;
}];
defaultGateway = "104.233.87.1";
nameservers = [ "8.8.8.8" ];
};
# based on ../../tv/2configs/CAC-Developer-2.nix
sound.enable = false;
}

1
makefu/2configs/Reaktor/random-emoji.nix
View File

@ -12,6 +12,7 @@ let
coreutils
gnused
gnugrep
xmlstarlet
curl]);
in {
# TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm

3
makefu/2configs/Reaktor/random-emoji.sh
View File

@ -2,4 +2,5 @@
curl http://emojicons.com/random -s | \
grep data-text | \
sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \
head -n 1
head -n 1 | \
xmlstarlet unesc

6
makefu/2configs/base-sources.nix
View File

@ -3,9 +3,9 @@
{
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
#url = https://github.com/makefu/nixpkgs;
rev = "dc18f39bfb2f9d1ba62c7e8ad98544bb15cb26b2"; # nixos-15.09
#url = https://github.com/NixOS/nixpkgs;
url = https://github.com/makefu/nixpkgs;
rev = "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picked iodine
};
dir.secrets = {

52
makefu/2configs/bepasty-dual.nix Normal file
View File

@ -0,0 +1,52 @@
{ config, lib, pkgs, ... }:
# 1systems should configure itself:
# krebs.bepasty.servers.internal.nginx.listen = [ "80" ]
# krebs.bepasty.servers.external.nginx.listen = [ "80" "443 ssl" ]
# 80 is redirected to 443 ssl
# secrets used:
# wildcard.krebsco.de.crt
# wildcard.krebsco.de.key
# bepasty-secret.nix <- contains single string
with lib;
{
krebs.nginx.enable = mkDefault true;
krebs.bepasty = {
enable = true;
serveNginx= true;
servers = {
internal = {
nginx = {
server-names = [ "paste.retiolum" "paste.${config.krebs.build.host.name}" ];
};
defaultPermissions = "admin,list,create,read,delete";
secretKey = import <secrets/bepasty-secret.nix>;
};
external = {
nginx = {
server-names = [ "paste.krebsco.de" ];
extraConfig = ''
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
ssl_certificate /root/secrets/wildcard.krebsco.de.crt;
ssl_certificate_key /root/secrets/wildcard.krebsco.de.key;
ssl_verify_client off;
proxy_ssl_session_reuse off;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers RC4:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
if ($scheme = http){
return 301 https://$server_name$request_uri;
}'';
};
defaultPermissions = "read";
secretKey = import <secrets/bepasty-secret.nix>;
};
};
};
}

4
makefu/2configs/exim-retiolum.nix
View File

@ -5,10 +5,6 @@ with lib;
krebs.exim-retiolum.enable = true;
environment.systemPackages = with pkgs; [
msmtp
mutt-kz
notmuch
# TODO: put this somewhere else
offlineimap
];
}

3
makefu/2configs/git/cgit-retiolum.nix
View File

@ -10,6 +10,9 @@ let
stockholm = {
desc = "Make all the systems into 1systems!";
};
tinc_graphs = {
desc = "Tinc Advanced Graph Generation";
};
};
priv-repos = mapAttrs make-priv-repo {

16
makefu/2configs/iodined.nix Normal file
View File

@ -0,0 +1,16 @@
{ services,builtins,environment,pkgs, ... }:
let
# TODO: make this a parameter
domain = "io.krebsco.de";
pw = import <secrets/iodinepw.nix>;
in {
services.iodined = {
enable = true;
domain = domain;
ip = "172.16.10.1/24";
extraConfig = "-P ${pw}";
};
}

12
makefu/2configs/mail-client.nix Normal file
View File

@ -0,0 +1,12 @@
{ config, lib, pkgs, ... }:
with lib;
{
environment.systemPackages = with pkgs; [
msmtp
mutt-kz
notmuch
offlineimap
];
}

19
makefu/2configs/unstable-sources.nix Normal file
View File

@ -0,0 +1,19 @@
{ config, lib, pkgs, ... }:
{
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/makefu/nixpkgs;
rev = "984d33884d63d404ff2da76920b8bc8b15471552";
};
dir.secrets = {
host = config.krebs.hosts.pornocauster;
path = "/home/makefu/secrets/${config.krebs.build.host.name}/";
};
dir.stockholm = {
host = config.krebs.hosts.pornocauster;
path = toString ../.. ;
};
};
}

1
makefu/3modules/default.nix
View File

@ -2,7 +2,6 @@ _:
{
imports = [
./tinc_graphs.nix
];
}

5
makefu/5pkgs/tinc_graphs/default.nix
View File

@ -2,20 +2,21 @@
python3Packages.buildPythonPackage rec {
name = "tinc_graphs-${version}";
version = "0.2.12";
version = "0.3.6";
propagatedBuildInputs = with pkgs;[
python3Packages.pygeoip
## ${geolite-legacy}/share/GeoIP/GeoIPCity.dat
];
src = fetchurl {
url = "https://pypi.python.org/packages/source/t/tinc_graphs/tinc_graphs-${version}.tar.gz";
sha256 = "03jxvxahpcbpnz4668x32b629dwaaz5jcjkyaijm0zzpgcn4cbgp";
sha256 = "0ghdx9aaipmppvc2b6cgks4nxw6zsb0fhjrmnisbx7rz0vjvzc74";
};
preFixup = with pkgs;''
wrapProgram $out/bin/build-graphs --prefix PATH : "$out/bin"
wrapProgram $out/bin/all-the-graphs --prefix PATH : "${imagemagick}/bin:${graphviz}/bin:$out/bin"
wrapProgram $out/bin/tinc-stats2json --prefix PATH : "${tinc}/bin"
'';
meta = {
homepage = http://krebsco.de/;
description = "Create Graphs from Tinc Stats";