various updates

This commit is contained in:
Jörg Thalheim 2018-09-13 07:43:21 +00:00
parent f1c4a08ae3
commit ae80866c76
31 changed files with 213 additions and 403 deletions

View File

@ -1,15 +1,16 @@
#!/usr/bin/env ruby
require 'json'
require 'pathname'
require 'fileutils'
require 'open3'
require "json"
require "pathname"
require "fileutils"
require "open3"
require 'socket'
LXC_PATH = Pathname.new("/data/containers")
BACKUP_LOCATIONS = %w{home srv etc usr/local}
CONFIG_PATH = "/etc/lxc/container.json"
BACKUP_PATH = "/mnt/backup/attic"
ATTIC_PATH = Pathname.new("/data/attic")
PASSWORD_FILE = ATTIC_PATH.join("passwordfile").to_s
LXC_PATH = Pathname.new("/lxc/")
BACKUP_LOCATIONS = %w{home srv etc usr/local var opt}
CONFIG_PATH = "/etc/nixos/lxc/container.json"
BACKUP_PATH = "eve-backup@backup:backup"
BORG_PATH = Pathname.new("/data/borg")
PASSWORD_FILE = BORG_PATH.join("passwordfile").to_s
KEEP_DAILY = 7
KEEP_WEEKLY = 4
KEEP_MONTHLY = 0
@ -67,7 +68,7 @@ class Container
if backupname.nil?
abort("backupname not set for backup-scripts for container '#{@name}'")
end
backupname = ATTIC_PATH.join(backupname.gsub("/", ""))
backupname = BORG_PATH.join(backupname.gsub("/", ""))
FileUtils.mkdir_p(backupname)
puts "cd #{backupname}"
Dir.chdir(backupname) do
@ -83,9 +84,7 @@ class Container
end
config = load_config
backup_paths = BACKUP_LOCATIONS.map do |location|
"/#{location}"
end
backup_paths = BACKUP_LOCATIONS.map { |location| "/#{location}" }
config["network"].each do |container, data|
next if data["lxc"] == false
container = Container.new(container, data["backup-paths"], data["backup-scripts"])
@ -93,15 +92,23 @@ config["network"].each do |container, data|
backup_paths += container.run_backup_scripts
end
env = { "ATTIC_PASSPHRASE" => File.read(PASSWORD_FILE).chomp }
env = { "BORG_PASSPHRASE" => File.read(PASSWORD_FILE).chomp }
now = Time.now.strftime("%Y-%m-%d-%H:%M:%S")
paths = backup_paths.map {|path| path.to_s }
sh("attic", env, "create", "--stats", "#{BACKUP_PATH}::eve-#{now}",
'--exclude', '*/srv/repo',
'--exclude', '*/home/joerg/git',
'--exclude', '*/home/joerg/login/git',
*paths)
sh("attic", env, "prune", "-v", BACKUP_PATH,
TCPSocket.open('home.devkid.net', 22198) do |socket|
socket.write(File.read("/etc/nixos/secrets/nas-wakeup-password"))
end
sh("borg", env, "create", "--stats",
"--compression", "zlib,9",
"--exclude", "*/srv/repo",
"--exclude", "*/srv/deluge",
"--exclude", "*/var/lib/lxcfs",
"--exclude", "*/joerg/git/openwrt",
"#{BACKUP_PATH}::eve-#{now}", *paths)
sh("borg", env, "prune", "-v",
"--keep-daily", KEEP_DAILY.to_s,
"--keep-weekly", KEEP_WEEKLY.to_s,
"--keep-monthly", KEEP_MONTHLY.to_s)
"--keep-monthly", KEEP_MONTHLY.to_s,
BACKUP_PATH)

View File

@ -1,109 +0,0 @@
#!/usr/bin/env ruby
require 'json'
require 'pathname'
require 'fileutils'
require 'open3'
LXC_PATH = Pathname.new("/data/containers")
BACKUP_LOCATIONS = %w{home srv etc usr/local}
CONFIG_PATH = "/etc/lxc/container.json"
BACKUP_PATH = "/mnt/backup/borg"
BORG_PATH = Pathname.new("/data/borg")
PASSWORD_FILE = BORG_PATH.join("passwordfile").to_s
KEEP_DAILY = 7
KEEP_WEEKLY = 4
KEEP_MONTHLY = 0
def load_config
return JSON.load(File.open(CONFIG_PATH))
rescue SystemCallError => e
abort "failed to open configuration '#{CONFIG_PATH}', #{e}"
rescue JSON::ParserError => e
abort "failed to parse configuration '#{CONFIG_PATH}', #{e}"
end
def sh(cmd, env={}, *args)
pretty_args = args.map {|arg| "'#{arg}'"}
puts ([cmd] + pretty_args).join(" ")
system(env, cmd, *args)
end
class Container
def initialize(name, backup_paths, backup_scripts)
@name = name
@backup_paths = backup_paths
@backup_scripts = backup_scripts
@path = LXC_PATH.join(name, "rootfs")
end
def backup_paths
paths = BACKUP_LOCATIONS
if @backup_paths.is_a?(Array)
paths += @backup_paths
end
paths.map do |relative_path|
@path.join(relative_path)
end
end
def run_backup_scripts
if @backup_scripts.is_a?(Array)
@backup_scripts.map do |script|
backup_script(script)
end
else
[]
end
end
private
def backup_script(script)
unless script.is_a?(Hash)
abort("backup-scripts: Expected an Object, got #{script.class}")
end
command = script["command"]
if command.nil?
abort("command not set for backup-scripts for container '#{@name}'")
end
backupname = script["backupname"]
if backupname.nil?
abort("backupname not set for backup-scripts for container '#{@name}'")
end
backupname = BORG_PATH.join(backupname.gsub("/", ""))
FileUtils.mkdir_p(backupname)
puts "cd #{backupname}"
Dir.chdir(backupname) do
sh(command)
end
backupname
end
def empty_directory?(path)
return false unless Dir.exists?(path)
return Dir.entries(path).size <= 2 # - [".", ".."]
end
end
config = load_config
backup_paths = BACKUP_LOCATIONS.map do |location|
"/#{location}"
end
config["network"].each do |container, data|
next if data["lxc"] == false
container = Container.new(container, data["backup-paths"], data["backup-scripts"])
backup_paths += container.backup_paths
backup_paths += container.run_backup_scripts
end
env = { "BORG_PASSPHRASE" => File.read(PASSWORD_FILE).chomp }
now = Time.now.strftime("%Y-%m-%d-%H:%M:%S")
paths = backup_paths.map {|path| path.to_s }
sh("borg", env, "create", "--stats", "#{BACKUP_PATH}::eve-#{now}",
'--compression', 'zlib,9',
'--exclude', '*/srv/repo',
'--exclude', '*/srv/deluge',
'--exclude', '*/home/joerg/git',
'--exclude', '*/home/joerg/login/git',
*paths)
sh("borg", env, "prune", "-v", BACKUP_PATH,
"--keep-daily", KEEP_DAILY.to_s,
"--keep-weekly", KEEP_WEEKLY.to_s,
"--keep-monthly", KEEP_MONTHLY.to_s)

View File

@ -2,14 +2,18 @@
# TARGET: Backup-Ziel
# IGNORE: Liste zu ignorierender Datenbanken (durch | getrennt)
IGNORE="mysql|information_schema|performance_schema|test"
PASSWORD="DtkXaU6ZeWeizvcZjRQJqY3no9dGf3ASa7N73Y8Z8PULxJVrvvrq7AAak4s2HvD2"
PASSWORD="$(cat /etc/nixos/secrets/mysql-password)"
DBS="$(/usr/bin/mysql --host="mysql" --user="root" --password="$PASSWORD" -Bse 'show databases' | /usr/bin/grep -Ev $IGNORE)"
set -eu -o pipefail
export PATH="/usr/bin/:$PATH"
DBS="$(lxc-attach -n mysql -- mysql --host="mysql" --user="root" --password="$PASSWORD" -Bse 'show databases' | grep -Ev $IGNORE)"
rm -f *.sql.bz2
for DB in $DBS; do
/usr/bin/mysqldump --host="mysql" --user="root" --password="$PASSWORD" --skip-extended-insert --skip-comments "$DB" | bzip2 -c > "$DB.sql.bz2"
lxc-attach -n mysql -- mysqldump --host="localhost" --user="root" --password="$PASSWORD" --skip-extended-insert --skip-comments "$DB" | bzip2 -c > "$DB.sql.bz2"
done
echo "$0 - Backup erfolgreich durchgefuehrt"

View File

@ -1,13 +1,12 @@
#!/bin/bash
#!/usr/bin/env bash
export PGPASS=/root/.pgpass
set -eu -o pipefail
# restore:
# psql -f $database.dump postgres
LIST=$(psql -h postgres -U postgres -At -c "select datname from pg_database order by datname;")
for d in $LIST
do
if [ "$d" != "template0" ]; then
pg_dump -h postgres -U postgres "$d" | gzip -c > "$d.dump.gz"
fi
export PATH="/usr/bin/:$PATH"
LIST=$(lxc-attach -n postgres -- psql -h postgres -U postgres -At -c "select datname from pg_database order by datname;")
for d in $LIST; do
[ "$d" == "template0" ] && continue
lxc-attach -n postgres -- pg_dump -h postgres -U postgres "$d" | gzip -c > "$d.dump.gz"
done

View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env bash
if [ $# -lt 1 ]
then

View File

@ -1,99 +0,0 @@
#!/bin/sh
set -u
bold=`tput bold`
normal=`tput sgr0`
if ! [ $EUID -eq 0 ]
then
echo "Must be root!" >&2
exit 1
fi
if [ $# -lt 2 ]
then
echo "Usage: $0 <name> <domain> [<features>]" >&2
exit 1
fi
NAME="$1"
DOMAIN="$2"
ROOTFS="/data/containers/$NAME/rootfs"
WEBFS="/data/containers/web/rootfs"
WEBPATH="/srv/http/$DOMAIN"
NGINX="$WEBFS/etc/nginx"
PHP_MODULES=('')
shift 2
# handle extra options
while (( "$#" ))
do
case $1 in
mysql)
PHP_MODULES=(mysqli mysql pdo_mysql)
;;
postgres)
PHP_MODULES=(pgsql pdo_pgsql)
;;
esac
shift
done
# clone container
echo "${bold}Cloning container ...$normal"
TEMPFILE=$(mktemp)
ruby -rjson -e 'puts ({php_extensions: ARGV}).to_json' "$PHP_MODULES[@]" > "$TEMPFILE"
lxc-clone -o base -n "$NAME" -- --group php --vars "$TEMPFILE"
rm "$TEMPFILE"
# configure bind mount
UNIT_NAME=$(systemd-escape --path --suffix=mount "${WEBFS}${WEBPATH}")
cat << EOF > "/etc/systemd/system/$UNIT_NAME"
[Mount]
What = ${ROOTFS}${WEBPATH}
Where = ${WEBFS}${WEBPATH}
Type = none
Options = bind,ro
[Install]
WantedBy=lxc-mount.target
EOF
systemctl enable "$UNIT_NAME"
systemctl start "$UNIT_NAME"
# configure nginx
echo "${bold}Configuring nginx ...$normal"
cat << EOF > "$NGINX/sites-available/$DOMAIN"
server {
listen 80;
listen 443 ssl;
index index.php index.html index.htm;
server_name $DOMAIN;
root /srv/http/$DOMAIN;
location ~ \.(php|php5)$ {
fastcgi_pass $NAME:9000;
fastcgi_index index.php;
include fastcgi.conf;
}
}
EOF
chroot "$WEBFS" nginx_ensite "$DOMAIN"
# restart nginx
echo "${bold}Reload nginx ...$normal"
lxc-attach -n web -- systemctl reload nginx
# configure filewall
echo "${bold}Configuring firewall ...$normal"
cat << EOF > "/etc/ferm.d/services/45-$NAME"
&def_service($NAME, $NAME, tcp, 9000);
&allow_service_for($NAME, web);
EOF
fw-apply
# start container
echo "${bold}Starting container ...$normal"
lxc-start -d -n "$NAME"

View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env bash
set -eu
@ -17,9 +17,8 @@ fi
USER="$1"
DBPASSWORD="$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)"
DATABASE="${2:-$USER}"
export PGPASSFILE=/root/.pgpass
psql --host postgres --user postgres <<EOF
lxc-attach -n postgres -- su postgres -c psql <<EOF
create user "$USER";
alter user "$USER" with password '$DBPASSWORD';
create database "$DATABASE" with owner "$USER";

View File

@ -1,38 +0,0 @@
#!/bin/bash
if ! [ "$EUID" -eq 0 ]
then
echo "This must be run as root!" >&2
exit 1
fi
set -e
echo -n "Password for data storage: "
read -s password
echo
echo $password | cryptsetup luksOpen /dev/sda3 zfs_hd_1
echo $password | cryptsetup luksOpen /dev/sdb3 zfs_hd_2
echo "import zfs pool"
retry=0
until [ $retry -ge 10 ]; do
zpool list | grep -q data && break || true
zpool import -f data
retry=$[$retry+1]
sleep 1
done
echo "mount zfs datasets"
zfs mount -a
echo "do bind mounts"
ruby /etc/lxc/hooks/update-mounts
mount -o bind /data/containers/login/rootfs/home/joerg /home/joerg/login
mount -o bind /data/containers/login/rootfs/home/devkid /home/devkid/login
mount -o bind /data/containers/pyload/rootfs/var/lib/pyload /data/pyload
mount -o bind /data/pacman/pkg /var/cache/pacman/pkg
mount -o bind /data/pacman/sync /var/lib/pacman/sync
systemctl start lxc.target

View File

@ -1,42 +0,0 @@
#!/bin/sh
bold=`tput bold`
normal=`tput sgr0`
if ! [ $EUID -eq 0 ]
then
echo "Must be root!" >&2
exit 1
fi
if [ $# -lt 2 ]
then
echo "Usage: $0 <name> <domain> [<features>]" >&2
exit 1
fi
NAME="$1"
DOMAIN="$2"
ROOTFS="/data/containers/$NAME/rootfs"
WEBFS="/data/containers/web/rootfs"
WEBPATH="/srv/http/$DOMAIN"
NGINX="$WEBFS/etc/nginx"
# destroy container
lxc-stop -n "$NAME"
/usr/bin/lxc-destroy -n "$NAME"
# unconfigure nginx
chroot "$WEBFS" nginx_dissite "$DOMAIN"
rmdir "$WEBFS$WEBPATH"
rm -f "$WEBFS/etc/nginx/sites-available/$DOMAIN"
# restart nginx
lxc-attach -n web -- systemctl restart nginx
# unconfigure firewall
rm -f "/etc/ferm.d/services/45-$NAME"
fw-apply
# clean up container.json
/etc/lxc/scripts/cleanup-container-file.rb

View File

@ -1,23 +0,0 @@
#!/usr/bin/env ruby
kernel_package = ARGV[0] || "linux"
content = ""
IO.popen(["pacman", "-Ql", kernel_package]) {|io| content = io.read }
kernel = /\/usr\/lib\/modules\/(?<version>.*)\/kernel/.match(content)
abort "no kernel version found in package" unless kernel
mods = Dir["/usr/src/*"].sort
mods.each do |mod|
match = /(?<name>[^\/-]+)-(?<version>.+)$/.match(mod)
unless match
puts "Skip module '#{mod}' (not following the name standard)"
next
end
args = ["dkms",
"install",
"-m", "#{match[:name]}/#{match[:version]}",
"-k", kernel[:version]]
puts "$ #{args.join(" ")}"
system(*args)
end

View File

@ -1,15 +0,0 @@
#!/bin/sh
if ! [ $EUID -eq 0 ]
then
echo "Must be root!" >&2
exit 1
fi
cd /etc
if ferm --interactive --timeout 10 ferm.conf
then
ferm -n -l --domain ip ferm.conf > /etc/iptables/iptables.rules
ferm -n -l --domain ip6 ferm.conf > /etc/iptables/ip6tables.rules
fi

View File

@ -1,7 +1,7 @@
#!/bin/sh
#!/usr/bin/env bash
(
for c in /data/containers/*
for c in /lxc/*
do
if [ -d "$c/rootfs" ]
then

9
lxc-attach Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
export HOME=/root
cd /root
exec /run/current-system/sw/bin/lxc-attach \
--clear-env \
--keep-var TERM \
--keep-var HOME \
"$@"

29
lxc-create-container Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env ruby
require "optparse"
def sh(cmd, *args)
puts "$ #{cmd} " + args.map {|a| "'#{a}'" }.join(" ")
system(cmd, *args) or abort "command failed"
end
options = {}
OptionParser.new do |opts|
opts.banner = "Usage: lxc-create-container [options]"
opts.on("-nNAME", "--name=NAME", "container name") do |n|
options[:name] = n
end
end.parse!
unless options[:name]
$stderr.puts "no option for --name supplied"
exit(1)
end
sh("systemctl", "stop", "lxc-base")
sh("/run/current-system/sw/bin/lxc-copy", "-n", "base", "-B", "zfs", "-N", options[:name])
puts "$ cd /etc/nixos/ansible"
Dir.chdir("/etc/nixos/ansible") do
sh("nix-shell", "--command", "ansible-playbook -i inventory site.yml --limit #{options[:name]}")
end
sh("lxc-info", "--name", "#{options[:name]}")

View File

@ -1,3 +1,32 @@
#!/bin/bash
#!/usr/bin/env ruby
echo use /usr/bin/lxc-destroy instead >&2
require "optparse"
def sh(*args)
puts("$ #{args.join(" ")}")
system(*args)
end
options = {}
OptionParser.new do |opts|
opts.banner = "Usage: lxc-destroy [options]"
opts.on("-nNAME", "--name=NAME", "container name") do |n|
options[:name] = n
end
end.parse!
unless options[:name]
$stderr.puts "no option for --name supplied"
exit(1)
end
print "enter the name of the container to delete: "
unless options[:name] == STDIN.gets.chomp
puts "does not match"
exit(1)
end
PREFIX="/run/current-system/sw/bin"
#sh "#{PREFIX}/lxc-stop", "-n", options[:name]
sh "#{PREFIX}/lxc-destroy", "-n", options[:name]

View File

@ -1,6 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
lxc_root=/data/containers
lxc_root=/lxc/
for n in `lxc-ls`; do
name=n
r=$lxc_root/$n/rootfs

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
if [ $EUID -ne 0 ]; then
echo "Must be root!" >&2

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -eu
@ -18,7 +18,7 @@ case "$1" in
;;
esac
CONTAINER_PATH="/data/containers/$CONTAINER"
CONTAINER_PATH="/lxc/$CONTAINER"
[ $EUID -eq 0 ] || die "Needs root permission"
[ -d "$CONTAINER_PATH" ] || die "No such container found: $CONTAINER"

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
if [ $EUID -ne 0 ]; then
echo "Must be root!" >&2

View File

@ -1,3 +1,3 @@
#!/bin/bash
#!/usr/bin/env bash
echo /data/containers/$1/rootfs/$2
echo /lxc/$1/rootfs/$2

View File

@ -1,11 +1,12 @@
#!/bin/bash
#!/usr/bin/env bash
if [ $EUID -ne 0 ]; then
echo "Must be root!" >&2
exit 1
fi
cd "${DIR:-/}"
cd /
export PATH=$PATH:/usr/local/bin CWD=/
lxc-attach --name login -- sudo -u aurrepo -- aurrepo --sign --verbose
lxc-attach --name login -- sudo -u aurrepo -- yaourt "$@"
lxc-attach --name login -- sudo -u aurrepo -- aurrepo --sign --verbose

45
matemat-stats.py Normal file
View File

@ -0,0 +1,45 @@
import requests
from datetime import datetime
from influxdb import InfluxDBClient
url = "http://matemat.hq.c3d2.de/{}"
def main():
resp = requests.get(url.format("backup/inventory.json"))
fields = {}
tags = {}
json_body = []
for article in resp.json():
json_body.append(dict(
measurement="inventory",
tags=dict(name=article["name"]),
fields=dict(filter(lambda t: t[0] not in ["artNr", "name"], article.items()))
))
client = InfluxDBClient('influxdb.thalheim.io',
port=8086,
ssl=True,
username="matemat",
password="eig0NaGoahCia5oo",
database="matemat")
client.write_points(json_body)
resp2 = requests.get(url.format("statistics.json"))
json_body = resp2.json()
if "total_balance" in json_body:
json_body["total_balance"] = float(json_body["total_balance"])
if "total_loss_retail_price" in json_body:
json_body["total_loss_retail_price"] = float(json_body["total_loss_retail_price"])
if "positive_balance" in json_body:
json_body["positive_balance"] = float(json_body["positive_balance"])
if "negative_balance" in json_body:
json_body["negative_balance"] = int(json_body["negative_balance"])
if "inactive_users_negative_balance" in json_body:
json_body["inactive_users_negative_balance"] = float(json_body["inactive_users_negative_balance"])
client.write_points([dict(measurement="statistics", fields=json_body, tags={})])
if __name__ == "__main__":
main()

9
netdata-rauter.rb Normal file
View File

@ -0,0 +1,9 @@
require 'open-uri'
req = open('http://rauter.evenet.dn42:19999/api/v1/allmetrics?format=prometheus')
req.each_line do |l|
l.gsub!(/#.*/,"")
if l =~ /([^{]+){[^}]+}\s+(\d+)\s+\d+/
metric, val = $1, $2
puts "#{metric.gsub("_", ".")} #{val} #{Time.new.utc.to_i}"
end
end

11
nixos-safe-rebuild Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env bash
if [ $EUID -ne 0 ]; then
echo "Must be root!" >&2
exit 1
fi
nixos-rebuild switch
echo "Press Ctrl-C to apply changes permanently (Rollback in 10s)"
sleep 10
nixos-rebuild switch --rollback

View File

@ -7,8 +7,9 @@ fi
pkgs=""
for pkg in "$@"
do
cp -- "$pkg" "/data/containers/login/rootfs/var/lib/aurrepo/"
cp -- "$pkg" "/lxc/login/rootfs/var/lib/aurrepo/"
name="$(basename $pkg)"
chown 102013:102013 "/lxc/login/rootfs/var/lib/aurrepo/$name"
lxc-attach -n login -- sudo -u aurrepo gpg --detach-sign --no-armor "/var/lib/aurrepo/$name"
pkgs="$pkgs /var/lib/aurrepo/$name"
done

View File

@ -1 +0,0 @@
/usr/bin/msmtp

View File

@ -1,7 +0,0 @@
#! /bin/sh
# Send mail
echo "$SMARTD_MESSAGE" | mail -s "$SMARTD_FAILTYPE" "$SMARTD_ADDRESS"
# Notify user
wall "$SMARTD_MESSAGE"

View File

@ -1,3 +0,0 @@
#!/bin/sh
pacman -S linux-lts linux-lts-headers spl-lts spl-utils-lts zfs-lts zfs-utils-lts

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
#Script to update motd with relevant information.
#Define output file
motd="/etc/motd"

1
vim
View File

@ -1 +0,0 @@
/usr/bin/nvim

5
wakeup-nas.rb Normal file
View File

@ -0,0 +1,5 @@
require 'socket'
TCPSocket.open('home.devkid.net', 22198) do |socket|
socket.write(File.read("/etc/nixos/secrets/nas-wakeup-password"))
end