get all basic services running

.dockerignore

@@ -1,3 +1,5 @@
 .envrc
 Dockerfile
+Makefile
 cache
+etc/secrets-template

.gitignore

@@ -1 +1,2 @@
 cache
+etc/secrets

Dockerfile

@@ -1,23 +1,39 @@
 FROM alpine:edge
 
+EXPOSE 22 80 443
+
 # 1. common tools
 # 2. programing languages
 # 3. icinga2 stuff
 # 4. letsencrypt deps
+ENV POSTGRES_VERSION=9.5.2-r1
 RUN apk add --no-cache --update \
   bind-tools tcpdump openssh lsof curl wget git mercurial strace tmux vim rsync htop bash openssl \
   ruby python go build-base \
-  nginx postgresql icinga2 \
+  nginx postgresql==$POSTGRES_VERSION icinga2 icinga2-bash-completion icingaweb2 icingaweb2-bash-completion nagios-plugins \
   py-requests py-setuptools && \
   mkdir -p /run/nginx
 
+# sshd
+RUN /usr/bin/ssh-keygen -A && \
+    sed -i -e "s!/root:/bin/ash!/root/:bin/bash!" /etc/passwd
+
+## temporary keys until letsencrypt certificate is generated
+RUN mkdir -p /etc/ssl/letsencrypt/nginx /etc/ssl/letsencrypt/influxdb && \
+    cd /etc/ssl/letsencrypt && \
+    openssl req -subj "/C=DE/ST=Saxony/L=Dresden/O=Evenet/OU=IT Department/CN=higgsboson.tk" \
+        -nodes -x509 -newkey rsa:2048 -keyout nginx/key.pem -out nginx/fullchain.pem -days 5 && \
+    cat nginx/key.pem nginx/fullchain.pem > influxdb/combined.pem
+
 ## S6 supervisor
 RUN curl -sSL https://github.com/just-containers/s6-overlay/releases/download/v1.17.2.0/s6-overlay-amd64.tar.gz | tar -xzf - -C /
+ENTRYPOINT ["/init"]
 
+ENV INFLUXDB_VERSION=0.12.2
 ## Influxdb
 RUN export GOPATH=/tmp/go \
   && mkdir -p /etc/influxdb/ $GOPATH/src/github.com/influxdata \
-  && curl -sSL https://github.com/influxdata/influxdb/archive/v0.12.2.tar.gz | tar -xzf - -C /tmp \
+  && curl -sSL https://github.com/influxdata/influxdb/archive/v${INFLUXDB_VERSION}.tar.gz | tar -xzf - -C /tmp \
   && mv /tmp/influxdb-* $GOPATH/src/github.com/influxdata/influxdb \
   && cd $GOPATH/src/github.com/influxdata/influxdb \
   && go get github.com/sparrc/gdm \
@@ -25,47 +41,58 @@ RUN export GOPATH=/tmp/go \
   && gdm restore \
   && go install ./... \
   && cp $GOPATH/bin/influx* /usr/bin/ \
-  && rm -rf $GOPATH
+  && rm -rf $GOPATH \
+  && addgroup -g 1001 influxdb \
+  && adduser -h /var/lib/influxdb -u 1001 -G influxdb -D influxdb \
+  && mkdir -p /backup/influxdb
 
 ## Grafana
 RUN apk add --no-cache nodejs \
    && export GOPATH=/tmp/go \
    && mkdir -p /opt/grafana/conf $GOPATH/src/github.com/grafana/ \
-   && curl -sSL https://github.com/grafana/grafana/archive/v3.0-beta5.tar.gz | tar -xzf - -C /tmp \
+   && curl -sSL https://github.com/grafana/grafana/archive/v3.0.0-beta6.tar.gz | tar -xzf - -C /tmp \
    && mv /tmp/grafana-* $GOPATH/src/github.com/grafana/grafana \
    && cd $GOPATH/src/github.com/grafana/grafana \
-   && go run build.go setup \
-   && $GOPATH/bin/godep restore \
-   && go run build.go build \
    && npm install \
    && npm install -g grunt-cli \
    && grunt \
    && npm uninstall -g grunt-cli \
-   && npm cache clear \
-   && cp -a $GOPATH/src/github.com/grafana/grafana/bin/* /usr/bin/ \
-   && cp -ra $GOPATH/src/github.com/grafana/grafana/public_gen /opt/grafana/public \
    && apk del --purge nodejs \
+   && go run build.go setup \
+   && $GOPATH/bin/godep restore \
+   && go run build.go build \
+   && mkdir -p /usr/share/grafana/conf \
+   && cp -a bin/grafana-server bin/grafana-cli /usr/bin/ \
+   && cp -ra public_gen /usr/share/grafana/public \
+   && cp conf/sample.ini /etc/grafana.ini \
+   && cp conf/defaults.ini /usr/share/grafana/conf/defaults.ini \
    && rm -rf $GOPATH /root/.npm* \
-   && addgroup -g 45555 grafana \
+   && addgroup -g 1002 grafana \
-   && adduser -u 45555 -G grafana -D grafana
+   && adduser -h /usr/share/grafana -u 1002 -G grafana -D grafana
 
 ## simp_le
 RUN apk --no-cache add libffi-dev python-dev openssl-dev && \
-  mkdir -p /tmp/simp_le /etc/ssl/letsencrypt && \
   curl -sSL https://github.com/kuba/simp_le/archive/master.tar.gz | tar -xzf - -C /tmp && \
   cd /tmp/simp_le-master && \
   python ./setup.py install && \
   rm -rf /tmp/simp_le-master && apk del git py-pip libffi-dev python-dev openssl-dev && \
   echo "3	10	*	*	*	/usr/bin/update-certs" >> /etc/crontabs/root
 
-## temporary keys until letsencrypt certificate is generated
+## postgres
-RUN cd /etc/ssl/letsencrypt && \
+RUN mkdir -p /backup/postgresql /var/lib/postgresql/data
-    openssl req -x509 -newkey rsa:2048 -keyout nginx/key.pem -out nginx/fullchain.pem -days 5 && \
-    cat nginx/key.pem nginx/fullchain.pem > influxdb/combined.pem
 
-RUN icinga2 feature enable ido-pgsql
+RUN icinga2 feature enable ido-pgsql graphite api checker command notification perfdata
 
-VOLUME [ "/var/lib/influxdb/data", "/var/lib/influxdb/meta", "/var/lib/postgres/", "/var/lib/grafana", "/etc/grafana", "/etc/icinga2", "/etc/icinga-web", "/etc/icingaweb2", "/var/lib/icinga2", "/etc/ssmtp"]
+VOLUME ["/var/lib/influxdb/data", \
-EXPOSE 22 80 443
+        "/var/lib/influxdb/meta", \
+        "/var/lib/postgresql/", \
+        "/var/lib/grafana", \
+        "/etc/icinga2", \
+        "/var/lib/icinga2", \
+        "/etc/icingaweb2", \
+        "/usr/share/icingaweb2", \
+        "/etc/ssmtp", \
+        "/backup"]
 ADD . /
-ENTRYPOINT ["/init"]
+RUN chown 400 /etc/secrets && . /etc/secrets && \
+    sed -i -e "s/admin_password = replaceme/admin_password = $GRAFANA_PASSWORD/" /etc/grafana.ini

Makefile

@@ -1,7 +1,17 @@
-all: run
+all: build
 
 build:
 	docker build -t monitoring .
 
 run: build
-	docker run --rm -ti monitoring sh
+	docker run --rm -ti \
+		-v influxdb-data:/var/lib/influxdb/data \
+		-v influxdb-meta:/var/lib/influxdb/meta \
+		-v postgresql:/var/lib/postgresql \
+		-v grafana-lib:/var/lib/grafana \
+		-v grafana-etc:/etc/grafana \
+		-v icinga2-etc:/etc/icinga2 \
+		-v icinga2-lib:/var/lib/icinga2 \
+		-v icingaweb2-etc:/etc/icingaweb2 \
+		-v ssmtp-etc:/etc/ssmtp \
+		monitoring sh

etc/cont-finish.d/backup-all

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+with_service() {
+  svc=$1
+  shift
+  s6-svc -wR -u "/var/run/s6/services/${svc}"
+  s6-setuidgid "$svc" $@
+  s6-svc -d "/var/run/s6/services/${svc}"
+}
+
+set -x
+
+echo backup influxdb
+with_service "influxdb" backup-influxdb
+
+echo backup postgres
+with_service "postgres" backup-postgres

etc/cont-init.d/influxdb

@@ -0,0 +1,43 @@
+#!/usr/bin/with-contenv /bin/sh
+
+set -eu
+
+VERSION_FILE=/var/lib/influxdb/meta/package-version
+
+. /etc/secrets
+
+influxql() {
+  s6-setuidgid influxdb influx \
+    -username "influxdb" \
+    -password "$INFLUXDB_PASSWORD" \
+    -unsafeSsl -ssl \
+    -execute="$1"
+}
+
+if [ -f $VERSION_FILE ]; then
+  [ ! -f /backup/influxdb/meta.00 ] || [ "$(cat $VERSION_FILE)" == "$INFLUXDB_VERSION" ] && exit
+  s6-setuidgid influxdb influxd restore -metadir=/var/lib/influxdb/meta /backup/influxdb/
+else
+  influxd -config /etc/influxdb/config.toml &
+  export pid=$!
+  trap "kill $pid" SIGINT SIGTERM
+  set +e
+  RET=1
+  while [[ $RET -ne 0 ]]; do
+    echo "=> Waiting for confirmation of InfluxDB service startup ..."
+    curl -k http://localhost:8086/ping 2> /dev/null
+    RET=$?
+    sleep 3
+  done
+  set -e
+
+  influxql "CREATE USER influxdb WITH PASSWORD '${INFLUXDB_PASSWORD}' WITH ALL PRIVILEGES"
+  influxql "CREATE DATABASE metrics"
+  influxql "CREATE USER icinga WITH PASSWORD '${INFLUXDB_ICINGA_PASSWORD}'"
+  influxql "GRANT ALL ON metrics TO icinga"
+  influxql "CREATE USER grafana WITH PASSWORD '${INFLUXDB_GRAFANA_PASSWORD}'"
+  influxql "GRANT READ ON metrics TO grafana"
+  kill $pid
+fi
+
+echo "$INFLUXDB_VERSION" > $VERSION_FILE

etc/cont-init.d/postgres

@@ -0,0 +1,32 @@
+#!/usr/bin/with-contenv /bin/sh
+
+set -eu
+
+PG_HOME=/var/lib/postgresql/data
+
+if [ -f $PG_HOME/package-version ]; then
+  [ ! -f /backup/postgresql/dump ] || [ "$(cat $PG_HOME/package-version)" == "$POSTGRES_VERSION" ] && exit
+  rm ${PG_HOME}.old || true
+  mv $PG_HOME ${PG_HOME}.old
+fi
+
+as_pg() {
+  s6-setuidgid postgres "$@"
+}
+
+as_pg initdb -D $PG_HOME
+as_pg pg_ctl -D $PG_HOME -w start
+
+if [ -f /backup/postgresql/dump ]; then
+  as_pg psql -d postgres -f /backup/postgresql/dump
+else
+  as_pg createuser icinga
+  as_pg createdb -O icinga -E UTF8 icinga
+  as_pg psql -d icinga < /usr/share/icinga2-ido-pgsql/schema/pgsql.sql
+
+  as_pg createuser grafana
+  as_pg createdb -O grafana -E UTF8 grafana
+fi
+
+echo "$POSTGRES_VERSION" > $PG_HOME/package-version
+as_pg pg_ctl -D $PG_HOME -w stop

etc/crontabs/influxdb

@@ -0,0 +1 @@
+5	10	*	*	*	/usr/bin/backup-influxdb

etc/crontabs/postgres

@@ -0,0 +1 @@
+3	10	*	*	*	/usr/bin/backup-postgres

etc/fix-attrs.d/grafana

@@ -0,0 +1,2 @@
+/var/lib/grafana true grafana 0644 0755
+/var/log/grafana true grafana 0644 0755

etc/fix-attrs.d/grafana-dir

@@ -1,2 +0,0 @@
-/var/lib/grafana true grafana 0755 0755
-/var/log/grafana true grafana 0755 0755

etc/fix-attrs.d/influxdb

@@ -0,0 +1,2 @@
+/var/lib/influxdb/ true influxdb 0644 0755
+/backup/influxdb true influxdb 0644 0755

etc/fix-attrs.d/postgres

@@ -0,0 +1,2 @@
+/var/lib/postgresql true postgres 0600 0700
+/backup/postgresql true postgres 0600 0700

etc/grafana.ini

@@ -0,0 +1,29 @@
+[paths]
+[server]
+root_url = https://stats.higgsboson.tk/
+[database]
+type = postgres
+host = /tmp
+name = grafana
+user = grafana
+password = ''
+[session]
+[analytics]
+check_for_updates = false
+[security]
+admin_user = grafana
+admin_password = replaceme
+[users]
+[auth.anonymous]
+[auth.github]
+[auth.google]
+[auth.proxy]
+[auth.basic]
+[auth.ldap]
+[smtp]
+[emails]
+[log]
+[log.console]
+[log.file]
+[event_publisher]
+[dashboards.json]

etc/icinga2/features-available/ido-pgsql.conf

@@ -0,0 +1,13 @@
+/**
+ * The db_ido_pgsql library implements IDO functionality
+ * for PostgreSQL.
+ */
+
+library "db_ido_pgsql"
+
+object IdoPgsqlConnection "ido-pgsql" {
+  user = "icinga",
+  password = "",
+  host = "/tmp",
+  database = "icinga"
+}

etc/influxdb/config.toml

@@ -1,4 +1,5 @@
-reporting-disabled = true
+[reporting]
+  reporting-disabled = true
 
 [meta]
   dir = "/var/lib/influxdb/meta"
@@ -6,6 +7,10 @@ reporting-disabled = true
   logging-enabled = true
   lease-duration = "1m0s"
 
+[data]
+  dir = "/var/lib/influxdb/data"
+  wal-dir = "/var/lib/influxdb/wal"
+
 [http]
   auth-enabled = true
   https-enabled = true

etc/secrets-template

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+GRAFANA_PASSWORD="<PASSWORD>"
+INFLUXDB_PASSWORD="<PASSWORD>"
+INFLUXDB_ICINGA_PASSWORD="<PASSWORD>"
+INFLUXDB_GRAFANA_PASSWORD="<PASSWORD>"

etc/services.d/grafana/run

@@ -1,4 +1,4 @@
-#!/usr/bin/execlineb -P
+#!/bin/sh
-cat
+
-#s6-setuidgid grafana
+cd /usr/share/grafana
-#/opt/grafana/bin/grafana-server -homepath /opt/grafana cfg:default.paths.data=/var/lib/grafana
+exec s6-setuidgid grafana grafana-server --config=/etc/grafana.ini

etc/services.d/influxdb/run

@@ -1,2 +1,3 @@
 #!/usr/bin/execlineb -P
+s6-setuidgid influxdb
 /usr/bin/influxd -config /etc/influxdb/config.toml

etc/services.d/postgres/run

@@ -1,2 +1,3 @@
 #!/usr/bin/execlineb -P
-cat
+s6-setuidgid postgres
+postgres -D /var/lib/postgresql/data

etc/services.d/sshd/run

@@ -1,2 +1,2 @@
 #!/usr/bin/execlineb -P
-cat
+/usr/sbin/sshd -D

root/.ssh/authorized_keys

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKbBp2dH2X3dcU1zh+xW3ZsdYROKpJd3n13ssOP092qE joerg@turingmachine

usr/bin/backup-influxdb

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+mv /backup/influxdb/tmp/meta.00 /backup/influxdb/ 2>/dev/null
+set -e
+influxd backup /backup/influxdb/tmp
+mv /backup/influxdb/tmp/meta.00 /backup/influxdb/

usr/bin/backup-postgres

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -eu
+cd /var/lib/postgresql/
+pg_dumpall > /backup/postgresql/dump.tmp
+mv /backup/postgresql/dump{.tmp,}

usr/bin/update-certs

@@ -16,10 +16,11 @@ gen_cert() {
     -f key.pem \
     $(for domain in "$@"; do echo "-d $domain"; done) \
     --default_root /var/www/letsenrypt
-  cat fullchain.pem key.pem > combined.pem
   local rc=$?
   [ $rc = 1 ] && $restart_action
   [ $rc != 0 ] && exit $rc
+  set -e
+  cat fullchain.pem key.pem > combined.pem
 }
 
 gen_cert "nginx" "nginx -s reload" "status.higgsboson.tk" "stats.higgsboson.tk"