get overlay based container working

hooks/lib/lxc/container.rb

@@ -15,6 +15,8 @@ module Lxc
       @ipv6_subnet = NetAddr::CIDR.create(zone["v6_subnet"] || "fd7d:aed0:18aa::/48")
       @ula_subnet  = NetAddr::CIDR.create(zone["ula_subnet"] || "fdc5:bdb8:b81::/48")
 
+      @container_root = Pathname.new(zone["lxc_root"]).join(name)
+
       network = data["network"]
       @name = name
       @ipv4 = ipv4 || find_address(@ipv4_subnet, collect_subnets(network, "ipv4"))
@@ -43,6 +45,10 @@ module Lxc
       end
       opts[:local_conf] = local_conf
 
+      fstab = @container_root.join("fstab")
+      opts[:fstab]  = fstab if File.exists?(fstab)
+      opts[:rootfs] = @data["zone"]["shared_rootfs"] || opts[:rootfs] || @container_root.join("rootfs")
+
       templ = Template.new(CONFIG_ROOT.join("hooks", "templates", "config.erb"))
       templ.write(config_path, opts)
     end

hooks/mount-base-rootfs

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-mount -o bind            /lxc/base/rootfs /lxc/${LXC_NAME}/rootfs
-mount -o remount,ro,bind /lxc/base/rootfs /lxc/${LXC_NAME}/rootfs

hooks/templates/config.erb

@@ -1,7 +1,10 @@
-lxc.include = /etc/lxc/default.conf
+lxc.include = /etc/lxc/overlay.conf
 lxc.include = <%= local_conf %>
 lxc.utsname = <%= name %>
 lxc.rootfs = <%= rootfs %>
+<%- if fstab -%>
+lxc.mount = <%= fstab %>
+<%- end -%>
 lxc.network.ipv4 = <%= ipv4 %>
 lxc.network.ipv6 = <%= ipv6 %>
 lxc.network.ipv6 = <%= ula %>

hooks/update-lxc-configs

@@ -2,19 +2,19 @@
 require "pathname"
 require_relative "lib/lxc"
 
-LXC_CONTAINER_ROOT = Pathname.new("/data/containers")
 
 registry = Lxc::Registry.new
+container_root = Pathname.new(registry.data["zone"]["lxc_root"])
 network = registry.data["network"] || {}
 network.each do |name, container|
   next if container["lxc"] == false
+  shared_rootfs = registry.data["zone"]["shared_rootfs"]
   container = Lxc::Container.new(registry.data,
                                  name: name,
                                  ipv4: container["ipv4"],
                                  ipv6: container["ipv6"],
                                  ula: container["ula"],
-                                 rootfs: LXC_CONTAINER_ROOT.join(name, "rootfs"),
                                  group: container["group"],
                                  vars: container["vars"])
-  container.write_config(LXC_CONTAINER_ROOT.join(name, "config"))
+  container.write_config(container_root.join(name, "config"))
 end

overlay.conf

@@ -1,5 +1,6 @@
 lxc.include = /etc/lxc/base.conf
-lxc.hook.pre-mount = /etc/lxc/hooks/mount-base-rootfs
+lxc.rootfs = /lxc/base/rootfs
+lxc.rootfs.options = ro
 
 ## this line is used by: /usr/share/lxc/templates/lxc-overlay
 # CREATE_HOOKS="setup-machine-id remove-journal cleanup-lxc-config create-lxc-config update-zone update-digitalocean-rnds"

templates/lxc-overlay

@@ -3,8 +3,10 @@
 set -ex
 
 add_overlayfs(){
-  local root_path="$1"
+  local base_rootfs="$1"
-  local rootfs="$root_path/rootfs" local path="$2"
+  local root_path="$2"
+  local rootfs="$root_path/rootfs"
+  local path="$3"
   local lower="$base_rootfs/${path}"
   local upper="$root_path/.${path}-delta"
   local work="$root_path/.${path}-work"
@@ -12,11 +14,14 @@ add_overlayfs(){
   [[ -d "$rootfs" ]] || mkdir -p "$rootfs"
   [[ -d "$work" ]] || mkdir -p "$work"
   [[ -d "$upper" ]] || mkdir -p "$upper"
+  [[ -d "$lower" ]] || mkdir -p "$lower"
 
-  echo "none $path overlay lowerdir=$lower,upperdir=$upper,workdir=$work 0 0" >> "$root_path/fstab"
+  echo "overlay $path overlay lowerdir=$lower,upperdir=$upper,workdir=$work 0 0" >> "$root_path/fstab"
 }
 
-options=$(getopt -o n:4:6: -l name:,ipv4:,ipv6:,path:,rootfs:,mapped-uid:,mapped-gid: -- "$@")
+echo args $@
+
+options=$(getopt -o n:4:6: -l name:,path:,rootfs:,mapped-uid:,mapped-gid: -- "$@")
 if [ $? -ne 0 ]; then
   usage $(basename $0)
   exit 1
@@ -27,10 +32,10 @@ while true
 do
   case "$1" in
     -n|--name)      name=$2; shift 2;;
-    -4|--ipv4)      ipv4=$2; shift 2;;
-    -6|--ipv6)      ipv6=$2; shift 2;;
     --path)         path=$2; shift 2;;
-    --rootfs)       base_rootfs=$2; shift 2;;
+    --rootfs)       rootfs=$2; shift 2;;
+    --mapped-uid)   uid=$2; shift 2;;
+    --mapped-gid)   gid=$2; shift 2;;
     --)             shift 1; break ;;
     *)              break ;;
   esac
@@ -46,25 +51,33 @@ if [ -z "${path}" ]; then
   exit 1
 fi
 
-if [ -z "${base_rootfs}" ]; then
+if [ -z "${rootfs}" ]; then
   echo "missing required 'rootfs' parameter"
   exit 1
 fi
 
+overlay_conf="/etc/lxc/overlay.conf"
+
+if [ ! -e "$overlay_conf" ]; then
+  echo "Configuration at /etc/lxc/overlay.conf does not exists"
+  exit 1
+fi
+
+base_rootfs=$(perl -ne 'print $1 if /lxc.rootfs\s*=\s*(.*)/' "$overlay_conf")
 config="${path}/config"
 
 touch "$path/fstab"
-add_overlayfs "$path" "var"
+add_overlayfs "$base_rootfs" "$path" "var"
-add_overlayfs "$path" "etc"
+add_overlayfs "$base_rootfs" "$path" "etc"
-add_overlayfs "$path" "home"
+add_overlayfs "$base_rootfs" "$path" "home"
-add_overlayfs "$path" "srv"
+add_overlayfs "$base_rootfs" "$path" "srv"
-add_overlayfs "$path" "mnt"
+add_overlayfs "$base_rootfs" "$path" "mnt"
-add_overlayfs "$path" "root"
+add_overlayfs "$base_rootfs" "$path" "root"
 
 cat > "$config" <<EOF
-lxc.include = /etc/lxc/default.conf
+lxc.include = $overlay_conf
 lxc.include = ${path}/local.conf
-lxc.rootfs = ${path}/rootfs
+lxc.mount = ${path}/fstab
 lxc.utsname = $name
 EOF
 touch "$path/local.conf"
@@ -72,5 +85,21 @@ touch "$path/local.conf"
 export LXC_ROOTFS_PATH="$path/rootfs"
 export LXC_NAME=$name
 export LXC_CONFIG_FILE="$config"
-perl -n -e'/CREATE_HOOKS="([^"]+)"/ && map { system("/etc/lxc/hooks/$_") == 0 or print("executing $_ failed\n") } split(/\s+/, $1)' \
+
-	/etc/lxc/default.conf
+cat > "/tmp/${LXC_NAME}-hooks" <<EOF
+#!/usr/bin/env bash
+mount --bind "$path/fstab" /etc/fstab
+mount --bind "$base_rootfs" "$LXC_ROOTFS_PATH" -o ro
+cd "$LXC_ROOTFS_PATH"
+mount -a
+perl -n \
+  -e'/CREATE_HOOKS="([^"]+)"/ && map { system("/etc/lxc/hooks/\$_") == 0 or print("executing \$_ failed\n") } split(/\s+/, \$1)' \
+  "$overlay_conf"
+EOF
+chmod +x "/tmp/${LXC_NAME}-hooks"
+unshare -m  "/tmp/${LXC_NAME}-hooks"
+rm "/tmp/${LXC_NAME}-hooks"
+
+chown -R "$uid:$gid" "${path}"/* "${path}"/.*
+chown -R "0:0" "${path}/config" "${path}/local.conf"
+chmod +x "${path}"