jailmaker/jlmkr.sh

#!/bin/bash

set -euo pipefail
shopt -s nullglob

ABSOLUTE_SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")"
SCRIPT_NAME=$(basename "${ABSOLUTE_SCRIPT_PATH}")
SCRIPT_DIR_PATH="$(dirname "${ABSOLUTE_SCRIPT_PATH}")"

# Only set a color if we have an interactive tty
[[ -t 1 ]] && BOLD=$(tput bold) || BOLD=
[[ -t 1 ]] && RED=$(tput setaf 1) || RED=
[[ -t 1 ]] && YELLOW=$(tput setaf 3) || YELLOW=
[[ -t 1 ]] && NORMAL=$(tput sgr0) || NORMAL=

DISCLAIMER="${YELLOW}${BOLD}USING THIS SCRIPT IS AT YOUR OWN RISK!
IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS.${NORMAL}"

JAILS_DIR_PATH='jails'
JAIL_ROOTFS_NAME='rootfs'
JAIL_CONFIG_NAME='config'

USAGE="${DISCLAIMER}

Version: 0.0.0

Usage: ./${SCRIPT_NAME} [COMMAND] [NAME]

Commands:
  create	Create new jail [NAME] in the ${JAILS_DIR_PATH} dir
  start		Start jail NAME from the ${JAILS_DIR_PATH} dir
"

error() {
	echo -e "${RED}${BOLD}${1}${NORMAL}" >&2
}

fail() {
	error "${1}" && exit 1
}

[[ -z "${BASH_VERSINFO+x}" ]] && fail "This script must run in bash..."
[[ $UID -ne 0 ]] && echo "${USAGE}" && fail "Run this script as root..."
cd "${SCRIPT_DIR_PATH}" || fail "Could not change working directory to ${SCRIPT_DIR_PATH}..."

trace() {
	# https://unix.stackexchange.com/a/504829/477308
	echo 'Error occurred:'
	awk 'NR>L-4 && NR<L+4 { printf "%-5d%3s%s\n",NR,(NR==L?">>>":""),$0 }' L="${1}" "${ABSOLUTE_SCRIPT_PATH}"
}

# Trap errors
trap 'trace $LINENO' ERR

#####################
# START FUNCTIONALITY
#####################

start_jail() {
	[[ -z "${1}" ]] && fail "Please specify the name of the jail to start."
	local jail_name="${1}"
	local jail_path="${JAILS_DIR_PATH}/${jail_name}"
	local jail_config_path="${jail_path}/${JAIL_CONFIG_NAME}"

	! [[ -f "${jail_config_path}" ]] && fail "ERROR: Couldn't find: ${jail_config_path}."

	echo 'Loading config...'

	local key value

	while read -r line || [ -n "$line" ]; do
		key="${line%%=*}"
		value="${line#*=}"

		case "${key}" in
		"DOCKER_COMPATIBLE") local docker_compatible="${value}" ;;
		"GPU_PASSTHROUGH") local gpu_passthrough="${value}" ;;
		"SYSTEMD_NSPAWN_USER_ARGS") local systemd_nspawn_user_args="${value}" ;;
		"SYSTEMD_RUN_DEFAULT_ARGS") local systemd_run_default_args="${value}" ;;
		"SYSTEMD_NSPAWN_DEFAULT_ARGS") local systemd_nspawn_default_args="${value}" ;;
		esac

	done <"${jail_config_path}"

	echo 'Config loaded!'

	local systemd_run_additional_args=("--unit=jlmkr-${jail_name}" "--working-directory=./${jail_path}" "--description=My nspawn jail ${jail_name} [created with jailmaker]")
	local systemd_nspawn_additional_args=("--machine=${jail_name}" "--directory=${JAIL_ROOTFS_NAME}")

	if [[ "${docker_compatible}" -eq 1 ]]; then
		# Enable ip forwarding on the host (docker needs it)
		echo 1 >/proc/sys/net/ipv4/ip_forward

		# To properly run docker inside the jail, we need to lift restrictions
		# Without DevicePolicy=auto images with device nodes may not be pulled
		# For example docker pull ljishen/sysbench would fail
		# Fortunately I didn't encounter many images with device nodes...
		#
		# Issue: https://github.com/moby/moby/issues/35245
		#
		# The systemd-nspawn manual explicitly mentions:
		# Device nodes may not be created
		# https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html
		#
		# Workaround: https://github.com/kinvolk/kube-spawn/pull/328
		#
		# However, it seems like the DeviceAllow= workaround may break in
		# a future Debian release with systemd version 250 or higher
		# https://github.com/systemd/systemd/issues/21987
		#
		# As of 29-1-2023 it still works with debian bookworm (nightly) and sid
		# using the latest systemd version 252.4-2 so I think we're good!
		#
		# Use SYSTEMD_SECCOMP=0: https://github.com/systemd/systemd/issues/18370
		systemd_run_additional_args+=(--setenv=SYSTEMD_SECCOMP=0 --property=DevicePolicy=auto)
		# Add additional flags required for docker
		systemd_nspawn_additional_args+=(--capability=all "--system-call-filter=add_key keyctl bpf")

		# # TODO: don't process these systemd_nspawn_user_args twice,
		# # it is done again below
		# while read -r arg; do
		# 	# TODO: does --network-macvlan also need this?
		# 	if [[ "${arg}" == "--network-bridge=*" ]]; then
		# 		echo "Enable br_netfilter, docker requires it when jail is connected to bridge."
		# 		# TODO: figure out what the consequence is when not using br_netfilter
		# 		# Can these warnings in `docker info` be safely ignored?
		# 		# WARNING: bridge-nf-call-iptables is disabled
		# 		# WARNING: bridge-nf-call-ip6tables is disabled
		# 		# https://unix.stackexchange.com/q/720105/477308
		# 		# https://github.com/moby/moby/issues/24809
		# 		# https://docs.oracle.com/en/operating-systems/oracle-linux/docker/docker-KnownIssues.html#docker-issues
		#		# https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf
		#		# https://serverfault.com/questions/963759/docker-breaks-libvirt-bridge-network
		# 		modprobe br_netfilter
		# 		sysctl net.bridge.bridge-nf-call-iptables=1
		# 		sysctl net.bridge.bridge-nf-call-ip6tables=1

		# 		break
		# 	fi
		# done < <(printf '%s' "${systemd_nspawn_user_args}" | xargs -n 1)
	fi

	if [[ "${gpu_passthrough}" -eq 1 ]]; then
		systemd_nspawn_additional_args+=("--property=DeviceAllow=char-drm rw")

		# Detect intel GPU device and if present add bind flag
		[[ -d /dev/dri ]] && systemd_nspawn_additional_args+=(--bind=/dev/dri)

		# Detect nvidia GPU
		if [[ -d /dev/nvidia ]]; then
			# Mount the nvidia driver files, so we are always in sync with the host
			while read -r line; do
				if [[ "${line}" == /dev/* ]]; then
					systemd_nspawn_additional_args+=("--bind='${line}'")
				else
					systemd_nspawn_additional_args+=("--bind-ro='${line}'")
				fi
			done < <(nvidia-container-cli list)
		fi
	fi

	local args=()

	# Build the array of arguments
	local arg
	# Append each argument, one at a time, to the array
	while read -r arg; do args+=("${arg}"); done < <(printf '%s' "${systemd_run_default_args}" | xargs -n 1)
	# Append each element in systemd_run_additional_args to the args array
	args+=("${systemd_run_additional_args[@]}")
	# Add two more args to the array
	args+=(-- systemd-nspawn)
	# Append each argument, one at a time, to the array
	while read -r arg; do args+=("${arg}"); done < <(printf '%s' "${systemd_nspawn_default_args}" | xargs -n 1)
	# Append each element in systemd_nspawn_additional_args to the args array
	args+=("${systemd_nspawn_additional_args[@]}")
	# Append each argument, one at a time, to the array
	while read -r arg; do args+=("${arg}"); done < <(printf '%s' "${systemd_nspawn_user_args}" | xargs -n 1)

	# Concat all arguments in the array into a single space separated string,
	# but use %q to output each argument in a format that can be reused as shell input
	# This escapes special characters for us, which were 'lost' when xargs read the input above
	# https://ss64.com/bash/printf.html
	args_string="$(printf '%q ' "${args[@]}")"

	echo
	echo "All the arguments to pass to systemd-run:"
	printf '%s' "${args_string}" | xargs -n 1
	echo
	echo "Starting jail with the following command:"
	echo
	echo "systemd-run ${args_string}"
	echo

	printf '%s' "${args_string}" | xargs systemd-run || {
		echo
		error 'Failed to start the jail...'
		fail "Please check and fix the config file with \"nano ${jail_config_path}\"."
	}

	echo
	echo "Check logging:"
	echo "journalctl -u jlmkr-${jail_name}"
	echo
	echo "Check status:"
	echo "systemctl status jlmkr-${jail_name}"
	echo
	echo "Stop the jail:"
	echo "machinectl stop ${jail_name}"
	echo
	echo "Get a shell:"
	echo "machinectl shell ${jail_name}"
}

######################
# CREATE FUNCTIONALITY
######################

cleanup() {
	# Remove the jail_path if it's a directory
	local jail_path="${1}"
	[[ -d "${jail_path}" ]] && echo -e "\n\nCleaning up: ${jail_path}\n" && rm -rf "${jail_path}"
}

stat_chmod() {
	# Only run chmod if mode is different from current mode
	if [[ "$(stat -c%a "${2}")" -ne "${1}" ]]; then chmod "${1}" "${2}"; fi
}

validate_download_script() {
	echo "6cca2eda73c7358c232fecb4e750b3bf0afa9636efb5de6a9517b7df78be12a4  ${1}" | sha256sum --check >/dev/null
}

create_jail() {
	echo -e "${DISCLAIMER}"
	echo

	local name_from_arg="${1}"
	local arch
	arch="$(dpkg --print-architecture)"
	local lxc_dir_path='.lxc'
	local lxc_cache_path="${lxc_dir_path}/cache"
	local lxc_download_script_path="${lxc_dir_path}/lxc-download.sh"

	[[ "$(basename "${SCRIPT_DIR_PATH}")" != 'jailmaker' ]] && {
		error "${SCRIPT_NAME} needs to create files."
		error "Currently it can't decide if it's safe to create files in:"
		error "${SCRIPT_DIR_PATH}"
		fail "Please create a dedicated directory called 'jailmaker', store ${SCRIPT_NAME} there and try again."
	}

	local reply

	if [[ $(findmnt --target . --output TARGET --noheadings --first-only) != /mnt/* ]]; then
		echo "${YELLOW}${BOLD}WARNING: BEWARE OF DATA LOSS${NORMAL}"
		echo
		echo "${SCRIPT_NAME} should be on a dataset mounted under /mnt (it currently isn't)."
		echo "Storing it on the boot-pool means losing all jails when updating TrueNAS."
		echo "If you continue, jails will be stored under:"
		echo "${SCRIPT_DIR_PATH}"
		echo
		read -p "Do you wish to ignore this warning and continue? [y/N] " -n 1 -r reply && echo
		# Enter accepts default (no)
		! [[ "${reply}" =~ ^[Yy]$ ]] && exit
	fi

	# Set appropriate permissions (if not already set) for this file, since it's executed as root
	stat_chmod 700 "${SCRIPT_NAME}"

	# Create the lxc dirs if nonexistent
	mkdir -p "${lxc_dir_path}"
	stat_chmod 700 "${lxc_dir_path}"
	mkdir -p "${lxc_cache_path}"
	stat_chmod 700 "${lxc_cache_path}"

	# Create the dir where to store the jails
	mkdir -p "${JAILS_DIR_PATH}"
	stat_chmod 700 "${JAILS_DIR_PATH}"

	# Fetch the lxc download script if not present locally (or hash doesn't match)
	if ! validate_download_script "${lxc_download_script_path}"; then
		curl -fSL https://raw.githubusercontent.com/Jip-Hop/lxc/58520263041b6864cadad96278848f9b8ce78ee9/templates/lxc-download.in -o "${lxc_download_script_path}"
		# Validate after download to prevent executing arbitrary code as root
		validate_download_script "${lxc_download_script_path}" || fail 'Abort! Downloaded script has unexpected contents.'
	fi

	stat_chmod 700 "${lxc_download_script_path}"

	local distro='debian' release='bullseye'

	read -p "Install the recommended distro (Debian 11)? [Y/n] " -n 1 -r reply && echo
	if ! [[ "${reply}" =~ ^([Yy]|)$ ]]; then
		echo
		echo "${YELLOW}${BOLD}WARNING: ADVANCED USAGE${NORMAL}"
		echo
		echo "You may now choose from a list which distro to install."
		echo "But not all of them will work with ${SCRIPT_NAME} since these images are made for LXC."
		echo "Distros based on systemd probably work (e.g. Ubuntu, Arch Linux and Rocky Linux)."
		echo "Others (Alpine, Devuan, Void Linux) probably won't."
		echo
		read -p "Press any key to continue: " -n 1 -r reply && echo
		echo
		lxc_cache_path=${lxc_cache_path} "${lxc_download_script_path}" --list --arch="${arch}" || :
		echo
		echo "Choose from the DIST column."
		echo
		read -e -r -p "Distribution: " distro && echo
		echo "Choose from the RELEASE column (or ARCH if RELEASE is empty)."
		echo
		read -e -r -p "Release: " release
	fi
	echo
	local jail_name jail_path

	while true; do
		read -e -r -p "Enter jail name: " -i "${name_from_arg}" jail_name && echo
		if ! [[ "${jail_name}" =~ ^[.a-zA-Z0-9-]{1,64}$ && "${jail_name}" != '.'* && "${jail_name}" != *'.' && "${jail_name}" != *'..'* ]]; then
			cat <<-EOF
				${YELLOW}${BOLD}WARNING: INVALID NAME${NORMAL}

				A valid name consists of:
				- allowed characters (alphanumeric, dash, dot)
				- no leading or trailing dots
				- no sequences of multiple dots
				- max 64 characters

			EOF
		else
			jail_path="${JAILS_DIR_PATH}/${jail_name}"

			if [[ -e "${jail_path}" ]]; then
				echo "A jail with this name already exists."
				echo
			else
				# Accept the name
				break
			fi
		fi
	done

	# Cleanup on exit, but only once the jail_path is final
	# Otherwise we may cleanup the wrong directory
	trap 'cleanup "${jail_path}"' EXIT

	local docker_compatible gpu_passthrough systemd_nspawn_user_args

	echo "Docker won't be installed by ${SCRIPT_NAME}."
	echo "But it can setup the jail with the capabilities required to run docker."
	echo "You can turn DOCKER_COMPATIBLE mode on/off post-install."
	echo
	read -p "Make jail docker compatible right now? [y/N] " -n 1 -r reply && echo
	# Enter accepts default (no)
	if ! [[ "${reply}" =~ ^[Yy]$ ]]; then docker_compatible=0; else docker_compatible=1; fi
	echo
	read -p "Give access to the GPU inside the jail? [y/N] " -n 1 -r reply && echo
	# Enter accepts default (no)
	if ! [[ "${reply}" =~ ^[Yy]$ ]]; then gpu_passthrough=0; else gpu_passthrough=1; fi
	echo
	echo "${YELLOW}${BOLD}WARNING: CHECK SYNTAX${NORMAL}"
	echo
	echo "You may pass additional flags to systemd-nspawn."
	echo "With incorrect flags the jail may not start."
	echo "It's possible to correct/add/remove flags post-install."
	echo
	read -p "Show the man page for systemd-nspawn? [y/N] " -n 1 -r reply && echo

	# Enter accepts default (no)
	if [[ "${reply}" =~ ^[Yy]$ ]]; then
		man systemd-nspawn
	else
		echo
		echo "You may read the systemd-nspawn manual online:"
		echo "https://manpages.debian.org/${distro}/systemd-container/systemd-nspawn.1.en.html"
	fi

	# Backslashes and colons need to be escaped in bind mount options:
	# e.g. to bind mount a file called:
	# weird chars :?\"
	# the corresponding command would be:
	# --bind-ro='/mnt/data/weird chars \:?\\"'

	echo
	echo "For example to mount directories inside the jail you may add:"
	echo "--bind=/mnt/a/readwrite/directory --bind-ro=/mnt/a/readonly/directory"
	echo
	read -e -r -p "Additional flags: " systemd_nspawn_user_args && echo

	# Create directory for rootfs
	JAIL_ROOTFS_PATH="${jail_path}/${JAIL_ROOTFS_NAME}"
	mkdir -p "${JAIL_ROOTFS_PATH}"

	local jail_config_path="${jail_path}/${JAIL_CONFIG_NAME}"
	# LXC download script needs to write to this file during install
	# but we don't need it so we will remove it later
	touch "${jail_config_path}"

	LXC_CACHE_PATH=${lxc_cache_path} "${lxc_download_script_path}" \
		--name="${jail_name}" --path="${jail_path}" --rootfs="${JAIL_ROOTFS_PATH}" \
		--arch="${arch}" --dist="${distro}" --release="${release}" ||
		fail "Aborted creating rootfs..."
	echo

	# Assuming the name of your jail is "myjail"
	# and "machinectl shell myjail" doesn't work
	# Try:
	#
	# Stop the jail with:
	# machinectl stop myjail
	# And start a shell inside the jail without the --boot option:
	# systemd-nspawn -q -D jails/myjail/rootfs /bin/sh
	# Then set a root password with:
	# In case of amazonlinux you may need to run:
	# yum update -y && yum install -y passwd
	# passwd
	# exit
	# Then you may login from the host via:
	# machinectl login myjail
	#
	# You could also enable SSH inside the jail to login
	#
	# Or if that doesn't work (e.g. for alpine) get a shell via:
	# nsenter -t $(machinectl show myjail -p Leader --value) -a /bin/sh -l
	# But alpine jails made with jailmaker have other issues
	# They don't shutdown cleanly via systemctl and machinectl...

	if [[ "$(basename "$(readlink -f "${JAIL_ROOTFS_PATH}/sbin/init")")" != systemd ]]; then
		echo "${YELLOW}${BOLD}WARNING: DISTRO NOT SUPPORTED${NORMAL}"
		echo
		echo "Chosen distro appears not to use systemd..."
		echo
		echo "You probably won't get a shell with:"
		echo "machinectl shell ${jail_name}"
		echo
		echo "You may get a shell with this command:"
		# About nsenter:
		# shellcheck disable=SC2016
		echo 'nsenter -t $(machinectl show '"${jail_name}"' -p Leader --value) -a /bin/sh -l'
		echo
		echo 'Read about the downsides of nsenter:'
		echo 'https://github.com/systemd/systemd/issues/12785#issuecomment-503019081'
		echo
		echo "${BOLD}Using this distro with ${SCRIPT_NAME} is NOT recommended.${NORMAL}"
		echo
		read -p "Abort creating jail? [Y/n] " -n 1 -r reply && echo
		# Enter accepts default (yes)
		[[ "${reply}" =~ ^([Yy]|)$ ]] && exit
		echo
	fi

	# Config which systemd handles for us
	rm -f "${JAIL_ROOTFS_PATH}/etc/machine-id"
	rm -f "${JAIL_ROOTFS_PATH}/etc/resolv.conf"
	rm -f "${JAIL_ROOTFS_PATH}/etc/resolv.conf"
	# https://github.com/systemd/systemd/issues/852
	printf 'pts/%d\n' $(seq 0 10) >"${JAIL_ROOTFS_PATH}/etc/securetty"

	local network_dir_path="${JAIL_ROOTFS_PATH}/etc/systemd/network/"

	# Check destination directory exists
	if [[ -d "${network_dir_path}" ]]; then
		local default_host0_network_file="${JAIL_ROOTFS_PATH}/lib/systemd/network/80-container-host0.network"

		# Check if default host0 network file exists
		if [[ -f "${default_host0_network_file}" ]]; then
			local override_network_file="${network_dir_path}/80-container-host0.network"

			# Override the default 80-container-host0.network file (by using the same name)
			# This config applies when using the --network-bridge option of systemd-nspawn
			# Disable LinkLocalAddressing or else the container won't get IP address via DHCP
			sed 's/LinkLocalAddressing=yes/LinkLocalAddressing=no/g' <"${default_host0_network_file}" >"${override_network_file}"
			# Enable DHCP only for ipv4 else systemd-networkd will complain that LinkLocalAddressing is disabled
			sed -i 's/DHCP=yes/DHCP=ipv4/g' "${override_network_file}"
		fi

		# Setup DHCP for macvlan network interfaces
		# This config applies when using the --network-macvlan option of systemd-nspawn
		# https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_modern_network_configuration_without_gui
		cat <<-'EOF' >"${network_dir_path}/mv-dhcp.network"
			[Match]
			Virtualization=container
			Name=mv-*

			[Network]
			DHCP=ipv4
			LinkLocalAddressing=no

			[DHCPv4]
			UseDNS=true
			UseTimezone=true
		EOF
	fi

	# Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file:
	# https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in
	# Use TasksMax=infinity since this is what docker does:
	# https://github.com/docker/engine/blob/master/contrib/init/systemd/docker.service

	# Use SYSTEMD_NSPAWN_LOCK=0: otherwise jail won't start jail after a shutdown (but why?)
	# Would give "directory tree currently busy" error and I'd have to run
	# `rm /run/systemd/nspawn/locks/*` and remove the .lck file from jail_path
	# Disabling locking isn't a big deal as systemd-nspawn will prevent starting a container
	# with the same name anyway: as long as we're starting jails using this script,
	# it won't be possible to start the same jail twice

	local systemd_run_default_args=(--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133
		--property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect
		--setenv=SYSTEMD_NSPAWN_LOCK=0)

	local systemd_nspawn_default_args=(--keep-unit --quiet --boot)

	{
		echo "DOCKER_COMPATIBLE=${docker_compatible}"
		echo "GPU_PASSTHROUGH=${gpu_passthrough}"
		echo "SYSTEMD_NSPAWN_USER_ARGS=${systemd_nspawn_user_args}"
		echo
		echo "# You generally won't need to change the options below"
		echo "SYSTEMD_RUN_DEFAULT_ARGS=${systemd_run_default_args[*]}"
		echo "SYSTEMD_NSPAWN_DEFAULT_ARGS=${systemd_nspawn_default_args[*]}"
	} >"${jail_config_path}"

	chmod 600 "${jail_config_path}"

	# Remove the cleanup trap on exit
	trap - EXIT
	echo "Done creating the jail."
	echo
	read -p "Start the jail now? [Y/n] " -n 1 -r reply && echo
	# Enter accepts default (yes)
	if [[ "${reply}" =~ ^([Yy]|)$ ]]; then
		start_jail "${jail_name}"
	else
		echo
		echo 'Skipped starting jail.'
	fi
}

#######################
# COMMAND LINE HANDLING
#######################

case "${1-""}" in

'')
	read -p "Create a new jail? [Y/n] " -n 1 -r reply && echo
	echo
	# Enter accepts default (yes)
	# https://stackoverflow.com/a/1885534
	if [[ "${reply}" =~ ^([Yy]|)$ ]]; then
		create_jail ""
	else

		echo "${USAGE}"
	fi
	;;

create)
	create_jail "${2-""}"
	;;

start)
	start_jail "${2-""}"
	;;

*)
	echo "${USAGE}"
	;;
esac