From b849cb787c41a9784f8f19719141cd0aaeed6903 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Wed, 8 May 2024 22:24:59 +0200 Subject: [PATCH] Update config templates Updated config templates to benefit from initial_setup running after jail has fully started. Added conditional nvidia-container-toolkit install during initial_setup. Config templates now default to bridge networking. --- templates/docker/README.md | 2 +- templates/docker/config | 26 ++++++++++++++++++++++---- templates/incus/README.md | 5 ++--- templates/incus/config | 10 +++++----- templates/lxd/README.md | 15 +-------------- templates/lxd/config | 10 ++++++---- templates/podman/config | 6 ++---- 7 files changed, 39 insertions(+), 35 deletions(-) diff --git a/templates/docker/README.md b/templates/docker/README.md index 17a5b5b..138478a 100644 --- a/templates/docker/README.md +++ b/templates/docker/README.md @@ -2,4 +2,4 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/docker/config mydockerjail`. \ No newline at end of file +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/docker/config mydockerjail`. If you want the `nvidia-container-toolkit` to be installed, ensure you set `gpu_passthrough_nvidia=1` when creating the jail. \ No newline at end of file diff --git a/templates/docker/config b/templates/docker/config index 8960ff5..51f7f1d 100644 --- a/templates/docker/config +++ b/templates/docker/config @@ -6,10 +6,10 @@ seccomp=1 # Use macvlan networking to provide an isolated network namespace, # so docker can manage firewall rules -# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts -systemd_nspawn_user_args=--network-macvlan=eno1 +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' @@ -29,8 +29,8 @@ release=bookworm # Install docker inside the jail: # https://docs.docker.com/engine/install/debian/#install-using-the-repository -# NOTE: this script will run in the host networking namespace and ignores -# all systemd_nspawn_user_args such as bind mounts +# Will also install the NVIDIA Container Toolkit if gpu_passthrough_nvidia=1 during initial setup +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html initial_setup=#!/usr/bin/bash set -euo pipefail @@ -43,8 +43,26 @@ initial_setup=#!/usr/bin/bash "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + # The /usr/bin/nvidia-smi will be present when gpu_passthrough_nvidia=1 + if [ -f /usr/bin/nvidia-smi ]; then + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey -o /etc/apt/keyrings/nvidia.asc + chmod a+r /etc/apt/keyrings/nvidia.asc + curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/etc/apt/keyrings/nvidia.asc] https://#g' | \ + tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + + apt-get update + apt-get install -y nvidia-container-toolkit + + nvidia-ctk runtime configure --runtime=docker + systemctl restart docker + fi + + docker info # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed diff --git a/templates/incus/README.md b/templates/incus/README.md index c8e3d26..ba8a9a2 100644 --- a/templates/incus/README.md +++ b/templates/incus/README.md @@ -8,11 +8,10 @@ Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/incus/config myincusjail`. -Unfortunately incus doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: +We manually finish the setup by running the following after creating and starting the jail: ```bash -jlmkr exec myincusjail bash -c 'apt-get -y install incus incus-ui-canonical && - incus admin init' +jlmkr exec myincusjail bash -c 'incus admin init' ``` Follow [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). diff --git a/templates/incus/config b/templates/incus/config index 644e71d..4234843 100644 --- a/templates/incus/config +++ b/templates/incus/config @@ -3,15 +3,16 @@ startup=0 gpu_passthrough_intel=0 gpu_passthrough_nvidia=0 # Turning off seccomp filtering improves performance at the expense of security -seccomp=1 +# TODO: don't disable seccomp but specify which syscalls should be allowed +seccomp=0 # Use macvlan networking to provide an isolated network namespace, # so incus can manage firewall rules -# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts # TODO: don't use --capability=all but specify only the required capabilities -systemd_nspawn_user_args=--network-macvlan=eno1 +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --capability=all --bind=/dev/fuse @@ -36,8 +37,6 @@ release=bookworm # Install incus according to: # https://github.com/zabbly/incus#installation -# NOTE: this script will run in the host networking namespace and ignores -# all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash set -euo pipefail apt-get update && apt-get -y install curl @@ -54,6 +53,7 @@ initial_setup=#!/usr/bin/bash EOF' apt-get update + apt-get -y install incus incus-ui-canonical # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed diff --git a/templates/lxd/README.md b/templates/lxd/README.md index 185a16f..49e90ec 100644 --- a/templates/lxd/README.md +++ b/templates/lxd/README.md @@ -8,20 +8,7 @@ Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/lxd/config mylxdjail`. -Unfortunately snapd doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: - -```bash -# Repeat listing the jail until you see it has an IPv4 address -jlmkr list - -# Install packages -jlmkr exec mylxdjail bash -c 'apt-get update && - apt-get install -y --no-install-recommends snapd && - snap install lxd' - -``` - -Choose the `dir` storage backend during `lxd init` and answer `yes` to "Would you like the LXD server to be available over the network?" +We manually finish the setup by running the command below after creating and starting the jail. Choose the `dir` storage backend during `lxd init` and answer `yes` to "Would you like the LXD server to be available over the network?" ```bash jlmkr exec mylxdjail bash -c 'lxd init && diff --git a/templates/lxd/config b/templates/lxd/config index 60cd8c1..6d22970 100644 --- a/templates/lxd/config +++ b/templates/lxd/config @@ -3,11 +3,12 @@ startup=0 gpu_passthrough_intel=0 gpu_passthrough_nvidia=0 # Turning off seccomp filtering improves performance at the expense of security -seccomp=1 +# TODO: don't disable seccomp but specify which syscalls should be allowed +seccomp=0 # Use macvlan networking to provide an isolated network namespace, # so lxd can manage firewall rules -# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts # TODO: don't use --capability=all but specify only the required capabilities @@ -34,12 +35,13 @@ pre_start_hook=#!/usr/bin/bash distro=ubuntu release=jammy -# NOTE: this script will run in the host networking namespace and ignores -# all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash set -euo pipefail # https://discuss.linuxcontainers.org/t/snap-inside-privileged-lxd-container/13691/8 ln -sf /bin/true /usr/local/bin/udevadm + apt-get update + apt-get install -y --no-install-recommends snapd + snap install lxd # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed diff --git a/templates/podman/config b/templates/podman/config index 989c686..e613779 100644 --- a/templates/podman/config +++ b/templates/podman/config @@ -6,10 +6,10 @@ seccomp=1 # Use macvlan networking to provide an isolated network namespace, # so podman can manage firewall rules -# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts -systemd_nspawn_user_args=--network-macvlan=eno1 +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' @@ -28,8 +28,6 @@ distro=fedora release=39 # Install podman inside the jail -# NOTE: this script will run in the host networking namespace and ignores -# all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash set -euo pipefail dnf -y install podman