From 38bf14d928232e7c21eddfc146fa23b633f5e98b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 20 Jan 2024 16:47:51 +0100 Subject: [PATCH 01/57] Fix f-string without any placeholders --- jlmkr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index 11d7f5e..9950df5 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -239,7 +239,7 @@ def start_jail(jail_name, check_startup_enabled=False): config = parse_config(jail_config_path) if not config: - fail(f'Aborting...') + fail("Aborting...") # Only start if the startup setting is enabled in the config if check_startup_enabled: From 3954acf75ba433f14d2146ea6218198b9568c879 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 12:55:11 +0100 Subject: [PATCH 02/57] Create rootless_podman_in_rootless_jail.md --- docs/rootless_podman_in_rootless_jail.md | 113 +++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 docs/rootless_podman_in_rootless_jail.md diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md new file mode 100644 index 0000000..c6f9c7d --- /dev/null +++ b/docs/rootless_podman_in_rootless_jail.md @@ -0,0 +1,113 @@ +# Rootless podman in rootless Fedora jail + +## Disclaimer + +**These notes are a work in progress. Using podman in this setup hasn't been extensively tested.** + +## Installation + +Prerequisites. Installed jailmaker and setup bridge networking. + +Run `jlmkr create rootless` to create a new jail. During jail creation choose fedora 39. This way we get the most recent version of podman available. Don't enable docker compatibility, we're going to enable only the required options manually. + +Add `systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:65536 --private-users-ownership=chown` during jail creation. + +We start at UID 524288, as this is the [systemd range used for containers](https://github.com/systemd/systemd/blob/main/docs/UIDS-GIDS.md#summary). + +The `--private-users-ownership=chown` option will ensure the rootfs ownership is corrected. + +After the jail has started run `jlmkr stop rootless && jlmkr edit rootless`, remove `--private-users-ownership=chown` and increase the UID range to `131072` to double the number of UIDs available in the jail. We need more than 65536 UIDs available in the jail, since rootless podman also needs to be able to map UIDs. If I leave the `--private-users-ownership=chown` option I get the following error: + +> systemd-nspawn[678877]: Automatic UID/GID adjusting is only supported for UID/GID ranges starting at multiples of 2^16 with a range of 2^16 + +The flags look like this now: + +``` +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf --private-users=524288:131072' +``` + +For some reason the network inside the jail doesn't come up by default. Correct this manually. + +Run the following from the TrueNAS host, from inside your jailmaker directory. + +```bash +nano jails/rootless/rootfs/lib/systemd/network/80-container-host0.network +# Manually set LinkLocalAddressing=yes to LinkLocalAddressing=ipv6 +``` + +Start the jail with `jlmkr start rootless` and open a shell session inside the jail (as the remapped root user) with `jlmkr shell rootless`. + +Then inside the jail start the network services (wait to get IP address via DHCP) and install podman: +```bash +systemctl enable systemd-networkd +systemctl start systemd-networkd + +# Add the required capabilities to the `newuidmap` and `newgidmap` binaries. +# https://github.com/containers/podman/issues/2788#issuecomment-1016301663 +# https://github.com/containers/podman/issues/2788#issuecomment-479972943 +# https://github.com/containers/podman/issues/12637#issuecomment-996524341 +setcap cap_setuid+eip /usr/bin/newuidmap +setcap cap_setgid+eip /usr/bin/newgidmap + +# Create new user +adduser rootless + +# Clear the subuids and subgids which have been assigned by default when creating the new user +usermod --del-subuids 0-4294967295 --del-subgids 0-4294967295 rootless +# Set a specific range, so it fits inside the number of available UIDs +usermod --add-subuids 65536-131071 --add-subgids 65536-131071 rootless +# Check the assigned range +cat /etc/subuid +# Check the available range +cat /proc/self/uid_map + +dnf -y install podman +exit +``` + +From the TrueNAS host, open a shell as the rootless user inside the jail. + +```bash +machinectl shell --uid 1000 rootless +``` + +Run rootless podman as user 1000. + +```bash +id +podman run hello-world +podman info +``` + +The output of podman info should contain: + +``` + graphDriverName: overlay + graphOptions: {} + graphRoot: /home/rootless/.local/share/containers/storage + [...] + graphStatus: + Backing Filesystem: zfs + Native Overlay Diff: "true" + Supports d_type: "true" + Supports shifting: "false" + Supports volatile: "true" + Using metacopy: "false" +``` + +## TODO: +On truenas host do: +sudo sysctl net.ipv4.ip_unprivileged_port_start=23 +> Which would prevent a process by your user impersonating the sshd daemon. +Actually make it persistent. + +## Additional resources: + +Resources mentioning `add_key keyctl bpf` +- https://bbs.archlinux.org/viewtopic.php?id=252840 +- https://wiki.archlinux.org/title/systemd-nspawn +- https://discourse.nixos.org/t/podman-docker-in-nixos-container-ideally-in-unprivileged-one/22909/12 +Resources mentioning `@keyring` +- https://github.com/systemd/systemd/issues/17606 +- https://github.com/systemd/systemd/blob/1c62c4fe0b54fb419b875cb2bae82a261518a745/src/shared/seccomp-util.c#L604 +`@keyring` also includes `request_key` but doesn't include `bpf` \ No newline at end of file From 83955dda447847a00cfdcea9faede1208b24a093 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 13:00:50 +0100 Subject: [PATCH 03/57] Update rootless_podman_in_rootless_jail.md --- docs/rootless_podman_in_rootless_jail.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index c6f9c7d..91208f1 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -10,7 +10,7 @@ Prerequisites. Installed jailmaker and setup bridge networking. Run `jlmkr create rootless` to create a new jail. During jail creation choose fedora 39. This way we get the most recent version of podman available. Don't enable docker compatibility, we're going to enable only the required options manually. -Add `systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:65536 --private-users-ownership=chown` during jail creation. +Add `--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:65536 --private-users-ownership=chown` when asked for additional systemd-nspawn flags during jail creation. We start at UID 524288, as this is the [systemd range used for containers](https://github.com/systemd/systemd/blob/main/docs/UIDS-GIDS.md#summary). From dc48a664933137d5f2c02309a5cebc84495ddfc8 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 13:02:18 +0100 Subject: [PATCH 04/57] Update rootless_podman_in_rootless_jail.md --- docs/rootless_podman_in_rootless_jail.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index 91208f1..deb88be 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -23,7 +23,7 @@ After the jail has started run `jlmkr stop rootless && jlmkr edit rootless`, rem The flags look like this now: ``` -systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf --private-users=524288:131072' +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:131072 ``` For some reason the network inside the jail doesn't come up by default. Correct this manually. From c952e1ac7d5e7bf3bda0c6517d9aae23cfb66e59 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 15:51:53 +0100 Subject: [PATCH 05/57] Remove unnecessary step --- docs/rootless_podman_in_rootless_jail.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index deb88be..72ad19b 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -26,15 +26,6 @@ The flags look like this now: systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:131072 ``` -For some reason the network inside the jail doesn't come up by default. Correct this manually. - -Run the following from the TrueNAS host, from inside your jailmaker directory. - -```bash -nano jails/rootless/rootfs/lib/systemd/network/80-container-host0.network -# Manually set LinkLocalAddressing=yes to LinkLocalAddressing=ipv6 -``` - Start the jail with `jlmkr start rootless` and open a shell session inside the jail (as the remapped root user) with `jlmkr shell rootless`. Then inside the jail start the network services (wait to get IP address via DHCP) and install podman: From 93190d453fee89dc2a135dee842658e6efffd734 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 18:10:09 +0100 Subject: [PATCH 06/57] Override systemd-networkd preset --- jlmkr.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jlmkr.py b/jlmkr.py index 9950df5..e374f38 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -787,6 +787,14 @@ def create_jail(jail_name, distro='debian', release='bookworm'): UseTimezone=true """), file=open(os.path.join(network_dir_path, "mv-dhcp.network"), "w")) + # Override preset which caused systemd-networkd to be disabled (e.g. fedora 39) + # https://www.freedesktop.org/software/systemd/man/latest/systemd.preset.html + # https://github.com/lxc/lxc-ci/blob/f632823ecd9b258ed42df40449ec54ed7ef8e77d/images/fedora.yaml#L312C5-L312C38 + + preset_path = os.path.join(jail_rootfs_path, 'etc/systemd/system-preset') + os.makedirs(preset_path, exist_ok=True) + print('enable systemd-networkd.service', file=open(os.path.join(preset_path, '00-jailmaker.preset'), "w")) + # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in # Use TasksMax=infinity since this is what docker does: From fd1617d1403324d4a1868ea7b101f11b357be847 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 18:11:09 +0100 Subject: [PATCH 07/57] Update rootless_podman_in_rootless_jail.md --- docs/rootless_podman_in_rootless_jail.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index 72ad19b..9ff804b 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -30,8 +30,8 @@ Start the jail with `jlmkr start rootless` and open a shell session inside the j Then inside the jail start the network services (wait to get IP address via DHCP) and install podman: ```bash -systemctl enable systemd-networkd -systemctl start systemd-networkd +# systemd-networkd should already be enabled when using jlmkr.py from the develop branch +systemctl --now enable systemd-networkd # Add the required capabilities to the `newuidmap` and `newgidmap` binaries. # https://github.com/containers/podman/issues/2788#issuecomment-1016301663 From 474faf6ede70a04a18779c8d8ff10af16e1e15c2 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 21 Jan 2024 18:21:24 +0100 Subject: [PATCH 08/57] Use ruff formatter --- jlmkr.py | 739 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 458 insertions(+), 281 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index e374f38..54bc067 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -25,32 +25,36 @@ from textwrap import dedent # Only set a color if we have an interactive tty if sys.stdout.isatty(): - BOLD = '\033[1m' - RED = '\033[91m' - YELLOW = '\033[93m' - UNDERLINE = '\033[4m' - NORMAL = '\033[0m' + BOLD = "\033[1m" + RED = "\033[91m" + YELLOW = "\033[93m" + UNDERLINE = "\033[4m" + NORMAL = "\033[0m" else: - BOLD = RED = YELLOW = UNDERLINE = NORMAL = '' + BOLD = RED = YELLOW = UNDERLINE = NORMAL = "" DISCLAIMER = f"""{YELLOW}{BOLD}USE THIS SCRIPT AT YOUR OWN RISK! IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS.{NORMAL}""" -DESCRIPTION = "Create persistent Linux 'jails' on TrueNAS SCALE, with full access to all files \ +DESCRIPTION = ( + "Create persistent Linux 'jails' on TrueNAS SCALE, with full access to all files \ via bind mounts, thanks to systemd-nspawn!" +) -VERSION = '1.0.1' +VERSION = "1.0.1" -JAILS_DIR_PATH = 'jails' -JAIL_CONFIG_NAME = 'config' -JAIL_ROOTFS_NAME = 'rootfs' -DOWNLOAD_SCRIPT_DIGEST = '6cca2eda73c7358c232fecb4e750b3bf0afa9636efb5de6a9517b7df78be12a4' +JAILS_DIR_PATH = "jails" +JAIL_CONFIG_NAME = "config" +JAIL_ROOTFS_NAME = "rootfs" +DOWNLOAD_SCRIPT_DIGEST = ( + "6cca2eda73c7358c232fecb4e750b3bf0afa9636efb5de6a9517b7df78be12a4" +) SCRIPT_PATH = os.path.realpath(__file__) SCRIPT_NAME = os.path.basename(SCRIPT_PATH) SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH) COMMAND_NAME = os.path.basename(__file__) -SYMLINK_NAME = 'jlmkr' -TEXT_EDITOR = 'nano' +SYMLINK_NAME = "jlmkr" +TEXT_EDITOR = "nano" def eprint(*args, **kwargs): @@ -81,24 +85,31 @@ def get_jail_rootfs_path(jail_name): def passthrough_intel(gpu_passthrough_intel, systemd_nspawn_additional_args): - if gpu_passthrough_intel != '1': + if gpu_passthrough_intel != "1": return - if not os.path.exists('/dev/dri'): - eprint(dedent(""" + if not os.path.exists("/dev/dri"): + eprint( + dedent( + """ No intel GPU seems to be present... - Skip passthrough of intel GPU.""")) + Skip passthrough of intel GPU.""" + ) + ) return - systemd_nspawn_additional_args.append('--bind=/dev/dri') + systemd_nspawn_additional_args.append("--bind=/dev/dri") -def passthrough_nvidia(gpu_passthrough_nvidia, systemd_nspawn_additional_args, jail_name): +def passthrough_nvidia( + gpu_passthrough_nvidia, systemd_nspawn_additional_args, jail_name +): jail_rootfs_path = get_jail_rootfs_path(jail_name) - ld_so_conf_path = Path(os.path.join(jail_rootfs_path), - f'etc/ld.so.conf.d/{SYMLINK_NAME}-nvidia.conf') + ld_so_conf_path = Path( + os.path.join(jail_rootfs_path), f"etc/ld.so.conf.d/{SYMLINK_NAME}-nvidia.conf" + ) - if gpu_passthrough_nvidia != '1': + if gpu_passthrough_nvidia != "1": # Cleanup the config file we made when passthrough was enabled ld_so_conf_path.unlink(missing_ok=True) return @@ -108,67 +119,93 @@ def passthrough_nvidia(gpu_passthrough_nvidia, systemd_nspawn_additional_args, j # If we can't run nvidia-smi successfully, # then nvidia-container-cli list will fail too: # we shouldn't continue with gpu passthrough - subprocess.run(['nvidia-smi', '-f', '/dev/null'], check=True) + subprocess.run(["nvidia-smi", "-f", "/dev/null"], check=True) except: eprint("Skip passthrough of nvidia GPU.") return try: - nvidia_files = set(([x for x in subprocess.check_output( - ['nvidia-container-cli', 'list']).decode().split('\n') if x])) + nvidia_files = set( + ( + [ + x + for x in subprocess.check_output(["nvidia-container-cli", "list"]) + .decode() + .split("\n") + if x + ] + ) + ) except: - eprint(dedent(""" + eprint( + dedent( + """ Unable to detect which nvidia driver files to mount. - Skip passthrough of nvidia GPU.""")) + Skip passthrough of nvidia GPU.""" + ) + ) return # Also make nvidia-smi available inside the path, # while mounting the symlink will be resolved and nvidia-smi will appear as a regular file - nvidia_files.add('/usr/bin/nvidia-smi') + nvidia_files.add("/usr/bin/nvidia-smi") nvidia_mounts = [] for file_path in nvidia_files: if not os.path.exists(file_path): # Don't try to mount files not present on the host - print( - f"Skipped mounting {file_path}, it doesn't exist on the host...") + print(f"Skipped mounting {file_path}, it doesn't exist on the host...") continue - if file_path.startswith('/dev/'): + if file_path.startswith("/dev/"): nvidia_mounts.append(f"--bind={file_path}") else: nvidia_mounts.append(f"--bind-ro={file_path}") # Check if the parent dir exists where we want to write our conf file if ld_so_conf_path.parent.exists(): - nvidia_libraries = set(Path(x) for x in subprocess.check_output( - ['nvidia-container-cli', 'list', '--libraries']).decode().split('\n') if x) + nvidia_libraries = set( + Path(x) + for x in subprocess.check_output( + ["nvidia-container-cli", "list", "--libraries"] + ) + .decode() + .split("\n") + if x + ) library_folders = set(str(x.parent) for x in nvidia_libraries) # Only write if the conf file doesn't yet exist or has different contents existing_conf_libraries = set() if ld_so_conf_path.exists(): existing_conf_libraries.update( - x for x in ld_so_conf_path.read_text().splitlines() if x) + x for x in ld_so_conf_path.read_text().splitlines() if x + ) if library_folders != existing_conf_libraries: - print("\n".join(x for x in library_folders), - file=ld_so_conf_path.open('w')) + print("\n".join(x for x in library_folders), file=ld_so_conf_path.open("w")) # Run ldconfig inside systemd-nspawn jail with nvidia mounts... subprocess.run( - ['systemd-nspawn', - '--quiet', + [ + "systemd-nspawn", + "--quiet", f"--machine={jail_name}", f"--directory={jail_rootfs_path}", *nvidia_mounts, - "ldconfig"]) + "ldconfig", + ] + ) else: - eprint(dedent(""" + eprint( + dedent( + """ Unable to write the ld.so.conf.d directory inside the jail (it doesn't exist). Skipping call to ldconfig. - The nvidia drivers will probably not be detected...""")) + The nvidia drivers will probably not be detected...""" + ) + ) systemd_nspawn_additional_args += nvidia_mounts @@ -177,8 +214,21 @@ def exec_jail(jail_name, cmd, args): """ Execute a command in the jail with given name. """ - subprocess.run(['systemd-run', '--machine', jail_name, '--quiet', '--pipe', - '--wait', '--collect', '--service-type=exec', cmd] + args, check=True) + subprocess.run( + [ + "systemd-run", + "--machine", + jail_name, + "--quiet", + "--pipe", + "--wait", + "--collect", + "--service-type=exec", + cmd, + ] + + args, + check=True, + ) def status_jail(jail_name): @@ -214,12 +264,12 @@ def parse_config(jail_config_path): config = configparser.ConfigParser() try: # Workaround to read config file without section headers - config.read_string('[DEFAULT]\n'+Path(jail_config_path).read_text()) + config.read_string("[DEFAULT]\n" + Path(jail_config_path).read_text()) except FileNotFoundError: - eprint(f'Unable to find config file: {jail_config_path}.') + eprint(f"Unable to find config file: {jail_config_path}.") return - config = dict(config['DEFAULT']) + config = dict(config["DEFAULT"]) return config @@ -228,7 +278,9 @@ def start_jail(jail_name, check_startup_enabled=False): """ Start jail with given name. """ - skip_start_message = f"Skipped starting jail {jail_name}. It appears to be running already..." + skip_start_message = ( + f"Skipped starting jail {jail_name}. It appears to be running already..." + ) if not check_startup_enabled and jail_is_running(jail_name): fail(skip_start_message) @@ -243,7 +295,7 @@ def start_jail(jail_name, check_startup_enabled=False): # Only start if the startup setting is enabled in the config if check_startup_enabled: - if config.get('startup') == '1': + if config.get("startup") == "1": # We should start this jail based on the startup config... if jail_is_running(jail_name): # ...but we can skip if it's already running @@ -264,9 +316,9 @@ def start_jail(jail_name, check_startup_enabled=False): f"--directory={JAIL_ROOTFS_NAME}", ] - if config.get('docker_compatible') == '1': + if config.get("docker_compatible") == "1": # Enable ip forwarding on the host (docker needs it) - print(1, file=open('/proc/sys/net/ipv4/ip_forward', 'w')) + print(1, file=open("/proc/sys/net/ipv4/ip_forward", "w")) # Load br_netfilter kernel module and enable bridge-nf-call to fix warning when running docker info: # WARNING: bridge-nf-call-iptables is disabled @@ -280,12 +332,16 @@ def start_jail(jail_name, check_startup_enabled=False): # https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf # https://serverfault.com/questions/963759/docker-breaks-libvirt-bridge-network - if subprocess.run(['modprobe', 'br_netfilter']).returncode == 0: - print(1, file=open('/proc/sys/net/bridge/bridge-nf-call-iptables', 'w')) - print(1, file=open('/proc/sys/net/bridge/bridge-nf-call-ip6tables', 'w')) + if subprocess.run(["modprobe", "br_netfilter"]).returncode == 0: + print(1, file=open("/proc/sys/net/bridge/bridge-nf-call-iptables", "w")) + print(1, file=open("/proc/sys/net/bridge/bridge-nf-call-ip6tables", "w")) else: - eprint(dedent(""" - Failed to load br_netfilter kernel module.""")) + eprint( + dedent( + """ + Failed to load br_netfilter kernel module.""" + ) + ) # To properly run docker inside the jail, we need to lift restrictions # Without DevicePolicy=auto images with device nodes may not be pulled @@ -310,56 +366,65 @@ def start_jail(jail_name, check_startup_enabled=False): # Use SYSTEMD_SECCOMP=0: https://github.com/systemd/systemd/issues/18370 systemd_run_additional_args += [ - '--setenv=SYSTEMD_SECCOMP=0', - '--property=DevicePolicy=auto', + "--setenv=SYSTEMD_SECCOMP=0", + "--property=DevicePolicy=auto", ] # Add additional flags required for docker systemd_nspawn_additional_args += [ - '--capability=all', - '--system-call-filter=add_key keyctl bpf', + "--capability=all", + "--system-call-filter=add_key keyctl bpf", ] # Legacy gpu_passthrough config setting - if config.get('gpu_passthrough') == '1': - gpu_passthrough_intel = '1' - gpu_passthrough_nvidia = '1' + if config.get("gpu_passthrough") == "1": + gpu_passthrough_intel = "1" + gpu_passthrough_nvidia = "1" else: - gpu_passthrough_intel = config.get('gpu_passthrough_intel') - gpu_passthrough_nvidia = config.get('gpu_passthrough_nvidia') + gpu_passthrough_intel = config.get("gpu_passthrough_intel") + gpu_passthrough_nvidia = config.get("gpu_passthrough_nvidia") - if gpu_passthrough_intel == '1' or gpu_passthrough_nvidia == '1': - systemd_nspawn_additional_args.append( - '--property=DeviceAllow=char-drm rw') + if gpu_passthrough_intel == "1" or gpu_passthrough_nvidia == "1": + systemd_nspawn_additional_args.append("--property=DeviceAllow=char-drm rw") passthrough_intel(gpu_passthrough_intel, systemd_nspawn_additional_args) - passthrough_nvidia(gpu_passthrough_nvidia, - systemd_nspawn_additional_args, jail_name) + passthrough_nvidia( + gpu_passthrough_nvidia, systemd_nspawn_additional_args, jail_name + ) - cmd = ['systemd-run', - *shlex.split(config.get('systemd_run_default_args', '')), - *systemd_run_additional_args, - '--', - 'systemd-nspawn', - *shlex.split(config.get('systemd_nspawn_default_args', '')), - *systemd_nspawn_additional_args, - *shlex.split(config.get('systemd_nspawn_user_args', '')) - ] + cmd = [ + "systemd-run", + *shlex.split(config.get("systemd_run_default_args", "")), + *systemd_run_additional_args, + "--", + "systemd-nspawn", + *shlex.split(config.get("systemd_nspawn_default_args", "")), + *systemd_nspawn_additional_args, + *shlex.split(config.get("systemd_nspawn_user_args", "")), + ] - print(dedent(f""" + print( + dedent( + f""" Starting jail {jail_name} with the following command: {shlex.join(cmd)} - """)) + """ + ) + ) try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError: - fail(dedent(f""" + fail( + dedent( + f""" Failed to start jail {jail_name}... In case of a config error, you may fix it with: {SYMLINK_NAME} edit {jail_name} - """)) + """ + ) + ) def cleanup(jail_path): @@ -387,19 +452,20 @@ def validate_sha256(file_path, digest): Validates if a file matches a sha256 digest. """ try: - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: file_hash = hashlib.sha256(f.read()).hexdigest() return file_hash == digest except FileNotFoundError: return False -def run_lxc_download_script(jail_name=None, jail_path=None, jail_rootfs_path=None, distro=None, release=None): - - arch = 'amd64' - lxc_dir = '.lxc' - lxc_cache = os.path.join(lxc_dir, 'cache') - lxc_download_script = os.path.join(lxc_dir, 'lxc-download.sh') +def run_lxc_download_script( + jail_name=None, jail_path=None, jail_rootfs_path=None, distro=None, release=None +): + arch = "amd64" + lxc_dir = ".lxc" + lxc_cache = os.path.join(lxc_dir, "cache") + lxc_download_script = os.path.join(lxc_dir, "lxc-download.sh") # Create the lxc dirs if nonexistent os.makedirs(lxc_dir, exist_ok=True) @@ -416,7 +482,9 @@ def run_lxc_download_script(jail_name=None, jail_path=None, jail_rootfs_path=Non # Fetch the lxc download script if not present locally (or hash doesn't match) if not validate_sha256(lxc_download_script, DOWNLOAD_SCRIPT_DIGEST): urllib.request.urlretrieve( - "https://raw.githubusercontent.com/Jip-Hop/lxc/58520263041b6864cadad96278848f9b8ce78ee9/templates/lxc-download.in", lxc_download_script) + "https://raw.githubusercontent.com/Jip-Hop/lxc/58520263041b6864cadad96278848f9b8ce78ee9/templates/lxc-download.in", + lxc_download_script, + ) if not validate_sha256(lxc_download_script, DOWNLOAD_SCRIPT_DIGEST): fail("Abort! Downloaded script has unexpected contents.") @@ -426,18 +494,29 @@ def run_lxc_download_script(jail_name=None, jail_path=None, jail_rootfs_path=Non if None not in [jail_name, jail_path, jail_rootfs_path, distro, release]: check_exit_code = True - cmd = [lxc_download_script, f'--name={jail_name}', f'--path={jail_path}', - f'--rootfs={jail_rootfs_path}', f'--arch={arch}', f'--dist={distro}', f'--release={release}'] + cmd = [ + lxc_download_script, + f"--name={jail_name}", + f"--path={jail_path}", + f"--rootfs={jail_rootfs_path}", + f"--arch={arch}", + f"--dist={distro}", + f"--release={release}", + ] else: - cmd = [lxc_download_script, '--list', f'--arch={arch}'] + cmd = [lxc_download_script, "--list", f"--arch={arch}"] - p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, env={ - "LXC_CACHE_PATH": lxc_cache}) + p1 = subprocess.Popen( + cmd, stdout=subprocess.PIPE, env={"LXC_CACHE_PATH": lxc_cache} + ) - for line in iter(p1.stdout.readline, b''): + for line in iter(p1.stdout.readline, b""): line = line.decode().strip() # Filter out the known incompatible distros - if not re.match(r"^(alpine|amazonlinux|busybox|devuan|funtoo|openwrt|plamo|voidlinux)\s", line): + if not re.match( + r"^(alpine|amazonlinux|busybox|devuan|funtoo|openwrt|plamo|voidlinux)\s", + line, + ): print(line) p1.wait() @@ -458,14 +537,13 @@ def agree(question, default=None): """ Ask user a yes/no question. """ - hint = '[Y/n]' if default == 'y' else ( - '[y/N]' if default == 'n' else '[y/n]') + hint = "[Y/n]" if default == "y" else ("[y/N]" if default == "n" else "[y/n]") while True: user_input = input(f"{question} {hint} ") or default - if user_input.lower() in ['y', 'n']: - return user_input.lower() == 'y' + if user_input.lower() in ["y", "n"]: + return user_input.lower() == "y" eprint("Invalid input. Please type 'y' for yes or 'n' for no and press enter.") @@ -484,18 +562,26 @@ def check_jail_name_valid(jail_name, warn=True): """ Return True if jail name matches the required format. """ - if re.match(r"^[.a-zA-Z0-9-]{1,64}$", jail_name) and not jail_name.startswith(".") and ".." not in jail_name: + if ( + re.match(r"^[.a-zA-Z0-9-]{1,64}$", jail_name) + and not jail_name.startswith(".") + and ".." not in jail_name + ): return True if warn: - eprint(dedent(f""" + eprint( + dedent( + f""" {YELLOW}{BOLD}WARNING: INVALID NAME{NORMAL} A valid name consists of: - allowed characters (alphanumeric, dash, dot) - no leading or trailing dots - no sequences of multiple dots - - max 64 characters""")) + - max 64 characters""" + ) + ) return False @@ -512,30 +598,38 @@ def check_jail_name_available(jail_name, warn=True): return False -def create_jail(jail_name, distro='debian', release='bookworm'): +def create_jail(jail_name, distro="debian", release="bookworm"): """ Create jail with given name. """ print(DISCLAIMER) - if os.path.basename(os.getcwd()) != 'jailmaker': - fail(dedent(f""" + if os.path.basename(os.getcwd()) != "jailmaker": + fail( + dedent( + f""" {COMMAND_NAME} needs to create files. Currently it can not decide if it is safe to create files in: {SCRIPT_DIR_PATH} - Please create a dedicated directory called 'jailmaker', store {SCRIPT_NAME} there and try again.""")) + Please create a dedicated directory called 'jailmaker', store {SCRIPT_NAME} there and try again.""" + ) + ) - if not PurePath(get_mount_point(os.getcwd())).is_relative_to('/mnt'): - print(dedent(f""" + if not PurePath(get_mount_point(os.getcwd())).is_relative_to("/mnt"): + print( + dedent( + f""" {YELLOW}{BOLD}WARNING: BEWARE OF DATA LOSS{NORMAL} {SCRIPT_NAME} should be on a dataset mounted under /mnt (it currently is not). Storing it on the boot-pool means losing all jails when updating TrueNAS. If you continue, jails will be stored under: {SCRIPT_DIR_PATH} - """)) - if not agree("Do you wish to ignore this warning and continue?", 'n'): + """ + ) + ) + if not agree("Do you wish to ignore this warning and continue?", "n"): fail("Aborting...") # Create the dir where to store the jails @@ -543,28 +637,40 @@ def create_jail(jail_name, distro='debian', release='bookworm'): stat_chmod(JAILS_DIR_PATH, 0o700) print() - if not agree(f"Install the recommended image ({distro} {release})?", 'y'): - print(dedent(f""" + if not agree(f"Install the recommended image ({distro} {release})?", "y"): + print( + dedent( + f""" {YELLOW}{BOLD}WARNING: ADVANCED USAGE{NORMAL} You may now choose from a list which distro to install. But not all of them may work with {COMMAND_NAME} since these images are made for LXC. Distros based on systemd probably work (e.g. Ubuntu, Arch Linux and Rocky Linux). - """)) + """ + ) + ) input("Press Enter to continue...") print() run_lxc_download_script() - print(dedent(""" + print( + dedent( + """ Choose from the DIST column. - """)) + """ + ) + ) distro = input("Distro: ") - print(dedent(""" + print( + dedent( + """ Choose from the RELEASE column (or ARCH if RELEASE is empty). - """)) + """ + ) + ) release = input("Release: ") @@ -580,58 +686,76 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # Cleanup in except, but only once the jail_path is final # Otherwise we may cleanup the wrong directory try: - print(dedent(f""" + print( + dedent( + f""" Docker won't be installed by {COMMAND_NAME}. But it can setup the jail with the capabilities required to run docker. You can turn DOCKER_COMPATIBLE mode on/off post-install. - """)) + """ + ) + ) docker_compatible = 0 - if agree('Make jail docker compatible right now?', 'n'): + if agree("Make jail docker compatible right now?", "n"): docker_compatible = 1 print() gpu_passthrough_intel = 0 - if os.path.exists('/dev/dri'): + if os.path.exists("/dev/dri"): print("Detected the presence of an intel GPU.\n") - if agree('Passthrough the intel GPU?', 'n'): + if agree("Passthrough the intel GPU?", "n"): gpu_passthrough_intel = 1 gpu_passthrough_nvidia = 0 try: - subprocess.run(['nvidia-smi'], check=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run( + ["nvidia-smi"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) nvidia_detected = True except: nvidia_detected = False if nvidia_detected: print("Detected the presence of an nvidia GPU.\n") - if agree('Passthrough the nvidia GPU?', 'n'): + if agree("Passthrough the nvidia GPU?", "n"): gpu_passthrough_nvidia = 1 - print(dedent(f""" + print( + dedent( + f""" {YELLOW}{BOLD}WARNING: CHECK SYNTAX{NORMAL} You may pass additional flags to systemd-nspawn. With incorrect flags the jail may not start. It is possible to correct/add/remove flags post-install. - """)) + """ + ) + ) - if agree('Show the man page for systemd-nspawn?', 'n'): - subprocess.run(['man', 'systemd-nspawn']) + if agree("Show the man page for systemd-nspawn?", "n"): + subprocess.run(["man", "systemd-nspawn"]) else: try: - base_os_version = platform.freedesktop_os_release().get('VERSION_CODENAME', release) + base_os_version = platform.freedesktop_os_release().get( + "VERSION_CODENAME", release + ) except AttributeError: base_os_version = release - print(dedent(f""" + print( + dedent( + f""" You may read the systemd-nspawn manual online: - https://manpages.debian.org/{base_os_version}/systemd-container/systemd-nspawn.1.en.html""")) + https://manpages.debian.org/{base_os_version}/systemd-container/systemd-nspawn.1.en.html""" + ) + ) # Backslashes and colons need to be escaped in bind mount options: # e.g. to bind mount a file called: @@ -639,7 +763,9 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # the corresponding command would be: # --bind-ro='/mnt/data/weird chars \:?\\"' - print(dedent(""" + print( + dedent( + """ Would you like to add additional systemd-nspawn flags? For example to mount directories inside the jail you may: Mount the TrueNAS location /mnt/pool/dataset to the /home directory of the jail with: @@ -648,20 +774,21 @@ def create_jail(jail_name, distro='debian', release='bookworm'): --bind-ro='/mnt/pool/dataset:/home' Or create MACVLAN interface for static IP, with: --network-macvlan=eno1 --resolv-conf=bind-host - """)) + """ + ) + ) # Enable tab auto completion of file paths after the = symbol - readline.set_completer_delims('=') - readline.parse_and_bind('tab: complete') + readline.set_completer_delims("=") + readline.parse_and_bind("tab: complete") readline_lib = ctypes.CDLL(readline.__file__) rl_completer_quote_characters = ctypes.c_char_p.in_dll( - readline_lib, - "rl_completer_quote_characters" + readline_lib, "rl_completer_quote_characters" ) # Let the readline library know about quote characters for completion - rl_completer_quote_characters.value = "\"'".encode('utf-8') + rl_completer_quote_characters.value = "\"'".encode("utf-8") # TODO: more robust tab completion of file paths with space or = character # Currently completing these only works when the path is quoted @@ -672,18 +799,26 @@ def create_jail(jail_name, distro='debian', release='bookworm'): systemd_nspawn_user_args = input("Additional flags: ") or "" # Disable tab auto completion - readline.parse_and_bind('tab: self-insert') + readline.parse_and_bind("tab: self-insert") - print(dedent(f""" + print( + dedent( + f""" The `{COMMAND_NAME} startup` command can automatically ensure {COMMAND_NAME} is installed properly and start a selection of jails. This comes in handy when you want to automatically start multiple jails after booting TrueNAS SCALE (e.g. from a Post Init Script). - """)) + """ + ) + ) - startup = int(agree( - f"Do you want to start this jail when running: {COMMAND_NAME} startup?", 'n')) + startup = int( + agree( + f"Do you want to start this jail when running: {COMMAND_NAME} startup?", + "n", + ) + ) print() - + jail_config_path = get_jail_config_path(jail_name) jail_rootfs_path = get_jail_rootfs_path(jail_name) @@ -693,8 +828,7 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # but we don't need it so we will remove it later open(jail_config_path, "a").close() - run_lxc_download_script(jail_name, jail_path, - jail_rootfs_path, distro, release) + run_lxc_download_script(jail_name, jail_path, jail_rootfs_path, distro, release) # Assuming the name of your jail is "myjail" # and "machinectl shell myjail" doesn't work @@ -719,9 +853,15 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # But alpine jails made with jailmaker have other issues # They don't shutdown cleanly via systemctl and machinectl... - if os.path.basename(os.path.realpath( - os.path.join(jail_rootfs_path, 'sbin/init'))) != "systemd": - print(dedent(f""" + if ( + os.path.basename( + os.path.realpath(os.path.join(jail_rootfs_path, "sbin/init")) + ) + != "systemd" + ): + print( + dedent( + f""" {YELLOW}{BOLD}WARNING: DISTRO NOT SUPPORTED{NORMAL} Chosen distro appears not to use systemd... @@ -736,44 +876,55 @@ def create_jail(jail_name, distro='debian', release='bookworm'): https://github.com/systemd/systemd/issues/12785#issuecomment-503019081 {BOLD}Using this distro with {COMMAND_NAME} is NOT recommended.{NORMAL} - """)) + """ + ) + ) - if agree("Abort creating jail?", 'y'): + if agree("Abort creating jail?", "y"): exit(1) with contextlib.suppress(FileNotFoundError): # Remove config which systemd handles for us - os.remove(os.path.join(jail_rootfs_path, 'etc/machine-id')) - os.remove(os.path.join(jail_rootfs_path, 'etc/resolv.conf')) + os.remove(os.path.join(jail_rootfs_path, "etc/machine-id")) + os.remove(os.path.join(jail_rootfs_path, "etc/resolv.conf")) # https://github.com/systemd/systemd/issues/852 - print('\n'.join([f"pts/{i}" for i in range(0, 11)]), - file=open(os.path.join(jail_rootfs_path, 'etc/securetty'), 'w')) + print( + "\n".join([f"pts/{i}" for i in range(0, 11)]), + file=open(os.path.join(jail_rootfs_path, "etc/securetty"), "w"), + ) - network_dir_path = os.path.join( - jail_rootfs_path, "etc/systemd/network") + network_dir_path = os.path.join(jail_rootfs_path, "etc/systemd/network") # Modify default network settings, if network_dir_path exists if os.path.isdir(network_dir_path): default_host0_network_file = os.path.join( - jail_rootfs_path, "lib/systemd/network/80-container-host0.network") + jail_rootfs_path, "lib/systemd/network/80-container-host0.network" + ) # Check if default host0 network file exists if os.path.isfile(default_host0_network_file): override_network_file = os.path.join( - network_dir_path, "80-container-host0.network") + network_dir_path, "80-container-host0.network" + ) # Override the default 80-container-host0.network file (by using the same name) # This config applies when using the --network-bridge option of systemd-nspawn # Disable LinkLocalAddressing on IPv4, or else the container won't get IP address via DHCP # But keep it enabled on IPv6, as SLAAC and DHCPv6 both require a local-link address to function - print(Path(default_host0_network_file).read_text().replace("LinkLocalAddressing=yes", - "LinkLocalAddressing=ipv6"), file=open(override_network_file, 'w')) + print( + Path(default_host0_network_file) + .read_text() + .replace("LinkLocalAddressing=yes", "LinkLocalAddressing=ipv6"), + file=open(override_network_file, "w"), + ) # Setup DHCP for macvlan network interfaces # This config applies when using the --network-macvlan option of systemd-nspawn # https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_modern_network_configuration_without_gui - print(cleandoc(""" + print( + cleandoc( + """ [Match] Virtualization=container Name=mv-* @@ -785,15 +936,21 @@ def create_jail(jail_name, distro='debian', release='bookworm'): [DHCPv4] UseDNS=true UseTimezone=true - """), file=open(os.path.join(network_dir_path, "mv-dhcp.network"), "w")) + """ + ), + file=open(os.path.join(network_dir_path, "mv-dhcp.network"), "w"), + ) # Override preset which caused systemd-networkd to be disabled (e.g. fedora 39) # https://www.freedesktop.org/software/systemd/man/latest/systemd.preset.html # https://github.com/lxc/lxc-ci/blob/f632823ecd9b258ed42df40449ec54ed7ef8e77d/images/fedora.yaml#L312C5-L312C38 - - preset_path = os.path.join(jail_rootfs_path, 'etc/systemd/system-preset') + + preset_path = os.path.join(jail_rootfs_path, "etc/systemd/system-preset") os.makedirs(preset_path, exist_ok=True) - print('enable systemd-networkd.service', file=open(os.path.join(preset_path, '00-jailmaker.preset'), "w")) + print( + "enable systemd-networkd.service", + file=open(os.path.join(preset_path, "00-jailmaker.preset"), "w"), + ) # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in @@ -808,23 +965,20 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # it won't be possible to start the same jail twice systemd_run_default_args = [ - '--property=KillMode=mixed', - '--property=Type=notify', - '--property=RestartForceExitStatus=133', - '--property=SuccessExitStatus=133', - '--property=Delegate=yes', - '--property=TasksMax=infinity', - '--collect', - '--setenv=SYSTEMD_NSPAWN_LOCK=0' + "--property=KillMode=mixed", + "--property=Type=notify", + "--property=RestartForceExitStatus=133", + "--property=SuccessExitStatus=133", + "--property=Delegate=yes", + "--property=TasksMax=infinity", + "--collect", + "--setenv=SYSTEMD_NSPAWN_LOCK=0", ] - systemd_nspawn_default_args = [ - '--keep-unit', - '--quiet', - '--boot' - ] + systemd_nspawn_default_args = ["--keep-unit", "--quiet", "--boot"] - config = cleandoc(f""" + config = cleandoc( + f""" startup={startup} docker_compatible={docker_compatible} gpu_passthrough_intel={gpu_passthrough_intel} @@ -833,9 +987,10 @@ def create_jail(jail_name, distro='debian', release='bookworm'): # You generally will not need to change the options below systemd_run_default_args={' '.join(systemd_run_default_args)} systemd_nspawn_default_args={' '.join(systemd_nspawn_default_args)} - """) + """ + ) - print(config, file=open(jail_config_path, 'w')) + print(config, file=open(jail_config_path, "w")) os.chmod(jail_config_path, 0o600) @@ -845,13 +1000,19 @@ def create_jail(jail_name, distro='debian', release='bookworm'): raise error print() - if agree(f"Do you want to start jail {jail_name} right now?", 'y'): + if agree(f"Do you want to start jail {jail_name} right now?", "y"): start_jail(jail_name) def jail_is_running(jail_name): - return subprocess.run(["machinectl", "show", jail_name], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL).returncode == 0 + return ( + subprocess.run( + ["machinectl", "show", jail_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + == 0 + ) def edit_jail(jail_name): @@ -881,8 +1042,10 @@ def remove_jail(jail_name): if check_jail_name_available(jail_name, False): eprint(f"A jail with name {jail_name} does not exist.") else: - check = input( - f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') or "" + check = ( + input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') + or "" + ) if check == jail_name: jail_path = get_jail_path(jail_name) if jail_is_running(jail_name): @@ -900,22 +1063,25 @@ def remove_jail(jail_name): def print_table(header, list_of_objects, empty_value_indicator): - # Find max width for each column widths = defaultdict(int) for obj in list_of_objects: for hdr in header: - widths[hdr] = max(widths[hdr], len( - str(obj.get(hdr))), len(str(hdr))) + widths[hdr] = max(widths[hdr], len(str(obj.get(hdr))), len(str(hdr))) # Print header - print(UNDERLINE + ' '.join(hdr.upper().ljust(widths[hdr]) - for hdr in header) + NORMAL) + print( + UNDERLINE + " ".join(hdr.upper().ljust(widths[hdr]) for hdr in header) + NORMAL + ) # Print rows for obj in list_of_objects: - print(' '.join(str(obj.get(hdr, empty_value_indicator)).ljust( - widths[hdr]) for hdr in header)) + print( + " ".join( + str(obj.get(hdr, empty_value_indicator)).ljust(widths[hdr]) + for hdr in header + ) + ) def run_command_and_parse_json(command): @@ -945,79 +1111,79 @@ def list_jails(): """ jails = {} - empty_value_indicator = '-' + empty_value_indicator = "-" jail_names = get_all_jail_names() if not jail_names: - print('No jails.') + print("No jails.") return for jail in jail_names: jails[jail] = {"name": jail, "running": False} # Get running jails from machinectl - running_machines = run_command_and_parse_json( - ['machinectl', 'list', '-o', 'json']) + running_machines = run_command_and_parse_json(["machinectl", "list", "-o", "json"]) # Augment the jails dict with output from machinectl for machine in running_machines: - machine_name = machine['machine'] + machine_name = machine["machine"] # We're only interested in the list of jails made with jailmaker - if machine['service'] == 'systemd-nspawn' and machine_name in jails: - - addresses = (machine.get('addresses') - or empty_value_indicator).split('\n') + if machine["service"] == "systemd-nspawn" and machine_name in jails: + addresses = (machine.get("addresses") or empty_value_indicator).split("\n") if len(addresses) > 1: - addresses = addresses[0] + '…' + addresses = addresses[0] + "…" else: addresses = addresses[0] jails[machine_name] = { "name": machine_name, "running": True, - "os": machine['os'], - "version": machine['version'], - "addresses": addresses + "os": machine["os"], + "version": machine["version"], + "addresses": addresses, } # TODO: add additional properties from the jails config file for jail_name in jails: - config = parse_config(get_jail_config_path(jail_name)) startup = False if config: - startup = bool(int(config.get('startup', '0'))) + startup = bool(int(config.get("startup", "0"))) # TODO: in case config is missing or parsing fails, # should an error message be thrown here? - jails[jail_name]['startup'] = startup + jails[jail_name]["startup"] = startup - print_table(["name", "running", "startup", "os", "version", "addresses"], - sorted(jails.values(), key=lambda x: x['name']), empty_value_indicator) + print_table( + ["name", "running", "startup", "os", "version", "addresses"], + sorted(jails.values(), key=lambda x: x["name"]), + empty_value_indicator, + ) def install_jailmaker(): # Check if command exists in path - if shutil.which('systemd-nspawn'): + if shutil.which("systemd-nspawn"): print("systemd-nspawn is already installed.") else: print("Installing jailmaker dependencies...") original_permissions = {} - print("Temporarily enable apt and dpkg (if not already enabled) to install systemd-nspawn.") + print( + "Temporarily enable apt and dpkg (if not already enabled) to install systemd-nspawn." + ) # Make /bin/apt* and /bin/dpkg* files executable - for file in (glob.glob('/bin/apt*') + (glob.glob('/bin/dpkg*'))): + for file in glob.glob("/bin/apt*") + (glob.glob("/bin/dpkg*")): original_permissions[file] = os.stat(file).st_mode stat_chmod(file, 0o755) - subprocess.run(['apt-get', 'update'], check=True) - subprocess.run(['apt-get', 'install', '-y', - 'systemd-container'], check=True) + subprocess.run(["apt-get", "update"], check=True) + subprocess.run(["apt-get", "install", "-y", "systemd-container"], check=True) # Restore original permissions print("Restore permissions of apt and dpkg.") @@ -1025,7 +1191,7 @@ def install_jailmaker(): for file, original_permission in original_permissions.items(): stat_chmod(file, original_permission) - target = f'/usr/local/sbin/{SYMLINK_NAME}' + target = f"/usr/local/sbin/{SYMLINK_NAME}" # Check if command exists in path if shutil.which(SYMLINK_NAME): @@ -1035,7 +1201,8 @@ def install_jailmaker(): os.symlink(SCRIPT_PATH, target) else: print( - f"File {target} already exists... Maybe it's a broken symlink from a previous install attempt?") + f"File {target} already exists... Maybe it's a broken symlink from a previous install attempt?" + ) print(f"Skipped creating new symlink {target} to {SCRIPT_PATH}.") print("Done installing jailmaker.") @@ -1050,64 +1217,74 @@ def startup_jails(): def main(): if os.stat(SCRIPT_PATH).st_uid != 0: fail( - f"This script should be owned by the root user... Fix it manually with: `chown root {SCRIPT_PATH}`.") + f"This script should be owned by the root user... Fix it manually with: `chown root {SCRIPT_PATH}`." + ) - parser = argparse.ArgumentParser( - description=DESCRIPTION, epilog=DISCLAIMER) + parser = argparse.ArgumentParser(description=DESCRIPTION, epilog=DISCLAIMER) - parser.add_argument('--version', action='version', version=VERSION) + parser.add_argument("--version", action="version", version=VERSION) - subparsers = parser.add_subparsers( - title='commands', dest='subcommand', metavar="") + subparsers = parser.add_subparsers(title="commands", dest="subcommand", metavar="") - subparsers.add_parser(name='install', epilog=DISCLAIMER, - help='install jailmaker dependencies and create symlink') + subparsers.add_parser( + name="install", + epilog=DISCLAIMER, + help="install jailmaker dependencies and create symlink", + ) - subparsers.add_parser(name='create', epilog=DISCLAIMER, - help='create a new jail').add_argument( - 'name', nargs='?', help='name of the jail') + subparsers.add_parser( + name="create", epilog=DISCLAIMER, help="create a new jail" + ).add_argument("name", nargs="?", help="name of the jail") - subparsers.add_parser(name='start', epilog=DISCLAIMER, - help='start a previously created jail').add_argument( - 'name', help='name of the jail') + subparsers.add_parser( + name="start", epilog=DISCLAIMER, help="start a previously created jail" + ).add_argument("name", help="name of the jail") - subparsers.add_parser(name='shell', epilog=DISCLAIMER, - help='open shell in running jail').add_argument( - 'name', help='name of the jail') + subparsers.add_parser( + name="shell", epilog=DISCLAIMER, help="open shell in running jail" + ).add_argument("name", help="name of the jail") - exec_parser = subparsers.add_parser(name='exec', epilog=DISCLAIMER, - help='execute a command in the jail') - exec_parser.add_argument('name', help='name of the jail') - exec_parser.add_argument('cmd', help='command to execute') + exec_parser = subparsers.add_parser( + name="exec", epilog=DISCLAIMER, help="execute a command in the jail" + ) + exec_parser.add_argument("name", help="name of the jail") + exec_parser.add_argument("cmd", help="command to execute") - subparsers.add_parser(name='status', epilog=DISCLAIMER, - help='show jail status').add_argument( - 'name', help='name of the jail') + subparsers.add_parser( + name="status", epilog=DISCLAIMER, help="show jail status" + ).add_argument("name", help="name of the jail") - subparsers.add_parser(name='log', epilog=DISCLAIMER, - help='show jail log').add_argument( - 'name', help='name of the jail') + subparsers.add_parser( + name="log", epilog=DISCLAIMER, help="show jail log" + ).add_argument("name", help="name of the jail") - subparsers.add_parser(name='stop', epilog=DISCLAIMER, - help='stop a running jail').add_argument( - 'name', help='name of the jail') + subparsers.add_parser( + name="stop", epilog=DISCLAIMER, help="stop a running jail" + ).add_argument("name", help="name of the jail") - subparsers.add_parser(name='edit', epilog=DISCLAIMER, - help=f'edit jail config with {TEXT_EDITOR} text editor').add_argument( - 'name', help='name of the jail to edit') + subparsers.add_parser( + name="edit", + epilog=DISCLAIMER, + help=f"edit jail config with {TEXT_EDITOR} text editor", + ).add_argument("name", help="name of the jail to edit") - subparsers.add_parser(name='remove', epilog=DISCLAIMER, - help='remove a previously created jail').add_argument( - 'name', help='name of the jail to remove') + subparsers.add_parser( + name="remove", epilog=DISCLAIMER, help="remove a previously created jail" + ).add_argument("name", help="name of the jail to remove") - subparsers.add_parser(name='list', epilog=DISCLAIMER, - help='list jails') + subparsers.add_parser(name="list", epilog=DISCLAIMER, help="list jails") - subparsers.add_parser(name='images', epilog=DISCLAIMER, - help='list available images to create jails from') + subparsers.add_parser( + name="images", + epilog=DISCLAIMER, + help="list available images to create jails from", + ) - subparsers.add_parser(name='startup', epilog=DISCLAIMER, - help=f'install {SYMLINK_NAME} and startup selected jails') + subparsers.add_parser( + name="startup", + epilog=DISCLAIMER, + help=f"install {SYMLINK_NAME} and startup selected jails", + ) if os.getuid() != 0: parser.print_usage() @@ -1121,54 +1298,54 @@ def main(): args, additional_args = parser.parse_known_args() - if args.subcommand == 'install': + if args.subcommand == "install": install_jailmaker() - elif args.subcommand == 'create': + elif args.subcommand == "create": create_jail(args.name) - elif args.subcommand == 'start': + elif args.subcommand == "start": start_jail(args.name) - elif args.subcommand == 'shell': + elif args.subcommand == "shell": shell_jail(args.name) - elif args.subcommand == 'exec': + elif args.subcommand == "exec": exec_jail(args.name, args.cmd, additional_args) - elif args.subcommand == 'status': + elif args.subcommand == "status": status_jail(args.name) - elif args.subcommand == 'log': + elif args.subcommand == "log": log_jail(args.name) - elif args.subcommand == 'stop': + elif args.subcommand == "stop": stop_jail(args.name) - elif args.subcommand == 'edit': + elif args.subcommand == "edit": edit_jail(args.name) - elif args.subcommand == 'remove': + elif args.subcommand == "remove": remove_jail(args.name) - elif args.subcommand == 'list': + elif args.subcommand == "list": list_jails() - elif args.subcommand == 'images': + elif args.subcommand == "images": run_lxc_download_script() - elif args.subcommand == 'startup': + elif args.subcommand == "startup": startup_jails() else: - if agree("Create a new jail?", 'y'): + if agree("Create a new jail?", "y"): print() create_jail("") else: parser.print_usage() -if __name__ == '__main__': +if __name__ == "__main__": try: main() except KeyboardInterrupt: From 89cc0d4fafe23c7bb787fa436f17c97d67b688fd Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:08:00 +0100 Subject: [PATCH 09/57] Add Cockpit management --- docs/rootless_podman_in_rootless_jail.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index 9ff804b..7add409 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -42,6 +42,8 @@ setcap cap_setgid+eip /usr/bin/newgidmap # Create new user adduser rootless +# Set password for user +passwd rootless # Clear the subuids and subgids which have been assigned by default when creating the new user usermod --del-subuids 0-4294967295 --del-subgids 0-4294967295 rootless @@ -86,6 +88,20 @@ The output of podman info should contain: Using metacopy: "false" ``` +## Cockpit management + +Inside the rootless jail run (as root user): + +```bash +dnf install cockpit cockpit-podman +systemctl enable --now cockpit.socket +ip a +``` + +Check the IP address of the jail and access the Cockpit web interface at https://0.0.0.0:9090 where 0.0.0.0 is the IP address you just found using `ip a`. + +Then login as user `rootless` with the password you've created earlier. Click on `Podman containers`. In case it shows `Podman service is not active` then click `Start podman`. You can now manage your rootless podman containers in the rootless jailmaker jail using the Cockpit web GUI. + ## TODO: On truenas host do: sudo sysctl net.ipv4.ip_unprivileged_port_start=23 From 9b307fd46b14422c88aa4da4cd11699bb13b9efd Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:57:13 +0100 Subject: [PATCH 10/57] Create incus_lxd_lxc_kvm.md --- docs/incus_lxd_lxc_kvm.md | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/incus_lxd_lxc_kvm.md diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md new file mode 100644 index 0000000..18daa07 --- /dev/null +++ b/docs/incus_lxd_lxc_kvm.md @@ -0,0 +1,78 @@ +# Incus / LXD / LXC / KVM inside jail + +## Disclaimer + +**These notes are a work in progress. Using Incus in this setup hasn't been extensively tested.** + +## Prerequisites + +- TrueNAS SCALE 23.10 installed bare metal (not inside VM) +- Jailmaker installed +- Setup bridge networking (see Advanced Networking in the readme) + +## Installation + +Create a debian 12 jail and [install incus](https://github.com/zabbly/incus#installation). Also install the `incus-ui-canonical` package to install the web interface. Ensure the config file looks like the below: + +``` +startup=0 +docker_compatible=1 +gpu_passthrough_intel=1 +gpu_passthrough_nvidia=0 +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock --bind-ro=/sys/module +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 +systemd_nspawn_default_args=--keep-unit --quiet --boot +``` + +Run `modprobe vhost_vsock` on the TrueNAS host. TODO: Check if this is really required. + +Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). + +## Create Ubuntu Desktop VM + +Incus web GUI should be running on port 8443. Create new instance, call it `dekstop`, and choose the `Ubuntu jammy desktop virtual-machine ubuntu/22.04/desktop` image. + +## Bind mount / virtiofs + +To access files from the TrueNAS host directly in a VM created with incus, we can use virtiofs. + +```bash +incus config device add desktop test disk source=/home/test/ path=/mnt/test +``` + +The command above (when ran as root user inside the incus jail) adds a new virtiofs mount of a test directory inside the jail to a VM named desktop. The `/home/test` dir resides in the jail, but you can first bind mount any directory from the TrueNAS host inside the incus jail and then forward this to the VM using virtiofs. This could be an alternative to NFS mounts. + +### Benchmarks + +#### Inside LXD ubuntu desktop VM with virtiofs mount +root@desktop:/mnt/test# mount | grep test +incus_test on /mnt/test type virtiofs (rw,relatime) +root@desktop:/mnt/test# time iozone -a +[...] +real 2m22.389s +user 0m2.222s +sys 0m59.275s + +#### In a jailmaker jail on the host: +root@incus:/home/test# time iozone -a +[...] +real 0m59.486s +user 0m1.468s +sys 0m25.458s + +#### Inside LXD ubuntu desktop VM with virtiofs mount +root@desktop:/mnt/test# dd if=/dev/random of=./test1.img bs=1G count=1 oflag=dsync +1+0 records in +1+0 records out +1073741824 bytes (1.1 GB, 1.0 GiB) copied, 36.321 s, 29.6 MB/s + +#### In a jailmaker jail on the host: +root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsync +1+0 records in +1+0 records out +1073741824 bytes (1.1 GB, 1.0 GiB) copied, 7.03723 s, 153 MB/s + +## Create Ubuntu container + +To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](rootless_podman_in_rootless_jail.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. \ No newline at end of file From 79bd8245051d93c7db5b6e5ea906156a4ecb04b9 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 20:56:25 +0100 Subject: [PATCH 11/57] Update comments --- jlmkr.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 54bc067..f8805b9 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -356,13 +356,13 @@ def start_jail(jail_name, check_startup_enabled=False): # # Workaround: https://github.com/kinvolk/kube-spawn/pull/328 # - # However, it seems like the DeviceAllow= workaround may break in - # a future Debian release with systemd version 250 or higher + # As of 26-3-2024 on TrueNAS-SCALE-23.10.1.1 it seems to no longer be + # required to use DevicePolicy=auto + # Docker can successfully pull the ljishen/sysbench test image + # Running mknod /dev/port c 1 4 manually works too... + # Unknown why this suddenly started working... # https://github.com/systemd/systemd/issues/21987 # - # As of 29-1-2023 it still works with debian bookworm (nightly) and sid - # using the latest systemd version 252.4-2 so I think we're good! - # # Use SYSTEMD_SECCOMP=0: https://github.com/systemd/systemd/issues/18370 systemd_run_additional_args += [ From da2c90374b105a72fc36e3d9cf402fdb65d75bed Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 22:33:33 +0100 Subject: [PATCH 12/57] Load nvidia kernel module --- README.md | 8 -------- jlmkr.py | 31 +++++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index dde93ff..2ee7007 100644 --- a/README.md +++ b/README.md @@ -135,14 +135,6 @@ See [Advanced Networking](./NETWORKING.md) for more. The `jailmaker` script won't install Docker for you, but it can setup the jail with the capabilities required to run docker. You can manually install Docker inside the jail using the [official installation guide](https://docs.docker.com/engine/install/#server) or use [convenience script](https://get.docker.com). -## Nvidia GPU - -To make passthrough of the nvidia GPU work, you need to schedule a Pre Init command. The reason is that TrueNAS SCALE by default doesn't load the nvidia kernel modules (and `jailmaker` doesn't do that either). [This screenshot](https://user-images.githubusercontent.com/1704047/222915803-d6dd51b0-c4dd-4189-84be-a04d38cca0b3.png) shows what the configuration should look like. - -``` -[ ! -f /dev/nvidia-uvm ] && modprobe nvidia-current-uvm && /usr/bin/nvidia-modprobe -c0 -u -``` - ## Documentation Additional documentation contributed by the community can be found in [the docs directory](./docs/). diff --git a/jlmkr.py b/jlmkr.py index f8805b9..384b79f 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -104,6 +104,17 @@ def passthrough_intel(gpu_passthrough_intel, systemd_nspawn_additional_args): def passthrough_nvidia( gpu_passthrough_nvidia, systemd_nspawn_additional_args, jail_name ): + # Load the nvidia kernel module + if subprocess.run(["modprobe", "nvidia-current-uvm"]).returncode != 0: + eprint( + dedent( + """ + Failed to load nvidia-current-uvm kernel module. + Skip passthrough of nvidia GPU.""" + ) + ) + return + jail_rootfs_path = get_jail_rootfs_path(jail_name) ld_so_conf_path = Path( os.path.join(jail_rootfs_path), f"etc/ld.so.conf.d/{SYMLINK_NAME}-nvidia.conf" @@ -114,13 +125,11 @@ def passthrough_nvidia( ld_so_conf_path.unlink(missing_ok=True) return - try: # Run nvidia-smi to initialize the nvidia driver # If we can't run nvidia-smi successfully, # then nvidia-container-cli list will fail too: # we shouldn't continue with gpu passthrough - subprocess.run(["nvidia-smi", "-f", "/dev/null"], check=True) - except: + if subprocess.run(["nvidia-smi", "-f", "/dev/null"]).returncode != 0: eprint("Skip passthrough of nvidia GPU.") return @@ -316,6 +325,11 @@ def start_jail(jail_name, check_startup_enabled=False): f"--directory={JAIL_ROOTFS_NAME}", ] + # TODO: split the docker_compatible option into separate options + # - privileged (to disable seccomp, set DevicePolicy=auto and add all capabilities) + # - how to call the option to enable ip_forward and bridge-nf-call? + # TODO: always add --bind-ro=/sys/module? Or only for privileged jails? + if config.get("docker_compatible") == "1": # Enable ip forwarding on the host (docker needs it) print(1, file=open("/proc/sys/net/ipv4/ip_forward", "w")) @@ -712,15 +726,16 @@ def create_jail(jail_name, distro="debian", release="bookworm"): gpu_passthrough_nvidia = 0 - try: + if ( subprocess.run( - ["nvidia-smi"], - check=True, + ["modprobe", "nvidia-current-uvm"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) + ).returncode + == 0 + ): nvidia_detected = True - except: + else: nvidia_detected = False if nvidia_detected: From bce6e3b43f4d00adefa3355a8c3894009501e730 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:19:48 +0100 Subject: [PATCH 13/57] Add reference --- docs/incus_lxd_lxc_kvm.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md index 18daa07..1750369 100644 --- a/docs/incus_lxd_lxc_kvm.md +++ b/docs/incus_lxd_lxc_kvm.md @@ -75,4 +75,8 @@ root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsyn ## Create Ubuntu container -To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](rootless_podman_in_rootless_jail.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. \ No newline at end of file +To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](rootless_podman_in_rootless_jail.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. + +## References + +- [Running QEMU/KVM Virtual Machines in Unprivileged LXD Containers](https://dshcherb.github.io/2017/12/04/qemu-kvm-virtual-machines-in-unprivileged-lxd.html) \ No newline at end of file From bf54fea9a148f530b2fbebbc56d97c8e6be8fcd4 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:19:57 +0100 Subject: [PATCH 14/57] Add reference --- jlmkr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index 384b79f..51b14cd 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -329,7 +329,8 @@ def start_jail(jail_name, check_startup_enabled=False): # - privileged (to disable seccomp, set DevicePolicy=auto and add all capabilities) # - how to call the option to enable ip_forward and bridge-nf-call? # TODO: always add --bind-ro=/sys/module? Or only for privileged jails? - + # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html + if config.get("docker_compatible") == "1": # Enable ip forwarding on the host (docker needs it) print(1, file=open("/proc/sys/net/ipv4/ip_forward", "w")) From ae23b1330103b1e7f89475fa6115a3931a5c334f Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 04:23:46 +0100 Subject: [PATCH 15/57] Just ask for GPU passthrough --- jlmkr.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 51b14cd..488db32 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -330,7 +330,7 @@ def start_jail(jail_name, check_startup_enabled=False): # - how to call the option to enable ip_forward and bridge-nf-call? # TODO: always add --bind-ro=/sys/module? Or only for privileged jails? # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html - + if config.get("docker_compatible") == "1": # Enable ip forwarding on the host (docker needs it) print(1, file=open("/proc/sys/net/ipv4/ip_forward", "w")) @@ -720,29 +720,15 @@ def create_jail(jail_name, distro="debian", release="bookworm"): gpu_passthrough_intel = 0 - if os.path.exists("/dev/dri"): - print("Detected the presence of an intel GPU.\n") - if agree("Passthrough the intel GPU?", "n"): - gpu_passthrough_intel = 1 + if agree("Passthrough the intel GPU (if present)?", "n"): + gpu_passthrough_intel = 1 + + print() gpu_passthrough_nvidia = 0 - if ( - subprocess.run( - ["modprobe", "nvidia-current-uvm"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ).returncode - == 0 - ): - nvidia_detected = True - else: - nvidia_detected = False - - if nvidia_detected: - print("Detected the presence of an nvidia GPU.\n") - if agree("Passthrough the nvidia GPU?", "n"): - gpu_passthrough_nvidia = 1 + if agree("Passthrough the nvidia GPU (if present)?", "n"): + gpu_passthrough_nvidia = 1 print( dedent( From 5021a060e53f779c06d889a10c0159fa1c68714a Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 04:40:14 +0100 Subject: [PATCH 16/57] Always bind /sys/module to make lsmod happy --- docs/incus_lxd_lxc_kvm.md | 2 +- jlmkr.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md index 1750369..dc61d3c 100644 --- a/docs/incus_lxd_lxc_kvm.md +++ b/docs/incus_lxd_lxc_kvm.md @@ -19,7 +19,7 @@ startup=0 docker_compatible=1 gpu_passthrough_intel=1 gpu_passthrough_nvidia=0 -systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock --bind-ro=/sys/module +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 systemd_nspawn_default_args=--keep-unit --quiet --boot diff --git a/jlmkr.py b/jlmkr.py index 488db32..760ad42 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -320,16 +320,18 @@ def start_jail(jail_name, check_startup_enabled=False): f"--description=My nspawn jail {jail_name} [created with jailmaker]", ] + # Always add --bind-ro=/sys/module to make lsmod happy + # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html systemd_nspawn_additional_args = [ f"--machine={jail_name}", + "--bind-ro=/sys/module", f"--directory={JAIL_ROOTFS_NAME}", ] # TODO: split the docker_compatible option into separate options # - privileged (to disable seccomp, set DevicePolicy=auto and add all capabilities) # - how to call the option to enable ip_forward and bridge-nf-call? - # TODO: always add --bind-ro=/sys/module? Or only for privileged jails? - # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html + # - add CSV value for preloading kernel modules like linux.kernel_modules in LXC if config.get("docker_compatible") == "1": # Enable ip forwarding on the host (docker needs it) From 2aba2c4a9dd7002e8fb549b05e7c033f1a1bf3a7 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 05:44:05 +0100 Subject: [PATCH 17/57] Load kernel module later --- jlmkr.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 760ad42..1dbf1c4 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -104,6 +104,16 @@ def passthrough_intel(gpu_passthrough_intel, systemd_nspawn_additional_args): def passthrough_nvidia( gpu_passthrough_nvidia, systemd_nspawn_additional_args, jail_name ): + jail_rootfs_path = get_jail_rootfs_path(jail_name) + ld_so_conf_path = Path( + os.path.join(jail_rootfs_path), f"etc/ld.so.conf.d/{SYMLINK_NAME}-nvidia.conf" + ) + + if gpu_passthrough_nvidia != "1": + # Cleanup the config file we made when passthrough was enabled + ld_so_conf_path.unlink(missing_ok=True) + return + # Load the nvidia kernel module if subprocess.run(["modprobe", "nvidia-current-uvm"]).returncode != 0: eprint( @@ -115,20 +125,10 @@ def passthrough_nvidia( ) return - jail_rootfs_path = get_jail_rootfs_path(jail_name) - ld_so_conf_path = Path( - os.path.join(jail_rootfs_path), f"etc/ld.so.conf.d/{SYMLINK_NAME}-nvidia.conf" - ) - - if gpu_passthrough_nvidia != "1": - # Cleanup the config file we made when passthrough was enabled - ld_so_conf_path.unlink(missing_ok=True) - return - - # Run nvidia-smi to initialize the nvidia driver - # If we can't run nvidia-smi successfully, - # then nvidia-container-cli list will fail too: - # we shouldn't continue with gpu passthrough + # Run nvidia-smi to initialize the nvidia driver + # If we can't run nvidia-smi successfully, + # then nvidia-container-cli list will fail too: + # we shouldn't continue with gpu passthrough if subprocess.run(["nvidia-smi", "-f", "/dev/null"]).returncode != 0: eprint("Skip passthrough of nvidia GPU.") return From 9dc1bc3b20ae2a512d940cb9e36fefb201dfce5d Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 12:21:23 +0100 Subject: [PATCH 18/57] Update incus_lxd_lxc_kvm.md --- docs/incus_lxd_lxc_kvm.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md index dc61d3c..abcb5ce 100644 --- a/docs/incus_lxd_lxc_kvm.md +++ b/docs/incus_lxd_lxc_kvm.md @@ -14,6 +14,8 @@ Create a debian 12 jail and [install incus](https://github.com/zabbly/incus#installation). Also install the `incus-ui-canonical` package to install the web interface. Ensure the config file looks like the below: +Run `modprobe vhost_vsock` on the TrueNAS host. + ``` startup=0 docker_compatible=1 @@ -25,8 +27,6 @@ systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --prop systemd_nspawn_default_args=--keep-unit --quiet --boot ``` -Run `modprobe vhost_vsock` on the TrueNAS host. TODO: Check if this is really required. - Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). ## Create Ubuntu Desktop VM From 303f79a3ae910243796ae901b5f50a482ca71f28 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 12:22:39 +0100 Subject: [PATCH 19/57] Add bind /sys/module to nspawn default args --- jlmkr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 1dbf1c4..32d9546 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -324,7 +324,6 @@ def start_jail(jail_name, check_startup_enabled=False): # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html systemd_nspawn_additional_args = [ f"--machine={jail_name}", - "--bind-ro=/sys/module", f"--directory={JAIL_ROOTFS_NAME}", ] @@ -979,7 +978,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): "--setenv=SYSTEMD_NSPAWN_LOCK=0", ] - systemd_nspawn_default_args = ["--keep-unit", "--quiet", "--boot"] + systemd_nspawn_default_args = ["--keep-unit", "--quiet", "--boot", "--bind-ro=/sys/module"] config = cleandoc( f""" From 53689df6456e629959b9c0ec19b09c2cb16424eb Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 12:23:45 +0100 Subject: [PATCH 20/57] Remove redundant system-call-filter Since SYSTEMD_SECCOMP=0 adding system-call-filter is redundant --- jlmkr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 32d9546..2281f19 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -389,7 +389,6 @@ def start_jail(jail_name, check_startup_enabled=False): # Add additional flags required for docker systemd_nspawn_additional_args += [ "--capability=all", - "--system-call-filter=add_key keyctl bpf", ] # Legacy gpu_passthrough config setting @@ -978,7 +977,12 @@ def create_jail(jail_name, distro="debian", release="bookworm"): "--setenv=SYSTEMD_NSPAWN_LOCK=0", ] - systemd_nspawn_default_args = ["--keep-unit", "--quiet", "--boot", "--bind-ro=/sys/module"] + systemd_nspawn_default_args = [ + "--keep-unit", + "--quiet", + "--boot", + "--bind-ro=/sys/module", + ] config = cleandoc( f""" From 1ac8bb8fc11eeb03db6241497cffd1ba7bc3929a Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 13:13:09 +0100 Subject: [PATCH 21/57] Update incus_lxd_lxc_kvm.md --- docs/incus_lxd_lxc_kvm.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md index abcb5ce..289de4f 100644 --- a/docs/incus_lxd_lxc_kvm.md +++ b/docs/incus_lxd_lxc_kvm.md @@ -77,6 +77,40 @@ root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsyn To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](rootless_podman_in_rootless_jail.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. +## Canonical LXD install via snap + +Installing the lxd snap is an alternative to Incus. But out of the box running `snap install lxd` will cause AppArmor issues when running inside a jailmaker jail on SCALE. + +### Workaround 1: Disable AppArmor kernel module + +[To my knowledge AppArmor is not uses on SCALE](https://github.com/truenas/charts/pull/428#issuecomment-1113936420). The AppArmor related packages aren't even installed. + +Ensure to add --bind=/dev/fuse and ensure using bridge or macvlan networking: + +``` +# On the host +cat /sys/module/apparmor/parameters/enabled +Y +midclt call system.advanced.update '{"kernel_extra_options": "apparmor=0"}' +reboot +cat /sys/module/apparmor/parameters/enabled + +# In Ubuntu jail +apt update +ln -s /bin/true /usr/local/bin/udevadm +apt install -y --no-install-recommends snapd +snap install lxd +lxd init +snap set lxd ui.enable=true +systemctl reload snap.lxd.daemon + +# Check out: https://example:8443 +``` + +### Workaround 2: inaccessible /sys/module/apparmor + +If I don't want to mess with kernel parameters, I can trick the jail into thinking the apparmor module is not loaded by mounting over /sys/module/apparmor: `mount -v -r -t tmpfs -o size=50m test /sys/module/apparmor`. Then `snap install lxd` completes! Best way to do this is to add `--inaccessible=/sys/module/apparmor` to the systemd_nspawn_user_args. + ## References - [Running QEMU/KVM Virtual Machines in Unprivileged LXD Containers](https://dshcherb.github.io/2017/12/04/qemu-kvm-virtual-machines-in-unprivileged-lxd.html) \ No newline at end of file From aee047cb8d54bfe6de971d3060257e2d82fb39e0 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 16:59:51 +0100 Subject: [PATCH 22/57] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ee7007..d36ddec 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ TrueNAS SCALE can create persistent Linux 'jails' with systemd-nspawn. This scri - Optional: GPU passthrough (including [nvidia GPU](README.md#nvidia-gpu) with the drivers bind mounted from the host) - Starting the jail with your config applied +## Security + +Despite what the word 'jail' implies, jailmaker's intended use case is to create one or more additional filesystems to run alongside SCALE with minimal isolation. By default the root user in the jail with uid 0 is mapped to the host's uid 0. This has [obvious security implications](https://linuxcontainers.org/lxc/security/#privileged-containers). If this is not acceptable to you, you may lock down the jails by [limiting capabilities](https://manpages.debian.org/bookworm/systemd-container/systemd-nspawn.1.en.html#Security_Options) and/or using [user namespacing](https://manpages.debian.org/bookworm/systemd-container/systemd-nspawn.1.en.html#User_Namespacing_Options) or use a VM instead. + ## Installation Create a new dataset called `jailmaker` with the default settings (from TrueNAS web interface). Then login as the root user and download `jlmkr.py`. @@ -123,7 +127,7 @@ jlmkr log myjail ### Additional Commands -Expert users may use the following additional commands to manage jails directly: `machinectl`, `systemd-nspawn`, `systemd-run`, `systemctl` and `journalctl`. The `jlmkr` script uses these commands under the hood and implements a subset of their capabilities. If you use them directly you will bypass any safety checks or configuration done by `jlmkr` and not everything will work in the context of TrueNAS SCALE. +Expert users may use the following additional commands to manage jails directly: `machinectl`, `systemd-nspawn`, `systemd-run`, `systemctl` and `journalctl`. The `jlmkr` script uses these commands under the hood and implements a subset of their functions. If you use them directly you will bypass any safety checks or configuration done by `jlmkr` and not everything will work in the context of TrueNAS SCALE. ## Networking From 0f22a569468ff031dfe6620744134625ebff351f Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 27 Jan 2024 17:30:31 +0100 Subject: [PATCH 23/57] Update jlmkr.py --- jlmkr.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jlmkr.py b/jlmkr.py index 2281f19..7a8e218 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -329,6 +329,13 @@ def start_jail(jail_name, check_startup_enabled=False): # TODO: split the docker_compatible option into separate options # - privileged (to disable seccomp, set DevicePolicy=auto and add all capabilities) + # "The bottom line is that using the --privileged flag does not tell the container + # engines to add additional security constraints. The --privileged flag does not add + # any privilege over what the processes launching the containers have." + # "Container engines user namespace is not affected by the --privileged flag" + # Meaning in the context of systemd-nspawn I could have a privileged option, + # which would also apply to jails with --private-users (user namespacing) + # https://www.redhat.com/sysadmin/privileged-flag-container-engines # - how to call the option to enable ip_forward and bridge-nf-call? # - add CSV value for preloading kernel modules like linux.kernel_modules in LXC From ad95fe7cab310a6235fb783b4447200ad8a26b50 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 29 Jan 2024 08:58:28 +0100 Subject: [PATCH 24/57] Update comments --- jlmkr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 7a8e218..6702f22 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -320,8 +320,6 @@ def start_jail(jail_name, check_startup_enabled=False): f"--description=My nspawn jail {jail_name} [created with jailmaker]", ] - # Always add --bind-ro=/sys/module to make lsmod happy - # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html systemd_nspawn_additional_args = [ f"--machine={jail_name}", f"--directory={JAIL_ROOTFS_NAME}", @@ -984,6 +982,8 @@ def create_jail(jail_name, distro="debian", release="bookworm"): "--setenv=SYSTEMD_NSPAWN_LOCK=0", ] + # Always add --bind-ro=/sys/module to make lsmod happy + # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html systemd_nspawn_default_args = [ "--keep-unit", "--quiet", From 4655f174b7cdac1308463f9b743a2f5b1cb9d048 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 29 Jan 2024 09:01:05 +0100 Subject: [PATCH 25/57] Add --inaccessible=/sys/module/apparmor To trick the jail into thinking the apparmor kernel module is not loaded. --- jlmkr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jlmkr.py b/jlmkr.py index 6702f22..1dc2e11 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -989,6 +989,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): "--quiet", "--boot", "--bind-ro=/sys/module", + "--inaccessible=/sys/module/apparmor", ] config = cleandoc( From 3940c87fdf6e41acf744bad675a24e2a1fa7ef2a Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 29 Jan 2024 09:10:45 +0100 Subject: [PATCH 26/57] Update incus_lxd_lxc_kvm.md --- docs/incus_lxd_lxc_kvm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/incus_lxd_lxc_kvm.md b/docs/incus_lxd_lxc_kvm.md index 289de4f..39c9491 100644 --- a/docs/incus_lxd_lxc_kvm.md +++ b/docs/incus_lxd_lxc_kvm.md @@ -24,7 +24,7 @@ gpu_passthrough_nvidia=0 systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 -systemd_nspawn_default_args=--keep-unit --quiet --boot +systemd_nspawn_default_args=--keep-unit --quiet --boot --bind-ro=/sys/module --inaccessible=/sys/module/apparmor ``` Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). From a5d53c0a7bf54711ec18a71b0667f37070f25989 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:12:34 +0100 Subject: [PATCH 27/57] Properly pass exit code from exec --- jlmkr.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 1dc2e11..6bb588c 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -223,20 +223,21 @@ def exec_jail(jail_name, cmd, args): """ Execute a command in the jail with given name. """ - subprocess.run( - [ - "systemd-run", - "--machine", - jail_name, - "--quiet", - "--pipe", - "--wait", - "--collect", - "--service-type=exec", - cmd, - ] - + args, - check=True, + sys.exit( + subprocess.run( + [ + "systemd-run", + "--machine", + jail_name, + "--quiet", + "--pipe", + "--wait", + "--collect", + "--service-type=exec", + cmd, + ] + + args + ).returncode ) From d94a2aac7d00edbb05b3974e1e533c025a2f5128 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:14:41 +0100 Subject: [PATCH 28/57] No need for try/except --- jlmkr.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 6bb588c..c0c72d7 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -434,9 +434,7 @@ def start_jail(jail_name, check_startup_enabled=False): ) ) - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError: + if subprocess.run(cmd).returncode != 0: fail( dedent( f""" From 00e98ac07d0cf53bc7ebc764816d10ff32bf0e25 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:15:38 +0100 Subject: [PATCH 29/57] Pass more status codes and arguments --- docs/rootless_podman_in_rootless_jail.md | 2 +- jlmkr.py | 231 ++++++++++++++--------- 2 files changed, 143 insertions(+), 90 deletions(-) diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md index 7add409..9ebdf4f 100644 --- a/docs/rootless_podman_in_rootless_jail.md +++ b/docs/rootless_podman_in_rootless_jail.md @@ -61,7 +61,7 @@ exit From the TrueNAS host, open a shell as the rootless user inside the jail. ```bash -machinectl shell --uid 1000 rootless +jlmkr shell --uid 1000 rootless ``` Run rootless podman as user 1000. diff --git a/jlmkr.py b/jlmkr.py index c0c72d7..4fbf0f8 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -223,22 +223,20 @@ def exec_jail(jail_name, cmd, args): """ Execute a command in the jail with given name. """ - sys.exit( - subprocess.run( - [ - "systemd-run", - "--machine", - jail_name, - "--quiet", - "--pipe", - "--wait", - "--collect", - "--service-type=exec", - cmd, - ] - + args - ).returncode - ) + return subprocess.run( + [ + "systemd-run", + "--machine", + jail_name, + "--quiet", + "--pipe", + "--wait", + "--collect", + "--service-type=exec", + cmd, + ] + + args + ).returncode def status_jail(jail_name): @@ -246,28 +244,32 @@ def status_jail(jail_name): Show the status of the systemd service wrapping the jail with given name. """ # Alternatively `machinectl status jail_name` could be used - subprocess.run(["systemctl", "status", f"{SYMLINK_NAME}-{jail_name}"]) + return subprocess.run( + ["systemctl", "status", f"{SYMLINK_NAME}-{jail_name}"] + ).returncode def log_jail(jail_name): """ Show the log file of the jail with given name. """ - subprocess.run(["journalctl", "-u", f"{SYMLINK_NAME}-{jail_name}"]) + return subprocess.run( + ["journalctl", "-u", f"{SYMLINK_NAME}-{jail_name}"] + ).returncode -def shell_jail(jail_name): +def shell_jail(args): """ Open a shell in the jail with given name. """ - subprocess.run(["machinectl", "shell", jail_name]) + return subprocess.run(["machinectl", "shell"] + args).returncode def stop_jail(jail_name): """ Stop jail with given name. """ - subprocess.run(["machinectl", "poweroff", jail_name]) + return subprocess.run(["machinectl", "poweroff", jail_name]).returncode def parse_config(jail_config_path): @@ -293,7 +295,8 @@ def start_jail(jail_name, check_startup_enabled=False): ) if not check_startup_enabled and jail_is_running(jail_name): - fail(skip_start_message) + eprint(skip_start_message) + return 1 jail_path = get_jail_path(jail_name) jail_config_path = get_jail_config_path(jail_name) @@ -301,7 +304,8 @@ def start_jail(jail_name, check_startup_enabled=False): config = parse_config(jail_config_path) if not config: - fail("Aborting...") + eprint("Aborting...") + return 1 # Only start if the startup setting is enabled in the config if check_startup_enabled: @@ -310,10 +314,10 @@ def start_jail(jail_name, check_startup_enabled=False): if jail_is_running(jail_name): # ...but we can skip if it's already running eprint(skip_start_message) - return + return 0 else: - # Skip starting this jail since the startup config setting isnot enabled - return + # Skip starting this jail since the startup config setting is not enabled + return 0 systemd_run_additional_args = [ f"--unit={SYMLINK_NAME}-{jail_name}", @@ -434,8 +438,9 @@ def start_jail(jail_name, check_startup_enabled=False): ) ) - if subprocess.run(cmd).returncode != 0: - fail( + returncode = subprocess.run(cmd).returncode + if returncode != 0: + eprint( dedent( f""" Failed to start jail {jail_name}... @@ -445,6 +450,8 @@ def start_jail(jail_name, check_startup_enabled=False): ) ) + return returncode + def cleanup(jail_path): """ @@ -505,7 +512,8 @@ def run_lxc_download_script( lxc_download_script, ) if not validate_sha256(lxc_download_script, DOWNLOAD_SCRIPT_DIGEST): - fail("Abort! Downloaded script has unexpected contents.") + eprint("Abort! Downloaded script has unexpected contents.") + return 1 stat_chmod(lxc_download_script, 0o700) @@ -541,7 +549,10 @@ def run_lxc_download_script( p1.wait() if check_exit_code and p1.returncode != 0: - fail("Aborting...") + eprint("Aborting...") + return p1.returncode + + return 0 def stat_chmod(file_path, mode): @@ -625,7 +636,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): print(DISCLAIMER) if os.path.basename(os.getcwd()) != "jailmaker": - fail( + eprint( dedent( f""" {COMMAND_NAME} needs to create files. @@ -634,6 +645,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): Please create a dedicated directory called 'jailmaker', store {SCRIPT_NAME} there and try again.""" ) ) + return 1 if not PurePath(get_mount_point(os.getcwd())).is_relative_to("/mnt"): print( @@ -649,7 +661,8 @@ def create_jail(jail_name, distro="debian", release="bookworm"): ) ) if not agree("Do you wish to ignore this warning and continue?", "n"): - fail("Aborting...") + eprint("Aborting...") + return 0 # Create the dir where to store the jails os.makedirs(JAILS_DIR_PATH, exist_ok=True) @@ -671,7 +684,9 @@ def create_jail(jail_name, distro="debian", release="bookworm"): input("Press Enter to continue...") print() - run_lxc_download_script() + returncode = run_lxc_download_script() + if returncode != 0: + return returncode print( dedent( @@ -834,7 +849,11 @@ def create_jail(jail_name, distro="debian", release="bookworm"): # but we don't need it so we will remove it later open(jail_config_path, "a").close() - run_lxc_download_script(jail_name, jail_path, jail_rootfs_path, distro, release) + returncode = run_lxc_download_script( + jail_name, jail_path, jail_rootfs_path, distro, release + ) + if returncode != 0: + return returncode # Assuming the name of your jail is "myjail" # and "machinectl shell myjail" doesn't work @@ -887,7 +906,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): ) if agree("Abort creating jail?", "y"): - exit(1) + return 1 with contextlib.suppress(FileNotFoundError): # Remove config which systemd handles for us @@ -1015,7 +1034,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): print() if agree(f"Do you want to start jail {jail_name} right now?", "y"): - start_jail(jail_name) + return start_jail(jail_name) def jail_is_running(jail_name): @@ -1033,18 +1052,34 @@ def edit_jail(jail_name): """ Edit jail with given name. """ - if check_jail_name_valid(jail_name): - if check_jail_name_available(jail_name, False): - eprint(f"A jail with name {jail_name} does not exist.") - else: - jail_config_path = get_jail_config_path(jail_name) - if not shutil.which(TEXT_EDITOR): - eprint(f"Unable to edit config file: {jail_config_path}.") - eprint(f"The {TEXT_EDITOR} text editor is not available.") - else: - subprocess.run([TEXT_EDITOR, get_jail_config_path(jail_name)]) - if jail_is_running(jail_name): - print("\nRestart the jail for edits to apply (if you made any).") + + if not check_jail_name_valid(jail_name): + return 1 + + if check_jail_name_available(jail_name, False): + eprint(f"A jail with name {jail_name} does not exist.") + return 1 + + jail_config_path = get_jail_config_path(jail_name) + if not shutil.which(TEXT_EDITOR): + eprint( + f"Unable to edit config file: {jail_config_path}.", + f"\nThe {TEXT_EDITOR} text editor is not available", + ) + return 1 + + returncode = subprocess.run( + [TEXT_EDITOR, get_jail_config_path(jail_name)] + ).returncode + + if returncode != 0: + eprint("An error occurred while editing the jail config.") + return returncode + + if jail_is_running(jail_name): + print("\nRestart the jail for edits to apply (if you made any).") + + return 0 def remove_jail(jail_name): @@ -1052,28 +1087,31 @@ def remove_jail(jail_name): Remove jail with given name. """ - if check_jail_name_valid(jail_name): - if check_jail_name_available(jail_name, False): - eprint(f"A jail with name {jail_name} does not exist.") - else: - check = ( - input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') - or "" - ) - if check == jail_name: - jail_path = get_jail_path(jail_name) - if jail_is_running(jail_name): - print(f"\nWait for {jail_name} to stop...", end="") - stop_jail(jail_name) - # Need to sleep since deleting immediately after stop causes problems... - while jail_is_running(jail_name): - time.sleep(1) - print(".", end="", flush=True) + if not check_jail_name_valid(jail_name): + return 1 - print(f"\nCleaning up: {jail_path}") - shutil.rmtree(jail_path) - else: - eprint("Wrong name, nothing happened.") + if check_jail_name_available(jail_name, False): + eprint(f"A jail with name {jail_name} does not exist.") + return 1 + + check = input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') + + if check == jail_name: + jail_path = get_jail_path(jail_name) + if jail_is_running(jail_name): + print(f"\nWait for {jail_name} to stop...", end="") + stop_jail(jail_name) + # Need to sleep since deleting immediately after stop causes problems... + while jail_is_running(jail_name): + time.sleep(1) + print(".", end="", flush=True) + + print(f"\nCleaning up: {jail_path}") + shutil.rmtree(jail_path) + return 0 + else: + eprint("Wrong name, nothing happened.") + return 1 def print_table(header, list_of_objects, empty_value_indicator): @@ -1106,7 +1144,7 @@ def run_command_and_parse_json(command): parsed_output = json.loads(output) return parsed_output except json.JSONDecodeError as e: - print(f"Error parsing JSON: {e}") + eprint(f"Error parsing JSON: {e}") return None @@ -1131,7 +1169,7 @@ def list_jails(): if not jail_names: print("No jails.") - return + return 0 for jail in jail_names: jails[jail] = {"name": jail, "running": False} @@ -1177,6 +1215,8 @@ def list_jails(): empty_value_indicator, ) + return 0 + def install_jailmaker(): # Check if command exists in path @@ -1220,12 +1260,23 @@ def install_jailmaker(): print(f"Skipped creating new symlink {target} to {SCRIPT_PATH}.") print("Done installing jailmaker.") + + return 0 def startup_jails(): - install_jailmaker() + returncode = install_jailmaker() + + if returncode != 0: + eprint("Failed to install jailmaker. Abort startup.") + return returncode + for jail_name in get_all_jail_names(): - start_jail(jail_name, True) + returncode = start_jail(jail_name, True) + eprint(f"Failed to start jail {jail_name}. Abort startup.") + return returncode + + return 0 def main(): @@ -1255,8 +1306,10 @@ def main(): ).add_argument("name", help="name of the jail") subparsers.add_parser( - name="shell", epilog=DISCLAIMER, help="open shell in running jail" - ).add_argument("name", help="name of the jail") + name="shell", + epilog=DISCLAIMER, + help="open shell in running jail (alias for machinectl shell)", + ) exec_parser = subparsers.add_parser( name="exec", epilog=DISCLAIMER, help="execute a command in the jail" @@ -1313,48 +1366,48 @@ def main(): args, additional_args = parser.parse_known_args() if args.subcommand == "install": - install_jailmaker() + sys.exit(install_jailmaker()) elif args.subcommand == "create": - create_jail(args.name) + sys.exit(create_jail(args.name)) elif args.subcommand == "start": - start_jail(args.name) + sys.exit(start_jail(args.name)) elif args.subcommand == "shell": - shell_jail(args.name) + sys.exit(shell_jail(additional_args)) elif args.subcommand == "exec": - exec_jail(args.name, args.cmd, additional_args) + sys.exit(exec_jail(args.name, args.cmd, additional_args)) elif args.subcommand == "status": - status_jail(args.name) + sys.exit(status_jail(args.name)) elif args.subcommand == "log": - log_jail(args.name) + sys.exit(log_jail(args.name)) elif args.subcommand == "stop": - stop_jail(args.name) + sys.exit(stop_jail(args.name)) elif args.subcommand == "edit": - edit_jail(args.name) + sys.exit(edit_jail(args.name)) elif args.subcommand == "remove": - remove_jail(args.name) + sys.exit(remove_jail(args.name)) elif args.subcommand == "list": - list_jails() + sys.exit(list_jails()) elif args.subcommand == "images": - run_lxc_download_script() + sys.exit(run_lxc_download_script()) elif args.subcommand == "startup": - startup_jails() + sys.exit(startup_jails()) else: if agree("Create a new jail?", "y"): print() - create_jail("") + sys.exit(create_jail("")) else: parser.print_usage() From ed7a883f63cf45a56bd6eb7ffec26b5cb558de2b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:33:27 +0100 Subject: [PATCH 30/57] Config file multiline formatting --- jlmkr.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 4fbf0f8..3582e75 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1010,17 +1010,25 @@ def create_jail(jail_name, distro="debian", release="bookworm"): "--inaccessible=/sys/module/apparmor", ] - config = cleandoc( - f""" - startup={startup} - docker_compatible={docker_compatible} - gpu_passthrough_intel={gpu_passthrough_intel} - gpu_passthrough_nvidia={gpu_passthrough_nvidia} - systemd_nspawn_user_args={systemd_nspawn_user_args} - # You generally will not need to change the options below - systemd_run_default_args={' '.join(systemd_run_default_args)} - systemd_nspawn_default_args={' '.join(systemd_nspawn_default_args)} - """ + systemd_nspawn_user_args_multiline = "\n\t".join( + shlex.split(systemd_nspawn_user_args) + ) + systemd_run_default_args_multiline = "\n\t".join(systemd_run_default_args) + systemd_nspawn_default_args_multiline = "\n\t".join(systemd_nspawn_default_args) + + config = "\n".join( + [ + f"startup={startup}", + f"docker_compatible={docker_compatible}", + f"gpu_passthrough_intel={gpu_passthrough_intel}", + f"gpu_passthrough_nvidia={gpu_passthrough_nvidia}", + f"systemd_nspawn_user_args={systemd_nspawn_user_args_multiline}", + "", + "# You generally will not need to change the options below", + f"systemd_run_default_args={systemd_run_default_args_multiline}", + "", + f"systemd_nspawn_default_args={systemd_nspawn_default_args_multiline}", + ] ) print(config, file=open(jail_config_path, "w")) @@ -1260,7 +1268,7 @@ def install_jailmaker(): print(f"Skipped creating new symlink {target} to {SCRIPT_PATH}.") print("Done installing jailmaker.") - + return 0 From 1c83bb2dc7811671f6060b3abdcd2e743ba5f98f Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 01:57:48 +0100 Subject: [PATCH 31/57] Add restart command --- jlmkr.py | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 3582e75..a2cfbaa 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -453,6 +453,18 @@ def start_jail(jail_name, check_startup_enabled=False): return returncode +def restart_jail(jail_name): + """ + Restart jail with given name. + """ + returncode = stop_jail_and_wait(jail_name) + if returncode != 0: + eprint("Abort restart.") + return returncode + + return start_jail(jail_name) + + def cleanup(jail_path): """ Cleanup after aborted jail creation. @@ -1090,6 +1102,25 @@ def edit_jail(jail_name): return 0 +def stop_jail_and_wait(jail_name): + """ + Wait for jail with given name to stop. + """ + + returncode = stop_jail(jail_name) + if returncode != 0: + eprint("Error while stopping jail.") + return returncode + + print(f"Wait for {jail_name} to stop", end="", flush=True) + # Need to sleep since deleting immediately after stop causes problems... + while jail_is_running(jail_name): + time.sleep(1) + print(".", end="", flush=True) + + return 0 + + def remove_jail(jail_name): """ Remove jail with given name. @@ -1107,12 +1138,10 @@ def remove_jail(jail_name): if check == jail_name: jail_path = get_jail_path(jail_name) if jail_is_running(jail_name): - print(f"\nWait for {jail_name} to stop...", end="") - stop_jail(jail_name) - # Need to sleep since deleting immediately after stop causes problems... - while jail_is_running(jail_name): - time.sleep(1) - print(".", end="", flush=True) + print() + returncode = stop_jail_and_wait(jail_name) + if returncode != 0: + return returncode print(f"\nCleaning up: {jail_path}") shutil.rmtree(jail_path) @@ -1313,6 +1342,10 @@ def main(): name="start", epilog=DISCLAIMER, help="start a previously created jail" ).add_argument("name", help="name of the jail") + subparsers.add_parser( + name="restart", epilog=DISCLAIMER, help="restart a running jail" + ).add_argument("name", help="name of the jail") + subparsers.add_parser( name="shell", epilog=DISCLAIMER, @@ -1382,6 +1415,9 @@ def main(): elif args.subcommand == "start": sys.exit(start_jail(args.name)) + elif args.subcommand == "restart": + sys.exit(restart_jail(args.name)) + elif args.subcommand == "shell": sys.exit(shell_jail(additional_args)) From c908b077c2880aeee21e7935a6e4bde82f1a8c43 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 07:39:54 +0100 Subject: [PATCH 32/57] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d36ddec..c10ef13 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,12 @@ jlmkr remove myjail jlmkr stop myjail ``` +### Restart Jail + +```shell +jlmkr restart myjail +``` + ### Jail Shell ```shell From c4a5dd1c75163f765c1b49c252624bc648bad116 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:51:14 +0100 Subject: [PATCH 33/57] Don't stop and wait if jail is not running --- jlmkr.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index a2cfbaa..3799e93 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -457,6 +457,7 @@ def restart_jail(jail_name): """ Restart jail with given name. """ + returncode = stop_jail_and_wait(jail_name) if returncode != 0: eprint("Abort restart.") @@ -470,7 +471,7 @@ def cleanup(jail_path): Cleanup after aborted jail creation. """ if os.path.isdir(jail_path): - eprint(f"Cleaning up: {jail_path}") + eprint(f"Cleaning up: {jail_path}.") shutil.rmtree(jail_path) @@ -1107,6 +1108,9 @@ def stop_jail_and_wait(jail_name): Wait for jail with given name to stop. """ + if not jail_is_running(jail_name): + return 0 + returncode = stop_jail(jail_name) if returncode != 0: eprint("Error while stopping jail.") @@ -1136,15 +1140,14 @@ def remove_jail(jail_name): check = input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') if check == jail_name: + print() jail_path = get_jail_path(jail_name) - if jail_is_running(jail_name): - print() - returncode = stop_jail_and_wait(jail_name) - if returncode != 0: - return returncode + returncode = stop_jail_and_wait(jail_name) + if returncode != 0: + return returncode - print(f"\nCleaning up: {jail_path}") - shutil.rmtree(jail_path) + print() + cleanup(jail_path) return 0 else: eprint("Wrong name, nothing happened.") From 6475b13f463f9852726ce6c339c99c054f673974 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:57:16 +0100 Subject: [PATCH 34/57] Add initial_rootfs_image to config for reference --- jlmkr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jlmkr.py b/jlmkr.py index 3799e93..a4353bc 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1041,6 +1041,9 @@ def create_jail(jail_name, distro="debian", release="bookworm"): f"systemd_run_default_args={systemd_run_default_args_multiline}", "", f"systemd_nspawn_default_args={systemd_nspawn_default_args_multiline}", + "", + "# The below is for reference only, currently not used", + f"initial_rootfs_image={distro} {release}", ] ) From f9730d3a321f4bcaeea3f1d116b7d1104c7ce1ec Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 9 Feb 2024 16:41:19 +0100 Subject: [PATCH 35/57] Add start/stop hooks --- jlmkr.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index a4353bc..3007d86 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -286,6 +286,26 @@ def parse_config(jail_config_path): return config +def add_hook(jail_path, systemd_run_additional_args, hook_command, hook_type): + if not hook_command: + return + + # Run the command directly if it doesn't start with a shebang + if not hook_command.startswith("#!"): + systemd_run_additional_args += [f"--property={hook_type}={hook_command}"] + return + + # Otherwise write a script file and call that + hook_file = os.path.abspath(os.path.join(jail_path, f".{hook_type}")) + + # Only write if contents are different + if not os.path.exists(hook_file) or Path(hook_file).read_text() != hook_command: + print(hook_command, file=open(hook_file, "w")) + + stat_chmod(hook_file, 0o700) + systemd_run_additional_args += [f"--property={hook_type}={hook_file}"] + + def start_jail(jail_name, check_startup_enabled=False): """ Start jail with given name. @@ -401,6 +421,21 @@ def start_jail(jail_name, check_startup_enabled=False): "--capability=all", ] + # Add hooks to execute commands on the host before starting and after stopping a jail + add_hook( + jail_path, + systemd_run_additional_args, + config.get("pre_start_hook"), + "ExecStartPre", + ) + + add_hook( + jail_path, + systemd_run_additional_args, + config.get("post_stop_hook"), + "ExecStopPost", + ) + # Legacy gpu_passthrough config setting if config.get("gpu_passthrough") == "1": gpu_passthrough_intel = "1" @@ -1029,13 +1064,31 @@ def create_jail(jail_name, distro="debian", release="bookworm"): systemd_run_default_args_multiline = "\n\t".join(systemd_run_default_args) systemd_nspawn_default_args_multiline = "\n\t".join(systemd_nspawn_default_args) - config = "\n".join( + config = cleandoc( + f""" + startup={startup} + docker_compatible={docker_compatible} + gpu_passthrough_intel={gpu_passthrough_intel} + gpu_passthrough_nvidia={gpu_passthrough_nvidia} + """ + ) + + config += f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" + + config += cleandoc( + """ + # Specify command/script to run on the HOST before starting the jail + pre_start_hook=echo 'PRE_START_HOOK' + + # Specify a command/script to run on the HOST after stopping the jail + post_stop_hook=#!/usr/bin/bash + echo 'POST STOP HOOK' + """ + ) + + config += "\n".join( [ - f"startup={startup}", - f"docker_compatible={docker_compatible}", - f"gpu_passthrough_intel={gpu_passthrough_intel}", - f"gpu_passthrough_nvidia={gpu_passthrough_nvidia}", - f"systemd_nspawn_user_args={systemd_nspawn_user_args_multiline}", + "", "", "# You generally will not need to change the options below", f"systemd_run_default_args={systemd_run_default_args_multiline}", From d7b30011b01bc4eb873eba16e7a26c29a4f85572 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 10 Feb 2024 13:04:18 +0100 Subject: [PATCH 36/57] Create jlmkr shell aliases --- README.md | 16 ++++----- jlmkr.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c10ef13..cf7a078 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Persistent Linux 'jails' on TrueNAS SCALE to install software (docker-compose, p TrueNAS SCALE can create persistent Linux 'jails' with systemd-nspawn. This script helps with the following: -- Installing the systemd-container package (which includes systemd-nspawn) - Setting up the jail so it won't be lost when you update SCALE - Choosing a distro (Debian 12 strongly recommended, but Ubuntu, Arch Linux or Rocky Linux seem good choices too) - Optional: configuring the jail so you can run Docker inside it @@ -23,7 +22,7 @@ Despite what the word 'jail' implies, jailmaker's intended use case is to create ## Installation -Create a new dataset called `jailmaker` with the default settings (from TrueNAS web interface). Then login as the root user and download `jlmkr.py`. +[Installation steps with screenshots](https://www.truenas.com/docs/scale/scaletutorials/apps/sandboxes/) are provided on the TrueNAS website. Start by creating a new dataset called `jailmaker` with the default settings (from TrueNAS web interface). Then login as the root user and download `jlmkr.py`. ```shell cd /mnt/mypool/jailmaker @@ -32,9 +31,9 @@ chmod +x jlmkr.py ./jlmkr.py install ``` -The `jlmkr.py` script (and the jails + config it creates) are now stored on the `jailmaker` dataset and will survive updates of TrueNAS SCALE. Additionally a symlink has been created so you can call `jlmkr` from anywhere. +The `jlmkr.py` script (and the jails + config it creates) are now stored on the `jailmaker` dataset and will survive updates of TrueNAS SCALE. Additionally a symlink has been created (if the boot pool is not readonly) so you can call `jlmkr` from anywhere. -After an update of TrueNAS SCALE the symlink will be lost and `systemd-nspawn` (the core package which makes `jailmaker` work) may be gone too. Not to worry, just run `./jlmkr.py install` again or use [the `./jlmkr.py startup` command](#startup-jails-on-boot). +After an update of TrueNAS SCALE the symlink will be lost (but the shell aliases will remain). To restore the symlink, just run `./jlmkr.py install` again or use [the `./jlmkr.py startup` command](#startup-jails-on-boot). ## Usage @@ -51,15 +50,12 @@ After answering a few questions you should have your first jail up and running! ### Startup Jails on Boot ```shell -# Best to call startup directly (not through the jlmkr symlink) +# Call startup using the absolute path to jlmkr.py +# The jlmkr shell alias doesn't work in Init/Shutdown Scripts /mnt/mypool/jailmaker/jlmkr.py startup - -# Can be called from the symlink too... -# But this may not be available after a TrueNAS SCALE update -jlmkr startup ``` -In order to start jails automatically after TrueNAS boots, run `/mnt/mypool/jailmaker/jlmkr.py startup` as Post Init Script with Type `Command` from the TrueNAS web interface. This will automatically fix the installation of `systemd-nspawn` and setup the `jlmkr` symlink, as well as start all the jails with `startup=1` in the config file. Running the `startup` command Post Init is recommended to keep `jailmaker` working after a TrueNAS SCALE update. +In order to start jails automatically after TrueNAS boots, run `/mnt/mypool/jailmaker/jlmkr.py startup` as Post Init Script with Type `Command` from the TrueNAS web interface. This creates the `jlmkr` symlink (if possible), as well as start all the jails with `startup=1` in the config file. ### Start Jail diff --git a/jlmkr.py b/jlmkr.py index 3007d86..1d53c3e 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -4,6 +4,7 @@ import argparse import configparser import contextlib import ctypes +import errno import glob import hashlib import json @@ -1073,7 +1074,9 @@ def create_jail(jail_name, distro="debian", release="bookworm"): """ ) - config += f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" + config += ( + f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" + ) config += cleandoc( """ @@ -1314,6 +1317,58 @@ def list_jails(): return 0 +def replace_or_add_string(file_path, regex, replacement_string): + """ + Replace all occurrences of a regular expression in a file with a given string. + Add the string to the end of the file if regex doesn't match. + + Args: + file_path (str): The path to the file. + regex (str): The regular expression to search for. + replacement_string (str): The string to replace the matches with. + """ + + with open(file_path, "a+") as f: + f.seek(0) + + updated = False + found = False + new_text = "" + replacement_line = f"{replacement_string}\n" + + for line in f: + if not re.match(regex, line): + new_text += line + continue + + found = True + new_text += replacement_line + + if replacement_line != line: + updated = True + + if not new_text.strip(): + # In case of an empty file just write the replacement_string + new_text = replacement_line + updated = True + elif not found: + # Add a newline to the end of the file in case it's not there + if not new_text.endswith("\n"): + new_text += "\n" + # Then add our replacement_string to the end of the file + new_text += replacement_line + updated = True + + # Only overwrite in case there are change to the file + if updated: + f.seek(0) + f.truncate() + f.write(new_text) + return True + + return False + + def install_jailmaker(): # Check if command exists in path if shutil.which("systemd-nspawn"): @@ -1341,19 +1396,41 @@ def install_jailmaker(): for file, original_permission in original_permissions.items(): stat_chmod(file, original_permission) - target = f"/usr/local/sbin/{SYMLINK_NAME}" + symlink = f"/usr/local/sbin/{SYMLINK_NAME}" - # Check if command exists in path - if shutil.which(SYMLINK_NAME): - print(f"The {SYMLINK_NAME} command is available.") - elif not os.path.lexists(target): - print(f"Creating symlink {target} to {SCRIPT_PATH}.") - os.symlink(SCRIPT_PATH, target) - else: + if os.path.lexists(symlink) and not os.path.islink(symlink): print( - f"File {target} already exists... Maybe it's a broken symlink from a previous install attempt?" + f"Unable to create symlink at {symlink}. File already exists but is not a symlink." ) - print(f"Skipped creating new symlink {target} to {SCRIPT_PATH}.") + # Check if the symlink is already pointing to the desired destination + elif os.path.realpath(symlink) != SCRIPT_PATH: + try: + Path(symlink).unlink(missing_ok=True) + os.symlink(SCRIPT_PATH, symlink) + print(f"Created symlink {symlink} to {SCRIPT_PATH}.") + except OSError as e: + if e.errno != errno.EROFS: + raise e + + print( + f"Cannot create symlink because {symlink} is on a readonly filesystem." + ) + + alias = f"alias jlmkr={shlex.quote(SCRIPT_PATH)} # managed by jailmaker" + alias_regex = re.compile(r"^\s*alias jlmkr=.*# managed by jailmaker\s*") + shell_env = os.getenv("SHELL") + + for shell_type in ["bash", "zsh"]: + file = "/root/.bashrc" if shell_type == "bash" else "/root/.zshrc" + + if replace_or_add_string(file, alias_regex, alias): + print(f"Created {shell_type} alias {SYMLINK_NAME}.") + if shell_env.endswith(shell_type): + print( + f"Please source {file} manually for the {SYMLINK_NAME} alias to become effective immediately." + ) + else: + print(f"The {shell_type} alias {SYMLINK_NAME} is already present.") print("Done installing jailmaker.") From a15d5d10f5252251ad80ecb4420858f53ba7d9ea Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 10 Feb 2024 13:21:49 +0100 Subject: [PATCH 37/57] Fix startup --- jlmkr.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 1d53c3e..9c7c164 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -307,7 +307,7 @@ def add_hook(jail_path, systemd_run_additional_args, hook_command, hook_type): systemd_run_additional_args += [f"--property={hook_type}={hook_file}"] -def start_jail(jail_name, check_startup_enabled=False): +def start_jail(jail_name): """ Start jail with given name. """ @@ -315,9 +315,9 @@ def start_jail(jail_name, check_startup_enabled=False): f"Skipped starting jail {jail_name}. It appears to be running already..." ) - if not check_startup_enabled and jail_is_running(jail_name): + if jail_is_running(jail_name): eprint(skip_start_message) - return 1 + return 0 jail_path = get_jail_path(jail_name) jail_config_path = get_jail_config_path(jail_name) @@ -328,18 +328,6 @@ def start_jail(jail_name, check_startup_enabled=False): eprint("Aborting...") return 1 - # Only start if the startup setting is enabled in the config - if check_startup_enabled: - if config.get("startup") == "1": - # We should start this jail based on the startup config... - if jail_is_running(jail_name): - # ...but we can skip if it's already running - eprint(skip_start_message) - return 0 - else: - # Skip starting this jail since the startup config setting is not enabled - return 0 - systemd_run_additional_args = [ f"--unit={SYMLINK_NAME}-{jail_name}", f"--working-directory=./{jail_path}", @@ -1443,12 +1431,17 @@ def startup_jails(): if returncode != 0: eprint("Failed to install jailmaker. Abort startup.") return returncode - + + start_failure = False for jail_name in get_all_jail_names(): - returncode = start_jail(jail_name, True) - eprint(f"Failed to start jail {jail_name}. Abort startup.") - return returncode - + config = parse_config(get_jail_config_path(jail_name)) + if config and config.get("startup") == "1": + if start_jail(jail_name) != 0: + start_failure = True + + if start_failure: + return 1 + return 0 From f16a91f81da70fa6ff387be1bed11cdd322ef660 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 10 Feb 2024 13:42:16 +0100 Subject: [PATCH 38/57] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf7a078..78ca7d7 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ chmod +x jlmkr.py ./jlmkr.py install ``` -The `jlmkr.py` script (and the jails + config it creates) are now stored on the `jailmaker` dataset and will survive updates of TrueNAS SCALE. Additionally a symlink has been created (if the boot pool is not readonly) so you can call `jlmkr` from anywhere. +The `jlmkr.py` script (and the jails + config it creates) are now stored on the `jailmaker` dataset and will survive updates of TrueNAS SCALE. A symlink has been created so you can call `jlmkr` from anywhere (unless the boot pool is readonly, which is the default since SCALE 24.04). Additionally shell aliases have been setup, so you can still call `jlmkr` in an interactive shell (even if the symlink couldn't be created). After an update of TrueNAS SCALE the symlink will be lost (but the shell aliases will remain). To restore the symlink, just run `./jlmkr.py install` again or use [the `./jlmkr.py startup` command](#startup-jails-on-boot). From 8571caa4316efc4cfd54afff381f7c3b5749b3c2 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 11 Feb 2024 18:30:47 +0100 Subject: [PATCH 39/57] Create from config template with initial_setup --- README.md | 8 +- docs/rootless_podman_in_rootless_jail.md | 120 ------ jlmkr.py | 396 ++++++++++++------ templates/docker/README.md | 3 + templates/docker/config | 59 +++ .../incus/README.md | 0 templates/lxd/README.md | 101 +++++ templates/lxd/config | 55 +++ templates/podman/README.md | 121 ++++++ templates/podman/config | 53 +++ 10 files changed, 656 insertions(+), 260 deletions(-) delete mode 100644 docs/rootless_podman_in_rootless_jail.md create mode 100644 templates/docker/README.md create mode 100644 templates/docker/config rename docs/incus_lxd_lxc_kvm.md => templates/incus/README.md (100%) create mode 100644 templates/lxd/README.md create mode 100644 templates/lxd/config create mode 100644 templates/podman/README.md create mode 100644 templates/podman/config diff --git a/README.md b/README.md index 78ca7d7..8d61936 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,13 @@ Creating a jail is interactive. You'll be presented with questions which guide y jlmkr create myjail ``` -After answering a few questions you should have your first jail up and running! +After answering some questions you should have your first jail up and running! + +You may also specify a path to a config template, for a quick and consistent jail creation process. + +```shell +jlmkr create myjail /path/to/config/template +``` ### Startup Jails on Boot diff --git a/docs/rootless_podman_in_rootless_jail.md b/docs/rootless_podman_in_rootless_jail.md deleted file mode 100644 index 9ebdf4f..0000000 --- a/docs/rootless_podman_in_rootless_jail.md +++ /dev/null @@ -1,120 +0,0 @@ -# Rootless podman in rootless Fedora jail - -## Disclaimer - -**These notes are a work in progress. Using podman in this setup hasn't been extensively tested.** - -## Installation - -Prerequisites. Installed jailmaker and setup bridge networking. - -Run `jlmkr create rootless` to create a new jail. During jail creation choose fedora 39. This way we get the most recent version of podman available. Don't enable docker compatibility, we're going to enable only the required options manually. - -Add `--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:65536 --private-users-ownership=chown` when asked for additional systemd-nspawn flags during jail creation. - -We start at UID 524288, as this is the [systemd range used for containers](https://github.com/systemd/systemd/blob/main/docs/UIDS-GIDS.md#summary). - -The `--private-users-ownership=chown` option will ensure the rootfs ownership is corrected. - -After the jail has started run `jlmkr stop rootless && jlmkr edit rootless`, remove `--private-users-ownership=chown` and increase the UID range to `131072` to double the number of UIDs available in the jail. We need more than 65536 UIDs available in the jail, since rootless podman also needs to be able to map UIDs. If I leave the `--private-users-ownership=chown` option I get the following error: - -> systemd-nspawn[678877]: Automatic UID/GID adjusting is only supported for UID/GID ranges starting at multiples of 2^16 with a range of 2^16 - -The flags look like this now: - -``` -systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --system-call-filter='add_key keyctl bpf' --private-users=524288:131072 -``` - -Start the jail with `jlmkr start rootless` and open a shell session inside the jail (as the remapped root user) with `jlmkr shell rootless`. - -Then inside the jail start the network services (wait to get IP address via DHCP) and install podman: -```bash -# systemd-networkd should already be enabled when using jlmkr.py from the develop branch -systemctl --now enable systemd-networkd - -# Add the required capabilities to the `newuidmap` and `newgidmap` binaries. -# https://github.com/containers/podman/issues/2788#issuecomment-1016301663 -# https://github.com/containers/podman/issues/2788#issuecomment-479972943 -# https://github.com/containers/podman/issues/12637#issuecomment-996524341 -setcap cap_setuid+eip /usr/bin/newuidmap -setcap cap_setgid+eip /usr/bin/newgidmap - -# Create new user -adduser rootless -# Set password for user -passwd rootless - -# Clear the subuids and subgids which have been assigned by default when creating the new user -usermod --del-subuids 0-4294967295 --del-subgids 0-4294967295 rootless -# Set a specific range, so it fits inside the number of available UIDs -usermod --add-subuids 65536-131071 --add-subgids 65536-131071 rootless -# Check the assigned range -cat /etc/subuid -# Check the available range -cat /proc/self/uid_map - -dnf -y install podman -exit -``` - -From the TrueNAS host, open a shell as the rootless user inside the jail. - -```bash -jlmkr shell --uid 1000 rootless -``` - -Run rootless podman as user 1000. - -```bash -id -podman run hello-world -podman info -``` - -The output of podman info should contain: - -``` - graphDriverName: overlay - graphOptions: {} - graphRoot: /home/rootless/.local/share/containers/storage - [...] - graphStatus: - Backing Filesystem: zfs - Native Overlay Diff: "true" - Supports d_type: "true" - Supports shifting: "false" - Supports volatile: "true" - Using metacopy: "false" -``` - -## Cockpit management - -Inside the rootless jail run (as root user): - -```bash -dnf install cockpit cockpit-podman -systemctl enable --now cockpit.socket -ip a -``` - -Check the IP address of the jail and access the Cockpit web interface at https://0.0.0.0:9090 where 0.0.0.0 is the IP address you just found using `ip a`. - -Then login as user `rootless` with the password you've created earlier. Click on `Podman containers`. In case it shows `Podman service is not active` then click `Start podman`. You can now manage your rootless podman containers in the rootless jailmaker jail using the Cockpit web GUI. - -## TODO: -On truenas host do: -sudo sysctl net.ipv4.ip_unprivileged_port_start=23 -> Which would prevent a process by your user impersonating the sshd daemon. -Actually make it persistent. - -## Additional resources: - -Resources mentioning `add_key keyctl bpf` -- https://bbs.archlinux.org/viewtopic.php?id=252840 -- https://wiki.archlinux.org/title/systemd-nspawn -- https://discourse.nixos.org/t/podman-docker-in-nixos-container-ideally-in-unprivileged-one/22909/12 -Resources mentioning `@keyring` -- https://github.com/systemd/systemd/issues/17606 -- https://github.com/systemd/systemd/blob/1c62c4fe0b54fb419b875cb2bae82a261518a745/src/shared/seccomp-util.c#L604 -`@keyring` also includes `request_key` but doesn't include `bpf` \ No newline at end of file diff --git a/jlmkr.py b/jlmkr.py index 9c7c164..a8e9fde 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -17,6 +17,7 @@ import shutil import stat import subprocess import sys +import tempfile import time import urllib.request from collections import defaultdict @@ -273,19 +274,21 @@ def stop_jail(jail_name): return subprocess.run(["machinectl", "poweroff", jail_name]).returncode -def parse_config(jail_config_path): +def parse_config_string(config_string): config = configparser.ConfigParser() + # Workaround to read config file without section headers + config.read_string("[DEFAULT]\n" + config_string) + config = dict(config["DEFAULT"]) + return config + + +def parse_config_file(jail_config_path): try: - # Workaround to read config file without section headers - config.read_string("[DEFAULT]\n" + Path(jail_config_path).read_text()) + return parse_config_string(Path(jail_config_path).read_text()) except FileNotFoundError: eprint(f"Unable to find config file: {jail_config_path}.") return - config = dict(config["DEFAULT"]) - - return config - def add_hook(jail_path, systemd_run_additional_args, hook_command, hook_type): if not hook_command: @@ -321,13 +324,53 @@ def start_jail(jail_name): jail_path = get_jail_path(jail_name) jail_config_path = get_jail_config_path(jail_name) + jail_rootfs_path = get_jail_rootfs_path(jail_name) - config = parse_config(jail_config_path) + config = parse_config_file(jail_config_path) if not config: eprint("Aborting...") return 1 + # Handle initial setup + initial_setup = config.get("initial_setup") + + # Alternative method to setup on first boot: + # https://www.undrground.org/2021/01/25/adding-a-single-run-task-via-systemd/ + # If there's no machine-id, then this the first time the jail is started + if initial_setup and not os.path.exists( + os.path.join(jail_rootfs_path, "etc/machine-id") + ): + # Run the command directly if it doesn't start with a shebang + if initial_setup.startswith("#!"): + # Write a script file and call that + initial_setup_file = os.path.abspath( + os.path.join(jail_path, ".initial_setup") + ) + print(initial_setup, file=open(initial_setup_file, "w")) + stat_chmod(initial_setup_file, 0o700) + cmd = [ + "systemd-nspawn", + "-q", + "-D", + jail_rootfs_path, + f"--bind-ro={initial_setup_file}:/root/initial_startup", + "/root/initial_startup", + ] + else: + cmd = ["systemd-nspawn", "-q", "-D", jail_rootfs_path, initial_setup] + + returncode = subprocess.run(cmd).returncode + if returncode != 0: + eprint("Failed to run initial setup:") + eprint(initial_setup) + eprint() + eprint("Abort starting jail.") + return returncode + + # Cleanup the initial_setup_file + Path(initial_setup_file).unlink(missing_ok=True) + systemd_run_additional_args = [ f"--unit={SYMLINK_NAME}-{jail_name}", f"--working-directory=./{jail_path}", @@ -665,11 +708,22 @@ def check_jail_name_available(jail_name, warn=True): return False -def create_jail(jail_name, distro="debian", release="bookworm"): +def ask_jail_name(jail_name=""): + while True: + print() + jail_name = input_with_default("Enter jail name: ", jail_name).strip() + if check_jail_name_valid(jail_name): + if check_jail_name_available(jail_name): + return jail_name + + +def create_jail(jail_name="", config_path=None, distro="debian", release="bookworm"): """ Create jail with given name. """ + config_string = "" + print(DISCLAIMER) if os.path.basename(os.getcwd()) != "jailmaker": @@ -705,58 +759,92 @@ def create_jail(jail_name, distro="debian", release="bookworm"): os.makedirs(JAILS_DIR_PATH, exist_ok=True) stat_chmod(JAILS_DIR_PATH, 0o700) - print() - if not agree(f"Install the recommended image ({distro} {release})?", "y"): - print( - dedent( - f""" - {YELLOW}{BOLD}WARNING: ADVANCED USAGE{NORMAL} + ################# + # Config handling + ################# - You may now choose from a list which distro to install. - But not all of them may work with {COMMAND_NAME} since these images are made for LXC. - Distros based on systemd probably work (e.g. Ubuntu, Arch Linux and Rocky Linux). - """ - ) - ) - input("Press Enter to continue...") + if config_path: + try: + config_string = Path(config_path).read_text() + except FileNotFoundError: + eprint(f"Unable to find file: {config_path}.") + return 1 + else: print() + if agree("Do you wish to create a jail from a config template?", "n"): + print( + dedent( + """ + A text editor will open so you can provide the config template. - returncode = run_lxc_download_script() - if returncode != 0: - return returncode - - print( - dedent( - """ - Choose from the DIST column. - """ + - please copy your config + - paste it into the text editor + - save and close the text editor + """ + ) ) - ) + input("Press Enter to open the text editor.") - distro = input("Distro: ") + with tempfile.NamedTemporaryFile() as f: + subprocess.call([TEXT_EDITOR, f.name]) + f.seek(0) + config_string = f.read().decode() - print( - dedent( - """ - Choose from the RELEASE column (or ARCH if RELEASE is empty). - """ - ) - ) - - release = input("Release: ") - - while True: + if config_string: + config = parse_config_string(config_string) + # Ask for jail name if not provided + if not ( + jail_name + and check_jail_name_valid(jail_name) + and check_jail_name_available(jail_name) + ): + jail_name = ask_jail_name(jail_name) + jail_path = get_jail_path(jail_name) + distro, release = config.get("initial_rootfs_image").split() + else: print() - jail_name = input_with_default("Enter jail name: ", jail_name).strip() - if check_jail_name_valid(jail_name): - if check_jail_name_available(jail_name): - break + if not agree(f"Install the recommended image ({distro} {release})?", "y"): + print( + dedent( + f""" + {YELLOW}{BOLD}WARNING: ADVANCED USAGE{NORMAL} - jail_path = get_jail_path(jail_name) + You may now choose from a list which distro to install. + But not all of them may work with {COMMAND_NAME} since these images are made for LXC. + Distros based on systemd probably work (e.g. Ubuntu, Arch Linux and Rocky Linux). + """ + ) + ) + input("Press Enter to continue...") + print() + + returncode = run_lxc_download_script() + if returncode != 0: + return returncode + + print( + dedent( + """ + Choose from the DIST column. + """ + ) + ) + + distro = input("Distro: ") + + print( + dedent( + """ + Choose from the RELEASE column (or ARCH if RELEASE is empty). + """ + ) + ) + + release = input("Release: ") + + jail_name = ask_jail_name(jail_name) + jail_path = get_jail_path(jail_name) - # Cleanup in except, but only once the jail_path is final - # Otherwise we may cleanup the wrong directory - try: print( dedent( f""" @@ -875,8 +963,106 @@ def create_jail(jail_name, distro="debian", release="bookworm"): ) ) + # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: + # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in + # Use TasksMax=infinity since this is what docker does: + # https://github.com/docker/engine/blob/master/contrib/init/systemd/docker.service + + # Use SYSTEMD_NSPAWN_LOCK=0: otherwise jail won't start jail after a shutdown (but why?) + # Would give "directory tree currently busy" error and I'd have to run + # `rm /run/systemd/nspawn/locks/*` and remove the .lck file from jail_path + # Disabling locking isn't a big deal as systemd-nspawn will prevent starting a container + # with the same name anyway: as long as we're starting jails using this script, + # it won't be possible to start the same jail twice + + systemd_run_default_args = [ + "--property=KillMode=mixed", + "--property=Type=notify", + "--property=RestartForceExitStatus=133", + "--property=SuccessExitStatus=133", + "--property=Delegate=yes", + "--property=TasksMax=infinity", + "--collect", + "--setenv=SYSTEMD_NSPAWN_LOCK=0", + ] + + # Always add --bind-ro=/sys/module to make lsmod happy + # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html + systemd_nspawn_default_args = [ + "--keep-unit", + "--quiet", + "--boot", + "--bind-ro=/sys/module", + "--inaccessible=/sys/module/apparmor", + ] + + systemd_nspawn_user_args_multiline = "\n\t".join( + shlex.split(systemd_nspawn_user_args) + ) + systemd_run_default_args_multiline = "\n\t".join(systemd_run_default_args) + systemd_nspawn_default_args_multiline = "\n\t".join(systemd_nspawn_default_args) + + config_string = cleandoc( + f""" + startup={startup} + docker_compatible={docker_compatible} + gpu_passthrough_intel={gpu_passthrough_intel} + gpu_passthrough_nvidia={gpu_passthrough_nvidia} + """ + ) + + config_string += ( + f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" + ) + + config_string += cleandoc( + """ + # # Specify command/script to run on the HOST before starting the jail + # # For example to load kernel modules and config kernel settings + # pre_start_hook=#!/usr/bin/bash + # echo 'PRE_START_HOOK' + # echo 1 > /proc/sys/net/ipv4/ip_forward + # modprobe br_netfilter + # echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + # echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + # + # # Specify a command/script to run on the HOST after stopping the jail + # post_stop_hook=echo 'POST_STOP_HOOK' + + # Specify command/script to run IN THE JAIL before starting it for the first time + # Useful to install packages on top of the base rootfs + # NOTE: this script will run in the host networking namespace and ignores + # all systemd_nspawn_user_args such as bind mounts + initial_setup=#!/usr/bin/bash + set -euo pipefail + apt-get update && apt-get -y install curl + curl -fsSL https://get.docker.com | sh + """ + ) + + config_string += "\n".join( + [ + "", + "", + "# You generally will not need to change the options below", + f"systemd_run_default_args={systemd_run_default_args_multiline}", + "", + f"systemd_nspawn_default_args={systemd_nspawn_default_args_multiline}", + "", + "# Used by jlmkr create", + f"initial_rootfs_image={distro} {release}", + ] + ) + print() + ############## + # Create start + ############## + + # Cleanup in except, but only once the jail_path is final + # Otherwise we may cleanup the wrong directory + try: jail_config_path = get_jail_config_path(jail_name) jail_rootfs_path = get_jail_rootfs_path(jail_name) @@ -1014,84 +1200,7 @@ def create_jail(jail_name, distro="debian", release="bookworm"): file=open(os.path.join(preset_path, "00-jailmaker.preset"), "w"), ) - # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: - # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in - # Use TasksMax=infinity since this is what docker does: - # https://github.com/docker/engine/blob/master/contrib/init/systemd/docker.service - - # Use SYSTEMD_NSPAWN_LOCK=0: otherwise jail won't start jail after a shutdown (but why?) - # Would give "directory tree currently busy" error and I'd have to run - # `rm /run/systemd/nspawn/locks/*` and remove the .lck file from jail_path - # Disabling locking isn't a big deal as systemd-nspawn will prevent starting a container - # with the same name anyway: as long as we're starting jails using this script, - # it won't be possible to start the same jail twice - - systemd_run_default_args = [ - "--property=KillMode=mixed", - "--property=Type=notify", - "--property=RestartForceExitStatus=133", - "--property=SuccessExitStatus=133", - "--property=Delegate=yes", - "--property=TasksMax=infinity", - "--collect", - "--setenv=SYSTEMD_NSPAWN_LOCK=0", - ] - - # Always add --bind-ro=/sys/module to make lsmod happy - # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html - systemd_nspawn_default_args = [ - "--keep-unit", - "--quiet", - "--boot", - "--bind-ro=/sys/module", - "--inaccessible=/sys/module/apparmor", - ] - - systemd_nspawn_user_args_multiline = "\n\t".join( - shlex.split(systemd_nspawn_user_args) - ) - systemd_run_default_args_multiline = "\n\t".join(systemd_run_default_args) - systemd_nspawn_default_args_multiline = "\n\t".join(systemd_nspawn_default_args) - - config = cleandoc( - f""" - startup={startup} - docker_compatible={docker_compatible} - gpu_passthrough_intel={gpu_passthrough_intel} - gpu_passthrough_nvidia={gpu_passthrough_nvidia} - """ - ) - - config += ( - f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" - ) - - config += cleandoc( - """ - # Specify command/script to run on the HOST before starting the jail - pre_start_hook=echo 'PRE_START_HOOK' - - # Specify a command/script to run on the HOST after stopping the jail - post_stop_hook=#!/usr/bin/bash - echo 'POST STOP HOOK' - """ - ) - - config += "\n".join( - [ - "", - "", - "# You generally will not need to change the options below", - f"systemd_run_default_args={systemd_run_default_args_multiline}", - "", - f"systemd_nspawn_default_args={systemd_nspawn_default_args_multiline}", - "", - "# The below is for reference only, currently not used", - f"initial_rootfs_image={distro} {release}", - ] - ) - - print(config, file=open(jail_config_path, "w")) + print(config_string, file=open(jail_config_path, "w")) os.chmod(jail_config_path, 0o600) @@ -1100,6 +1209,13 @@ def create_jail(jail_name, distro="debian", release="bookworm"): cleanup(jail_path) raise error + # In case you want to create a jail without any user interaction, + # you need to skip this final question + # echo 'y' | jlmkr create test testconfig + # TODO: make jlmkr create work cleanly without user interaction. + # Current echo 'y' workaround may cause problems when the jail name already exists + # You'd end up with a new jail called 'y' + # and the script will crash at the agree statement below print() if agree(f"Do you want to start jail {jail_name} right now?", "y"): return start_jail(jail_name) @@ -1286,7 +1402,7 @@ def list_jails(): # TODO: add additional properties from the jails config file for jail_name in jails: - config = parse_config(get_jail_config_path(jail_name)) + config = parse_config_file(get_jail_config_path(jail_name)) startup = False if config: @@ -1431,17 +1547,17 @@ def startup_jails(): if returncode != 0: eprint("Failed to install jailmaker. Abort startup.") return returncode - + start_failure = False for jail_name in get_all_jail_names(): - config = parse_config(get_jail_config_path(jail_name)) + config = parse_config_file(get_jail_config_path(jail_name)) if config and config.get("startup") == "1": if start_jail(jail_name) != 0: start_failure = True - + if start_failure: return 1 - + return 0 @@ -1463,9 +1579,11 @@ def main(): help="install jailmaker dependencies and create symlink", ) - subparsers.add_parser( + create_parser = subparsers.add_parser( name="create", epilog=DISCLAIMER, help="create a new jail" - ).add_argument("name", nargs="?", help="name of the jail") + ) + create_parser.add_argument("name", nargs="?", help="name of the jail") + create_parser.add_argument("config", nargs="?", help="path to config file template") subparsers.add_parser( name="start", epilog=DISCLAIMER, help="start a previously created jail" @@ -1539,7 +1657,7 @@ def main(): sys.exit(install_jailmaker()) elif args.subcommand == "create": - sys.exit(create_jail(args.name)) + sys.exit(create_jail(args.name, args.config)) elif args.subcommand == "start": sys.exit(start_jail(args.name)) @@ -1580,7 +1698,7 @@ def main(): else: if agree("Create a new jail?", "y"): print() - sys.exit(create_jail("")) + sys.exit(create_jail()) else: parser.print_usage() diff --git a/templates/docker/README.md b/templates/docker/README.md new file mode 100644 index 0000000..fc61bea --- /dev/null +++ b/templates/docker/README.md @@ -0,0 +1,3 @@ +# Debian Docker Jail Template + +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mydockerjail /mnt/tank/path/to/docker/config`. \ No newline at end of file diff --git a/templates/docker/config b/templates/docker/config new file mode 100644 index 0000000..9c7e428 --- /dev/null +++ b/templates/docker/config @@ -0,0 +1,59 @@ +startup=0 +gpu_passthrough_intel=0 +gpu_passthrough_nvidia=0 + +# Use macvlan networking to provide an isolated network namespace, +# so docker can manage firewall rules +# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Ensure to change eno1/br1 to the interface name you want to use +# You may want to add additional options here, e.g. bind mounts +systemd_nspawn_user_args=--network-macvlan=eno1 + --resolv-conf=bind-host + --system-call-filter='add_key keyctl bpf' + +# Script to run on the HOST before starting the jail +# Load kernel module and config kernel settings required for docker +pre_start_hook=#!/usr/bin/bash + echo 'PRE_START_HOOK' + echo 1 > /proc/sys/net/ipv4/ip_forward + modprobe br_netfilter + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + +# Install docker inside the jail: +# https://docs.docker.com/engine/install/debian/#install-using-the-repository +# NOTE: this script will run in the host networking namespace and ignores +# all systemd_nspawn_user_args such as bind mounts +initial_setup=#!/usr/bin/bash + set -euo pipefail + + apt-get update && apt-get -y install ca-certificates curl + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 + +systemd_nspawn_default_args=--keep-unit + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor + +# Used by jlmkr create +initial_rootfs_image=debian bookworm \ No newline at end of file diff --git a/docs/incus_lxd_lxc_kvm.md b/templates/incus/README.md similarity index 100% rename from docs/incus_lxd_lxc_kvm.md rename to templates/incus/README.md diff --git a/templates/lxd/README.md b/templates/lxd/README.md new file mode 100644 index 0000000..b69c2e2 --- /dev/null +++ b/templates/lxd/README.md @@ -0,0 +1,101 @@ +# Ubuntu LXD Jail Template + +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mylxdjail /mnt/tank/path/to/lxd/config`. + +Unfortunately snapd doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: + +```bash +jlmkr exec mylxdjail bash -c 'apt-get update && + apt-get install -y --no-install-recommends snapd && + snap install lxd' + +# Answer yes when asked the following: +# Would you like the LXD server to be available over the network? (yes/no) [default=no]: yes +# TODO: fix ZFS +jlmkr exec mylxdjail bash -c 'lxd init && + snap set lxd ui.enable=true && + systemctl reload snap.lxd.daemon' +``` + +Then visit the `lxd` GUI inside the browser https://0.0.0.0:8443. To find out which IP address to use instead of 0.0.0.0, check the IP address for your jail with `jlmkr list`. + +## Disclaimer + +**These notes are a work in progress. Using Incus in this setup hasn't been extensively tested.** + +## Installation + +Create a debian 12 jail and [install incus](https://github.com/zabbly/incus#installation). Also install the `incus-ui-canonical` package to install the web interface. Ensure the config file looks like the below: + +Run `modprobe vhost_vsock` on the TrueNAS host. + +``` +startup=0 +docker_compatible=1 +gpu_passthrough_intel=1 +gpu_passthrough_nvidia=0 +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 +systemd_nspawn_default_args=--keep-unit --quiet --boot --bind-ro=/sys/module --inaccessible=/sys/module/apparmor +``` + +Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). + +## Create Ubuntu Desktop VM + +Incus web GUI should be running on port 8443. Create new instance, call it `dekstop`, and choose the `Ubuntu jammy desktop virtual-machine ubuntu/22.04/desktop` image. + +## Bind mount / virtiofs + +To access files from the TrueNAS host directly in a VM created with incus, we can use virtiofs. + +```bash +incus config device add desktop test disk source=/home/test/ path=/mnt/test +``` + +The command above (when ran as root user inside the incus jail) adds a new virtiofs mount of a test directory inside the jail to a VM named desktop. The `/home/test` dir resides in the jail, but you can first bind mount any directory from the TrueNAS host inside the incus jail and then forward this to the VM using virtiofs. This could be an alternative to NFS mounts. + +### Benchmarks + +#### Inside LXD ubuntu desktop VM with virtiofs mount +root@desktop:/mnt/test# mount | grep test +incus_test on /mnt/test type virtiofs (rw,relatime) +root@desktop:/mnt/test# time iozone -a +[...] +real 2m22.389s +user 0m2.222s +sys 0m59.275s + +#### In a jailmaker jail on the host: +root@incus:/home/test# time iozone -a +[...] +real 0m59.486s +user 0m1.468s +sys 0m25.458s + +#### Inside LXD ubuntu desktop VM with virtiofs mount +root@desktop:/mnt/test# dd if=/dev/random of=./test1.img bs=1G count=1 oflag=dsync +1+0 records in +1+0 records out +1073741824 bytes (1.1 GB, 1.0 GiB) copied, 36.321 s, 29.6 MB/s + +#### In a jailmaker jail on the host: +root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsync +1+0 records in +1+0 records out +1073741824 bytes (1.1 GB, 1.0 GiB) copied, 7.03723 s, 153 MB/s + +## Create Ubuntu container + +To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](../podman/README.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. + +## Canonical LXD install via snap + +Installing the lxd snap is an alternative to Incus. But out of the box running `snap install lxd` will cause AppArmor issues when running inside a jailmaker jail on SCALE. + + + +## References + +- [Running QEMU/KVM Virtual Machines in Unprivileged LXD Containers](https://dshcherb.github.io/2017/12/04/qemu-kvm-virtual-machines-in-unprivileged-lxd.html) \ No newline at end of file diff --git a/templates/lxd/config b/templates/lxd/config new file mode 100644 index 0000000..1ec066e --- /dev/null +++ b/templates/lxd/config @@ -0,0 +1,55 @@ +startup=0 +gpu_passthrough_intel=1 +gpu_passthrough_nvidia=0 + +# Use macvlan networking to provide an isolated network namespace, +# so lxd can manage firewall rules +# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Ensure to change eno1/br1 to the interface name you want to use +# You may want to add additional options here, e.g. bind mounts +# TODO: don't use --capability=all but specify only the required capabilities +systemd_nspawn_user_args=--network-macvlan=eno1 + --resolv-conf=bind-host + --capability=all + --bind=/dev/fuse + --bind=/dev/kvm + --bind=/dev/vsock + --bind=/dev/vhost-vsock + +# Script to run on the HOST before starting the jail +# Load kernel module and config kernel settings required for lxd +pre_start_hook=#!/usr/bin/bash + echo 'PRE_START_HOOK' + echo 1 > /proc/sys/net/ipv4/ip_forward + modprobe br_netfilter + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + modprobe vhost_vsock + +# NOTE: this script will run in the host networking namespace and ignores +# all systemd_nspawn_user_args such as bind mounts +initial_setup=#!/usr/bin/bash + # https://discuss.linuxcontainers.org/t/snap-inside-privileged-lxd-container/13691/8 + ln -s /bin/true /usr/local/bin/udevadm + +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 +# TODO: check if the below 2 are required +# --setenv=SYSTEMD_SECCOMP=0 +# --property=DevicePolicy=auto + +systemd_nspawn_default_args=--keep-unit + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor + +# Used by jlmkr create +initial_rootfs_image=ubuntu jammy \ No newline at end of file diff --git a/templates/podman/README.md b/templates/podman/README.md new file mode 100644 index 0000000..dae1c6f --- /dev/null +++ b/templates/podman/README.md @@ -0,0 +1,121 @@ +# Fedora Podman Jail Template + +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mypodmanjail /mnt/tank/path/to/podman/config`. + +## Rootless + +### Disclaimer + +**These notes are a work in progress. Using podman in this setup hasn't been extensively tested.** + +### Installation + +Prerequisites: created a jail using the [config](./config) template file. + +Run `jlmkr edit mypodmanjail` and add `--private-users=524288:65536 --private-users-ownership=chown` to `systemd_nspawn_user_args`. We start at UID 524288, as this is the [systemd range used for containers](https://github.com/systemd/systemd/blob/main/docs/UIDS-GIDS.md#summary). + +The `--private-users-ownership=chown` option will ensure the rootfs ownership is corrected. + +After the jail has started run `jlmkr stop mypodmanjail && jlmkr edit mypodmanjail`, remove `--private-users-ownership=chown` and increase the UID range to `131072` to double the number of UIDs available in the jail. We need more than 65536 UIDs available in the jail, since rootless podman also needs to be able to map UIDs. If I leave the `--private-users-ownership=chown` option I get the following error: + +> systemd-nspawn[678877]: Automatic UID/GID adjusting is only supported for UID/GID ranges starting at multiples of 2^16 with a range of 2^16 + +The flags look like this now: + +``` +systemd_nspawn_user_args=--network-macvlan=eno1 + --resolv-conf=bind-host + --system-call-filter='add_key keyctl bpf' + --private-users=524288:131072 +``` + +Start the jail with `jlmkr start mypodmanjail` and open a shell session inside the jail (as the remapped root user) with `jlmkr shell mypodmanjail`. + +Then inside the jail setup the new rootless user: + +```bash +# Create new user +adduser rootless +# Set password for user +passwd rootless + +# Clear the subuids and subgids which have been assigned by default when creating the new user +usermod --del-subuids 0-4294967295 --del-subgids 0-4294967295 rootless +# Set a specific range, so it fits inside the number of available UIDs +usermod --add-subuids 65536-131071 --add-subgids 65536-131071 rootless + +# Check the assigned range +cat /etc/subuid +# Check the available range +cat /proc/self/uid_map + +exit +``` + +From the TrueNAS host, open a shell as the rootless user inside the jail. + +```bash +jlmkr shell --uid 1000 mypodmanjail +``` + +Run rootless podman as user 1000. + +```bash +id +podman run hello-world +podman info +``` + +The output of podman info should contain: + +``` + graphDriverName: overlay + graphOptions: {} + graphRoot: /home/rootless/.local/share/containers/storage + [...] + graphStatus: + Backing Filesystem: zfs + Native Overlay Diff: "true" + Supports d_type: "true" + Supports shifting: "false" + Supports volatile: "true" + Using metacopy: "false" +``` + +### Binding to Privileged Ports: + +Add `sysctl net.ipv4.ip_unprivileged_port_start=23` to the `pre_start_hook` inside the config to lower the range of privileged ports. This will still prevent an unprivileged process from impersonating the sshd daemon. Since this lowers the range globally on the TrueNAS host, a better solution would be to specifically add the capability to bind to privileged ports. + +## Cockpit Management + +Install and enable cockpit: + +```bash +jlmkr exec mypodmanjail bash -c "dnf -y install cockpit cockpit-podman && \ + systemctl enable --now cockpit.socket && \ + ip a && + ip route | awk '/default/ { print \$9 }'" +``` + +Check the IP address of the jail and access the Cockpit web interface at https://0.0.0.0:9090 where 0.0.0.0 is the IP address you just found using `ip a`. + +If you've setup the `rootless` user, you may login with the password you've created earlier. Otherwise you'd have to add an admin user first: + +```bash +jlmkr exec podmantest bash -c 'adduser admin +passwd admin +usermod -aG wheel admin' +``` + +Click on `Podman containers`. In case it shows `Podman service is not active` then click `Start podman`. You can now manage your (rootless) podman containers in the (rootless) jailmaker jail using the Cockpit web GUI. + +## Additional Resources: + +Resources mentioning `add_key keyctl bpf` +- https://bbs.archlinux.org/viewtopic.php?id=252840 +- https://wiki.archlinux.org/title/systemd-nspawn +- https://discourse.nixos.org/t/podman-docker-in-nixos-container-ideally-in-unprivileged-one/22909/12 +Resources mentioning `@keyring` +- https://github.com/systemd/systemd/issues/17606 +- https://github.com/systemd/systemd/blob/1c62c4fe0b54fb419b875cb2bae82a261518a745/src/shared/seccomp-util.c#L604 +`@keyring` also includes `request_key` but doesn't include `bpf` \ No newline at end of file diff --git a/templates/podman/config b/templates/podman/config new file mode 100644 index 0000000..ee8f2c7 --- /dev/null +++ b/templates/podman/config @@ -0,0 +1,53 @@ +startup=0 +gpu_passthrough_intel=0 +gpu_passthrough_nvidia=0 + +# Use macvlan networking to provide an isolated network namespace, +# so podman can manage firewall rules +# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Ensure to change eno1/br1 to the interface name you want to use +# You may want to add additional options here, e.g. bind mounts +systemd_nspawn_user_args=--network-macvlan=eno1 + --resolv-conf=bind-host + --system-call-filter='add_key keyctl bpf' + +# Script to run on the HOST before starting the jail +# Load kernel module and config kernel settings required for podman +pre_start_hook=#!/usr/bin/bash + echo 'PRE_START_HOOK' + echo 1 > /proc/sys/net/ipv4/ip_forward + modprobe br_netfilter + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + +# Install podman inside the jail +# NOTE: this script will run in the host networking namespace and ignores +# all systemd_nspawn_user_args such as bind mounts + +initial_setup=#!/usr/bin/bash + set -euo pipefail + dnf -y install podman + # Add the required capabilities to the `newuidmap` and `newgidmap` binaries + # https://github.com/containers/podman/issues/2788#issuecomment-1016301663 + # https://github.com/containers/podman/issues/12637#issuecomment-996524341 + setcap cap_setuid+eip /usr/bin/newuidmap + setcap cap_setgid+eip /usr/bin/newgidmap + +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 + +systemd_nspawn_default_args=--keep-unit + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor + +# Used by jlmkr create +initial_rootfs_image=fedora 39 \ No newline at end of file From ba74d5d3bd1af502a7fc6c46c2024d96554e919b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 11 Feb 2024 18:32:33 +0100 Subject: [PATCH 40/57] Update README.md --- docs/{wikimain.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{wikimain.md => README.md} (100%) diff --git a/docs/wikimain.md b/docs/README.md similarity index 100% rename from docs/wikimain.md rename to docs/README.md From 0d742e8a904a7fa9359004243f264c02ae11717f Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Tue, 13 Feb 2024 21:01:28 +0100 Subject: [PATCH 41/57] Add Incus template --- templates/incus/README.md | 67 +++---------------------------------- templates/incus/config | 70 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 62 deletions(-) create mode 100644 templates/incus/config diff --git a/templates/incus/README.md b/templates/incus/README.md index 39c9491..043dc02 100644 --- a/templates/incus/README.md +++ b/templates/incus/README.md @@ -1,37 +1,14 @@ -# Incus / LXD / LXC / KVM inside jail +# Debian Incus Jail Template (LXD / LXC / KVM) + +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create myincusjail /mnt/tank/path/to/incus/config`. Then check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). ## Disclaimer **These notes are a work in progress. Using Incus in this setup hasn't been extensively tested.** -## Prerequisites - -- TrueNAS SCALE 23.10 installed bare metal (not inside VM) -- Jailmaker installed -- Setup bridge networking (see Advanced Networking in the readme) - -## Installation - -Create a debian 12 jail and [install incus](https://github.com/zabbly/incus#installation). Also install the `incus-ui-canonical` package to install the web interface. Ensure the config file looks like the below: - -Run `modprobe vhost_vsock` on the TrueNAS host. - -``` -startup=0 -docker_compatible=1 -gpu_passthrough_intel=1 -gpu_passthrough_nvidia=0 -systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock -# You generally will not need to change the options below -systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 -systemd_nspawn_default_args=--keep-unit --quiet --boot --bind-ro=/sys/module --inaccessible=/sys/module/apparmor -``` - -Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). - ## Create Ubuntu Desktop VM -Incus web GUI should be running on port 8443. Create new instance, call it `dekstop`, and choose the `Ubuntu jammy desktop virtual-machine ubuntu/22.04/desktop` image. +Incus web GUI should be running on port 8443. Create new instance, call it `desktop`, and choose the `Ubuntu jammy desktop virtual-machine ubuntu/22.04/desktop` image. ## Bind mount / virtiofs @@ -75,41 +52,7 @@ root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsyn ## Create Ubuntu container -To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](rootless_podman_in_rootless_jail.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. - -## Canonical LXD install via snap - -Installing the lxd snap is an alternative to Incus. But out of the box running `snap install lxd` will cause AppArmor issues when running inside a jailmaker jail on SCALE. - -### Workaround 1: Disable AppArmor kernel module - -[To my knowledge AppArmor is not uses on SCALE](https://github.com/truenas/charts/pull/428#issuecomment-1113936420). The AppArmor related packages aren't even installed. - -Ensure to add --bind=/dev/fuse and ensure using bridge or macvlan networking: - -``` -# On the host -cat /sys/module/apparmor/parameters/enabled -Y -midclt call system.advanced.update '{"kernel_extra_options": "apparmor=0"}' -reboot -cat /sys/module/apparmor/parameters/enabled - -# In Ubuntu jail -apt update -ln -s /bin/true /usr/local/bin/udevadm -apt install -y --no-install-recommends snapd -snap install lxd -lxd init -snap set lxd ui.enable=true -systemctl reload snap.lxd.daemon - -# Check out: https://example:8443 -``` - -### Workaround 2: inaccessible /sys/module/apparmor - -If I don't want to mess with kernel parameters, I can trick the jail into thinking the apparmor module is not loaded by mounting over /sys/module/apparmor: `mount -v -r -t tmpfs -o size=50m test /sys/module/apparmor`. Then `snap install lxd` completes! Best way to do this is to add `--inaccessible=/sys/module/apparmor` to the systemd_nspawn_user_args. +To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](../podman/README.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. ## References diff --git a/templates/incus/config b/templates/incus/config new file mode 100644 index 0000000..94092f5 --- /dev/null +++ b/templates/incus/config @@ -0,0 +1,70 @@ +startup=0 +gpu_passthrough_intel=1 +gpu_passthrough_nvidia=0 + +# Use macvlan networking to provide an isolated network namespace, +# so incus can manage firewall rules +# Alternatively use --network-bridge=br1 instead of --network-macvlan +# Ensure to change eno1/br1 to the interface name you want to use +# You may want to add additional options here, e.g. bind mounts +# TODO: don't use --capability=all but specify only the required capabilities +systemd_nspawn_user_args=--network-macvlan=eno1 + --resolv-conf=bind-host + --capability=all + --bind=/dev/fuse + --bind=/dev/kvm + --bind=/dev/vsock + --bind=/dev/vhost-vsock + +# Script to run on the HOST before starting the jail +# Load kernel module and config kernel settings required for incus +pre_start_hook=#!/usr/bin/bash + echo 'PRE_START_HOOK' + echo 1 > /proc/sys/net/ipv4/ip_forward + modprobe br_netfilter + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + modprobe vhost_vsock + +# Install incus according to: +# https://github.com/zabbly/incus#installation +# NOTE: this script will run in the host networking namespace and ignores +# all systemd_nspawn_user_args such as bind mounts +initial_setup=#!/usr/bin/bash + mkdir -p /etc/apt/keyrings/ + curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc + sh -c 'cat < /etc/apt/sources.list.d/zabbly-incus-stable.sources + Enabled: yes + Types: deb + URIs: https://pkgs.zabbly.com/incus/stable + Suites: $(. /etc/os-release && echo ${VERSION_CODENAME}) + Components: main + Architectures: $(dpkg --print-architecture) + Signed-By: /etc/apt/keyrings/zabbly.asc + + EOF' + apt-get update + apt-get -y install incus incus-ui-canonical + +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 +# TODO: check if the below 2 are required +# --setenv=SYSTEMD_SECCOMP=0 +# --property=DevicePolicy=auto +# TODO: add and use privileged flag? + +systemd_nspawn_default_args=--keep-unit + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor + +# Used by jlmkr create +initial_rootfs_image=debian bookworm \ No newline at end of file From dd60c6a6f6fd2e5f89596746518e9f84b58eef97 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 15 Feb 2024 17:23:57 +0100 Subject: [PATCH 42/57] Update templates --- templates/docker/README.md | 2 + templates/incus/README.md | 29 +++++++++-- templates/incus/config | 8 +-- templates/lxd/README.md | 100 +++++++++++++++---------------------- templates/lxd/config | 9 ++-- templates/podman/README.md | 4 +- 6 files changed, 81 insertions(+), 71 deletions(-) diff --git a/templates/docker/README.md b/templates/docker/README.md index fc61bea..29212ce 100644 --- a/templates/docker/README.md +++ b/templates/docker/README.md @@ -1,3 +1,5 @@ # Debian Docker Jail Template +## Setup + Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mydockerjail /mnt/tank/path/to/docker/config`. \ No newline at end of file diff --git a/templates/incus/README.md b/templates/incus/README.md index 043dc02..36d0add 100644 --- a/templates/incus/README.md +++ b/templates/incus/README.md @@ -1,10 +1,33 @@ # Debian Incus Jail Template (LXD / LXC / KVM) -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create myincusjail /mnt/tank/path/to/incus/config`. Then check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). - ## Disclaimer -**These notes are a work in progress. Using Incus in this setup hasn't been extensively tested.** +**Experimental. Using Incus in this setup hasn't been extensively tested and has [known issues](#known-issues).** + +## Setup + +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create myincusjail /mnt/tank/path/to/incus/config`. + +Unfortunately incus doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: + +```bash +jlmkr exec myincusjail bash -c 'apt-get -y install incus incus-ui-canonical && + incus admin init' +``` + +Follow [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). + +Then visit the Incus GUI inside the browser https://0.0.0.0:8443. To find out which IP address to use instead of 0.0.0.0, check the IP address for your jail with `jlmkr list`. + +## Known Issues + +Using Incus in the jail will cause the following error when starting a VM from the TrueNAS SCALE web GUI: + +``` +[EFAULT] internal error: process exited while connecting to monitor: Could not access KVM kernel module: Permission denied 2024-02-16T14:40:14.886658Z qemu-system-x86_64: -accel kvm: failed to initialize kvm: Permission denied +``` + +A reboot will resolve the issue (until you start the Incus jail again). ## Create Ubuntu Desktop VM diff --git a/templates/incus/config b/templates/incus/config index 94092f5..0c19790 100644 --- a/templates/incus/config +++ b/templates/incus/config @@ -8,6 +8,7 @@ gpu_passthrough_nvidia=0 # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts # TODO: don't use --capability=all but specify only the required capabilities +# TODO: or add and use privileged flag? systemd_nspawn_user_args=--network-macvlan=eno1 --resolv-conf=bind-host --capability=all @@ -31,6 +32,8 @@ pre_start_hook=#!/usr/bin/bash # NOTE: this script will run in the host networking namespace and ignores # all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash + set -euo pipefail + apt-get update && apt-get -y install curl mkdir -p /etc/apt/keyrings/ curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc sh -c 'cat < /etc/apt/sources.list.d/zabbly-incus-stable.sources @@ -44,7 +47,6 @@ initial_setup=#!/usr/bin/bash EOF' apt-get update - apt-get -y install incus incus-ui-canonical # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed @@ -55,10 +57,8 @@ systemd_run_default_args=--property=KillMode=mixed --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 -# TODO: check if the below 2 are required -# --setenv=SYSTEMD_SECCOMP=0 +# TODO: add below if required: # --property=DevicePolicy=auto -# TODO: add and use privileged flag? systemd_nspawn_default_args=--keep-unit --quiet diff --git a/templates/lxd/README.md b/templates/lxd/README.md index b69c2e2..03e3026 100644 --- a/templates/lxd/README.md +++ b/templates/lxd/README.md @@ -1,17 +1,29 @@ # Ubuntu LXD Jail Template +## Disclaimer + +**Experimental. Using LXD in this setup hasn't been extensively tested and has [known issues](#known-issues).** + +## Setup + Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mylxdjail /mnt/tank/path/to/lxd/config`. Unfortunately snapd doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: ```bash +# Repeat listing the jail until you see it has an IPv4 address +jlmkr list + +# Install packages jlmkr exec mylxdjail bash -c 'apt-get update && apt-get install -y --no-install-recommends snapd && snap install lxd' -# Answer yes when asked the following: -# Would you like the LXD server to be available over the network? (yes/no) [default=no]: yes -# TODO: fix ZFS +``` + +Choose the `dir` storage backend during `lxd init` and answer `yes` to "Would you like the LXD server to be available over the network?" + +```bash jlmkr exec mylxdjail bash -c 'lxd init && snap set lxd ui.enable=true && systemctl reload snap.lxd.daemon' @@ -19,82 +31,52 @@ jlmkr exec mylxdjail bash -c 'lxd init && Then visit the `lxd` GUI inside the browser https://0.0.0.0:8443. To find out which IP address to use instead of 0.0.0.0, check the IP address for your jail with `jlmkr list`. -## Disclaimer +## Known Issues -**These notes are a work in progress. Using Incus in this setup hasn't been extensively tested.** +### Instance creation failed -## Installation - -Create a debian 12 jail and [install incus](https://github.com/zabbly/incus#installation). Also install the `incus-ui-canonical` package to install the web interface. Ensure the config file looks like the below: - -Run `modprobe vhost_vsock` on the TrueNAS host. +[LXD no longer has access to the LinuxContainers image server](https://discuss.linuxcontainers.org/t/important-notice-for-lxd-users-image-server/18479). ``` -startup=0 -docker_compatible=1 -gpu_passthrough_intel=1 -gpu_passthrough_nvidia=0 -systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --bind=/dev/fuse --bind=/dev/kvm --bind=/dev/vsock --bind=/dev/vhost-vsock -# You generally will not need to change the options below -systemd_run_default_args=--property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Delegate=yes --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 -systemd_nspawn_default_args=--keep-unit --quiet --boot --bind-ro=/sys/module --inaccessible=/sys/module/apparmor +Failed getting remote image info: Failed getting image: The requested image couldn't be found for fingerprint "ubuntu/focal/desktop" ``` -Check out [First steps with Incus](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/). +### SCALE Virtual Machines +Using LXD in the jail will cause the following error when starting a VM from the TrueNAS SCALE web GUI: -## Create Ubuntu Desktop VM - -Incus web GUI should be running on port 8443. Create new instance, call it `dekstop`, and choose the `Ubuntu jammy desktop virtual-machine ubuntu/22.04/desktop` image. - -## Bind mount / virtiofs - -To access files from the TrueNAS host directly in a VM created with incus, we can use virtiofs. - -```bash -incus config device add desktop test disk source=/home/test/ path=/mnt/test +``` +[EFAULT] internal error: process exited while connecting to monitor: Could not access KVM kernel module: Permission denied 2024-02-16T14:40:14.886658Z qemu-system-x86_64: -accel kvm: failed to initialize kvm: Permission denied ``` -The command above (when ran as root user inside the incus jail) adds a new virtiofs mount of a test directory inside the jail to a VM named desktop. The `/home/test` dir resides in the jail, but you can first bind mount any directory from the TrueNAS host inside the incus jail and then forward this to the VM using virtiofs. This could be an alternative to NFS mounts. +A reboot will resolve the issue (until you start the LXD jail again). -### Benchmarks +### ZFS Issues -#### Inside LXD ubuntu desktop VM with virtiofs mount -root@desktop:/mnt/test# mount | grep test -incus_test on /mnt/test type virtiofs (rw,relatime) -root@desktop:/mnt/test# time iozone -a -[...] -real 2m22.389s -user 0m2.222s -sys 0m59.275s +If you create a new dataset on your pool (e.g. `tank`) called `lxd` from the TrueNAS SCALE web GUI and tell LXD to use it during `lxd init`, then you will run into issues. Firstly you'd have to run `apt-get install -y --no-install-recommends zfsutils-linux` inside the jail to install the ZFS userspace utils and you've have to add `--bind=/dev/zfs` to the `systemd_nspawn_user_args` in the jail config. By mounting `/dev/zfs` into this jail, **it will have total control of the storage on the host!** -#### In a jailmaker jail on the host: -root@incus:/home/test# time iozone -a -[...] -real 0m59.486s -user 0m1.468s -sys 0m25.458s +But then SCALE doesn't seem to like the ZFS datasets created by LXD. I get the following errors when browsing the sub-datasets: -#### Inside LXD ubuntu desktop VM with virtiofs mount -root@desktop:/mnt/test# dd if=/dev/random of=./test1.img bs=1G count=1 oflag=dsync -1+0 records in -1+0 records out -1073741824 bytes (1.1 GB, 1.0 GiB) copied, 36.321 s, 29.6 MB/s +``` +[EINVAL] legacy: path must be absolute +``` -#### In a jailmaker jail on the host: -root@incus:/home/test# dd if=/dev/random of=./test2.img bs=1G count=1 oflag=dsync -1+0 records in -1+0 records out -1073741824 bytes (1.1 GB, 1.0 GiB) copied, 7.03723 s, 153 MB/s +``` +[EFAULT] Failed retreiving USER quotas for tank/lxd/virtual-machines +``` -## Create Ubuntu container +As long as you don't operate on these datasets in the SCALE GUI this may not be a real problem... -To be able to create unprivileged (rootless) containers with incus inside the jail, you need to increase the amount of UIDs available inside the jail. Please refer to the [Podman instructions](../podman/README.md) for more information. If you don't increase the UIDs you can only create privileged containers. You'd have to change `Privileged` to `Allow` in `Security policies` in this case. +However, creating an LXD VM doesn't work with the ZFS storage backend (creating a container works though): -## Canonical LXD install via snap +``` +Failed creating instance from image: Could not locate a zvol for tank/lxd/images/1555b13f0e89bfcf516bd0090eee6f73a0db5f4d0d36c38cae94316de82bf817.block +``` -Installing the lxd snap is an alternative to Incus. But out of the box running `snap install lxd` will cause AppArmor issues when running inside a jailmaker jail on SCALE. +Could this be the same issue as [Instance creation failed](#instance-creation-failed)? +## More info +Refer to the [Incus README](../incus/README.md) as a lot of it applies to LXD too. ## References diff --git a/templates/lxd/config b/templates/lxd/config index 1ec066e..db6c1fc 100644 --- a/templates/lxd/config +++ b/templates/lxd/config @@ -8,7 +8,8 @@ gpu_passthrough_nvidia=0 # Ensure to change eno1/br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts # TODO: don't use --capability=all but specify only the required capabilities -systemd_nspawn_user_args=--network-macvlan=eno1 +# TODO: or add and use privileged flag? +systemd_nspawn_user_args=--network-bridge=br1 --resolv-conf=bind-host --capability=all --bind=/dev/fuse @@ -29,8 +30,9 @@ pre_start_hook=#!/usr/bin/bash # NOTE: this script will run in the host networking namespace and ignores # all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash + set -euo pipefail # https://discuss.linuxcontainers.org/t/snap-inside-privileged-lxd-container/13691/8 - ln -s /bin/true /usr/local/bin/udevadm + ln -sf /bin/true /usr/local/bin/udevadm # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed @@ -41,8 +43,7 @@ systemd_run_default_args=--property=KillMode=mixed --property=TasksMax=infinity --collect --setenv=SYSTEMD_NSPAWN_LOCK=0 -# TODO: check if the below 2 are required -# --setenv=SYSTEMD_SECCOMP=0 +# TODO: add below if required: # --property=DevicePolicy=auto systemd_nspawn_default_args=--keep-unit diff --git a/templates/podman/README.md b/templates/podman/README.md index dae1c6f..b27682e 100644 --- a/templates/podman/README.md +++ b/templates/podman/README.md @@ -1,12 +1,14 @@ # Fedora Podman Jail Template +## Setup + Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mypodmanjail /mnt/tank/path/to/podman/config`. ## Rootless ### Disclaimer -**These notes are a work in progress. Using podman in this setup hasn't been extensively tested.** +**Experimental. Using podman in this setup hasn't been extensively tested.** ### Installation From f37f6df7f70ccc91852bf99bbb562f6850b794d2 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Thu, 15 Feb 2024 17:40:13 +0100 Subject: [PATCH 43/57] Update jlmkr.py --- jlmkr.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index a8e9fde..cb8e93c 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -855,6 +855,11 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) + # TODO: ask to setup hooks and initial_setup + # Open text editor with current config file + # Or don't ask and make this a template-only feature, + # make it possible to override values in the template during jlmkr create with cli args + docker_compatible = 0 if agree("Make jail docker compatible right now?", "n"): @@ -1029,14 +1034,14 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo # # Specify a command/script to run on the HOST after stopping the jail # post_stop_hook=echo 'POST_STOP_HOOK' - # Specify command/script to run IN THE JAIL before starting it for the first time - # Useful to install packages on top of the base rootfs - # NOTE: this script will run in the host networking namespace and ignores - # all systemd_nspawn_user_args such as bind mounts - initial_setup=#!/usr/bin/bash - set -euo pipefail - apt-get update && apt-get -y install curl - curl -fsSL https://get.docker.com | sh + # # Specify command/script to run IN THE JAIL before starting it for the first time + # # Useful to install packages on top of the base rootfs + # # NOTE: this script will run in the host networking namespace and ignores + # # all systemd_nspawn_user_args such as bind mounts + # initial_setup=#!/usr/bin/bash + # set -euo pipefail + # apt-get update && apt-get -y install curl + # curl -fsSL https://get.docker.com | sh """ ) From 7be1c7c7d205e1102ee0e742ef1985675fa5907e Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:39:48 +0100 Subject: [PATCH 44/57] Always wait until jail stopped --- jlmkr.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index cb8e93c..4c09210 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -266,14 +266,6 @@ def shell_jail(args): """ return subprocess.run(["machinectl", "shell"] + args).returncode - -def stop_jail(jail_name): - """ - Stop jail with given name. - """ - return subprocess.run(["machinectl", "poweroff", jail_name]).returncode - - def parse_config_string(config_string): config = configparser.ConfigParser() # Workaround to read config file without section headers @@ -525,7 +517,7 @@ def restart_jail(jail_name): Restart jail with given name. """ - returncode = stop_jail_and_wait(jail_name) + returncode = stop_jail(jail_name) if returncode != 0: eprint("Abort restart.") return returncode @@ -1221,6 +1213,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo # Current echo 'y' workaround may cause problems when the jail name already exists # You'd end up with a new jail called 'y' # and the script will crash at the agree statement below + # TODO: move this question higher, above the actual creating bit print() if agree(f"Do you want to start jail {jail_name} right now?", "y"): return start_jail(jail_name) @@ -1271,21 +1264,21 @@ def edit_jail(jail_name): return 0 -def stop_jail_and_wait(jail_name): +def stop_jail(jail_name): """ - Wait for jail with given name to stop. + Stop jail with given name and wait until stopped. """ if not jail_is_running(jail_name): return 0 - returncode = stop_jail(jail_name) + returncode = subprocess.run(["machinectl", "poweroff", jail_name]).returncode if returncode != 0: eprint("Error while stopping jail.") return returncode print(f"Wait for {jail_name} to stop", end="", flush=True) - # Need to sleep since deleting immediately after stop causes problems... + while jail_is_running(jail_name): time.sleep(1) print(".", end="", flush=True) @@ -1310,7 +1303,7 @@ def remove_jail(jail_name): if check == jail_name: print() jail_path = get_jail_path(jail_name) - returncode = stop_jail_and_wait(jail_name) + returncode = stop_jail(jail_name) if returncode != 0: return returncode From 930e9568d4097cf131c7f097f3dc56c7c9822d59 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:46:51 +0100 Subject: [PATCH 45/57] Ask startup question earlier --- jlmkr.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 4c09210..dd9166f 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -266,6 +266,7 @@ def shell_jail(args): """ return subprocess.run(["machinectl", "shell"] + args).returncode + def parse_config_string(config_string): config = configparser.ConfigParser() # Workaround to read config file without section headers @@ -326,7 +327,7 @@ def start_jail(jail_name): # Handle initial setup initial_setup = config.get("initial_setup") - + # Alternative method to setup on first boot: # https://www.undrground.org/2021/01/25/adding-a-single-run-task-via-systemd/ # If there's no machine-id, then this the first time the jail is started @@ -359,7 +360,7 @@ def start_jail(jail_name): eprint() eprint("Abort starting jail.") return returncode - + # Cleanup the initial_setup_file Path(initial_setup_file).unlink(missing_ok=True) @@ -960,6 +961,11 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) + print() + start_now = agree( + "Do you want to start this jail now (when create is done)?", "y" + ) + # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in # Use TasksMax=infinity since this is what docker does: @@ -1206,16 +1212,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo cleanup(jail_path) raise error - # In case you want to create a jail without any user interaction, - # you need to skip this final question - # echo 'y' | jlmkr create test testconfig - # TODO: make jlmkr create work cleanly without user interaction. - # Current echo 'y' workaround may cause problems when the jail name already exists - # You'd end up with a new jail called 'y' - # and the script will crash at the agree statement below - # TODO: move this question higher, above the actual creating bit - print() - if agree(f"Do you want to start jail {jail_name} right now?", "y"): + if start_now: return start_jail(jail_name) From aa0d0c6d1db21d4bda26fe3f24dd1066a8246f05 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:52:29 +0100 Subject: [PATCH 46/57] Cleanup if create is aborted --- jlmkr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index dd9166f..9d4d13a 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1131,7 +1131,8 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) - if agree("Abort creating jail?", "y"): + if agree("Continue?", "n"): + cleanup(jail_path) return 1 with contextlib.suppress(FileNotFoundError): From f030606c83bf1031b9f9bd3e3e1a51730911f9c0 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:11:20 +0100 Subject: [PATCH 47/57] Don't auto start jail without systemd --- jlmkr.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 9d4d13a..2499e28 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1007,7 +1007,6 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo config_string = cleandoc( f""" - startup={startup} docker_compatible={docker_compatible} gpu_passthrough_intel={gpu_passthrough_intel} gpu_passthrough_nvidia={gpu_passthrough_nvidia} @@ -1130,10 +1129,13 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo """ ) ) + + print("Autostart has been disabled.") + print("You need to start this jail manually.") + startup = 0 + start_now = False - if agree("Continue?", "n"): - cleanup(jail_path) - return 1 + config_string = f"startup={startup}\n" + config_string with contextlib.suppress(FileNotFoundError): # Remove config which systemd handles for us From 21eef74929b02cb3f9b39b054e7e5fee02d48fec Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 16 Feb 2024 13:41:12 +0100 Subject: [PATCH 48/57] List more jail details --- jlmkr.py | 74 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 2499e28..867bb87 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1129,7 +1129,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo """ ) ) - + print("Autostart has been disabled.") print("You need to start this jail manually.") startup = 0 @@ -1372,46 +1372,56 @@ def list_jails(): print("No jails.") return 0 - for jail in jail_names: - jails[jail] = {"name": jail, "running": False} - # Get running jails from machinectl running_machines = run_command_and_parse_json(["machinectl", "list", "-o", "json"]) - # Augment the jails dict with output from machinectl - for machine in running_machines: - machine_name = machine["machine"] - # We're only interested in the list of jails made with jailmaker - if machine["service"] == "systemd-nspawn" and machine_name in jails: - addresses = (machine.get("addresses") or empty_value_indicator).split("\n") - if len(addresses) > 1: - addresses = addresses[0] + "…" - else: - addresses = addresses[0] + for jail_name in jail_names: + jails[jail_name] = {"name": jail_name, "running": False} + jail = jails[jail_name] - jails[machine_name] = { - "name": machine_name, - "running": True, - "os": machine["os"], - "version": machine["version"], - "addresses": addresses, - } - - # TODO: add additional properties from the jails config file - - for jail_name in jails: config = parse_config_file(get_jail_config_path(jail_name)) - - startup = False if config: - startup = bool(int(config.get("startup", "0"))) - # TODO: in case config is missing or parsing fails, - # should an error message be thrown here? + # TODO: also list privileged once this setting is implemented + jail["startup"] = bool(int(config.get("startup", "0"))) + jail["gpu_intel"] = bool(int(config.get("gpu_passthrough_intel", "0"))) + jail["gpu_nvidia"] = bool(int(config.get("gpu_passthrough_nvidia", "0"))) + initial_rootfs_image = config.get("initial_rootfs_image") + if initial_rootfs_image: + distro, release = config.get("initial_rootfs_image").split() + jail["os"] = distro + jail["version"] = release - jails[jail_name]["startup"] = startup + if jail_name in running_machines: + machine = running_machines[jail_name] + + # We're only interested in the list of jails made with jailmaker + if machine["service"] == "systemd-nspawn": + # Augment the jails dict with output from machinectl + jail["running"] = True + # Override os and version we got from the config file + jail["os"] = machine["os"] + jail["version"] = machine["version"] + + addresses = machine.get("addresses") + if not addresses: + jail["addresses"] = empty_value_indicator + else: + addresses = addresses.split("\n") + jail["addresses"] = addresses[0] + if len(addresses) > 1: + jail["addresses"] += "…" print_table( - ["name", "running", "startup", "os", "version", "addresses"], + [ + "name", + "running", + "startup", + "gpu_intel", + "gpu_nvidia", + "os", + "version", + "addresses", + ], sorted(jails.values(), key=lambda x: x["name"]), empty_value_indicator, ) From dc928ecd9665c91d36243c45a374d7389ea0f1df Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 25 Feb 2024 16:19:48 +0100 Subject: [PATCH 49/57] Format config templates --- templates/docker/config | 36 +++++++++++++++++++----------------- templates/incus/config | 33 ++++++++++++++++++--------------- templates/lxd/config | 33 ++++++++++++++++++--------------- templates/podman/config | 35 ++++++++++++++++++----------------- 4 files changed, 73 insertions(+), 64 deletions(-) diff --git a/templates/docker/config b/templates/docker/config index 9c7e428..c141ec8 100644 --- a/templates/docker/config +++ b/templates/docker/config @@ -1,6 +1,6 @@ startup=0 -gpu_passthrough_intel=0 -gpu_passthrough_nvidia=0 +gpu_passthrough_intel=1 +gpu_passthrough_nvidia=0 # Use macvlan networking to provide an isolated network namespace, # so docker can manage firewall rules @@ -14,18 +14,23 @@ systemd_nspawn_user_args=--network-macvlan=eno1 # Script to run on the HOST before starting the jail # Load kernel module and config kernel settings required for docker pre_start_hook=#!/usr/bin/bash + set -euo pipefail echo 'PRE_START_HOOK' echo 1 > /proc/sys/net/ipv4/ip_forward modprobe br_netfilter echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables +# Only used while creating the jail +distro=debian +release=bookworm + # Install docker inside the jail: # https://docs.docker.com/engine/install/debian/#install-using-the-repository # NOTE: this script will run in the host networking namespace and ignores # all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash - set -euo pipefail + set -euo pipefail apt-get update && apt-get -y install ca-certificates curl install -m 0755 -d /etc/apt/keyrings @@ -41,19 +46,16 @@ initial_setup=#!/usr/bin/bash # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed - --property=Type=notify - --property=RestartForceExitStatus=133 - --property=SuccessExitStatus=133 - --property=Delegate=yes - --property=TasksMax=infinity - --collect - --setenv=SYSTEMD_NSPAWN_LOCK=0 + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 systemd_nspawn_default_args=--keep-unit - --quiet - --boot - --bind-ro=/sys/module - --inaccessible=/sys/module/apparmor - -# Used by jlmkr create -initial_rootfs_image=debian bookworm \ No newline at end of file + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor \ No newline at end of file diff --git a/templates/incus/config b/templates/incus/config index 0c19790..2b82dd8 100644 --- a/templates/incus/config +++ b/templates/incus/config @@ -1,6 +1,7 @@ +# WARNING: EXPERIMENTAL CONFIG TEMPLATE! startup=0 gpu_passthrough_intel=1 -gpu_passthrough_nvidia=0 +gpu_passthrough_nvidia=0 # Use macvlan networking to provide an isolated network namespace, # so incus can manage firewall rules @@ -20,6 +21,7 @@ systemd_nspawn_user_args=--network-macvlan=eno1 # Script to run on the HOST before starting the jail # Load kernel module and config kernel settings required for incus pre_start_hook=#!/usr/bin/bash + set -euo pipefail echo 'PRE_START_HOOK' echo 1 > /proc/sys/net/ipv4/ip_forward modprobe br_netfilter @@ -27,6 +29,10 @@ pre_start_hook=#!/usr/bin/bash echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables modprobe vhost_vsock +# Only used while creating the jail +distro=debian +release=bookworm + # Install incus according to: # https://github.com/zabbly/incus#installation # NOTE: this script will run in the host networking namespace and ignores @@ -50,21 +56,18 @@ initial_setup=#!/usr/bin/bash # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed - --property=Type=notify - --property=RestartForceExitStatus=133 - --property=SuccessExitStatus=133 - --property=Delegate=yes - --property=TasksMax=infinity - --collect - --setenv=SYSTEMD_NSPAWN_LOCK=0 + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 # TODO: add below if required: # --property=DevicePolicy=auto systemd_nspawn_default_args=--keep-unit - --quiet - --boot - --bind-ro=/sys/module - --inaccessible=/sys/module/apparmor - -# Used by jlmkr create -initial_rootfs_image=debian bookworm \ No newline at end of file + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor \ No newline at end of file diff --git a/templates/lxd/config b/templates/lxd/config index db6c1fc..2c1e46e 100644 --- a/templates/lxd/config +++ b/templates/lxd/config @@ -1,6 +1,7 @@ +# WARNING: EXPERIMENTAL CONFIG TEMPLATE! startup=0 gpu_passthrough_intel=1 -gpu_passthrough_nvidia=0 +gpu_passthrough_nvidia=0 # Use macvlan networking to provide an isolated network namespace, # so lxd can manage firewall rules @@ -20,6 +21,7 @@ systemd_nspawn_user_args=--network-bridge=br1 # Script to run on the HOST before starting the jail # Load kernel module and config kernel settings required for lxd pre_start_hook=#!/usr/bin/bash + set -euo pipefail echo 'PRE_START_HOOK' echo 1 > /proc/sys/net/ipv4/ip_forward modprobe br_netfilter @@ -27,6 +29,10 @@ pre_start_hook=#!/usr/bin/bash echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables modprobe vhost_vsock +# Only used while creating the jail +distro=ubuntu +release=jammy + # NOTE: this script will run in the host networking namespace and ignores # all systemd_nspawn_user_args such as bind mounts initial_setup=#!/usr/bin/bash @@ -36,21 +42,18 @@ initial_setup=#!/usr/bin/bash # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed - --property=Type=notify - --property=RestartForceExitStatus=133 - --property=SuccessExitStatus=133 - --property=Delegate=yes - --property=TasksMax=infinity - --collect - --setenv=SYSTEMD_NSPAWN_LOCK=0 + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 # TODO: add below if required: # --property=DevicePolicy=auto systemd_nspawn_default_args=--keep-unit - --quiet - --boot - --bind-ro=/sys/module - --inaccessible=/sys/module/apparmor - -# Used by jlmkr create -initial_rootfs_image=ubuntu jammy \ No newline at end of file + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor \ No newline at end of file diff --git a/templates/podman/config b/templates/podman/config index ee8f2c7..4675e07 100644 --- a/templates/podman/config +++ b/templates/podman/config @@ -1,6 +1,6 @@ startup=0 gpu_passthrough_intel=0 -gpu_passthrough_nvidia=0 +gpu_passthrough_nvidia=0 # Use macvlan networking to provide an isolated network namespace, # so podman can manage firewall rules @@ -14,18 +14,22 @@ systemd_nspawn_user_args=--network-macvlan=eno1 # Script to run on the HOST before starting the jail # Load kernel module and config kernel settings required for podman pre_start_hook=#!/usr/bin/bash + set -euo pipefail echo 'PRE_START_HOOK' echo 1 > /proc/sys/net/ipv4/ip_forward modprobe br_netfilter echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables +# Only used while creating the jail +distro=fedora +release=39 + # Install podman inside the jail # NOTE: this script will run in the host networking namespace and ignores # all systemd_nspawn_user_args such as bind mounts - initial_setup=#!/usr/bin/bash - set -euo pipefail + set -euo pipefail dnf -y install podman # Add the required capabilities to the `newuidmap` and `newgidmap` binaries # https://github.com/containers/podman/issues/2788#issuecomment-1016301663 @@ -35,19 +39,16 @@ initial_setup=#!/usr/bin/bash # You generally will not need to change the options below systemd_run_default_args=--property=KillMode=mixed - --property=Type=notify - --property=RestartForceExitStatus=133 - --property=SuccessExitStatus=133 - --property=Delegate=yes - --property=TasksMax=infinity - --collect - --setenv=SYSTEMD_NSPAWN_LOCK=0 + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 systemd_nspawn_default_args=--keep-unit - --quiet - --boot - --bind-ro=/sys/module - --inaccessible=/sys/module/apparmor - -# Used by jlmkr create -initial_rootfs_image=fedora 39 \ No newline at end of file + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor \ No newline at end of file From fc38d0108213717e54af81fa720587a758f92794 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 25 Feb 2024 16:31:34 +0100 Subject: [PATCH 50/57] Improved config parsing and bug fixes Fixed failing cleanup of initial_setup_file when initial_setup is a command instead of a file --- jlmkr.py | 641 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 398 insertions(+), 243 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 867bb87..2aac834 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 +"""Create persistent Linux 'jails' on TrueNAS SCALE, \ +with full access to all files via bind mounts, \ +thanks to systemd-nspawn!""" + +__version__ = "1.0.1" + +__disclaimer__ = """USE THIS SCRIPT AT YOUR OWN RISK! +IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS.""" + import argparse import configparser import contextlib @@ -7,6 +16,7 @@ import ctypes import errno import glob import hashlib +import io import json import os import platform @@ -25,25 +35,79 @@ from inspect import cleandoc from pathlib import Path, PurePath from textwrap import dedent -# Only set a color if we have an interactive tty -if sys.stdout.isatty(): - BOLD = "\033[1m" - RED = "\033[91m" - YELLOW = "\033[93m" - UNDERLINE = "\033[4m" - NORMAL = "\033[0m" -else: - BOLD = RED = YELLOW = UNDERLINE = NORMAL = "" +DEFAULT_CONFIG = """startup=0 +gpu_passthrough_intel=0 +gpu_passthrough_nvidia=0 +docker_compatible=0 -DISCLAIMER = f"""{YELLOW}{BOLD}USE THIS SCRIPT AT YOUR OWN RISK! -IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS.{NORMAL}""" +# Add additional systemd-nspawn flags +# E.g. to mount host storage in the jail (--bind-ro for readonly): +# --bind='/mnt/pool/dataset:/home' --bind-ro=/etc/certificates +# E.g. macvlan networking: +# --network-macvlan=eno1 --resolv-conf=bind-host +# E.g. bridge networking: +# --network-bridge=br1 --resolv-conf=bind-host +# E.g. add capabilities required by docker: +# --system-call-filter='add_key keyctl bpf' +systemd_nspawn_user_args= -DESCRIPTION = ( - "Create persistent Linux 'jails' on TrueNAS SCALE, with full access to all files \ - via bind mounts, thanks to systemd-nspawn!" -) +# Specify command/script to run on the HOST before starting the jail +# For example to load kernel modules and config kernel settings +pre_start_hook= +# pre_start_hook=#!/usr/bin/bash +# set -euo pipefail +# echo 'PRE_START_HOOK_EXAMPLE' +# echo 1 > /proc/sys/net/ipv4/ip_forward +# modprobe br_netfilter +# echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables +# echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables -VERSION = "1.0.1" +# Specify a command/script to run on the HOST after stopping the jail +post_stop_hook= +# post_stop_hook=echo 'POST_STOP_HOOK_EXAMPLE' + +# Only used while creating the jail +distro=debian +release=bookworm + +# Specify command/script to run IN THE JAIL before the first start +# Useful to install packages on top of the base rootfs +# NOTE: this script will run in the host networking namespace and +# ignores all systemd_nspawn_user_args such as bind mounts +initial_setup= +# initial_setup=bash -c 'apt-get update && apt-get -y upgrade' + +# Usually no need to change systemd_run_default_args +systemd_run_default_args=--collect + --property=Delegate=yes + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=TasksMax=infinity + --property=Type=notify + --setenv=SYSTEMD_NSPAWN_LOCK=0 + --property=KillMode=mixed + +# Usually no need to change systemd_nspawn_default_args +systemd_nspawn_default_args=--bind-ro=/sys/module + --boot + --inaccessible=/sys/module/apparmor + --quiet + --keep-unit""" + +# Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: +# https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in +# Use TasksMax=infinity since this is what docker does: +# https://github.com/docker/engine/blob/master/contrib/init/systemd/docker.service + +# Use SYSTEMD_NSPAWN_LOCK=0: otherwise jail won't start jail after a shutdown (but why?) +# Would give "directory tree currently busy" error and I'd have to run +# `rm /run/systemd/nspawn/locks/*` and remove the .lck file from jail_path +# Disabling locking isn't a big deal as systemd-nspawn will prevent starting a container +# with the same name anyway: as long as we're starting jails using this script, +# it won't be possible to start the same jail twice + +# Always add --bind-ro=/sys/module to make lsmod happy +# https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html JAILS_DIR_PATH = "jails" JAIL_CONFIG_NAME = "config" @@ -58,6 +122,141 @@ COMMAND_NAME = os.path.basename(__file__) SYMLINK_NAME = "jlmkr" TEXT_EDITOR = "nano" +# Only set a color if we have an interactive tty +if sys.stdout.isatty(): + BOLD = "\033[1m" + RED = "\033[91m" + YELLOW = "\033[93m" + UNDERLINE = "\033[4m" + NORMAL = "\033[0m" +else: + BOLD = RED = YELLOW = UNDERLINE = NORMAL = "" + +DISCLAIMER = f"""{YELLOW}{BOLD}{__disclaimer__}{NORMAL}""" + +# Used in parser getters to indicate the default behavior when a specific +# option is not found it to raise an exception. Created to enable `None` as +# a valid fallback value. +_UNSET = object() + + +class KeyValueParser(configparser.ConfigParser): + """Simple comment preserving parser based on ConfigParser. + Reads a file containing key/value pairs and/or comments. + Values can span multiple lines, as long as they are indented + deeper than the first line of the value. Comments or keys + must NOT be indented. + """ + + def __init__(self, *args, **kwargs): + # Set defaults if not specified by user + if "interpolation" not in kwargs: + kwargs["interpolation"] = None + if "allow_no_value" not in kwargs: + kwargs["allow_no_value"] = True + if "comment_prefixes" not in kwargs: + kwargs["comment_prefixes"] = "#" + + super().__init__(*args, **kwargs) + + # Backup _comment_prefixes + self._comment_prefixes_backup = self._comment_prefixes + # Unset _comment_prefixes so comments won't be skipped + self._comment_prefixes = () + # Starting point for the comment IDs + self._comment_id = 0 + # Default delimiter to use + delimiter = self._delimiters[0] + # Template to store comments as key value pair + self._comment_template = "#{0} " + delimiter + " {1}" + # Regex to match the comment prefix + self._comment_regex = re.compile(f"^#\d+\s*{re.escape(delimiter)}[^\S\n]*") + # Regex to match cosmetic newlines (skips newlines in multiline values): + # consecutive whitespace from start of line followed by a line not starting with whitespace + self._cosmetic_newlines_regex = re.compile(r"^(\s+)(?=^\S)", re.MULTILINE) + # Dummy section name + self._section_name = "a" + + def _find_cosmetic_newlines(self, text): + # Indices of the lines containing cosmetic newlines + cosmetic_newline_indices = set() + for match in re.finditer(self._cosmetic_newlines_regex, text): + start_index = text.count("\n", 0, match.start()) + end_index = start_index + text.count("\n", match.start(), match.end()) + cosmetic_newline_indices.update(range(start_index, end_index)) + + return cosmetic_newline_indices + + def _read(self, fp, fpname): + lines = fp.readlines() + cosmetic_newline_indices = self._find_cosmetic_newlines("".join(lines)) + # Preprocess config file to preserve comments + for i, line in enumerate(lines): + if i in cosmetic_newline_indices or line.startswith( + self._comment_prefixes_backup + ): + # Store cosmetic newline or comment with unique key + lines[i] = self._comment_template.format(self._comment_id, line) + self._comment_id += 1 + + # Convert to in-memory file and prepend a dummy section header + lines = io.StringIO(f"[{self._section_name}]\n" + "".join(lines)) + # Feed preprocessed file to original _read method + return super()._read(lines, fpname) + + def read_default_string(self, string, source=""): + # Ignore all comments when parsing default key/values + string = "\n".join( + [ + line + for line in string.splitlines() + if not line.startswith(self._comment_prefixes_backup) + ] + ) + # Feed preprocessed file to original _read method + return super()._read(io.StringIO("[DEFAULT]\n" + string), source) + + def write(self, fp, space_around_delimiters=False): + # Write the config to an in-memory file + with io.StringIO() as sfile: + super().write(sfile, space_around_delimiters) + # Start from the beginning of sfile + sfile.seek(0) + + line = sfile.readline() + # Throw away lines until we reach the dummy section header + while line.strip() != f"[{self._section_name}]": + line = sfile.readline() + + lines = sfile.readlines() + + for i, line in enumerate(lines): + # Remove the comment id prefix + lines[i] = self._comment_regex.sub("", line, 1) + + fp.write("".join(lines).rstrip()) + + # Set value for specified option key + def my_set(self, option, value): + if isinstance(value, bool): + value = str(int(value)) + elif not isinstance(value, str): + value = str(value) + + super().set(self._section_name, option, value) + + # Return value for specified option key + def my_get(self, option): + return super().get(self._section_name, option) + + # Return value converted to boolean for specified option key + def my_getboolean(self, option, fallback=_UNSET): + return super().getboolean(self._section_name, option, fallback=fallback) + + # # Return all keys inside our only section + # def my_options(self): + # return super().options(self._section_name) + def eprint(*args, **kwargs): """ @@ -267,17 +466,15 @@ def shell_jail(args): return subprocess.run(["machinectl", "shell"] + args).returncode -def parse_config_string(config_string): - config = configparser.ConfigParser() - # Workaround to read config file without section headers - config.read_string("[DEFAULT]\n" + config_string) - config = dict(config["DEFAULT"]) - return config - - def parse_config_file(jail_config_path): + config = KeyValueParser() + # Read default config to fallback to default values + # for keys not found in the jail_config_path file + config.read_default_string(DEFAULT_CONFIG) try: - return parse_config_string(Path(jail_config_path).read_text()) + with open(jail_config_path, "r") as fp: + config.read_file(fp) + return config except FileNotFoundError: eprint(f"Unable to find config file: {jail_config_path}.") return @@ -326,7 +523,7 @@ def start_jail(jail_name): return 1 # Handle initial setup - initial_setup = config.get("initial_setup") + initial_setup = config.my_get("initial_setup") # Alternative method to setup on first boot: # https://www.undrground.org/2021/01/25/adding-a-single-run-task-via-systemd/ @@ -334,7 +531,8 @@ def start_jail(jail_name): if initial_setup and not os.path.exists( os.path.join(jail_rootfs_path, "etc/machine-id") ): - # Run the command directly if it doesn't start with a shebang + initial_setup_file = None + if initial_setup.startswith("#!"): # Write a script file and call that initial_setup_file = os.path.abspath( @@ -351,9 +549,21 @@ def start_jail(jail_name): "/root/initial_startup", ] else: - cmd = ["systemd-nspawn", "-q", "-D", jail_rootfs_path, initial_setup] + # Run the command directly if it doesn't start with a shebang + cmd = [ + "systemd-nspawn", + "-q", + "-D", + jail_rootfs_path, + *shlex.split(initial_setup), + ] returncode = subprocess.run(cmd).returncode + + # Cleanup the initial_setup_file + if initial_setup_file: + Path(initial_setup_file).unlink(missing_ok=True) + if returncode != 0: eprint("Failed to run initial setup:") eprint(initial_setup) @@ -361,9 +571,6 @@ def start_jail(jail_name): eprint("Abort starting jail.") return returncode - # Cleanup the initial_setup_file - Path(initial_setup_file).unlink(missing_ok=True) - systemd_run_additional_args = [ f"--unit={SYMLINK_NAME}-{jail_name}", f"--working-directory=./{jail_path}", @@ -387,7 +594,7 @@ def start_jail(jail_name): # - how to call the option to enable ip_forward and bridge-nf-call? # - add CSV value for preloading kernel modules like linux.kernel_modules in LXC - if config.get("docker_compatible") == "1": + if config.my_getboolean("docker_compatible"): # Enable ip forwarding on the host (docker needs it) print(1, file=open("/proc/sys/net/ipv4/ip_forward", "w")) @@ -450,26 +657,27 @@ def start_jail(jail_name): add_hook( jail_path, systemd_run_additional_args, - config.get("pre_start_hook"), + config.my_get("pre_start_hook"), "ExecStartPre", ) add_hook( jail_path, systemd_run_additional_args, - config.get("post_stop_hook"), + config.my_get("post_stop_hook"), "ExecStopPost", ) # Legacy gpu_passthrough config setting - if config.get("gpu_passthrough") == "1": - gpu_passthrough_intel = "1" - gpu_passthrough_nvidia = "1" + # TODO: deprecate this and stop supporting it + if config.my_getboolean("gpu_passthrough", False): + gpu_passthrough_intel = True + gpu_passthrough_nvidia = True else: - gpu_passthrough_intel = config.get("gpu_passthrough_intel") - gpu_passthrough_nvidia = config.get("gpu_passthrough_nvidia") + gpu_passthrough_intel = config.my_getboolean("gpu_passthrough_intel") + gpu_passthrough_nvidia = config.my_getboolean("gpu_passthrough_nvidia") - if gpu_passthrough_intel == "1" or gpu_passthrough_nvidia == "1": + if gpu_passthrough_intel or gpu_passthrough_nvidia: systemd_nspawn_additional_args.append("--property=DeviceAllow=char-drm rw") passthrough_intel(gpu_passthrough_intel, systemd_nspawn_additional_args) @@ -479,13 +687,13 @@ def start_jail(jail_name): cmd = [ "systemd-run", - *shlex.split(config.get("systemd_run_default_args", "")), + *shlex.split(config.my_get("systemd_run_default_args")), *systemd_run_additional_args, "--", "systemd-nspawn", - *shlex.split(config.get("systemd_nspawn_default_args", "")), + *shlex.split(config.my_get("systemd_nspawn_default_args")), *systemd_nspawn_additional_args, - *shlex.split(config.get("systemd_nspawn_user_args", "")), + *shlex.split(config.my_get("systemd_nspawn_user_args")), ] print( @@ -710,12 +918,21 @@ def ask_jail_name(jail_name=""): return jail_name -def create_jail(jail_name="", config_path=None, distro="debian", release="bookworm"): +def agree_with_default(config, key, question): + default_answer = "y" if config.my_getboolean(key) else "n" + config.my_set(key, agree(question, default_answer)) + + +def create_jail_interactive(): """ Create jail with given name. """ - config_string = "" + config = KeyValueParser() + config.read_string(DEFAULT_CONFIG) + + recommended_distro = config.my_get("distro") + recommended_release = config.my_get("release") print(DISCLAIMER) @@ -755,48 +972,39 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ################# # Config handling ################# + jail_name = "" - if config_path: - try: - config_string = Path(config_path).read_text() - except FileNotFoundError: - eprint(f"Unable to find file: {config_path}.") - return 1 - else: - print() - if agree("Do you wish to create a jail from a config template?", "n"): - print( - dedent( - """ - A text editor will open so you can provide the config template. + print() + if agree("Do you wish to create a jail from a config template?", "n"): + print( + dedent( + """ + A text editor will open so you can provide the config template. - - please copy your config - - paste it into the text editor - - save and close the text editor - """ - ) + 1. Please copy your config + 2. Paste it into the text editor + 3. Save and close the text editor + """ ) - input("Press Enter to open the text editor.") + ) + input("Press Enter to open the text editor.") - with tempfile.NamedTemporaryFile() as f: - subprocess.call([TEXT_EDITOR, f.name]) - f.seek(0) - config_string = f.read().decode() + with tempfile.NamedTemporaryFile(mode="w+t") as f: + subprocess.call([TEXT_EDITOR, f.name]) + f.seek(0) + # Start over with a new KeyValueParser to parse user config + config = KeyValueParser() + config.read_file(f) - if config_string: - config = parse_config_string(config_string) - # Ask for jail name if not provided - if not ( - jail_name - and check_jail_name_valid(jail_name) - and check_jail_name_available(jail_name) - ): - jail_name = ask_jail_name(jail_name) + # Ask for jail name + jail_name = ask_jail_name(jail_name) jail_path = get_jail_path(jail_name) - distro, release = config.get("initial_rootfs_image").split() else: print() - if not agree(f"Install the recommended image ({distro} {release})?", "y"): + if not agree( + f"Install the recommended image ({recommended_distro} {recommended_release})?", + "y", + ): print( dedent( f""" @@ -823,7 +1031,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) - distro = input("Distro: ") + config.my_set("distro", input("Distro: ")) print( dedent( @@ -833,7 +1041,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) - release = input("Release: ") + config.my_set("release", input("Release: ")) jail_name = ask_jail_name(jail_name) jail_path = get_jail_path(jail_name) @@ -848,29 +1056,17 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) - # TODO: ask to setup hooks and initial_setup - # Open text editor with current config file - # Or don't ask and make this a template-only feature, - # make it possible to override values in the template during jlmkr create with cli args - - docker_compatible = 0 - - if agree("Make jail docker compatible right now?", "n"): - docker_compatible = 1 - + agree_with_default( + config, "docker_compatible", "Make jail docker compatible right now?" + ) print() - - gpu_passthrough_intel = 0 - - if agree("Passthrough the intel GPU (if present)?", "n"): - gpu_passthrough_intel = 1 - + agree_with_default( + config, "gpu_passthrough_intel", "Passthrough the intel GPU (if present)?" + ) print() - - gpu_passthrough_nvidia = 0 - - if agree("Passthrough the nvidia GPU (if present)?", "n"): - gpu_passthrough_nvidia = 1 + agree_with_default( + config, "gpu_passthrough_nvidia", "Passthrough the nvidia GPU (if present)?" + ) print( dedent( @@ -889,10 +1085,10 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo else: try: base_os_version = platform.freedesktop_os_release().get( - "VERSION_CODENAME", release + "VERSION_CODENAME", recommended_release ) except AttributeError: - base_os_version = release + base_os_version = recommended_release print( dedent( f""" @@ -916,7 +1112,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo --bind='/mnt/pool/dataset:/home' Or the same, but readonly, with: --bind-ro='/mnt/pool/dataset:/home' - Or create MACVLAN interface for static IP, with: + Or create macvlan interface with: --network-macvlan=eno1 --resolv-conf=bind-host """ ) @@ -941,7 +1137,11 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo # https://github.com/python-cmd2/cmd2/blob/ee7599f9ac0dbb6ce3793f6b665ba1200d3ef9a3/cmd2/cmd2.py # https://stackoverflow.com/a/40152927 - systemd_nspawn_user_args = input("Additional flags: ") or "" + config.my_set( + "systemd_nspawn_user_args", + "\n ".join(shlex.split(input("Additional flags: ") or "")), + ) + # Disable tab auto completion readline.parse_and_bind("tab: self-insert") @@ -954,114 +1154,48 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo ) ) - startup = int( + config.my_set( + "startup", agree( f"Do you want to start this jail when running: {COMMAND_NAME} startup?", "n", - ) + ), ) - print() - start_now = agree( - "Do you want to start this jail now (when create is done)?", "y" - ) + print() + start_now = agree("Do you want to start this jail now (when create is done)?", "y") - # Use mostly default settings for systemd-nspawn but with systemd-run instead of a service file: - # https://github.com/systemd/systemd/blob/main/units/systemd-nspawn%40.service.in - # Use TasksMax=infinity since this is what docker does: - # https://github.com/docker/engine/blob/master/contrib/init/systemd/docker.service - - # Use SYSTEMD_NSPAWN_LOCK=0: otherwise jail won't start jail after a shutdown (but why?) - # Would give "directory tree currently busy" error and I'd have to run - # `rm /run/systemd/nspawn/locks/*` and remove the .lck file from jail_path - # Disabling locking isn't a big deal as systemd-nspawn will prevent starting a container - # with the same name anyway: as long as we're starting jails using this script, - # it won't be possible to start the same jail twice - - systemd_run_default_args = [ - "--property=KillMode=mixed", - "--property=Type=notify", - "--property=RestartForceExitStatus=133", - "--property=SuccessExitStatus=133", - "--property=Delegate=yes", - "--property=TasksMax=infinity", - "--collect", - "--setenv=SYSTEMD_NSPAWN_LOCK=0", - ] - - # Always add --bind-ro=/sys/module to make lsmod happy - # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html - systemd_nspawn_default_args = [ - "--keep-unit", - "--quiet", - "--boot", - "--bind-ro=/sys/module", - "--inaccessible=/sys/module/apparmor", - ] - - systemd_nspawn_user_args_multiline = "\n\t".join( - shlex.split(systemd_nspawn_user_args) - ) - systemd_run_default_args_multiline = "\n\t".join(systemd_run_default_args) - systemd_nspawn_default_args_multiline = "\n\t".join(systemd_nspawn_default_args) - - config_string = cleandoc( - f""" - docker_compatible={docker_compatible} - gpu_passthrough_intel={gpu_passthrough_intel} - gpu_passthrough_nvidia={gpu_passthrough_nvidia} - """ - ) - - config_string += ( - f"\n\nsystemd_nspawn_user_args={systemd_nspawn_user_args_multiline}\n\n" - ) - - config_string += cleandoc( - """ - # # Specify command/script to run on the HOST before starting the jail - # # For example to load kernel modules and config kernel settings - # pre_start_hook=#!/usr/bin/bash - # echo 'PRE_START_HOOK' - # echo 1 > /proc/sys/net/ipv4/ip_forward - # modprobe br_netfilter - # echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables - # echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables - # - # # Specify a command/script to run on the HOST after stopping the jail - # post_stop_hook=echo 'POST_STOP_HOOK' - - # # Specify command/script to run IN THE JAIL before starting it for the first time - # # Useful to install packages on top of the base rootfs - # # NOTE: this script will run in the host networking namespace and ignores - # # all systemd_nspawn_user_args such as bind mounts - # initial_setup=#!/usr/bin/bash - # set -euo pipefail - # apt-get update && apt-get -y install curl - # curl -fsSL https://get.docker.com | sh - """ - ) - - config_string += "\n".join( - [ - "", - "", - "# You generally will not need to change the options below", - f"systemd_run_default_args={systemd_run_default_args_multiline}", - "", - f"systemd_nspawn_default_args={systemd_nspawn_default_args_multiline}", - "", - "# Used by jlmkr create", - f"initial_rootfs_image={distro} {release}", - ] - ) - - print() + print() ############## # Create start ############## + create_options = { + "jail_name": jail_name, + "jail_path": jail_path, + "start_now": start_now, + "config": config, + } + + rc = write_jail(create_options) + if rc != 0: + return rc + + if create_options["start_now"]: + return start_jail(jail_name) + + return 0 + + +def write_jail(create_options): + jail_name = create_options["jail_name"] + jail_path = create_options["jail_path"] + config = create_options["config"] + + distro = config.my_get("distro") + release = config.my_get("release") + # Cleanup in except, but only once the jail_path is final # Otherwise we may cleanup the wrong directory try: @@ -1132,10 +1266,8 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo print("Autostart has been disabled.") print("You need to start this jail manually.") - startup = 0 - start_now = False - - config_string = f"startup={startup}\n" + config_string + config.my_set("startup", 0) + create_options["start_now"] = False with contextlib.suppress(FileNotFoundError): # Remove config which systemd handles for us @@ -1206,7 +1338,8 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo file=open(os.path.join(preset_path, "00-jailmaker.preset"), "w"), ) - print(config_string, file=open(jail_config_path, "w")) + with open(jail_config_path, "w") as fp: + config.write(fp) os.chmod(jail_config_path, 0o600) @@ -1215,8 +1348,7 @@ def create_jail(jail_name="", config_path=None, distro="debian", release="bookwo cleanup(jail_path) raise error - if start_now: - return start_jail(jail_name) + return 0 def jail_is_running(jail_name): @@ -1358,6 +1490,17 @@ def get_all_jail_names(): return jail_names +def parse_os_release(candidates): + for candidate in candidates: + try: + with open(candidate, encoding="utf-8") as f: + return platform._parse_os_release(f) + except OSError: + # Silently ignore failing to read os release info + pass + return {} + + def list_jails(): """ List all available and running jails. @@ -1374,42 +1517,58 @@ def list_jails(): # Get running jails from machinectl running_machines = run_command_and_parse_json(["machinectl", "list", "-o", "json"]) + # Index running_machines by machine name + # We're only interested in systemd-nspawn machines + running_machines = { + item["machine"]: item + for item in running_machines + if item["service"] == "systemd-nspawn" + } for jail_name in jail_names: + jail_rootfs_path = get_jail_rootfs_path(jail_name) jails[jail_name] = {"name": jail_name, "running": False} jail = jails[jail_name] config = parse_config_file(get_jail_config_path(jail_name)) if config: # TODO: also list privileged once this setting is implemented - jail["startup"] = bool(int(config.get("startup", "0"))) - jail["gpu_intel"] = bool(int(config.get("gpu_passthrough_intel", "0"))) - jail["gpu_nvidia"] = bool(int(config.get("gpu_passthrough_nvidia", "0"))) - initial_rootfs_image = config.get("initial_rootfs_image") - if initial_rootfs_image: - distro, release = config.get("initial_rootfs_image").split() - jail["os"] = distro - jail["version"] = release + jail["startup"] = config.my_getboolean("startup") + + # TODO: deprecate gpu_passthrough and stop supporting it + if config.my_getboolean("gpu_passthrough", False): + jail["gpu_intel"] = True + jail["gpu_nvidia"] = True + else: + jail["gpu_intel"] = config.my_getboolean("gpu_passthrough_intel") + jail["gpu_nvidia"] = config.my_getboolean("gpu_passthrough_nvidia") if jail_name in running_machines: machine = running_machines[jail_name] + # Augment the jails dict with output from machinectl + jail["running"] = True + jail["os"] = machine["os"] + jail["version"] = machine["version"] - # We're only interested in the list of jails made with jailmaker - if machine["service"] == "systemd-nspawn": - # Augment the jails dict with output from machinectl - jail["running"] = True - # Override os and version we got from the config file - jail["os"] = machine["os"] - jail["version"] = machine["version"] + addresses = machine.get("addresses") + if not addresses: + jail["addresses"] = empty_value_indicator + else: + addresses = addresses.split("\n") + jail["addresses"] = addresses[0] + if len(addresses) > 1: + jail["addresses"] += "…" + else: + # Parse os-release info ourselves + jail_platform = parse_os_release( + ( + os.path.join(jail_rootfs_path, "etc/os-release"), + os.path.join(jail_rootfs_path, "usr/lib/os-release"), + ) + ) - addresses = machine.get("addresses") - if not addresses: - jail["addresses"] = empty_value_indicator - else: - addresses = addresses.split("\n") - jail["addresses"] = addresses[0] - if len(addresses) > 1: - jail["addresses"] += "…" + jail["os"] = jail_platform.get("ID") + jail["version"] = jail_platform.get("VERSION_ID") print_table( [ @@ -1559,7 +1718,7 @@ def startup_jails(): start_failure = False for jail_name in get_all_jail_names(): config = parse_config_file(get_jail_config_path(jail_name)) - if config and config.get("startup") == "1": + if config and config.my_getboolean("startup"): if start_jail(jail_name) != 0: start_failure = True @@ -1575,9 +1734,9 @@ def main(): f"This script should be owned by the root user... Fix it manually with: `chown root {SCRIPT_PATH}`." ) - parser = argparse.ArgumentParser(description=DESCRIPTION, epilog=DISCLAIMER) + parser = argparse.ArgumentParser(description=__doc__, epilog=DISCLAIMER) - parser.add_argument("--version", action="version", version=VERSION) + parser.add_argument("--version", action="version", version=__version__) subparsers = parser.add_subparsers(title="commands", dest="subcommand", metavar="") @@ -1587,11 +1746,7 @@ def main(): help="install jailmaker dependencies and create symlink", ) - create_parser = subparsers.add_parser( - name="create", epilog=DISCLAIMER, help="create a new jail" - ) - create_parser.add_argument("name", nargs="?", help="name of the jail") - create_parser.add_argument("config", nargs="?", help="path to config file template") + subparsers.add_parser(name="create", epilog=DISCLAIMER, help="create a new jail") subparsers.add_parser( name="start", epilog=DISCLAIMER, help="start a previously created jail" @@ -1665,7 +1820,7 @@ def main(): sys.exit(install_jailmaker()) elif args.subcommand == "create": - sys.exit(create_jail(args.name, args.config)) + sys.exit(create_jail_interactive()) elif args.subcommand == "start": sys.exit(start_jail(args.name)) @@ -1706,7 +1861,7 @@ def main(): else: if agree("Create a new jail?", "y"): print() - sys.exit(create_jail()) + sys.exit(create_jail_interactive()) else: parser.print_usage() From fe00c3cf37bb0840c660f00f861d25e070f7f346 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:35:05 +0100 Subject: [PATCH 51/57] Non-interactive jail create --- README.md | 14 +- jlmkr.py | 465 ++++++++++++++++++++++++++----------- templates/docker/README.md | 2 +- templates/incus/README.md | 2 +- templates/lxd/README.md | 2 +- templates/podman/README.md | 2 +- 6 files changed, 344 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index 8d61936..5f862c1 100644 --- a/README.md +++ b/README.md @@ -39,20 +39,26 @@ After an update of TrueNAS SCALE the symlink will be lost (but the shell aliases ### Create Jail -Creating a jail is interactive. You'll be presented with questions which guide you through the process. +Creating jail with the default settings is as simple as: ```shell jlmkr create myjail ``` -After answering some questions you should have your first jail up and running! - You may also specify a path to a config template, for a quick and consistent jail creation process. ```shell -jlmkr create myjail /path/to/config/template +jlmkr create --config /path/to/config/template myjail ``` +If you omit the jail name, the create process is interactive. You'll be presented with questions which guide you through the process. + +```shell +jlmkr create +``` + +After answering some questions you should have your first jail up and running! + ### Startup Jails on Boot ```shell diff --git a/jlmkr.py b/jlmkr.py index 2aac834..159f777 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -240,6 +240,8 @@ class KeyValueParser(configparser.ConfigParser): def my_set(self, option, value): if isinstance(value, bool): value = str(int(value)) + elif isinstance(value, list): + value = str("\n ".join(value)) elif not isinstance(value, str): value = str(value) @@ -253,9 +255,23 @@ class KeyValueParser(configparser.ConfigParser): def my_getboolean(self, option, fallback=_UNSET): return super().getboolean(self._section_name, option, fallback=fallback) - # # Return all keys inside our only section - # def my_options(self): - # return super().options(self._section_name) + +class ExceptionWithParser(Exception): + def __init__(self, parser, message): + self.parser = parser + self.message = message + super().__init__(message) + + +# Workaround for exit_on_error=False not applying to: +# "error: the following arguments are required" +# https://github.com/python/cpython/issues/103498 +class CustomSubParser(argparse.ArgumentParser): + def error(self, message): + if self.exit_on_error: + super().error(message) + else: + raise ExceptionWithParser(self, message) def eprint(*args, **kwargs): @@ -420,7 +436,7 @@ def passthrough_nvidia( systemd_nspawn_additional_args += nvidia_mounts -def exec_jail(jail_name, cmd, args): +def exec_jail(jail_name, cmd): """ Execute a command in the jail with given name. """ @@ -434,9 +450,8 @@ def exec_jail(jail_name, cmd, args): "--wait", "--collect", "--service-type=exec", - cmd, + *cmd, ] - + args ).returncode @@ -923,11 +938,7 @@ def agree_with_default(config, key, question): config.my_set(key, agree(question, default_answer)) -def create_jail_interactive(): - """ - Create jail with given name. - """ - +def interactive_config(): config = KeyValueParser() config.read_string(DEFAULT_CONFIG) @@ -998,7 +1009,6 @@ def create_jail_interactive(): # Ask for jail name jail_name = ask_jail_name(jail_name) - jail_path = get_jail_path(jail_name) else: print() if not agree( @@ -1044,7 +1054,6 @@ def create_jail_interactive(): config.my_set("release", input("Release: ")) jail_name = ask_jail_name(jail_name) - jail_path = get_jail_path(jail_name) print( dedent( @@ -1167,31 +1176,67 @@ def create_jail_interactive(): print() - ############## - # Create start - ############## - - create_options = { - "jail_name": jail_name, - "jail_path": jail_path, - "start_now": start_now, - "config": config, - } - - rc = write_jail(create_options) - if rc != 0: - return rc - - if create_options["start_now"]: - return start_jail(jail_name) - - return 0 + return jail_name, config, start_now -def write_jail(create_options): - jail_name = create_options["jail_name"] - jail_path = create_options["jail_path"] - config = create_options["config"] +def create_jail(**kwargs): + jail_name = kwargs.pop("jail_name", None) + start_now = False + + # Non-interactive create + if jail_name: + if not check_jail_name_valid(jail_name): + return 1 + + if not check_jail_name_available(jail_name): + return 1 + + jail_config_path = kwargs.pop("config") + + config = KeyValueParser() + + if jail_config_path: + print(f"Creating jail {jail_name} from config template {jail_config_path}.") + if jail_config_path not in config.read(jail_config_path): + eprint(f"Failed to read config config template {jail_config_path}.") + return 1 + else: + print(f"Creating jail {jail_name} with default config.") + config.read_string(DEFAULT_CONFIG) + + user_overridden = False + + for option in [ + "distro", + "docker_compatible", + "gpu_passthrough_intel", + "gpu_passthrough_nvidia", + "release", + "startup", + "systemd_nspawn_user_args", + ]: + value = kwargs.pop(option) + if value: + # TODO: this will wipe all systemd_nspawn_user_args from the template... + # Should there be an option to append them instead? + print(f"Overriding {option} config value with {value}.") + config.my_set(option, value) + user_overridden = True + + if not user_overridden: + print( + dedent( + f""" + TIP: Run `{SYMLINK_NAME} create` without any arguments for interactive config. + Or use CLI args to override the default options. + For more info, run: `{SYMLINK_NAME} create --help` + """ + ) + ) + else: + jail_name, config, start_now = interactive_config() + + jail_path = get_jail_path(jail_name) distro = config.my_get("distro") release = config.my_get("release") @@ -1267,7 +1312,7 @@ def write_jail(create_options): print("Autostart has been disabled.") print("You need to start this jail manually.") config.my_set("startup", 0) - create_options["start_now"] = False + start_now = False with contextlib.suppress(FileNotFoundError): # Remove config which systemd handles for us @@ -1348,6 +1393,9 @@ def write_jail(create_options): cleanup(jail_path) raise error + if start_now: + return start_jail(jail_name) + return 0 @@ -1728,84 +1776,205 @@ def startup_jails(): return 0 +def split_at_string(lst, string): + try: + index = lst.index(string) + return lst[:index], lst[index + 1 :] + except ValueError: + return lst, [] + + +def add_parser(subparser, **kwargs): + if kwargs.get("add_help") is False: + # Don't add help if explicitly disabled + add_help = False + else: + # Never add help with the built in add_help + kwargs["add_help"] = False + add_help = True + + kwargs["epilog"] = DISCLAIMER + kwargs["exit_on_error"] = False + parser = subparser.add_parser(**kwargs) + + if add_help: + parser.add_argument( + "-h", "--help", help="show this help message and exit", action="store_true" + ) + + # Setting the add_help after the parser has been created with add_parser has no effect, + # but it allows us to look up if this parser has a help message available + parser.add_help = add_help + + return parser + + def main(): if os.stat(SCRIPT_PATH).st_uid != 0: fail( f"This script should be owned by the root user... Fix it manually with: `chown root {SCRIPT_PATH}`." ) - parser = argparse.ArgumentParser(description=__doc__, epilog=DISCLAIMER) + parser = argparse.ArgumentParser( + description=__doc__, epilog=DISCLAIMER, allow_abbrev=False + ) parser.add_argument("--version", action="version", version=__version__) - subparsers = parser.add_subparsers(title="commands", dest="subcommand", metavar="") - - subparsers.add_parser( - name="install", - epilog=DISCLAIMER, - help="install jailmaker dependencies and create symlink", + subparsers = parser.add_subparsers( + title="commands", dest="command", metavar="", parser_class=CustomSubParser ) - subparsers.add_parser(name="create", epilog=DISCLAIMER, help="create a new jail") + split_commands = ["create", "exec"] + commands = {} - subparsers.add_parser( - name="start", epilog=DISCLAIMER, help="start a previously created jail" - ).add_argument("name", help="name of the jail") + for d in [ + dict( + name="create", # + help="create a new jail", + ), + dict( + name="edit", + help=f"edit jail config with {TEXT_EDITOR} text editor", + ), + dict( + name="exec", + help="execute a command in the jail", + ), + dict( + name="images", + help="list available images to create jails from", + ), + dict( + name="install", + help="install jailmaker dependencies and create symlink", + ), + dict( + name="list", # + help="list jails", + ), + dict( + name="log", # + help="show jail log", + ), + dict( + name="remove", + help="remove previously created jail", + ), + dict( + name="restart", # + help="restart a running jail", + ), + dict( + name="shell", + help="open shell in running jail (alias for machinectl shell)", + add_help=False, + ), + dict( + name="start", + help="start previously created jail", + ), + dict( + name="startup", + help=f"install {SYMLINK_NAME} and startup selected jails", + ), + dict( + name="status", # + help="show jail status", + ), + dict( + name="stop", # + help="stop a running jail", + ), + ]: + commands[d["name"]] = add_parser(subparsers, **d) - subparsers.add_parser( - name="restart", epilog=DISCLAIMER, help="restart a running jail" - ).add_argument("name", help="name of the jail") + # Install parser + commands["install"].set_defaults(func=install_jailmaker) - subparsers.add_parser( - name="shell", - epilog=DISCLAIMER, - help="open shell in running jail (alias for machinectl shell)", + # Create parser + commands["create"].add_argument("jail_name", nargs="?", help="name of the jail") + commands["create"].add_argument("--distro") + commands["create"].add_argument("--release") + commands["create"].add_argument( + "--startup", + type=int, + choices=[0, 1], + help=f"start this jail when running: {SCRIPT_NAME} startup", ) - - exec_parser = subparsers.add_parser( - name="exec", epilog=DISCLAIMER, help="execute a command in the jail" + commands["create"].add_argument("--docker_compatible", type=int, choices=[0, 1]) + commands["create"].add_argument( + "-c", "--config", help="path to config file template" ) - exec_parser.add_argument("name", help="name of the jail") - exec_parser.add_argument("cmd", help="command to execute") - - subparsers.add_parser( - name="status", epilog=DISCLAIMER, help="show jail status" - ).add_argument("name", help="name of the jail") - - subparsers.add_parser( - name="log", epilog=DISCLAIMER, help="show jail log" - ).add_argument("name", help="name of the jail") - - subparsers.add_parser( - name="stop", epilog=DISCLAIMER, help="stop a running jail" - ).add_argument("name", help="name of the jail") - - subparsers.add_parser( - name="edit", - epilog=DISCLAIMER, - help=f"edit jail config with {TEXT_EDITOR} text editor", - ).add_argument("name", help="name of the jail to edit") - - subparsers.add_parser( - name="remove", epilog=DISCLAIMER, help="remove a previously created jail" - ).add_argument("name", help="name of the jail to remove") - - subparsers.add_parser(name="list", epilog=DISCLAIMER, help="list jails") - - subparsers.add_parser( - name="images", - epilog=DISCLAIMER, - help="list available images to create jails from", + commands["create"].add_argument( + "-gi", "--gpu_passthrough_intel", type=int, choices=[0, 1] ) - - subparsers.add_parser( - name="startup", - epilog=DISCLAIMER, - help=f"install {SYMLINK_NAME} and startup selected jails", + commands["create"].add_argument( + "-gn", "--gpu_passthrough_nvidia", type=int, choices=[0, 1] ) + commands["create"].add_argument( + "systemd_nspawn_user_args", + nargs="*", + help="add additional systemd-nspawn flags", + ) + commands["create"].set_defaults(func=create_jail) + + # Start parser + commands["start"].add_argument("jail_name", help="name of the jail") + commands["start"].set_defaults(func=start_jail) + + # Restart parser + commands["restart"].add_argument("jail_name", help="name of the jail") + commands["restart"].set_defaults(func=restart_jail) + + # Shell parser + commands["shell"].add_argument( + "args", + nargs="*", + help="args to pass to machinectl shell", + ) + commands["shell"].set_defaults(func=shell_jail) + + # Exec parser + commands["exec"].add_argument("jail_name", help="name of the jail") + commands["exec"].add_argument( + "cmd", + nargs="*", + help="command to execute", + ) + commands["exec"].set_defaults(func=exec_jail) + + # Status parser + commands["status"].add_argument("jail_name", help="name of the jail") + commands["status"].set_defaults(func=status_jail) + + # Log parser + commands["log"].add_argument("jail_name", help="name of the jail") + commands["log"].set_defaults(func=log_jail) + + # Stop parser + commands["stop"].add_argument("jail_name", help="name of the jail") + commands["stop"].set_defaults(func=stop_jail) + + # Edit parser + commands["edit"].add_argument("jail_name", help="name of the jail to edit") + commands["edit"].set_defaults(func=edit_jail) + + # Remove parser + commands["remove"].add_argument("jail_name", help="name of the jail to remove") + commands["remove"].set_defaults(func=remove_jail) + + # List parser + commands["list"].set_defaults(func=list_jails) + + # Images parser + commands["images"].set_defaults(func=run_lxc_download_script) + + # Startup parser + commands["startup"].set_defaults(func=startup_jails) if os.getuid() != 0: - parser.print_usage() + parser.print_help() fail("Run this script as root...") # Set appropriate permissions (if not already set) for this file, since it's executed as root @@ -1814,56 +1983,82 @@ def main(): # Work relative to this script os.chdir(SCRIPT_DIR_PATH) - args, additional_args = parser.parse_known_args() + # Ignore all args after the first "--" + args_to_parse = split_at_string(sys.argv[1:], "--")[0] + # Check for help + if any(item in args_to_parse for item in ["-h", "--help"]): + # Likely we need to show help output... + try: + args = vars(parser.parse_known_args(args_to_parse)[0]) + # We've exited by now if not invoking a subparser: jlmkr.py --help + if args.get("help"): + need_help = True + command = args.get("command") - if args.subcommand == "install": - sys.exit(install_jailmaker()) + # Edge case for some commands + if command in split_commands and args["jail_name"]: + # Ignore all args after the jail name + args_to_parse = split_at_string(args_to_parse, args["jail_name"])[0] + # Add back the jail_name as it may be a required positional and we + # don't want to end up in the except clause below + args_to_parse += [args["jail_name"]] + # Parse one more time... + args = vars(parser.parse_known_args(args_to_parse)[0]) + # ...and check if help is still in the remaining args + need_help = args.get("help") + print(need_help) - elif args.subcommand == "create": - sys.exit(create_jail_interactive()) + if need_help: + commands[command].print_help() + sys.exit() + except ExceptionWithParser as e: + # Print help output on error, e.g. due to: + # "error: the following arguments are required" + if e.parser.add_help: + e.parser.print_help() + sys.exit() - elif args.subcommand == "start": - sys.exit(start_jail(args.name)) + # Exit on parse errors (e.g. missing positional args) + for command in commands: + commands[command].exit_on_error = True - elif args.subcommand == "restart": - sys.exit(restart_jail(args.name)) + # Parse to find command and function and ignore unknown args which may be present + # such as args intended to pass through to systemd-run + args = vars(parser.parse_known_args()[0]) + command = args.pop("command", None) - elif args.subcommand == "shell": - sys.exit(shell_jail(additional_args)) + # Start over with original args + args_to_parse = sys.argv[1:] - elif args.subcommand == "exec": - sys.exit(exec_jail(args.name, args.cmd, additional_args)) + if not command: + # Parse args and show error for unknown args + parser.parse_args(args_to_parse) - elif args.subcommand == "status": - sys.exit(status_jail(args.name)) - - elif args.subcommand == "log": - sys.exit(log_jail(args.name)) - - elif args.subcommand == "stop": - sys.exit(stop_jail(args.name)) - - elif args.subcommand == "edit": - sys.exit(edit_jail(args.name)) - - elif args.subcommand == "remove": - sys.exit(remove_jail(args.name)) - - elif args.subcommand == "list": - sys.exit(list_jails()) - - elif args.subcommand == "images": - sys.exit(run_lxc_download_script()) - - elif args.subcommand == "startup": - sys.exit(startup_jails()) - - else: if agree("Create a new jail?", "y"): print() - sys.exit(create_jail_interactive()) + sys.exit(create_jail()) else: - parser.print_usage() + parser.print_help() + sys.exit() + + elif command == "shell": + # Pass anything after the "shell" command to machinectl + _, shell_args = split_at_string(args_to_parse, command) + sys.exit(args["func"](shell_args)) + elif command in split_commands and args["jail_name"]: + jlmkr_args, remaining_args = split_at_string(args_to_parse, args["jail_name"]) + if remaining_args and remaining_args[0] != "--": + # Add "--" after the jail name to ensure further args, e.g. + # --help or --version, are captured as systemd_nspawn_user_args + args_to_parse = jlmkr_args + [args["jail_name"], "--"] + remaining_args + + # Parse args again, but show error for unknown args + args = vars(parser.parse_args(args_to_parse)) + # Clean the args + args.pop("help") + args.pop("command", None) + func = args.pop("func") + sys.exit(func(**args)) if __name__ == "__main__": diff --git a/templates/docker/README.md b/templates/docker/README.md index 29212ce..b7e0764 100644 --- a/templates/docker/README.md +++ b/templates/docker/README.md @@ -2,4 +2,4 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mydockerjail /mnt/tank/path/to/docker/config`. \ No newline at end of file +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/docker/config mydockerjail`. \ No newline at end of file diff --git a/templates/incus/README.md b/templates/incus/README.md index 36d0add..70cc801 100644 --- a/templates/incus/README.md +++ b/templates/incus/README.md @@ -6,7 +6,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create myincusjail /mnt/tank/path/to/incus/config`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/incus/config myincusjail`. Unfortunately incus doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: diff --git a/templates/lxd/README.md b/templates/lxd/README.md index 03e3026..baa24cb 100644 --- a/templates/lxd/README.md +++ b/templates/lxd/README.md @@ -6,7 +6,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mylxdjail /mnt/tank/path/to/lxd/config`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/lxd/config mylxdjail`. Unfortunately snapd doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: diff --git a/templates/podman/README.md b/templates/podman/README.md index b27682e..dba7fbd 100644 --- a/templates/podman/README.md +++ b/templates/podman/README.md @@ -2,7 +2,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create mypodmanjail /mnt/tank/path/to/podman/config`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/podman/config mypodmanjail`. ## Rootless From 98f812be8b66e39bec29a24e2c7df7c385ff462b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 17:20:47 +0100 Subject: [PATCH 52/57] Cosmetic changes --- jlmkr.py | 121 +++++++++++++++++++++++++------------------------------ 1 file changed, 55 insertions(+), 66 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 159f777..1613c98 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -187,6 +187,7 @@ class KeyValueParser(configparser.ConfigParser): return cosmetic_newline_indices + # TODO: can I create a solution which not depends on the internal _read method? def _read(self, fp, fpname): lines = fp.readlines() cosmetic_newline_indices = self._find_cosmetic_newlines("".join(lines)) @@ -1795,7 +1796,9 @@ def add_parser(subparser, **kwargs): kwargs["epilog"] = DISCLAIMER kwargs["exit_on_error"] = False + func = kwargs.pop("func") parser = subparser.add_parser(**kwargs) + parser.set_defaults(func=func) if add_help: parser.add_argument( @@ -1832,68 +1835,97 @@ def main(): dict( name="create", # help="create a new jail", + func=create_jail, ), dict( name="edit", help=f"edit jail config with {TEXT_EDITOR} text editor", + func=edit_jail, ), dict( - name="exec", + name="exec", # help="execute a command in the jail", + func=exec_jail, ), dict( name="images", help="list available images to create jails from", + func=run_lxc_download_script, ), dict( name="install", help="install jailmaker dependencies and create symlink", + func=install_jailmaker, ), dict( name="list", # help="list jails", + func=list_jails, ), dict( name="log", # help="show jail log", + func=log_jail, ), dict( - name="remove", + name="remove", # help="remove previously created jail", + func=remove_jail, ), dict( name="restart", # help="restart a running jail", + func=restart_jail, ), dict( name="shell", help="open shell in running jail (alias for machinectl shell)", + func=shell_jail, add_help=False, ), dict( - name="start", + name="start", # help="start previously created jail", + func=start_jail, ), dict( name="startup", help=f"install {SYMLINK_NAME} and startup selected jails", + func=startup_jails, ), dict( name="status", # help="show jail status", + func=status_jail, ), dict( name="stop", # help="stop a running jail", + func=stop_jail, ), ]: commands[d["name"]] = add_parser(subparsers, **d) - # Install parser - commands["install"].set_defaults(func=install_jailmaker) + for cmd in ["edit", "exec", "log", "remove", "restart", "start", "status", "stop"]: + commands[cmd].add_argument("jail_name", help="name of the jail") - # Create parser - commands["create"].add_argument("jail_name", nargs="?", help="name of the jail") + commands["exec"].add_argument( + "cmd", + nargs="*", + help="command to execute", + ) + + commands["shell"].add_argument( + "args", + nargs="*", + help="args to pass to machinectl shell", + ) + + commands["create"].add_argument( + "jail_name", # + nargs="?", + help="name of the jail", + ) commands["create"].add_argument("--distro") commands["create"].add_argument("--release") commands["create"].add_argument( @@ -1902,76 +1934,33 @@ def main(): choices=[0, 1], help=f"start this jail when running: {SCRIPT_NAME} startup", ) - commands["create"].add_argument("--docker_compatible", type=int, choices=[0, 1]) commands["create"].add_argument( - "-c", "--config", help="path to config file template" + "--docker_compatible", # + type=int, + choices=[0, 1], ) commands["create"].add_argument( - "-gi", "--gpu_passthrough_intel", type=int, choices=[0, 1] + "-c", # + "--config", + help="path to config file template", ) commands["create"].add_argument( - "-gn", "--gpu_passthrough_nvidia", type=int, choices=[0, 1] + "-gi", # + "--gpu_passthrough_intel", + type=int, + choices=[0, 1], + ) + commands["create"].add_argument( + "-gn", # + "--gpu_passthrough_nvidia", + type=int, + choices=[0, 1], ) commands["create"].add_argument( "systemd_nspawn_user_args", nargs="*", help="add additional systemd-nspawn flags", ) - commands["create"].set_defaults(func=create_jail) - - # Start parser - commands["start"].add_argument("jail_name", help="name of the jail") - commands["start"].set_defaults(func=start_jail) - - # Restart parser - commands["restart"].add_argument("jail_name", help="name of the jail") - commands["restart"].set_defaults(func=restart_jail) - - # Shell parser - commands["shell"].add_argument( - "args", - nargs="*", - help="args to pass to machinectl shell", - ) - commands["shell"].set_defaults(func=shell_jail) - - # Exec parser - commands["exec"].add_argument("jail_name", help="name of the jail") - commands["exec"].add_argument( - "cmd", - nargs="*", - help="command to execute", - ) - commands["exec"].set_defaults(func=exec_jail) - - # Status parser - commands["status"].add_argument("jail_name", help="name of the jail") - commands["status"].set_defaults(func=status_jail) - - # Log parser - commands["log"].add_argument("jail_name", help="name of the jail") - commands["log"].set_defaults(func=log_jail) - - # Stop parser - commands["stop"].add_argument("jail_name", help="name of the jail") - commands["stop"].set_defaults(func=stop_jail) - - # Edit parser - commands["edit"].add_argument("jail_name", help="name of the jail to edit") - commands["edit"].set_defaults(func=edit_jail) - - # Remove parser - commands["remove"].add_argument("jail_name", help="name of the jail to remove") - commands["remove"].set_defaults(func=remove_jail) - - # List parser - commands["list"].set_defaults(func=list_jails) - - # Images parser - commands["images"].set_defaults(func=run_lxc_download_script) - - # Startup parser - commands["startup"].set_defaults(func=startup_jails) if os.getuid() != 0: parser.print_help() From 2b104682b54cb128c19b6fa6569989a07d653998 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 17:20:50 +0100 Subject: [PATCH 53/57] Update README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f862c1..ae483bb 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,12 @@ You may also specify a path to a config template, for a quick and consistent jai jlmkr create --config /path/to/config/template myjail ``` +Or you can override the default config by using flags. See `jlmkr create --help` for the available options. Anything passed after the jail name will be passed to `systemd-nspawn` when starting the jail. See the `systemd-nspawn` manual for available options, specifically [Mount Options](https://manpages.debian.org/bookworm/systemd-container/systemd-nspawn.1.en.html#Mount_Options) and [Networking Options](https://manpages.debian.org/bookworm/systemd-container/systemd-nspawn.1.en.html#Networking_Options) are frequently used. + +```shell +jlmkr create --distro=ubuntu --release=jammy myjail --bind-ro=/mnt +``` + If you omit the jail name, the create process is interactive. You'll be presented with questions which guide you through the process. ```shell @@ -151,7 +157,7 @@ See [Advanced Networking](./NETWORKING.md) for more. ## Docker -The `jailmaker` script won't install Docker for you, but it can setup the jail with the capabilities required to run docker. You can manually install Docker inside the jail using the [official installation guide](https://docs.docker.com/engine/install/#server) or use [convenience script](https://get.docker.com). +The `jailmaker` script won't install Docker for you, but it can setup the jail with the capabilities required to run docker. You can manually install Docker inside the jail using the [official installation guide](https://docs.docker.com/engine/install/#server) or use [convenience script](https://get.docker.com). Additionally you may use the [docker config template](./templates/docker/README.md). ## Documentation From ac75cd3c284d7d711cbeda5eee56735bdccebd2f Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 17:34:55 +0100 Subject: [PATCH 54/57] Add --start flag for create command --- jlmkr.py | 6 ++++++ templates/docker/README.md | 2 +- templates/incus/README.md | 2 +- templates/lxd/README.md | 2 +- templates/podman/README.md | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 1613c98..5d62e66 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1192,6 +1192,7 @@ def create_jail(**kwargs): if not check_jail_name_available(jail_name): return 1 + start_now = kwargs.pop("start", start_now) jail_config_path = kwargs.pop("config") config = KeyValueParser() @@ -1928,6 +1929,11 @@ def main(): ) commands["create"].add_argument("--distro") commands["create"].add_argument("--release") + commands["create"].add_argument( + "--start", # + help="start jail after create", + action="store_true", + ) commands["create"].add_argument( "--startup", type=int, diff --git a/templates/docker/README.md b/templates/docker/README.md index b7e0764..17a5b5b 100644 --- a/templates/docker/README.md +++ b/templates/docker/README.md @@ -2,4 +2,4 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/docker/config mydockerjail`. \ No newline at end of file +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/docker/config mydockerjail`. \ No newline at end of file diff --git a/templates/incus/README.md b/templates/incus/README.md index 70cc801..72d7339 100644 --- a/templates/incus/README.md +++ b/templates/incus/README.md @@ -6,7 +6,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/incus/config myincusjail`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/incus/config myincusjail`. Unfortunately incus doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: diff --git a/templates/lxd/README.md b/templates/lxd/README.md index baa24cb..185a16f 100644 --- a/templates/lxd/README.md +++ b/templates/lxd/README.md @@ -6,7 +6,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/lxd/config mylxdjail`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/lxd/config mylxdjail`. Unfortunately snapd doesn't want to install from the `initial_setup` script inside the config file. So we manually finish the setup by running the following after creating and starting the jail: diff --git a/templates/podman/README.md b/templates/podman/README.md index dba7fbd..4e28d24 100644 --- a/templates/podman/README.md +++ b/templates/podman/README.md @@ -2,7 +2,7 @@ ## Setup -Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --config /mnt/tank/path/to/podman/config mypodmanjail`. +Check out the [config](./config) template file. You may provide it when asked during `jlmkr create` or, if you have the template file stored on your NAS, you may provide it directly by running `jlmkr create --start --config /mnt/tank/path/to/podman/config mypodmanjail`. ## Rootless From 9e160c8a524278669196e161aaa2a14c16884074 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:20:25 +0100 Subject: [PATCH 55/57] Fix list fallback value --- jlmkr.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 5d62e66..e61f19b 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1502,7 +1502,10 @@ def print_table(header, list_of_objects, empty_value_indicator): widths = defaultdict(int) for obj in list_of_objects: for hdr in header: - widths[hdr] = max(widths[hdr], len(str(obj.get(hdr))), len(str(hdr))) + value = obj.get(hdr) + if value is None: + obj[hdr] = value = empty_value_indicator + widths[hdr] = max(widths[hdr], len(str(value)), len(str(hdr))) # Print header print( @@ -1511,12 +1514,7 @@ def print_table(header, list_of_objects, empty_value_indicator): # Print rows for obj in list_of_objects: - print( - " ".join( - str(obj.get(hdr, empty_value_indicator)).ljust(widths[hdr]) - for hdr in header - ) - ) + print(" ".join(str(obj.get(hdr)).ljust(widths[hdr]) for hdr in header)) def run_command_and_parse_json(command): @@ -1544,6 +1542,7 @@ def parse_os_release(candidates): for candidate in candidates: try: with open(candidate, encoding="utf-8") as f: + # TODO: can I create a solution which not depends on the internal _parse_os_release method? return platform._parse_os_release(f) except OSError: # Silently ignore failing to read os release info @@ -1597,8 +1596,8 @@ def list_jails(): machine = running_machines[jail_name] # Augment the jails dict with output from machinectl jail["running"] = True - jail["os"] = machine["os"] - jail["version"] = machine["version"] + jail["os"] = machine["os"] or None + jail["version"] = machine["version"] or None addresses = machine.get("addresses") if not addresses: @@ -1618,7 +1617,9 @@ def list_jails(): ) jail["os"] = jail_platform.get("ID") - jail["version"] = jail_platform.get("VERSION_ID") + jail["version"] = jail_platform.get("VERSION_ID") or jail_platform.get( + "VERSION_CODENAME" + ) print_table( [ From 2740a4323a678a1c90debb41889ba42945e4fa22 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:22:46 +0100 Subject: [PATCH 56/57] Update compatibility.md --- docs/compatibility.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index b4ddb3a..90559de 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1,15 +1,20 @@ # TrueNAS Compatibility -TrueNAS Core ❌ -TrueNAS 22.12 ✅ -TrueNAS 23.10-BETA1 ✅ -TrueNAS 23.10-RC1 ✅ +| | | +|---|---| +|TrueNAS Core|❌| +|TrueNAS 22.12|✅| +|TrueNAS 23.10|✅| +|TrueNAS 24.04 nightly|✅| # Distro Compatibility -Debian 11 Bullseye ✅ -Debian 12 Bookworm ✅ -Arch 🟨 -Ubuntu 🟨 -Alpine ❌ +| | | +|---|---| +|Debian 11 Bullseye|✅| +|Debian 12 Bookworm|✅| +|Ubuntu Jammy|✅| +|Fedora 39|✅| +|Arch|🟨| +|Alpine|❌| ✅ = Personally tested and working 🟨 = Haven't personally tested From cd067d705832b02b1a30ce420b6b212aabbc66b3 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:38:06 +0100 Subject: [PATCH 57/57] Bump version to 1.1.0 --- jlmkr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index e61f19b..7a43719 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -4,7 +4,7 @@ with full access to all files via bind mounts, \ thanks to systemd-nspawn!""" -__version__ = "1.0.1" +__version__ = "1.1.0" __disclaimer__ = """USE THIS SCRIPT AT YOUR OWN RISK! IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS."""