From f87e372daea70ecb33e32dc4303e51fdaecc6ef5 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 23 Jun 2024 11:27:48 +0200 Subject: [PATCH 01/37] Fix removal of resolv.conf --- jlmkr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index 3e6a9ed..262fab3 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1379,9 +1379,10 @@ def create_jail(**kwargs): config.my_set("startup", 0) start_now = False + # Remove config which systemd handles for us with contextlib.suppress(FileNotFoundError): - # Remove config which systemd handles for us os.remove(os.path.join(jail_rootfs_path, "etc/machine-id")) + with contextlib.suppress(FileNotFoundError): os.remove(os.path.join(jail_rootfs_path, "etc/resolv.conf")) # https://github.com/systemd/systemd/issues/852 From 2ce89c2945b46680827e328c13e5f838790c7ce2 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 23 Jun 2024 13:01:21 +0200 Subject: [PATCH 02/37] Fix removal of immutable/append-only files --- jlmkr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index 262fab3..e745866 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -775,7 +775,12 @@ def cleanup(jail_path): # Should be fixed in Python 3.13 https://stackoverflow.com/a/70549000 def _onerror(func, path, exc_info): exc_type, exc_value, exc_traceback = exc_info - if not issubclass(exc_type, FileNotFoundError): + if issubclass(exc_type, PermissionError): + # Update the file permissions with the immutable and append-only bit cleared + subprocess.run(["chattr", "-i", "-a", path]) + # Reattempt the removal + func(path) + elif not issubclass(exc_type, FileNotFoundError): raise exc_value eprint(f"Cleaning up: {jail_path}.") From 74e717a23be84b96afd6515c6115c2b1f88d2129 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sun, 23 Jun 2024 18:33:20 +0200 Subject: [PATCH 03/37] Fix comment in config templates --- templates/docker/config | 2 +- templates/incus/config | 2 +- templates/k3s/config | 2 +- templates/lxd/config | 2 +- templates/podman/config | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/docker/config b/templates/docker/config index 51f7f1d..5d72f87 100644 --- a/templates/docker/config +++ b/templates/docker/config @@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0 # Turning off seccomp filtering improves performance at the expense of security seccomp=1 -# Use macvlan networking to provide an isolated network namespace, +# Use bridge networking to provide an isolated network namespace, # so docker can manage firewall rules # Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use diff --git a/templates/incus/config b/templates/incus/config index 4234843..9485b91 100644 --- a/templates/incus/config +++ b/templates/incus/config @@ -6,7 +6,7 @@ gpu_passthrough_nvidia=0 # TODO: don't disable seccomp but specify which syscalls should be allowed seccomp=0 -# Use macvlan networking to provide an isolated network namespace, +# Use bridge networking to provide an isolated network namespace, # so incus can manage firewall rules # Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use diff --git a/templates/k3s/config b/templates/k3s/config index 84eabfe..5853e1b 100644 --- a/templates/k3s/config +++ b/templates/k3s/config @@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0 # Turning off seccomp filtering improves performance at the expense of security seccomp=1 -# Use macvlan networking to provide an isolated network namespace, +# Use bridge networking to provide an isolated network namespace, # so kubernetes can manage firewall rules # Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use diff --git a/templates/lxd/config b/templates/lxd/config index 36a3af5..0b068b3 100644 --- a/templates/lxd/config +++ b/templates/lxd/config @@ -6,7 +6,7 @@ gpu_passthrough_nvidia=0 # TODO: don't disable seccomp but specify which syscalls should be allowed seccomp=0 -# Use macvlan networking to provide an isolated network namespace, +# Use bridge networking to provide an isolated network namespace, # so lxd can manage firewall rules # Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use diff --git a/templates/podman/config b/templates/podman/config index b19106d..6aa1cfc 100644 --- a/templates/podman/config +++ b/templates/podman/config @@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0 # Turning off seccomp filtering improves performance at the expense of security seccomp=1 -# Use macvlan networking to provide an isolated network namespace, +# Use bridge networking to provide an isolated network namespace, # so podman can manage firewall rules # Alternatively use --network-macvlan=eno1 instead of --network-bridge # Ensure to change eno1/br1 to the interface name you want to use From 804be6d7602f4d7ae14ef77a5dc512a86ce267a2 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:41:24 +0200 Subject: [PATCH 04/37] Put initial_setup script in jail root --- jlmkr.py | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index e745866..d4ba0a6 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -485,9 +485,7 @@ def log_jail(jail_name): """ Show the log file of the jail with given name. """ - return subprocess.run( - ["journalctl", "-u", f"{SHORTNAME}-{jail_name}"] - ).returncode + return subprocess.run(["journalctl", "-u", f"{SHORTNAME}-{jail_name}"]).returncode def shell_jail(args): @@ -658,18 +656,7 @@ def start_jail(jail_name): if not os.path.exists(os.path.join(jail_rootfs_path, "etc/machine-id")) and ( initial_setup := config.my_get("initial_setup") ): - if not initial_setup.startswith("#!"): - initial_setup = "#!/bin/sh\n" + initial_setup - - initial_setup_file_jailed_path = "/root/jlmkr-initial-setup" - initial_setup_file_host_path = os.path.abspath( - jail_rootfs_path + initial_setup_file_jailed_path - ) - - # Write a script file to call during initial setup - print(initial_setup, file=open(initial_setup_file_host_path, "w")) - stat_chmod(initial_setup_file_host_path, 0o700) - + # initial_setup has been assigned due to := expression above # Ensure the jail init system is ready before we start the initial_setup systemd_nspawn_additional_args += [ "--notify-ready=yes", @@ -712,38 +699,56 @@ def start_jail(jail_name): # Handle initial setup after jail is up and running (for the first time) if initial_setup: - print("About to run the initial setup.") + if not initial_setup.startswith("#!"): + initial_setup = "#!/bin/sh\n" + initial_setup + + with tempfile.NamedTemporaryFile( + mode="w+t", + prefix="jlmkr-initial-setup.", + dir=jail_rootfs_path, + delete=False, + ) as initial_setup_file: + # Write a script file to call during initial setup + initial_setup_file.write(initial_setup) + + initial_setup_file_name = os.path.basename(initial_setup_file.name) + initial_setup_file_host_path = os.path.abspath(initial_setup_file.name) + stat_chmod(initial_setup_file_host_path, 0o700) + + print(f"About to run the initial setup script: {initial_setup_file_name}.") print("Waiting for networking in the jail to be ready.") - print("Please wait (this may take 90s in case of bridge networking with STP is enabled)...") + print( + "Please wait (this may take 90s in case of bridge networking with STP is enabled)..." + ) returncode = exec_jail( jail_name, [ "--", "systemd-run", - f"--unit={os.path.basename(initial_setup_file_jailed_path)}", + f"--unit={initial_setup_file_name}", "--quiet", "--pipe", "--wait", "--service-type=exec", "--property=After=network-online.target", "--property=Wants=network-online.target", - initial_setup_file_jailed_path, + "/" + initial_setup_file_name, ], ) - # Cleanup the initial_setup_file_host_path - if initial_setup_file_host_path: - Path(initial_setup_file_host_path).unlink(missing_ok=True) - if returncode != 0: eprint("Tried to run the following commands inside the jail:") eprint(initial_setup) eprint() + eprint(f"{RED}{BOLD}Failed to run initial setup...") eprint( - f"""{RED}{BOLD}Failed to run initial setup... you may want to stop and remove the jail and try again.{NORMAL}""" + f"You may want to manually run /{initial_setup_file_name} inside the jail for debugging purposes." ) + eprint(f"Or stop and remove the jail and try again.{NORMAL}") return returncode else: + # Cleanup the initial_setup_file_host_path + Path(initial_setup_file_host_path).unlink(missing_ok=True) print(f"Done with initial setup of jail {jail_name}!") return returncode From 49d65c10731669bb88991ec8ad7a993f01ae35b5 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:55:41 +0200 Subject: [PATCH 05/37] Fix case --- jlmkr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index d4ba0a6..9cb8358 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1289,7 +1289,7 @@ def create_jail(**kwargs): print( dedent( f""" - TIP: Run `{COMMAND_NAME} create` without any arguments for interactive config. + Hint: run `{COMMAND_NAME} create` without any arguments for interactive config. Or use CLI args to override the default options. For more info, run: `{COMMAND_NAME} create --help` """ From 51884e215c3cc7fc6b556dc62c6934afec05463b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:17:33 +0200 Subject: [PATCH 06/37] Parse os-release in chroot To parse the right os-release file case of absolute symlinks --- jlmkr.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 9cb8358..4525dd2 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -125,6 +125,8 @@ SCRIPT_PATH = os.path.realpath(__file__) SCRIPT_NAME = os.path.basename(SCRIPT_PATH) SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH) COMMAND_NAME = os.path.basename(__file__) +INITIAL_ROOT = os.open("/", os.O_PATH) +CWD_BEFORE_CHROOT = None SHORTNAME = "jlmkr" # Only set a color if we have an interactive tty @@ -294,6 +296,20 @@ def fail(*args, **kwargs): sys.exit(1) +def enter_chroot(new_root): + global CWD_BEFORE_CHROOT + CWD_BEFORE_CHROOT = os.path.abspath(os.getcwd()) + os.chdir(new_root) + os.chroot(".") + + +# https://stackoverflow.com/a/61533559 +def exit_chroot(): + os.chdir(INITIAL_ROOT) + os.chroot(".") + os.chdir(CWD_BEFORE_CHROOT) + + def get_jail_path(jail_name): return os.path.join(JAILS_DIR_PATH, jail_name) @@ -1627,16 +1643,21 @@ def get_all_jail_names(): return jail_names -def parse_os_release(candidates): - for candidate in candidates: +def parse_os_release(new_rootfs): + enter_chroot(new_rootfs) + result = {} + for candidate in ["/etc/os-release", "/usr/lib/os-release"]: try: with open(candidate, encoding="utf-8") as f: # TODO: can I create a solution which not depends on the internal _parse_os_release method? - return platform._parse_os_release(f) + result = platform._parse_os_release(f) + break except OSError: # Silently ignore failing to read os release info pass - return {} + + exit_chroot() + return result def list_jails(): @@ -1691,13 +1712,7 @@ def list_jails(): jail["addresses"] += "…" else: # Parse os-release info ourselves - jail_platform = parse_os_release( - ( - os.path.join(jail_rootfs_path, "etc/os-release"), - os.path.join(jail_rootfs_path, "usr/lib/os-release"), - ) - ) - + jail_platform = parse_os_release(jail_rootfs_path) jail["os"] = jail_platform.get("ID") jail["version"] = jail_platform.get("VERSION_ID") or jail_platform.get( "VERSION_CODENAME" From 7832b17ae193a39d61baa13b79cfffeed0cbdc21 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:24:33 +0200 Subject: [PATCH 07/37] Detect init system in chroot --- jlmkr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 4525dd2..9f6faee 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1373,11 +1373,12 @@ def create_jail(**kwargs): # But alpine jails made with jailmaker have other issues # They don't shutdown cleanly via systemctl and machinectl... + enter_chroot(jail_rootfs_path) + init_system_name = os.path.basename(os.path.realpath("/sbin/init")) + exit_chroot() + if ( - os.path.basename( - os.path.realpath(os.path.join(jail_rootfs_path, "sbin/init")) - ) - != "systemd" + init_system_name != "systemd" ): print( dedent( From 32143a99075bebf1f7dfe57cbdaa120d4bc00cf3 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:27:33 +0200 Subject: [PATCH 08/37] Add support for nixos --- docs/compatibility.md | 1 + jlmkr.py | 1 + templates/nixos/README.md | 35 +++++++++++++++++++++++++ templates/nixos/config | 54 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 templates/nixos/README.md create mode 100644 templates/nixos/config diff --git a/docs/compatibility.md b/docs/compatibility.md index c4b35ea..70f6223 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -15,6 +15,7 @@ |Debian 12 Bookworm|✅| |Ubuntu Jammy|✅| |Fedora 39|✅| +|Nixos 24.05|✅| |Arch|🟨| |Alpine|❌| diff --git a/jlmkr.py b/jlmkr.py index 9f6faee..ad60a5d 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1379,6 +1379,7 @@ def create_jail(**kwargs): if ( init_system_name != "systemd" + and parse_os_release(jail_rootfs_path).get("ID") != "nixos" ): print( dedent( diff --git a/templates/nixos/README.md b/templates/nixos/README.md new file mode 100644 index 0000000..9181243 --- /dev/null +++ b/templates/nixos/README.md @@ -0,0 +1,35 @@ +# Nixos Jail Template + +## Disclaimer + +**Experimental. Using nixos in this setup hasn't been extensively tested and has [known issues](#known-issues).** + +## Setup + +Check out the [config](./config) template file. You may provide it when asked during `./jlmkr.py create` or, if you have the template file stored on your NAS, you may provide it directly by running `./jlmkr.py create --start --config /mnt/tank/path/to/nixos/config mynixosjail`. + +## Manual Setup + +```bash +# Create the jail without starting +./jlmkr.py create --distro=nixos --release=24.05 nixos --network-bridge=br1 --resolv-conf=bind-host --bind-ro=./lxd.nix:/etc/nixos/lxd.nix +# Create empty nix module to satisfy import in default lxc configuration.nix +echo '{ ... }:{}' > ./jails/nixos/lxd.nix +# Start the nixos jail +./jlmkr.py start nixos +sleep 90 +# Network should be up by now +./jlmkr.py shell nixos /bin/sh -c 'ifconfig' +# Try to rebuild the system +./jlmkr.py shell nixos /bin/sh -c 'nixos-rebuild switch' +``` + +## Known Issues + +### Environment jlmkr exec + +Running `./jlmkr.py exec mynixosjail ifconfig` doesn't work because the shell environment isn't setup properly. You can run `./jlmkr.py shell mynixosjail /bin/sh -c 'ifconfig'` or `./jlmkr.py exec mynixosjail /bin/sh -c '. /etc/bashrc; ifconfig'` instead. + +### Bridge networking only + +This setup has NOT been tested with macvlan networking. \ No newline at end of file diff --git a/templates/nixos/config b/templates/nixos/config new file mode 100644 index 0000000..4411aa9 --- /dev/null +++ b/templates/nixos/config @@ -0,0 +1,54 @@ +startup=0 +gpu_passthrough_intel=0 +gpu_passthrough_nvidia=0 +# Turning off seccomp filtering improves performance at the expense of security +seccomp=1 + +# Use bridge networking to provide an isolated network namespace, +# so nixos can manage firewall rules +# Ensure to change br1 to the interface name you want to use +# You may want to add additional options here, e.g. bind mounts +systemd_nspawn_user_args=--network-bridge=br1 + --resolv-conf=bind-host + --bind-ro=./lxd.nix:/etc/nixos/lxd.nix + +# Script to run on the HOST before starting the jail +pre_start_hook=#!/usr/bin/env bash + set -euo pipefail + echo 'PRE_START_HOOK' + + # If there's no machine-id then this we're about to start the jail for the first time + if [ ! -e ./rootfs/etc/machine-id ]; then + echo 'BEFORE_FIRST_BOOT' + # Create empty nix module to satisfy import in default lxc configuration.nix + echo '{ ... }:{}' > ./lxd.nix + fi + +# Only used while creating the jail +distro=nixos +release=24.05 + +# # Example initial_setup which rebuild the system, +# # for when you mount your own /etc/nixos/configuration.nix inside the jail +# initial_setup=#!/run/current-system/sw/bin/bash +# . /etc/bashrc +# set -x +# ifconfig +# nixos-rebuild switch +# echo "All Done" + +# You generally will not need to change the options below +systemd_run_default_args=--property=KillMode=mixed + --property=Type=notify + --property=RestartForceExitStatus=133 + --property=SuccessExitStatus=133 + --property=Delegate=yes + --property=TasksMax=infinity + --collect + --setenv=SYSTEMD_NSPAWN_LOCK=0 + +systemd_nspawn_default_args=--keep-unit + --quiet + --boot + --bind-ro=/sys/module + --inaccessible=/sys/module/apparmor \ No newline at end of file From 1fa69d6bcc49c63490f87db3435e0213251b24a0 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:29:51 +0200 Subject: [PATCH 09/37] Bump version to 2.1.0 --- jlmkr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index ad60a5d..5a95c1e 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -4,7 +4,7 @@ with full access to all files via bind mounts, \ thanks to systemd-nspawn!""" -__version__ = "2.0.1" +__version__ = "2.1.0" __author__ = "Jip-Hop" __copyright__ = "Copyright (C) 2023, Jip-Hop" __license__ = "LGPL-3.0-only" From dd30ffe255fe82d3e03126ba7e98e2d361cca97c Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:04:40 +0200 Subject: [PATCH 10/37] Chroot with contextmanager --- jlmkr.py | 66 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 5a95c1e..ac9d2d0 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -125,8 +125,6 @@ SCRIPT_PATH = os.path.realpath(__file__) SCRIPT_NAME = os.path.basename(SCRIPT_PATH) SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH) COMMAND_NAME = os.path.basename(__file__) -INITIAL_ROOT = os.open("/", os.O_PATH) -CWD_BEFORE_CHROOT = None SHORTNAME = "jlmkr" # Only set a color if we have an interactive tty @@ -281,6 +279,25 @@ class CustomSubParser(argparse.ArgumentParser): raise ExceptionWithParser(self, message) +class Chroot: + def __init__(self, new_root): + self.new_root = new_root + self.old_root = None + self.initial_cwd = None + + def __enter__(self): + self.old_root = os.open("/", os.O_PATH) + self.initial_cwd = os.path.abspath(os.getcwd()) + os.chdir(self.new_root) + os.chroot(".") + + def __exit__(self, exc_type, exc_value, traceback): + os.chdir(self.old_root) + os.chroot(".") + os.close(self.old_root) + os.chdir(self.initial_cwd) + + def eprint(*args, **kwargs): """ Print to stderr. @@ -296,20 +313,6 @@ def fail(*args, **kwargs): sys.exit(1) -def enter_chroot(new_root): - global CWD_BEFORE_CHROOT - CWD_BEFORE_CHROOT = os.path.abspath(os.getcwd()) - os.chdir(new_root) - os.chroot(".") - - -# https://stackoverflow.com/a/61533559 -def exit_chroot(): - os.chdir(INITIAL_ROOT) - os.chroot(".") - os.chdir(CWD_BEFORE_CHROOT) - - def get_jail_path(jail_name): return os.path.join(JAILS_DIR_PATH, jail_name) @@ -787,6 +790,7 @@ def cleanup(jail_path): """ Cleanup jail. """ + if get_zfs_dataset(jail_path): eprint(f"Cleaning up: {jail_path}.") remove_zfs_dataset(jail_path) @@ -1373,9 +1377,9 @@ def create_jail(**kwargs): # But alpine jails made with jailmaker have other issues # They don't shutdown cleanly via systemctl and machinectl... - enter_chroot(jail_rootfs_path) - init_system_name = os.path.basename(os.path.realpath("/sbin/init")) - exit_chroot() + with Chroot(jail_rootfs_path): + # Use chroot to correctly resolve absolute /sbin/init symlink + init_system_name = os.path.basename(os.path.realpath("/sbin/init")) if ( init_system_name != "systemd" @@ -1645,20 +1649,20 @@ def get_all_jail_names(): return jail_names -def parse_os_release(new_rootfs): - enter_chroot(new_rootfs) +def parse_os_release(new_root): result = {} - for candidate in ["/etc/os-release", "/usr/lib/os-release"]: - try: - with open(candidate, encoding="utf-8") as f: - # TODO: can I create a solution which not depends on the internal _parse_os_release method? - result = platform._parse_os_release(f) - break - except OSError: - # Silently ignore failing to read os release info - pass + with Chroot(new_root): + # Use chroot to correctly resolve os-release symlink (for nixos) + for candidate in ["/etc/os-release", "/usr/lib/os-release"]: + try: + with open(candidate, encoding="utf-8") as f: + # TODO: can I create a solution which not depends on the internal _parse_os_release method? + result = platform._parse_os_release(f) + break + except OSError: + # Silently ignore failing to read os release info + pass - exit_chroot() return result From 2841137177ebc99795b95d3f33da35f9c1b9aed3 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:40:45 +0200 Subject: [PATCH 11/37] Copy resolv.conf on first start --- templates/nixos/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nixos/config b/templates/nixos/config index 4411aa9..fb8f92d 100644 --- a/templates/nixos/config +++ b/templates/nixos/config @@ -9,7 +9,6 @@ seccomp=1 # Ensure to change br1 to the interface name you want to use # You may want to add additional options here, e.g. bind mounts systemd_nspawn_user_args=--network-bridge=br1 - --resolv-conf=bind-host --bind-ro=./lxd.nix:/etc/nixos/lxd.nix # Script to run on the HOST before starting the jail @@ -22,6 +21,7 @@ pre_start_hook=#!/usr/bin/env bash echo 'BEFORE_FIRST_BOOT' # Create empty nix module to satisfy import in default lxc configuration.nix echo '{ ... }:{}' > ./lxd.nix + cp /etc/resolv.conf ./rootfs/etc/resolv.conf fi # Only used while creating the jail From 1b796ca2cf58ca54a5546516bba462a642ce61d8 Mon Sep 17 00:00:00 2001 From: Lockszmith Date: Thu, 27 Jun 2024 18:18:09 -0400 Subject: [PATCH 12/37] Added passing arguments to log and status This allows automated tests to run smoother. Currently there are 2 tests that block the console for input: * edit * shell --- jlmkr.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index ac9d2d0..ee61b7c 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -490,21 +490,23 @@ def exec_jail(jail_name, cmd): ).returncode -def status_jail(jail_name): +def status_jail(jail_name, cmd_args): """ Show the status of the systemd service wrapping the jail with given name. """ # Alternatively `machinectl status jail_name` could be used + if not cmd_args: cmd_args = [] return subprocess.run( - ["systemctl", "status", f"{SHORTNAME}-{jail_name}"] + ["systemctl", "status", f"{SHORTNAME}-{jail_name}", *cmd_args, ] ).returncode -def log_jail(jail_name): +def log_jail(jail_name, cmd_args): """ Show the log file of the jail with given name. """ - return subprocess.run(["journalctl", "-u", f"{SHORTNAME}-{jail_name}"]).returncode + if not cmd_args: cmd_args = ["-xe"] + return subprocess.run(["journalctl", *cmd_args, "-u", f"{SHORTNAME}-{jail_name}"]).returncode def shell_jail(args): @@ -1807,7 +1809,7 @@ def main(): title="commands", dest="command", metavar="", parser_class=CustomSubParser ) - split_commands = ["create", "exec"] + split_commands = ["create", "exec", "log", "status"] commands = {} for d in [ @@ -1883,6 +1885,13 @@ def main(): for cmd in ["edit", "exec", "log", "remove", "restart", "start", "status", "stop"]: commands[cmd].add_argument("jail_name", help="name of the jail") + for cmd in ["log", "status"]: + commands[cmd].add_argument( + "cmd_args", + nargs="*", + help="journalctl arguments", + ) + commands["exec"].add_argument( "cmd", nargs="*", From 1bd58c951ef796e819290bbf9037e7e4d09b88db Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:09:16 +0200 Subject: [PATCH 13/37] Cleanup log and status code --- jlmkr.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index ee61b7c..f00c77f 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -490,23 +490,23 @@ def exec_jail(jail_name, cmd): ).returncode -def status_jail(jail_name, cmd_args): +def status_jail(jail_name, args): """ Show the status of the systemd service wrapping the jail with given name. """ # Alternatively `machinectl status jail_name` could be used - if not cmd_args: cmd_args = [] return subprocess.run( - ["systemctl", "status", f"{SHORTNAME}-{jail_name}", *cmd_args, ] + ["systemctl", "status", f"{SHORTNAME}-{jail_name}", *args] ).returncode -def log_jail(jail_name, cmd_args): +def log_jail(jail_name, args): """ Show the log file of the jail with given name. """ - if not cmd_args: cmd_args = ["-xe"] - return subprocess.run(["journalctl", *cmd_args, "-u", f"{SHORTNAME}-{jail_name}"]).returncode + return subprocess.run( + ["journalctl", "-u", f"{SHORTNAME}-{jail_name}", *args] + ).returncode def shell_jail(args): @@ -1885,13 +1885,6 @@ def main(): for cmd in ["edit", "exec", "log", "remove", "restart", "start", "status", "stop"]: commands[cmd].add_argument("jail_name", help="name of the jail") - for cmd in ["log", "status"]: - commands[cmd].add_argument( - "cmd_args", - nargs="*", - help="journalctl arguments", - ) - commands["exec"].add_argument( "cmd", nargs="*", @@ -1904,6 +1897,18 @@ def main(): help="args to pass to machinectl shell", ) + commands["log"].add_argument( + "args", + nargs="*", + help="args to pass to journalctl", + ) + + commands["status"].add_argument( + "args", + nargs="*", + help="args to pass to systemctl", + ) + commands["create"].add_argument( "jail_name", # nargs="?", From e1f1d078724a1d3ca63f4490e1a412dfe0d4067c Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 28 Jun 2024 13:58:32 +0200 Subject: [PATCH 14/37] Remove debug logging --- jlmkr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index f00c77f..a90267e 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1989,7 +1989,6 @@ def main(): args = vars(parser.parse_known_args(args_to_parse)[0]) # ...and check if help is still in the remaining args need_help = args.get("help") - print(need_help) if need_help: commands[command].print_help() From 01e1156832e7df049f94b764dd35f091ca76571c Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:51:01 +0200 Subject: [PATCH 15/37] Accept config template from stdin Closes #208 --- jlmkr.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index a90267e..748111a 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1276,10 +1276,14 @@ def create_jail(**kwargs): if jail_config_path: # TODO: fallback to default values for e.g. distro and release if they are not in the config file - print(f"Creating jail {jail_name} from config template {jail_config_path}.") - if jail_config_path not in config.read(jail_config_path): - eprint(f"Failed to read config template {jail_config_path}.") - return 1 + if jail_config_path == "-": + print(f"Creating jail {jail_name} from config template passed via stdin.") + config.read_string(sys.stdin.read()) + else: + print(f"Creating jail {jail_name} from config template {jail_config_path}.") + if jail_config_path not in config.read(jail_config_path): + eprint(f"Failed to read config template {jail_config_path}.") + return 1 else: print(f"Creating jail {jail_name} with default config.") config.read_string(DEFAULT_CONFIG) @@ -1936,7 +1940,7 @@ def main(): commands["create"].add_argument( "-c", # "--config", - help="path to config file template", + help="path to config file template or - for stdin", ) commands["create"].add_argument( "-gi", # From 21efe90062d7acaa2abca0a6b6d74ed1582f667a Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Sat, 29 Jun 2024 16:43:17 +0200 Subject: [PATCH 16/37] Fix Python 3.12 SyntaxWarning SyntaxWarning: invalid escape sequence --- jlmkr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 748111a..55b8bf2 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -174,7 +174,7 @@ class KeyValueParser(configparser.ConfigParser): # Template to store comments as key value pair self._comment_template = "#{0} " + delimiter + " {1}" # Regex to match the comment prefix - self._comment_regex = re.compile(f"^#\d+\s*{re.escape(delimiter)}[^\S\n]*") + self._comment_regex = re.compile(r"^#\d+\s*" + re.escape(delimiter) + r"[^\S\n]*") # Regex to match cosmetic newlines (skips newlines in multiline values): # consecutive whitespace from start of line followed by a line not starting with whitespace self._cosmetic_newlines_regex = re.compile(r"^(\s+)(?=^\S)", re.MULTILINE) @@ -538,7 +538,7 @@ def systemd_escape_path(path): """ return "".join( map( - lambda char: "\s" if char == " " else "\\\\" if char == "\\" else char, path + lambda char: r"\s" if char == " " else "\\\\" if char == "\\" else char, path ) ) From 13f8a670b15f315ecdc335cea7d7a3a8cfbc8867 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:29:31 +0200 Subject: [PATCH 17/37] Cleanup networking docs --- docs/network.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/network.md b/docs/network.md index 3558936..16fb283 100644 --- a/docs/network.md +++ b/docs/network.md @@ -23,7 +23,7 @@ Add the `--network-bridge=br1 --resolv-conf=bind-host` systemd-nspawn flag when The TrueNAS host and the jail will be able to communicate with each other as if the jail was just another device on the LAN. It will use the same DNS servers as the TrueNAS host because the `--resolv-conf=bind-host` option bind mounts the `/etc/resolv.conf` file from the host inside the jail. If you want to use the DNS servers advertised via DHCP, then check [DNS via DHCP](#dns-via-dhcp). ### Bridge Static IP -To configure a static IP with our bridge interface, we need to edit the `/etc/systemd/network/80-container-host0.network` file. Change the [Network] section to look like this: +To configure a static IP with our bridge interface, we need to edit the `/etc/systemd/network/80-container-host0.network` file. Change the [Network] section to look like this: ```ini [Network] @@ -43,14 +43,14 @@ ifconfig ``` ### Multiple Bridge Interfaces -[Systemd-nspawn](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html), the technology on which jailmaker is built, [currently](https://github.com/systemd/systemd/issues/11087) only supports the definition and automatic configuration of a single bridge interface via the [`--network-bridge`](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html#--network-bridge=) argument. In some cases however, for instance when trying to utilize different vlan interfaces, it can be useful to configure multiple bridge interfaces within a jail. It is possible to create extra interfaces and join them to host bridges manually with systemd-nspwan using a combination of the [`--network-veth-extra`](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html#--network-veth-extra=) argument and a service config containing `ExecStartPost` commands as outlined [here](https://wiki.csclub.uwaterloo.ca/Systemd-nspawn#Multiple_network_interfaces). +[Systemd-nspawn](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html), the technology on which jailmaker is built, [currently](https://github.com/systemd/systemd/issues/11087) only supports the definition and automatic configuration of a single bridge interface via the [`--network-bridge`](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html#--network-bridge=) argument. In some cases however, for instance when trying to utilize different vlan interfaces, it can be useful to configure multiple bridge interfaces within a jail. It is possible to create extra interfaces and join them to host bridges manually with systemd-nspwan using a combination of the [`--network-veth-extra`](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html#--network-veth-extra=) argument and a service config containing `ExecStartPost` commands as outlined [here](https://wiki.csclub.uwaterloo.ca/Systemd-nspawn#Multiple_network_interfaces). The `--network-veth-extra` argument instructs system-nspawn to create an addition linked interface between the host and jail and uses a syntax of ``` --network-veth-extra=: ``` -The service config constaining `ExecStartPost` commands is then used to add the host side of the interface link to an existing host bridge and bring the interface up. Jailmaker has simplified this process by including a `post_start_hook` configuration parameter which can automate the creation of the service config by including the `ExecStartPost` commands as below. +The service config `ExecStartPost` commands is then used to add the host side of the interface link to an existing host bridge and bring the interface up. Jailmaker has simplified this process by including a `post_start_hook` configuration parameter which can automate the creation of the service config by including the `ExecStartPost` commands as below. ``` post_start_hook=#!/usr/bin/bash @@ -62,7 +62,7 @@ post_start_hook=#!/usr/bin/bash ip link set dev ve-docker-2 up ``` -With the new `--network-veth-extra` interface link created and the host side added to an existing host bridge, the jail side of the link still needs to be configured. Jailmaker provides a network file in the form of `/etc/systemd/network/vee-dhcp.network` which will automatically perform this configuration. In order for `vee-dhcp.network` to successfully match and configure the link's jail side interface, the `` must begin with a ***vee-*** prefix. An example jailmaker config with properly named `--network-veth-extra` interfaces and `post_start_hook` commands is available [here](https://github.com/Jip-Hop/jailmaker/discussions/179#discussioncomment-9499289). +With the new `--network-veth-extra` interface link created and the host side added to an existing host bridge, the jail side of the link still needs to be configured. Jailmaker provides a network file in the form of `/etc/systemd/network/vee-dhcp.network` which will automatically perform this configuration. In order for `vee-dhcp.network` to successfully match and configure the link's jail side interface, the `` must begin with a ***vee-*** prefix. An example jailmaker config with properly named `--network-veth-extra` interfaces and `post_start_hook` commands is available [here](https://github.com/Jip-Hop/jailmaker/discussions/179#discussioncomment-9499289). ## Macvlan Networking From 742a70b3d057ceb238560abbb0942f9411e19c37 Mon Sep 17 00:00:00 2001 From: "Jon C. Thomason" <2807816+jonct@users.noreply.github.com> Date: Fri, 5 Jul 2024 05:21:49 -0400 Subject: [PATCH 18/37] Add a simple router template (#216) * Add simple router example --------- Co-authored-by: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> --- templates/router/README.md | 44 ++++++++++++++ templates/router/config | 82 +++++++++++++++++++++++++++ templates/router/dnsmasq-example.conf | 50 ++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 templates/router/README.md create mode 100644 templates/router/config create mode 100644 templates/router/dnsmasq-example.conf diff --git a/templates/router/README.md b/templates/router/README.md new file mode 100644 index 0000000..70912c5 --- /dev/null +++ b/templates/router/README.md @@ -0,0 +1,44 @@ +# Router Jail Template + +Host a subordinate LAN using nftables and dnsmasq for DHCP, DNS, routing, and netboot infrastructure. +``` +router laptop desktop + | | | + +-- LAN --+-------+ + | + { TrueNAS SCALE } + | + +-----+-----+-- LAN2 --+------+------+-------+ + | | | | | | | + RPi1 RPi2 RPi3 NUC01 NUC02 NUC03 CrayYMP +``` +*Example usage*: deploy a flock of headless/diskless Raspberry Pi worker nodes for Kubernetes; each netbooting into an iSCSI or NFS root volume. + +## Setup + +Use the TrueNAS SCALE administrative UI to create a network bridge interface. Assign to that bridge a physical interface that's not shared with the host network. + +Use the `dnsmasq-example.conf` file as a starting point for your own dnsmasq settings file(s). Copy or mount them inside `/etc/dnsmasq.d/` within the jail. + +Optional: place assets in the mounted `/tftp/` directory for netbooting clients. + +Optional: attach more jails to this same bridge to host e.g. a K3s control plane, an nginx load balancer, a PostgreSQL database... + +Check out the [config](./config) template file. You may provide it when asked during `./jlmkr.py create` or, if you have the template file stored on your NAS, you may provide it directly by running `./jlmkr.py create --start --config /mnt/tank/path/to/router/config myrouterjail`. + +## Additional Resources + +There are as many reasons to host LAN infrastructure as there are to connect a LAN. This template can help you kick-start such a leaf network, using a TrueNAS jail as its gateway host. + +For those specifically interested in *netbooting Raspberry Pi*, the following **external** links might help you get started. + +* [Network Booting a Raspberry Pi 4 with an iSCSI Root via FreeNAS][G1]; the title says it all +* [Raspberry Pi Network Boot Guide][G2] covers more Raspberry Pi models; written for Synology users +* [pi_iscsi_netboot][s1] and [prep-netboot-storage][s2] are scripts showing preparation of boot assets and iSCSI root volumes + +Good luck! + +[G1]: https://shawnwilsher.com/2020/05/network-booting-a-raspberry-pi-4-with-an-iscsi-root-via-freenas/ +[G2]: https://warmestrobot.com/blog/2021/6/21/raspberry-pi-network-boot-guide +[s1]: https://github.com/tjpetz/pi_iscsi_netboot +[s2]: https://gitlab.com/jnicpon/rpi-prep/-/blob/main/scripts/prep-netboot-storage.fish?ref_type=heads diff --git a/templates/router/config b/templates/router/config new file mode 100644 index 0000000..65b1cfa --- /dev/null +++ b/templates/router/config @@ -0,0 +1,82 @@ +# See also: +# +# +startup=0 +gpu_passthrough_intel=0 +gpu_passthrough_nvidia=0 +# Turning off seccomp filtering improves performance at the expense of security +seccomp=1 + +# Use bridge networking to provide an isolated network namespace +# Alternatively use --network-macvlan=eno1 instead of --network-bridge +# Ensure to change br0 to the HOST interface name you want to use +# and br1 to the SECONDARY interface name you want to prepare +# Substitute your own dnsmasq.d and TFTP dataset bindings +systemd_nspawn_user_args=--network-bridge=br0 + --network-veth-extra=ve-router-1:vee-1 + --resolv-conf=bind-host + --system-call-filter='add_key keyctl bpf' + --bind=/mnt/pool/subnet/dnsmasq.d:/etc/dnsmasq.d + --bind-ro=/mnt/pool/subnet/tftpboot:/tftp + +# Script to run on the HOST before starting the jail +# Load kernel module and config kernel settings required for podman +pre_start_hook=#!/usr/bin/bash + set -euo pipefail + echo 'PRE_START_HOOK' + echo 1 > /proc/sys/net/ipv4/ip_forward + modprobe br_netfilter + echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables + echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables + modprobe iptable_nat + modprobe iptable_filter + +# Script to run on the HOST after starting the jail +# For example to attach to multiple bridge interfaces +post_start_hook=#!/usr/bin/bash + set -euo pipefail + echo 'POST_START_HOOK' + ip link set dev ve-router-1 master br1 + ip link set dev ve-router-1 up + #ip link set dev eth2 master br1 + +# Only used while creating the jail +distro=debian +release=bookworm + +# Install and configure within the jail +initial_setup=#!/usr/bin/bash + set -euo pipefail + + # Catch up on updates + apt-get update && apt-get full-upgrade -y + + # Configure worker LAN interface with static IP + sh -c 'cat < /etc/systemd/network/80-container-vee-1.network + [Match] + Virtualization=container + Name=vee-1 + + [Network] + DHCP=false + Address=10.3.14.202/24 + EOF' + systemctl restart systemd-networkd.service + + # Configure routing from LAN clients + apt-get install nftables -y + nft add table nat + nft add chain nat prerouting { type nat hook prerouting priority 0 \; } + nft add chain nat postrouting { type nat hook postrouting priority 100 \; } + nft add rule nat postrouting masquerade + mkdir -p /etc/nftables.d + nft list table nat >/etc/nftables.d/nat.conf + ( echo ; echo 'include "/etc/nftables.d/*.conf"' ) >>/etc/nftables.conf + + # Install dnsmasq alongside local resolver + sed -i -e 's/^#DNSStubListener=yes$/DNSStubListener=no/' /etc/systemd/resolved.conf + systemctl restart systemd-resolved.service + apt-get install dnsmasq -y + sed -i -e 's/^#DNS=$/DNS=127.0.0.1/' /etc/systemd/resolved.conf + systemctl restart systemd-resolved.service + systemctl restart dnsmasq.service diff --git a/templates/router/dnsmasq-example.conf b/templates/router/dnsmasq-example.conf new file mode 100644 index 0000000..fee9a4e --- /dev/null +++ b/templates/router/dnsmasq-example.conf @@ -0,0 +1,50 @@ +# customize and place this file inside /etc/dnsmasq.d + +# serve only Raspberry Pi network; don't backfeed to the host LAN +no-dhcp-interface=host0 +interface=vee-1 +bind-interfaces + +# designated upstream query servers +server=1.1.1.1 +server=1.0.0.1 + +# pirate TLD for the Democratic Republic of Raspberry Pi +domain=pi,10.3.14.0/24 + +# enable DHCP services +dhcp-authoritative +dhcp-rapid-commit +dhcp-range=10.3.14.101,10.3.14.199 + +# meet the 'berries +dhcp-host=e4:5f:01:da:da:b1,rpi1,10.3.14.11,infinite,set:rpi +dhcp-host=e4:5f:01:da:da:b2,rpi2,10.3.14.12,infinite,set:rpi +dhcp-host=e4:5f:01:da:da:b3,rpi3,10.3.14.13,infinite,set:rpi +dhcp-host=e4:5f:01:da:da:b4,rpi4,10.3.14.14,infinite,set:rpi +dhcp-host=e4:5f:01:da:da:b5,rpi5,10.3.14.15,infinite,set:rpi +dhcp-host=e4:5f:01:da:*:*,set:rpicube + +# PXE +dhcp-option-force=66,10.3.14.202 +# magic number +dhcp-option-force=208,f1:00:74:7e +# config filename +dhcp-option-force=209,configs/common +# path prefix +dhcp-option-force=210,/boot/ +# reboot time (i -> 32 bit) +dhcp-option-force=211,30i + +dhcp-boot=bootcode.bin + +#dhcp-match=set:ipxe,175 +#dhcp-boot=tag:ipxe,http://boot.netboot.xyz/ipxe/netboot.xyz.efi + +# TFTP +enable-tftp +tftp-root=/tftp + +#debugging +#log-queries +#log-dhcp From 9da33ab2b015f3aa48074e0c813ff233ff8bc53b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:27:30 +0200 Subject: [PATCH 19/37] Stay in workdir Fixes #209 --- jlmkr.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/jlmkr.py b/jlmkr.py index 55b8bf2..7dc8906 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -115,9 +115,6 @@ systemd_nspawn_default_args=--bind-ro=/sys/module # Always add --bind-ro=/sys/module to make lsmod happy # https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html -JAILS_DIR_PATH = "jails" -JAIL_CONFIG_NAME = "config" -JAIL_ROOTFS_NAME = "rootfs" DOWNLOAD_SCRIPT_DIGEST = ( "cfcb5d08b24187d108f2ab0d21a6cc4b73dcd7f5d7dfc80803bfd7f1642d638d" ) @@ -125,6 +122,9 @@ SCRIPT_PATH = os.path.realpath(__file__) SCRIPT_NAME = os.path.basename(SCRIPT_PATH) SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH) COMMAND_NAME = os.path.basename(__file__) +JAILS_DIR_PATH = os.path.join(SCRIPT_DIR_PATH, "jails") +JAIL_CONFIG_NAME = "config" +JAIL_ROOTFS_NAME = "rootfs" SHORTNAME = "jlmkr" # Only set a color if we have an interactive tty @@ -591,7 +591,7 @@ def start_jail(jail_name): systemd_run_additional_args = [ f"--unit={SHORTNAME}-{jail_name}", - f"--working-directory=./{jail_path}", + f"--working-directory={jail_path}", f"--description=My nspawn jail {jail_name} [created with jailmaker]", ] @@ -940,6 +940,8 @@ def get_mount_point(path): path = os.path.dirname(path) return path +def get_relative_path_in_jailmaker_dir(absolute_path): + return PurePath(absolute_path).relative_to(SCRIPT_DIR_PATH) def get_zfs_dataset(path): """ @@ -970,21 +972,23 @@ def get_zfs_base_path(): return zfs_base_path -def create_zfs_dataset(relative_path): +def create_zfs_dataset(absolute_path): """ - Create a ZFS Dataset. - Receives the dataset to be created relative to the jailmaker script (e.g. "jails" or "jails/newjail"). + Create a ZFS Dataset inside the jailmaker directory at the provided absolute path. + E.g. "/mnt/mypool/jailmaker/jails" or "/mnt/mypool/jailmaker/jails/newjail"). """ + relative_path = get_relative_path_in_jailmaker_dir(absolute_path) dataset_to_create = os.path.join(get_zfs_base_path(), relative_path) eprint(f"Creating ZFS Dataset {dataset_to_create}") subprocess.run(["zfs", "create", dataset_to_create], check=True) -def remove_zfs_dataset(relative_path): +def remove_zfs_dataset(absolute_path): """ - Remove a ZFS Dataset. - Receives the dataset to be removed relative to the jailmaker script (e.g. "jails/oldjail"). + Remove a ZFS Dataset inside the jailmaker directory at the provided absolute path. + E.g. "/mnt/mypool/jailmaker/jails/oldjail". """ + relative_path = get_relative_path_in_jailmaker_dir(absolute_path) dataset_to_remove = os.path.join((get_zfs_base_path()), relative_path) eprint(f"Removing ZFS Dataset {dataset_to_remove}") subprocess.run(["zfs", "destroy", "-r", dataset_to_remove], check=True) @@ -1232,7 +1236,7 @@ def interactive_config(): def create_jail(**kwargs): print(DISCLAIMER) - if os.path.basename(os.getcwd()) != "jailmaker": + if os.path.basename(SCRIPT_DIR_PATH) != "jailmaker": eprint( dedent( f""" @@ -1244,7 +1248,7 @@ def create_jail(**kwargs): ) return 1 - if not PurePath(get_mount_point(os.getcwd())).is_relative_to("/mnt"): + if not PurePath(get_mount_point(SCRIPT_DIR_PATH)).is_relative_to("/mnt"): print( dedent( f""" @@ -1967,9 +1971,6 @@ def main(): # Set appropriate permissions (if not already set) for this file, since it's executed as root stat_chmod(SCRIPT_PATH, 0o700) - # Work relative to this script - os.chdir(SCRIPT_DIR_PATH) - # Ignore all args after the first "--" args_to_parse = split_at_string(sys.argv[1:], "--")[0] # Check for help From 2d3ae20cd79746ca7517a43d473fc96757ea82ba Mon Sep 17 00:00:00 2001 From: "Jon C. Thomason" <2807816+jonct@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:59:39 -0400 Subject: [PATCH 20/37] Prepare resources in the GitHub action for test scripts (#220) --------- Co-authored-by: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> --- .github/workflows/test.yml | 74 ++++++++++++++++++++++++++++++++++---- test/test.sh | 30 ++-------------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4279132..d339374 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,9 +6,9 @@ name: CI on: # Triggers the workflow on push or pull request events for any branch push: - branches: [ "**" ] + branches: ["**"] pull_request: - branches: [ "**" ] + branches: ["**"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -16,7 +16,7 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" - build: + test: # The type of runner that the job will run on runs-on: ubuntu-24.04 @@ -25,6 +25,68 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - # Runs a single command using the runners shell - - name: Run a one-line script - run: sudo ./test/test.sh \ No newline at end of file + - name: Tune GitHub-hosted runner network + uses: smorimoto/tune-github-hosted-runner-network@v1 + + # Create a network namespace in the GitHub-hosted runner VM, + # simulating a primary bridge network on TrueNAS SCALE + - name: Set up networking resources + run: | + sudo -s < /etc/resolv.conf + + apt-get install -qq -y systemd-container + + cat </etc/systemd/network/10-br1.network + [Match] + Kind=bridge + Name=br1 + + [Network] + # Default to using a /24 prefix, giving up to 253 addresses per virtual network. + Address=0.0.0.0/24 + LinkLocalAddressing=yes + DHCPServer=yes + IPMasquerade=both + LLDP=yes + EmitLLDP=customer-bridge + IPv6AcceptRA=no + IPv6SendRA=yes + NETWORKCONFIG + + systemctl restart systemd-networkd + ip link add name br1 type bridge + + iptables -I DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + iptables -I DOCKER-USER -i br1 -o eth0 -j ACCEPT + END + + - name: Examine the GitHub-hosted runner environment + run: | + uname -r + cat /etc/os-release + python3 --version + ip addr + + # # TODO: create zpool with virtual disks, create jailmaker dataset and test jlmkr.py from there + # # https://medium.com/@abaddonsd/zfs-usage-with-virtual-disks-62898064a29b + # - name: Create a parent ZFS dataset + # run: | + # sudo -s < Date: Tue, 9 Jul 2024 04:20:31 +0200 Subject: [PATCH 21/37] Cast int to string Closed #224 --- jlmkr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlmkr.py b/jlmkr.py index 7dc8906..150d8ed 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1306,7 +1306,7 @@ def create_jail(**kwargs): value = kwargs.pop(option) if ( value is not None - and len(value) + and len(str(value)) and value is not config.my_get(option, None) ): # TODO: this will wipe all systemd_nspawn_user_args from the template... From 9fcb5d52a1d2267637179d2126c72ec496e95406 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Tue, 9 Jul 2024 04:27:34 +0200 Subject: [PATCH 22/37] Add TODO --- jlmkr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jlmkr.py b/jlmkr.py index 150d8ed..b339c00 100755 --- a/jlmkr.py +++ b/jlmkr.py @@ -1601,6 +1601,7 @@ def remove_jail(jail_name): return 1 # TODO: print which dataset is about to be removed before the user confirmation + # TODO: print that all zfs snapshots will be removed if jail has it's own zfs dataset check = input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n') if check == jail_name: From ef595e576acfe219bddee496a0721e66aabb0631 Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Tue, 9 Jul 2024 04:30:57 +0200 Subject: [PATCH 23/37] Test podman --- test/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index 04a2aef..26f1ebc 100755 --- a/test/test.sh +++ b/test/test.sh @@ -8,5 +8,5 @@ set -euo pipefail # TODO: test jlmkr.py from inside another working directory, with a relative path to a config file to test if it uses the config file (and doesn't look for it relative to the jlmkr.py file itself) -./jlmkr.py create --start --config=./templates/docker/config test -./jlmkr.py exec test docker run hello-world +./jlmkr.py create --start --config=./templates/podman/config test +./jlmkr.py exec test podman run hello-world From 434e195ce9eee5b54d0698ff62bd44b0bd2facd5 Mon Sep 17 00:00:00 2001 From: Gal Szkolnik Date: Mon, 8 Jul 2024 22:58:07 -0400 Subject: [PATCH 24/37] Automated testing script (#215) --- .github/workflows/test.yml | 5 +- test/test-jlmkr | 192 +++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 3 deletions(-) create mode 100755 test/test-jlmkr diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d339374..127ac8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,6 +87,5 @@ jobs: env: PYTHONUNBUFFERED: 1 run: | - sudo chown 0:0 jlmkr.py test/test.sh - sudo chmod +x jlmkr.py test/test.sh - sudo ./test/test.sh + sudo chown 0:0 jlmkr.py ./test/test-jlmkr + sudo bash ./test/test-jlmkr diff --git a/test/test-jlmkr b/test/test-jlmkr new file mode 100755 index 0000000..bc87295 --- /dev/null +++ b/test/test-jlmkr @@ -0,0 +1,192 @@ +#! /usr/bin/env bash + +# Example invokation +# sudo SCALE_POOL_ROOT=$SCALE_POOL_ROOT $SCALE_POOL_ROOT/jailmaker/test/test-jlmkr docker + +set -e + +#### Global variables + +if [[ -z "$JLMKR_PATH" && -n "$SCALE_POOL_ROOT" ]]; then + SCALE_POOL_ROOT=${SCALE_POOL_ROOT:?must be exported before you can run test suite} + JLMKR_PATH=${SCALE_POOL_ROOT}/jailmaker +elif [[ -z "$JLMKR_PATH" ]]; then + JLMKR_PATH=${PWD:-.} +fi +if [[ ! -r "$JLMKR_PATH/jlmkr.py" ]]; then + >&2 printf "%s\n" \ + "couldn't find jlmkr.py. Are you running from the jailmaker directory?" \ + "If not, setup either JLMKR_PATH or SCALE_POOL_ROOT" \ + "" + + >&2 printf "\tPWD: %s\n\tJLMKR_PATH: %s\n\tSCALE_POOL_ROOT: %s\n" \ + "$PWD" "$JLMKR_PATH" "$SCALE_POOL_ROOT" + + >&2 printf "\n" + exit 2 +fi +JAIL_TYPE="${1}" +if [ -n "${JAIL_TYPE}" ]; then + if [ -r "${JAIL_TYPE}" ]; then + JAIL_CONFIG="$JAIL_TYPE" + JAIL_TYPE= + # Can't perform full test with config path + FULL_TEST=0 + else + JAIL_CONFIG="$JLMKR_PATH/templates/$JAIL_TYPE/config" + # Full test is an option when using JAIL_TYPE + FULL_TEST=${FULL_TEST:-0} + fi + if [ ! -r "${JAIL_CONFIG}" ]; then + >&2 printf "Must supply a valid jail type or config path\n" + exit 2 + fi +fi + +# shellcheck disable=SC2034 # JAIL is used inside perform_test_suite +JAIL="${2:-${JAIL_TYPE:-default}-test}" + +# STOP=0 - (default) perform all tests, in non-blocking mode +# STOP=l - only list and images, nothing else +# STOP=i - interactive test, includ console-blocking waiting for input tests (edit and shell) +STOP=${STOP:-0} + +REPORT=(create edit exec images list log remove restart shell start status stop) + +WAIT_FOR_JAIL=${WAIT_FOR_JAIL:-4s} + +#### Functions +jlmkr () { + /usr/bin/env python3 "$JLMKR_PATH/jlmkr.py" "${@:---help}" +} + +iterate () { + # shellcheck disable=SC2206 # $1 will pass multiple values, we want splitting here + local SET=($1) DO=("${@:2}") + local x j _x x_STATUS + + for j in "${SET[@]}"; do + for _x in "${DO[@]}"; do + x="${_x//\(Jv)/JLMKR_"$j"}" + # echo "$x" >&2 + ${NO_EVAL:+:} eval "echo \$JLMKR_${j} \"$x\"" + x_STATUS=✅ + ${NO_EVAL:+:} eval "JLMKR_$j=$x_STATUS" + ${NO_EVAL:+:} eval "JLMKR_${j}_x=\"${x//\"/\'}\"" + + if [[ -n "$DELAY" ]]; then + printf "Waiting %s seconds before test...\n" "${DELAY}" + sleep "${DELAY}" + fi + + set +e + eval "$x" || x_STATUS=$? + set -e + if [[ "$x_STATUS" != "✅" ]]; then + ${NO_EVAL:+:} eval "JLMKR_${j}_x=\"($x_STATUS) ${x//\"/\'}\"" + ${NO_EVAL:+:} eval "JLMKR_$j=❌" + STOP=E:$x_STATUS + return + fi + done + done +} + +perform_test_suite() { + # shellcheck disable=SC2016 # function relies heavily on single quotes preventing expansion + + if [[ "$STOP" =~ ^(0|l|i)$ ]]; then + # Initialize REPORT with empty checkboxes - NO_EVAL=1 is important here, otherwise Status will be evaluated + NO_EVAL=1 iterate "${REPORT[*]}" '(Jv)="🔳"' + + TESTS=(list images) + iterate "${TESTS[*]}" 'jlmkr $j' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(create) \ + && iterate "${TESTS[*]}" 'jlmkr $j ${JAIL_CONFIG:+--config} $JAIL_CONFIG $JAIL' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(start) \ + && iterate "${TESTS[*]}" 'jlmkr $j $JAIL' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(restart) \ + && DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL' + + # If this is an interactive test, edit and shell will wait for input + [[ "$STOP" == "i" ]] && TESTS=(edit shell) \ + && DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL' + + # This is the non-interactive test for edit + [[ "$STOP" == "0" ]] && TESTS=(edit) \ + && DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'EDITOR=cat jlmkr $j $JAIL' + + # This is the non-interactive test for shell + [[ "$STOP" == "0" ]] && TESTS=(shell) \ + && DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL /bin/sh -c "echo shell called successful"' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(exec) \ + && DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL /bin/sh -c "echo exec called successful"' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(status) \ + && iterate "${TESTS[*]}" 'jlmkr $j $JAIL --no-pager' + + [[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(log) \ + && iterate "${TESTS[*]}" 'jlmkr $j $JAIL -n 10' + + [[ "$STOP" =~ ^(0|l|i)$ ]] || >&2 printf "Had an Error. Cleanup up and stopping.\n" + + # Always perform these cleanup steps, even if something failed + [[ "$STOP" != l ]] \ + && TESTS=(stop) \ + && iterate "${TESTS[*]}" 'jlmkr $j $JAIL' \ + + [[ "$STOP" != l ]] \ + && TESTS=(remove) \ + && iterate "${TESTS[*]}" 'jlmkr $j $JAIL <<<"$JAIL" ' + + fi + printf '\n\nReport for:\n\tCWD: %s\t\tJAIL_CONFIG: %s\n\n' "$(pwd)" "${JAIL_CONFIG}" + + # shellcheck disable=SC2016 + NO_EVAL=1 iterate "${REPORT[*]}" 'echo "$(Jv) ${(Jv)_x:-$j}"' +} + +#### Execution starts here + +perform_test_suite + +if [[ "$STOP" =~ ^(0|i)$ ]]; then + [[ "$FULL_TEST" == 1 ]] || STOP="Single Test" +fi + +if [[ "$STOP" =~ ^(0|i)$ ]]; then + pushd "$JLMKR_PATH" > /dev/null || STOP=pushd + STOP=0 # The following test suite should only be run non-interactivley + + JAIL_CONFIG="./templates/$JAIL_TYPE/config" + perform_test_suite + + JAIL_CONFIG="templates/$JAIL_TYPE/config" + perform_test_suite + + JAIL_CONFIG="$JLMKR_PATH/$JAIL_CONFIG" + perform_test_suite + + cd ~ + perform_test_suite + + TMP_JAIL_CFG=$(mktemp) \ + && cp "$JAIL_CONFIG" "$TMP_JAIL_CFG" \ + && JAIL_CONFIG="${TMP_JAIL_CFG}" perform_test_suite \ + && rm "$TMP_JAIL_CFG" \ + || STOP=E:temp_file + + popd > /dev/null || STOP=popd +fi + +if [[ "$STOP" = 0 ]]; then + printf "All tests completed\n" +else + printf "Stopped: %s\n" "$STOP" + [[ "$STOP" = "Single Test" ]] || exit 1 +fi + From 49b5bf2e703bde6b8a68ed92eddd2c0e6974ae7b Mon Sep 17 00:00:00 2001 From: Jip-Hop <2871973+Jip-Hop@users.noreply.github.com> Date: Tue, 9 Jul 2024 04:59:41 +0200 Subject: [PATCH 25/37] Add readme --- test/README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 test/README.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..0653693 --- /dev/null +++ b/test/README.md @@ -0,0 +1,77 @@ +# Jailmaker Testing + +This readme documents the [test-jlmkr](./test-jlmkr) script. + +The script has 2 optional parameter invocation sets: +* `` [``] +* `