Jip-Hop
GitHub
commit
b221bf04ac
|
|
@ -16,7 +16,7 @@ on:
|
|||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
jobs:
|
||||
# This workflow contains a single job called "build"
|
||||
build:
|
||||
test:
|
||||
# The type of runner that the job will run on
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
|
|
@ -25,6 +25,68 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# Runs a single command using the runners shell
|
||||
- name: Run a one-line script
|
||||
run: sudo ./test/test.sh
|
||||
- name: Tune GitHub-hosted runner network
|
||||
uses: smorimoto/tune-github-hosted-runner-network@v1
|
||||
|
||||
# Create a network namespace in the GitHub-hosted runner VM,
|
||||
# simulating a primary bridge network on TrueNAS SCALE
|
||||
- name: Set up networking resources
|
||||
run: |
|
||||
sudo -s <<END
|
||||
|
||||
systemctl disable systemd-resolved --now
|
||||
rm /etc/resolv.conf
|
||||
echo 'nameserver 1.1.1.1' > /etc/resolv.conf
|
||||
|
||||
apt-get install -qq -y systemd-container
|
||||
|
||||
cat <<NETWORKCONFIG >/etc/systemd/network/10-br1.network
|
||||
[Match]
|
||||
Kind=bridge
|
||||
Name=br1
|
||||
|
||||
[Network]
|
||||
# Default to using a /24 prefix, giving up to 253 addresses per virtual network.
|
||||
Address=0.0.0.0/24
|
||||
LinkLocalAddressing=yes
|
||||
DHCPServer=yes
|
||||
IPMasquerade=both
|
||||
LLDP=yes
|
||||
EmitLLDP=customer-bridge
|
||||
IPv6AcceptRA=no
|
||||
IPv6SendRA=yes
|
||||
NETWORKCONFIG
|
||||
|
||||
systemctl restart systemd-networkd
|
||||
ip link add name br1 type bridge
|
||||
|
||||
iptables -I DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
||||
iptables -I DOCKER-USER -i br1 -o eth0 -j ACCEPT
|
||||
END
|
||||
|
||||
- name: Examine the GitHub-hosted runner environment
|
||||
run: |
|
||||
uname -r
|
||||
cat /etc/os-release
|
||||
python3 --version
|
||||
ip addr
|
||||
|
||||
# # TODO: create zpool with virtual disks, create jailmaker dataset and test jlmkr.py from there
|
||||
# # https://medium.com/@abaddonsd/zfs-usage-with-virtual-disks-62898064a29b
|
||||
# - name: Create a parent ZFS dataset
|
||||
# run: |
|
||||
# sudo -s <<END
|
||||
# apt-get install -y -qq zfsutils-linux
|
||||
# modinfo zfs | grep version
|
||||
# zfs --version
|
||||
# zpool --version
|
||||
# END
|
||||
|
||||
# Run multiple commands using the runners shell
|
||||
- name: Run the test script
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
run: |
|
||||
sudo chown 0:0 jlmkr.py ./test/test-jlmkr
|
||||
sudo bash ./test/test-jlmkr
|
||||
sudo ./test/test.sh
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|Debian 12 Bookworm|✅|
|
||||
|Ubuntu Jammy|✅|
|
||||
|Fedora 39|✅|
|
||||
|Nixos 24.05|✅|
|
||||
|Arch|🟨|
|
||||
|Alpine|❌|
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ The `--network-veth-extra` argument instructs system-nspawn to create an additio
|
|||
--network-veth-extra=<host_interface_name>:<jail_interface_name>
|
||||
```
|
||||
|
||||
The service config constaining `ExecStartPost` commands is then used to add the host side of the interface link to an existing host bridge and bring the interface up. Jailmaker has simplified this process by including a `post_start_hook` configuration parameter which can automate the creation of the service config by including the `ExecStartPost` commands as below.
|
||||
The service config `ExecStartPost` commands is then used to add the host side of the interface link to an existing host bridge and bring the interface up. Jailmaker has simplified this process by including a `post_start_hook` configuration parameter which can automate the creation of the service config by including the `ExecStartPost` commands as below.
|
||||
|
||||
```
|
||||
post_start_hook=#!/usr/bin/bash
|
||||
|
|
|
|||
223
jlmkr.py
223
jlmkr.py
|
|
@ -4,7 +4,7 @@
|
|||
with full access to all files via bind mounts, \
|
||||
thanks to systemd-nspawn!"""
|
||||
|
||||
__version__ = "2.0.1"
|
||||
__version__ = "2.1.0"
|
||||
__author__ = "Jip-Hop"
|
||||
__copyright__ = "Copyright (C) 2023, Jip-Hop"
|
||||
__license__ = "LGPL-3.0-only"
|
||||
|
|
@ -115,9 +115,6 @@ systemd_nspawn_default_args=--bind-ro=/sys/module
|
|||
# Always add --bind-ro=/sys/module to make lsmod happy
|
||||
# https://manpages.debian.org/bookworm/manpages/sysfs.5.en.html
|
||||
|
||||
JAILS_DIR_PATH = "jails"
|
||||
JAIL_CONFIG_NAME = "config"
|
||||
JAIL_ROOTFS_NAME = "rootfs"
|
||||
DOWNLOAD_SCRIPT_DIGEST = (
|
||||
"cfcb5d08b24187d108f2ab0d21a6cc4b73dcd7f5d7dfc80803bfd7f1642d638d"
|
||||
)
|
||||
|
|
@ -125,6 +122,9 @@ SCRIPT_PATH = os.path.realpath(__file__)
|
|||
SCRIPT_NAME = os.path.basename(SCRIPT_PATH)
|
||||
SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH)
|
||||
COMMAND_NAME = os.path.basename(__file__)
|
||||
JAILS_DIR_PATH = os.path.join(SCRIPT_DIR_PATH, "jails")
|
||||
JAIL_CONFIG_NAME = "config"
|
||||
JAIL_ROOTFS_NAME = "rootfs"
|
||||
SHORTNAME = "jlmkr"
|
||||
|
||||
# Only set a color if we have an interactive tty
|
||||
|
|
@ -174,7 +174,9 @@ class KeyValueParser(configparser.ConfigParser):
|
|||
# Template to store comments as key value pair
|
||||
self._comment_template = "#{0} " + delimiter + " {1}"
|
||||
# Regex to match the comment prefix
|
||||
self._comment_regex = re.compile(f"^#\d+\s*{re.escape(delimiter)}[^\S\n]*")
|
||||
self._comment_regex = re.compile(
|
||||
r"^#\d+\s*" + re.escape(delimiter) + r"[^\S\n]*"
|
||||
)
|
||||
# Regex to match cosmetic newlines (skips newlines in multiline values):
|
||||
# consecutive whitespace from start of line followed by a line not starting with whitespace
|
||||
self._cosmetic_newlines_regex = re.compile(r"^(\s+)(?=^\S)", re.MULTILINE)
|
||||
|
|
@ -279,6 +281,25 @@ class CustomSubParser(argparse.ArgumentParser):
|
|||
raise ExceptionWithParser(self, message)
|
||||
|
||||
|
||||
class Chroot:
|
||||
def __init__(self, new_root):
|
||||
self.new_root = new_root
|
||||
self.old_root = None
|
||||
self.initial_cwd = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old_root = os.open("/", os.O_PATH)
|
||||
self.initial_cwd = os.path.abspath(os.getcwd())
|
||||
os.chdir(self.new_root)
|
||||
os.chroot(".")
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
os.chdir(self.old_root)
|
||||
os.chroot(".")
|
||||
os.close(self.old_root)
|
||||
os.chdir(self.initial_cwd)
|
||||
|
||||
|
||||
def eprint(*args, **kwargs):
|
||||
"""
|
||||
Print to stderr.
|
||||
|
|
@ -471,22 +492,22 @@ def exec_jail(jail_name, cmd):
|
|||
).returncode
|
||||
|
||||
|
||||
def status_jail(jail_name):
|
||||
def status_jail(jail_name, args):
|
||||
"""
|
||||
Show the status of the systemd service wrapping the jail with given name.
|
||||
"""
|
||||
# Alternatively `machinectl status jail_name` could be used
|
||||
return subprocess.run(
|
||||
["systemctl", "status", f"{SHORTNAME}-{jail_name}"]
|
||||
["systemctl", "status", f"{SHORTNAME}-{jail_name}", *args]
|
||||
).returncode
|
||||
|
||||
|
||||
def log_jail(jail_name):
|
||||
def log_jail(jail_name, args):
|
||||
"""
|
||||
Show the log file of the jail with given name.
|
||||
"""
|
||||
return subprocess.run(
|
||||
["journalctl", "-u", f"{SHORTNAME}-{jail_name}"]
|
||||
["journalctl", "-u", f"{SHORTNAME}-{jail_name}", *args]
|
||||
).returncode
|
||||
|
||||
|
||||
|
|
@ -519,7 +540,8 @@ def systemd_escape_path(path):
|
|||
"""
|
||||
return "".join(
|
||||
map(
|
||||
lambda char: "\s" if char == " " else "\\\\" if char == "\\" else char, path
|
||||
lambda char: r"\s" if char == " " else "\\\\" if char == "\\" else char,
|
||||
path,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -572,7 +594,7 @@ def start_jail(jail_name):
|
|||
|
||||
systemd_run_additional_args = [
|
||||
f"--unit={SHORTNAME}-{jail_name}",
|
||||
f"--working-directory=./{jail_path}",
|
||||
f"--working-directory={jail_path}",
|
||||
f"--description=My nspawn jail {jail_name} [created with jailmaker]",
|
||||
]
|
||||
|
||||
|
|
@ -658,18 +680,7 @@ def start_jail(jail_name):
|
|||
if not os.path.exists(os.path.join(jail_rootfs_path, "etc/machine-id")) and (
|
||||
initial_setup := config.my_get("initial_setup")
|
||||
):
|
||||
if not initial_setup.startswith("#!"):
|
||||
initial_setup = "#!/bin/sh\n" + initial_setup
|
||||
|
||||
initial_setup_file_jailed_path = "/root/jlmkr-initial-setup"
|
||||
initial_setup_file_host_path = os.path.abspath(
|
||||
jail_rootfs_path + initial_setup_file_jailed_path
|
||||
)
|
||||
|
||||
# Write a script file to call during initial setup
|
||||
print(initial_setup, file=open(initial_setup_file_host_path, "w"))
|
||||
stat_chmod(initial_setup_file_host_path, 0o700)
|
||||
|
||||
# initial_setup has been assigned due to := expression above
|
||||
# Ensure the jail init system is ready before we start the initial_setup
|
||||
systemd_nspawn_additional_args += [
|
||||
"--notify-ready=yes",
|
||||
|
|
@ -712,38 +723,56 @@ def start_jail(jail_name):
|
|||
|
||||
# Handle initial setup after jail is up and running (for the first time)
|
||||
if initial_setup:
|
||||
print("About to run the initial setup.")
|
||||
if not initial_setup.startswith("#!"):
|
||||
initial_setup = "#!/bin/sh\n" + initial_setup
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w+t",
|
||||
prefix="jlmkr-initial-setup.",
|
||||
dir=jail_rootfs_path,
|
||||
delete=False,
|
||||
) as initial_setup_file:
|
||||
# Write a script file to call during initial setup
|
||||
initial_setup_file.write(initial_setup)
|
||||
|
||||
initial_setup_file_name = os.path.basename(initial_setup_file.name)
|
||||
initial_setup_file_host_path = os.path.abspath(initial_setup_file.name)
|
||||
stat_chmod(initial_setup_file_host_path, 0o700)
|
||||
|
||||
print(f"About to run the initial setup script: {initial_setup_file_name}.")
|
||||
print("Waiting for networking in the jail to be ready.")
|
||||
print("Please wait (this may take 90s in case of bridge networking with STP is enabled)...")
|
||||
print(
|
||||
"Please wait (this may take 90s in case of bridge networking with STP is enabled)..."
|
||||
)
|
||||
returncode = exec_jail(
|
||||
jail_name,
|
||||
[
|
||||
"--",
|
||||
"systemd-run",
|
||||
f"--unit={os.path.basename(initial_setup_file_jailed_path)}",
|
||||
f"--unit={initial_setup_file_name}",
|
||||
"--quiet",
|
||||
"--pipe",
|
||||
"--wait",
|
||||
"--service-type=exec",
|
||||
"--property=After=network-online.target",
|
||||
"--property=Wants=network-online.target",
|
||||
initial_setup_file_jailed_path,
|
||||
"/" + initial_setup_file_name,
|
||||
],
|
||||
)
|
||||
|
||||
# Cleanup the initial_setup_file_host_path
|
||||
if initial_setup_file_host_path:
|
||||
Path(initial_setup_file_host_path).unlink(missing_ok=True)
|
||||
|
||||
if returncode != 0:
|
||||
eprint("Tried to run the following commands inside the jail:")
|
||||
eprint(initial_setup)
|
||||
eprint()
|
||||
eprint(f"{RED}{BOLD}Failed to run initial setup...")
|
||||
eprint(
|
||||
f"""{RED}{BOLD}Failed to run initial setup... you may want to stop and remove the jail and try again.{NORMAL}"""
|
||||
f"You may want to manually run /{initial_setup_file_name} inside the jail for debugging purposes."
|
||||
)
|
||||
eprint(f"Or stop and remove the jail and try again.{NORMAL}")
|
||||
return returncode
|
||||
else:
|
||||
# Cleanup the initial_setup_file_host_path
|
||||
Path(initial_setup_file_host_path).unlink(missing_ok=True)
|
||||
print(f"Done with initial setup of jail {jail_name}!")
|
||||
|
||||
return returncode
|
||||
|
|
@ -766,6 +795,7 @@ def cleanup(jail_path):
|
|||
"""
|
||||
Cleanup jail.
|
||||
"""
|
||||
|
||||
if get_zfs_dataset(jail_path):
|
||||
eprint(f"Cleaning up: {jail_path}.")
|
||||
remove_zfs_dataset(jail_path)
|
||||
|
|
@ -775,7 +805,12 @@ def cleanup(jail_path):
|
|||
# Should be fixed in Python 3.13 https://stackoverflow.com/a/70549000
|
||||
def _onerror(func, path, exc_info):
|
||||
exc_type, exc_value, exc_traceback = exc_info
|
||||
if not issubclass(exc_type, FileNotFoundError):
|
||||
if issubclass(exc_type, PermissionError):
|
||||
# Update the file permissions with the immutable and append-only bit cleared
|
||||
subprocess.run(["chattr", "-i", "-a", path])
|
||||
# Reattempt the removal
|
||||
func(path)
|
||||
elif not issubclass(exc_type, FileNotFoundError):
|
||||
raise exc_value
|
||||
|
||||
eprint(f"Cleaning up: {jail_path}.")
|
||||
|
|
@ -838,10 +873,7 @@ def run_lxc_download_script(
|
|||
|
||||
stat_chmod(lxc_download_script, 0o700)
|
||||
|
||||
check_exit_code = False
|
||||
|
||||
if None not in [jail_name, jail_path, jail_rootfs_path, distro, release]:
|
||||
check_exit_code = True
|
||||
cmd = [
|
||||
lxc_download_script,
|
||||
f"--name={jail_name}",
|
||||
|
|
@ -851,7 +883,13 @@ def run_lxc_download_script(
|
|||
f"--dist={distro}",
|
||||
f"--release={release}",
|
||||
]
|
||||
|
||||
if rc := subprocess.run(cmd, env={"LXC_CACHE_PATH": lxc_cache}).returncode != 0:
|
||||
eprint("Aborting...")
|
||||
return rc
|
||||
|
||||
else:
|
||||
# List images
|
||||
cmd = [lxc_download_script, "--list", f"--arch={arch}"]
|
||||
|
||||
p1 = subprocess.Popen(
|
||||
|
|
@ -867,11 +905,12 @@ def run_lxc_download_script(
|
|||
):
|
||||
print(line)
|
||||
|
||||
p1.wait()
|
||||
rc = p1.wait()
|
||||
# Currently --list will always return a non-zero exit code, even when listing the images was successful
|
||||
# https://github.com/lxc/lxc/pull/4462
|
||||
# Therefore we must currently return 0, to prevent aborting the interactive create process
|
||||
|
||||
if check_exit_code and p1.returncode != 0:
|
||||
eprint("Aborting...")
|
||||
return p1.returncode
|
||||
# return rc
|
||||
|
||||
return 0
|
||||
|
||||
|
|
@ -909,6 +948,10 @@ def get_mount_point(path):
|
|||
return path
|
||||
|
||||
|
||||
def get_relative_path_in_jailmaker_dir(absolute_path):
|
||||
return PurePath(absolute_path).relative_to(SCRIPT_DIR_PATH)
|
||||
|
||||
|
||||
def get_zfs_dataset(path):
|
||||
"""
|
||||
Get ZFS dataset path.
|
||||
|
|
@ -938,21 +981,23 @@ def get_zfs_base_path():
|
|||
return zfs_base_path
|
||||
|
||||
|
||||
def create_zfs_dataset(relative_path):
|
||||
def create_zfs_dataset(absolute_path):
|
||||
"""
|
||||
Create a ZFS Dataset.
|
||||
Receives the dataset to be created relative to the jailmaker script (e.g. "jails" or "jails/newjail").
|
||||
Create a ZFS Dataset inside the jailmaker directory at the provided absolute path.
|
||||
E.g. "/mnt/mypool/jailmaker/jails" or "/mnt/mypool/jailmaker/jails/newjail").
|
||||
"""
|
||||
relative_path = get_relative_path_in_jailmaker_dir(absolute_path)
|
||||
dataset_to_create = os.path.join(get_zfs_base_path(), relative_path)
|
||||
eprint(f"Creating ZFS Dataset {dataset_to_create}")
|
||||
subprocess.run(["zfs", "create", dataset_to_create], check=True)
|
||||
|
||||
|
||||
def remove_zfs_dataset(relative_path):
|
||||
def remove_zfs_dataset(absolute_path):
|
||||
"""
|
||||
Remove a ZFS Dataset.
|
||||
Receives the dataset to be removed relative to the jailmaker script (e.g. "jails/oldjail").
|
||||
Remove a ZFS Dataset inside the jailmaker directory at the provided absolute path.
|
||||
E.g. "/mnt/mypool/jailmaker/jails/oldjail".
|
||||
"""
|
||||
relative_path = get_relative_path_in_jailmaker_dir(absolute_path)
|
||||
dataset_to_remove = os.path.join((get_zfs_base_path()), relative_path)
|
||||
eprint(f"Removing ZFS Dataset {dataset_to_remove}")
|
||||
subprocess.run(["zfs", "destroy", "-r", dataset_to_remove], check=True)
|
||||
|
|
@ -1082,9 +1127,8 @@ def interactive_config():
|
|||
input("Press Enter to continue...")
|
||||
print()
|
||||
|
||||
returncode = run_lxc_download_script()
|
||||
if returncode != 0:
|
||||
return returncode
|
||||
if run_lxc_download_script() != 0:
|
||||
fail("Failed to list images. Aborting...")
|
||||
|
||||
print(
|
||||
dedent(
|
||||
|
|
@ -1200,7 +1244,7 @@ def interactive_config():
|
|||
def create_jail(**kwargs):
|
||||
print(DISCLAIMER)
|
||||
|
||||
if os.path.basename(os.getcwd()) != "jailmaker":
|
||||
if os.path.basename(SCRIPT_DIR_PATH) != "jailmaker":
|
||||
eprint(
|
||||
dedent(
|
||||
f"""
|
||||
|
|
@ -1212,7 +1256,7 @@ def create_jail(**kwargs):
|
|||
)
|
||||
return 1
|
||||
|
||||
if not PurePath(get_mount_point(os.getcwd())).is_relative_to("/mnt"):
|
||||
if not PurePath(get_mount_point(SCRIPT_DIR_PATH)).is_relative_to("/mnt"):
|
||||
print(
|
||||
dedent(
|
||||
f"""
|
||||
|
|
@ -1244,7 +1288,15 @@ def create_jail(**kwargs):
|
|||
|
||||
if jail_config_path:
|
||||
# TODO: fallback to default values for e.g. distro and release if they are not in the config file
|
||||
print(f"Creating jail {jail_name} from config template {jail_config_path}.")
|
||||
if jail_config_path == "-":
|
||||
print(
|
||||
f"Creating jail {jail_name} from config template passed via stdin."
|
||||
)
|
||||
config.read_string(sys.stdin.read())
|
||||
else:
|
||||
print(
|
||||
f"Creating jail {jail_name} from config template {jail_config_path}."
|
||||
)
|
||||
if jail_config_path not in config.read(jail_config_path):
|
||||
eprint(f"Failed to read config template {jail_config_path}.")
|
||||
return 1
|
||||
|
|
@ -1266,7 +1318,8 @@ def create_jail(**kwargs):
|
|||
value = kwargs.pop(option)
|
||||
if (
|
||||
value is not None
|
||||
and len(value)
|
||||
# String, non-empty list of args or int
|
||||
and (isinstance(value, int) or len(value))
|
||||
and value is not config.my_get(option, None)
|
||||
):
|
||||
# TODO: this will wipe all systemd_nspawn_user_args from the template...
|
||||
|
|
@ -1279,7 +1332,7 @@ def create_jail(**kwargs):
|
|||
print(
|
||||
dedent(
|
||||
f"""
|
||||
TIP: Run `{COMMAND_NAME} create` without any arguments for interactive config.
|
||||
Hint: run `{COMMAND_NAME} create` without any arguments for interactive config.
|
||||
Or use CLI args to override the default options.
|
||||
For more info, run: `{COMMAND_NAME} create --help`
|
||||
"""
|
||||
|
|
@ -1318,10 +1371,13 @@ def create_jail(**kwargs):
|
|||
# but we don't need it so we will remove it later
|
||||
open(jail_config_path, "a").close()
|
||||
|
||||
returncode = run_lxc_download_script(
|
||||
if (
|
||||
returncode := run_lxc_download_script(
|
||||
jail_name, jail_path, jail_rootfs_path, distro, release
|
||||
)
|
||||
if returncode != 0:
|
||||
!= 0
|
||||
):
|
||||
cleanup(jail_path)
|
||||
return returncode
|
||||
|
||||
# Assuming the name of your jail is "myjail"
|
||||
|
|
@ -1347,11 +1403,13 @@ def create_jail(**kwargs):
|
|||
# But alpine jails made with jailmaker have other issues
|
||||
# They don't shutdown cleanly via systemctl and machinectl...
|
||||
|
||||
with Chroot(jail_rootfs_path):
|
||||
# Use chroot to correctly resolve absolute /sbin/init symlink
|
||||
init_system_name = os.path.basename(os.path.realpath("/sbin/init"))
|
||||
|
||||
if (
|
||||
os.path.basename(
|
||||
os.path.realpath(os.path.join(jail_rootfs_path, "sbin/init"))
|
||||
)
|
||||
!= "systemd"
|
||||
init_system_name != "systemd"
|
||||
and parse_os_release(jail_rootfs_path).get("ID") != "nixos"
|
||||
):
|
||||
print(
|
||||
dedent(
|
||||
|
|
@ -1379,9 +1437,10 @@ def create_jail(**kwargs):
|
|||
config.my_set("startup", 0)
|
||||
start_now = False
|
||||
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
# Remove config which systemd handles for us
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.remove(os.path.join(jail_rootfs_path, "etc/machine-id"))
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.remove(os.path.join(jail_rootfs_path, "etc/resolv.conf"))
|
||||
|
||||
# https://github.com/systemd/systemd/issues/852
|
||||
|
|
@ -1558,6 +1617,7 @@ def remove_jail(jail_name):
|
|||
return 1
|
||||
|
||||
# TODO: print which dataset is about to be removed before the user confirmation
|
||||
# TODO: print that all zfs snapshots will be removed if jail has it's own zfs dataset
|
||||
check = input(f'\nCAUTION: Type "{jail_name}" to confirm jail deletion!\n\n')
|
||||
|
||||
if check == jail_name:
|
||||
|
|
@ -1616,16 +1676,21 @@ def get_all_jail_names():
|
|||
return jail_names
|
||||
|
||||
|
||||
def parse_os_release(candidates):
|
||||
for candidate in candidates:
|
||||
def parse_os_release(new_root):
|
||||
result = {}
|
||||
with Chroot(new_root):
|
||||
# Use chroot to correctly resolve os-release symlink (for nixos)
|
||||
for candidate in ["/etc/os-release", "/usr/lib/os-release"]:
|
||||
try:
|
||||
with open(candidate, encoding="utf-8") as f:
|
||||
# TODO: can I create a solution which not depends on the internal _parse_os_release method?
|
||||
return platform._parse_os_release(f)
|
||||
result = platform._parse_os_release(f)
|
||||
break
|
||||
except OSError:
|
||||
# Silently ignore failing to read os release info
|
||||
pass
|
||||
return {}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def list_jails():
|
||||
|
|
@ -1680,13 +1745,7 @@ def list_jails():
|
|||
jail["addresses"] += "…"
|
||||
else:
|
||||
# Parse os-release info ourselves
|
||||
jail_platform = parse_os_release(
|
||||
(
|
||||
os.path.join(jail_rootfs_path, "etc/os-release"),
|
||||
os.path.join(jail_rootfs_path, "usr/lib/os-release"),
|
||||
)
|
||||
)
|
||||
|
||||
jail_platform = parse_os_release(jail_rootfs_path)
|
||||
jail["os"] = jail_platform.get("ID")
|
||||
jail["version"] = jail_platform.get("VERSION_ID") or jail_platform.get(
|
||||
"VERSION_CODENAME"
|
||||
|
|
@ -1775,7 +1834,7 @@ def main():
|
|||
title="commands", dest="command", metavar="", parser_class=CustomSubParser
|
||||
)
|
||||
|
||||
split_commands = ["create", "exec"]
|
||||
split_commands = ["create", "exec", "log", "status"]
|
||||
commands = {}
|
||||
|
||||
for d in [
|
||||
|
|
@ -1863,6 +1922,18 @@ def main():
|
|||
help="args to pass to machinectl shell",
|
||||
)
|
||||
|
||||
commands["log"].add_argument(
|
||||
"args",
|
||||
nargs="*",
|
||||
help="args to pass to journalctl",
|
||||
)
|
||||
|
||||
commands["status"].add_argument(
|
||||
"args",
|
||||
nargs="*",
|
||||
help="args to pass to systemctl",
|
||||
)
|
||||
|
||||
commands["create"].add_argument(
|
||||
"jail_name", #
|
||||
nargs="?",
|
||||
|
|
@ -1890,7 +1961,7 @@ def main():
|
|||
commands["create"].add_argument(
|
||||
"-c", #
|
||||
"--config",
|
||||
help="path to config file template",
|
||||
help="path to config file template or - for stdin",
|
||||
)
|
||||
commands["create"].add_argument(
|
||||
"-gi", #
|
||||
|
|
@ -1917,9 +1988,6 @@ def main():
|
|||
# Set appropriate permissions (if not already set) for this file, since it's executed as root
|
||||
stat_chmod(SCRIPT_PATH, 0o700)
|
||||
|
||||
# Work relative to this script
|
||||
os.chdir(SCRIPT_DIR_PATH)
|
||||
|
||||
# Ignore all args after the first "--"
|
||||
args_to_parse = split_at_string(sys.argv[1:], "--")[0]
|
||||
# Check for help
|
||||
|
|
@ -1943,7 +2011,6 @@ def main():
|
|||
args = vars(parser.parse_known_args(args_to_parse)[0])
|
||||
# ...and check if help is still in the remaining args
|
||||
need_help = args.get("help")
|
||||
print(need_help)
|
||||
|
||||
if need_help:
|
||||
commands[command].print_help()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0
|
|||
# Turning off seccomp filtering improves performance at the expense of security
|
||||
seccomp=1
|
||||
|
||||
# Use macvlan networking to provide an isolated network namespace,
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so docker can manage firewall rules
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change eno1/br1 to the interface name you want to use
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ gpu_passthrough_nvidia=0
|
|||
# TODO: don't disable seccomp but specify which syscalls should be allowed
|
||||
seccomp=0
|
||||
|
||||
# Use macvlan networking to provide an isolated network namespace,
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so incus can manage firewall rules
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change eno1/br1 to the interface name you want to use
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0
|
|||
# Turning off seccomp filtering improves performance at the expense of security
|
||||
seccomp=1
|
||||
|
||||
# Use macvlan networking to provide an isolated network namespace,
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so kubernetes can manage firewall rules
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change eno1/br1 to the interface name you want to use
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ gpu_passthrough_nvidia=0
|
|||
# TODO: don't disable seccomp but specify which syscalls should be allowed
|
||||
seccomp=0
|
||||
|
||||
# Use macvlan networking to provide an isolated network namespace,
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so lxd can manage firewall rules
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change eno1/br1 to the interface name you want to use
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
# Nixos Jail Template
|
||||
|
||||
## Disclaimer
|
||||
|
||||
**Experimental. Using nixos in this setup hasn't been extensively tested and has [known issues](#known-issues).**
|
||||
|
||||
## Setup
|
||||
|
||||
Check out the [config](./config) template file. You may provide it when asked during `./jlmkr.py create` or, if you have the template file stored on your NAS, you may provide it directly by running `./jlmkr.py create --start --config /mnt/tank/path/to/nixos/config mynixosjail`.
|
||||
|
||||
## Manual Setup
|
||||
|
||||
```bash
|
||||
# Create the jail without starting
|
||||
./jlmkr.py create --distro=nixos --release=24.05 nixos --network-bridge=br1 --resolv-conf=bind-host --bind-ro=./lxd.nix:/etc/nixos/lxd.nix
|
||||
# Create empty nix module to satisfy import in default lxc configuration.nix
|
||||
echo '{ ... }:{}' > ./jails/nixos/lxd.nix
|
||||
# Start the nixos jail
|
||||
./jlmkr.py start nixos
|
||||
sleep 90
|
||||
# Network should be up by now
|
||||
./jlmkr.py shell nixos /bin/sh -c 'ifconfig'
|
||||
# Try to rebuild the system
|
||||
./jlmkr.py shell nixos /bin/sh -c 'nixos-rebuild switch'
|
||||
```
|
||||
|
||||
## Known Issues
|
||||
|
||||
### Environment jlmkr exec
|
||||
|
||||
Running `./jlmkr.py exec mynixosjail ifconfig` doesn't work because the shell environment isn't setup properly. You can run `./jlmkr.py shell mynixosjail /bin/sh -c 'ifconfig'` or `./jlmkr.py exec mynixosjail /bin/sh -c '. /etc/bashrc; ifconfig'` instead.
|
||||
|
||||
### Bridge networking only
|
||||
|
||||
This setup has NOT been tested with macvlan networking.
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
startup=0
|
||||
gpu_passthrough_intel=0
|
||||
gpu_passthrough_nvidia=0
|
||||
# Turning off seccomp filtering improves performance at the expense of security
|
||||
seccomp=1
|
||||
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so nixos can manage firewall rules
|
||||
# Ensure to change br1 to the interface name you want to use
|
||||
# You may want to add additional options here, e.g. bind mounts
|
||||
systemd_nspawn_user_args=--network-bridge=br1
|
||||
--bind-ro=./lxd.nix:/etc/nixos/lxd.nix
|
||||
|
||||
# Script to run on the HOST before starting the jail
|
||||
pre_start_hook=#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
echo 'PRE_START_HOOK'
|
||||
|
||||
# If there's no machine-id then this we're about to start the jail for the first time
|
||||
if [ ! -e ./rootfs/etc/machine-id ]; then
|
||||
echo 'BEFORE_FIRST_BOOT'
|
||||
# Create empty nix module to satisfy import in default lxc configuration.nix
|
||||
echo '{ ... }:{}' > ./lxd.nix
|
||||
cp /etc/resolv.conf ./rootfs/etc/resolv.conf
|
||||
fi
|
||||
|
||||
# Only used while creating the jail
|
||||
distro=nixos
|
||||
release=24.05
|
||||
|
||||
# # Example initial_setup which rebuild the system,
|
||||
# # for when you mount your own /etc/nixos/configuration.nix inside the jail
|
||||
# initial_setup=#!/run/current-system/sw/bin/bash
|
||||
# . /etc/bashrc
|
||||
# set -x
|
||||
# ifconfig
|
||||
# nixos-rebuild switch
|
||||
# echo "All Done"
|
||||
|
||||
# You generally will not need to change the options below
|
||||
systemd_run_default_args=--property=KillMode=mixed
|
||||
--property=Type=notify
|
||||
--property=RestartForceExitStatus=133
|
||||
--property=SuccessExitStatus=133
|
||||
--property=Delegate=yes
|
||||
--property=TasksMax=infinity
|
||||
--collect
|
||||
--setenv=SYSTEMD_NSPAWN_LOCK=0
|
||||
|
||||
systemd_nspawn_default_args=--keep-unit
|
||||
--quiet
|
||||
--boot
|
||||
--bind-ro=/sys/module
|
||||
--inaccessible=/sys/module/apparmor
|
||||
|
|
@ -4,7 +4,7 @@ gpu_passthrough_nvidia=0
|
|||
# Turning off seccomp filtering improves performance at the expense of security
|
||||
seccomp=1
|
||||
|
||||
# Use macvlan networking to provide an isolated network namespace,
|
||||
# Use bridge networking to provide an isolated network namespace,
|
||||
# so podman can manage firewall rules
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change eno1/br1 to the interface name you want to use
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
# Router Jail Template
|
||||
|
||||
Host a subordinate LAN using nftables and dnsmasq for DHCP, DNS, routing, and netboot infrastructure.
|
||||
```
|
||||
router laptop desktop
|
||||
| | |
|
||||
+-- LAN --+-------+
|
||||
|
|
||||
{ TrueNAS SCALE }
|
||||
|
|
||||
+-----+-----+-- LAN2 --+------+------+-------+
|
||||
| | | | | | |
|
||||
RPi1 RPi2 RPi3 NUC01 NUC02 NUC03 CrayYMP
|
||||
```
|
||||
*Example usage*: deploy a flock of headless/diskless Raspberry Pi worker nodes for Kubernetes; each netbooting into an iSCSI or NFS root volume.
|
||||
|
||||
## Setup
|
||||
|
||||
Use the TrueNAS SCALE administrative UI to create a network bridge interface. Assign to that bridge a physical interface that's not shared with the host network.
|
||||
|
||||
Use the `dnsmasq-example.conf` file as a starting point for your own dnsmasq settings file(s). Copy or mount them inside `/etc/dnsmasq.d/` within the jail.
|
||||
|
||||
Optional: place assets in the mounted `/tftp/` directory for netbooting clients.
|
||||
|
||||
Optional: attach more jails to this same bridge to host e.g. a K3s control plane, an nginx load balancer, a PostgreSQL database...
|
||||
|
||||
Check out the [config](./config) template file. You may provide it when asked during `./jlmkr.py create` or, if you have the template file stored on your NAS, you may provide it directly by running `./jlmkr.py create --start --config /mnt/tank/path/to/router/config myrouterjail`.
|
||||
|
||||
## Additional Resources
|
||||
|
||||
There are as many reasons to host LAN infrastructure as there are to connect a LAN. This template can help you kick-start such a leaf network, using a TrueNAS jail as its gateway host.
|
||||
|
||||
For those specifically interested in *netbooting Raspberry Pi*, the following **external** links might help you get started.
|
||||
|
||||
* [Network Booting a Raspberry Pi 4 with an iSCSI Root via FreeNAS][G1]; the title says it all
|
||||
* [Raspberry Pi Network Boot Guide][G2] covers more Raspberry Pi models; written for Synology users
|
||||
* [pi_iscsi_netboot][s1] and [prep-netboot-storage][s2] are scripts showing preparation of boot assets and iSCSI root volumes
|
||||
|
||||
Good luck!
|
||||
|
||||
[G1]: https://shawnwilsher.com/2020/05/network-booting-a-raspberry-pi-4-with-an-iscsi-root-via-freenas/
|
||||
[G2]: https://warmestrobot.com/blog/2021/6/21/raspberry-pi-network-boot-guide
|
||||
[s1]: https://github.com/tjpetz/pi_iscsi_netboot
|
||||
[s2]: https://gitlab.com/jnicpon/rpi-prep/-/blob/main/scripts/prep-netboot-storage.fish?ref_type=heads
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# See also:
|
||||
# <https://github.com/Jip-Hop/jailmaker/tree/main/templates/router>
|
||||
#
|
||||
startup=0
|
||||
gpu_passthrough_intel=0
|
||||
gpu_passthrough_nvidia=0
|
||||
# Turning off seccomp filtering improves performance at the expense of security
|
||||
seccomp=1
|
||||
|
||||
# Use bridge networking to provide an isolated network namespace
|
||||
# Alternatively use --network-macvlan=eno1 instead of --network-bridge
|
||||
# Ensure to change br0 to the HOST interface name you want to use
|
||||
# and br1 to the SECONDARY interface name you want to prepare
|
||||
# Substitute your own dnsmasq.d and TFTP dataset bindings
|
||||
systemd_nspawn_user_args=--network-bridge=br0
|
||||
--network-veth-extra=ve-router-1:vee-1
|
||||
--resolv-conf=bind-host
|
||||
--system-call-filter='add_key keyctl bpf'
|
||||
--bind=/mnt/pool/subnet/dnsmasq.d:/etc/dnsmasq.d
|
||||
--bind-ro=/mnt/pool/subnet/tftpboot:/tftp
|
||||
|
||||
# Script to run on the HOST before starting the jail
|
||||
# Load kernel module and config kernel settings required for podman
|
||||
pre_start_hook=#!/usr/bin/bash
|
||||
set -euo pipefail
|
||||
echo 'PRE_START_HOOK'
|
||||
echo 1 > /proc/sys/net/ipv4/ip_forward
|
||||
modprobe br_netfilter
|
||||
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
|
||||
echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables
|
||||
modprobe iptable_nat
|
||||
modprobe iptable_filter
|
||||
|
||||
# Script to run on the HOST after starting the jail
|
||||
# For example to attach to multiple bridge interfaces
|
||||
post_start_hook=#!/usr/bin/bash
|
||||
set -euo pipefail
|
||||
echo 'POST_START_HOOK'
|
||||
ip link set dev ve-router-1 master br1
|
||||
ip link set dev ve-router-1 up
|
||||
#ip link set dev eth2 master br1
|
||||
|
||||
# Only used while creating the jail
|
||||
distro=debian
|
||||
release=bookworm
|
||||
|
||||
# Install and configure within the jail
|
||||
initial_setup=#!/usr/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Catch up on updates
|
||||
apt-get update && apt-get full-upgrade -y
|
||||
|
||||
# Configure worker LAN interface with static IP
|
||||
sh -c 'cat <<EOF > /etc/systemd/network/80-container-vee-1.network
|
||||
[Match]
|
||||
Virtualization=container
|
||||
Name=vee-1
|
||||
|
||||
[Network]
|
||||
DHCP=false
|
||||
Address=10.3.14.202/24
|
||||
EOF'
|
||||
systemctl restart systemd-networkd.service
|
||||
|
||||
# Configure routing from LAN clients
|
||||
apt-get install nftables -y
|
||||
nft add table nat
|
||||
nft add chain nat prerouting { type nat hook prerouting priority 0 \; }
|
||||
nft add chain nat postrouting { type nat hook postrouting priority 100 \; }
|
||||
nft add rule nat postrouting masquerade
|
||||
mkdir -p /etc/nftables.d
|
||||
nft list table nat >/etc/nftables.d/nat.conf
|
||||
( echo ; echo 'include "/etc/nftables.d/*.conf"' ) >>/etc/nftables.conf
|
||||
|
||||
# Install dnsmasq alongside local resolver
|
||||
sed -i -e 's/^#DNSStubListener=yes$/DNSStubListener=no/' /etc/systemd/resolved.conf
|
||||
systemctl restart systemd-resolved.service
|
||||
apt-get install dnsmasq -y
|
||||
sed -i -e 's/^#DNS=$/DNS=127.0.0.1/' /etc/systemd/resolved.conf
|
||||
systemctl restart systemd-resolved.service
|
||||
systemctl restart dnsmasq.service
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
# customize and place this file inside /etc/dnsmasq.d
|
||||
|
||||
# serve only Raspberry Pi network; don't backfeed to the host LAN
|
||||
no-dhcp-interface=host0
|
||||
interface=vee-1
|
||||
bind-interfaces
|
||||
|
||||
# designated upstream query servers
|
||||
server=1.1.1.1
|
||||
server=1.0.0.1
|
||||
|
||||
# pirate TLD for the Democratic Republic of Raspberry Pi
|
||||
domain=pi,10.3.14.0/24
|
||||
|
||||
# enable DHCP services
|
||||
dhcp-authoritative
|
||||
dhcp-rapid-commit
|
||||
dhcp-range=10.3.14.101,10.3.14.199
|
||||
|
||||
# meet the 'berries
|
||||
dhcp-host=e4:5f:01:da:da:b1,rpi1,10.3.14.11,infinite,set:rpi
|
||||
dhcp-host=e4:5f:01:da:da:b2,rpi2,10.3.14.12,infinite,set:rpi
|
||||
dhcp-host=e4:5f:01:da:da:b3,rpi3,10.3.14.13,infinite,set:rpi
|
||||
dhcp-host=e4:5f:01:da:da:b4,rpi4,10.3.14.14,infinite,set:rpi
|
||||
dhcp-host=e4:5f:01:da:da:b5,rpi5,10.3.14.15,infinite,set:rpi
|
||||
dhcp-host=e4:5f:01:da:*:*,set:rpicube
|
||||
|
||||
# PXE
|
||||
dhcp-option-force=66,10.3.14.202
|
||||
# magic number
|
||||
dhcp-option-force=208,f1:00:74:7e
|
||||
# config filename
|
||||
dhcp-option-force=209,configs/common
|
||||
# path prefix
|
||||
dhcp-option-force=210,/boot/
|
||||
# reboot time (i -> 32 bit)
|
||||
dhcp-option-force=211,30i
|
||||
|
||||
dhcp-boot=bootcode.bin
|
||||
|
||||
#dhcp-match=set:ipxe,175
|
||||
#dhcp-boot=tag:ipxe,http://boot.netboot.xyz/ipxe/netboot.xyz.efi
|
||||
|
||||
# TFTP
|
||||
enable-tftp
|
||||
tftp-root=/tftp
|
||||
|
||||
#debugging
|
||||
#log-queries
|
||||
#log-dhcp
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
# Jailmaker Testing
|
||||
|
||||
This readme documents the [test-jlmkr](./test-jlmkr) script.
|
||||
|
||||
The script has 2 optional parameter invocation sets:
|
||||
* `<jail type>` [`<jail name>`]
|
||||
* `<template/path>` `<jail name>`
|
||||
|
||||
If the script is invoked without arguments, it will use the default configuration and name the test jail `default-jail`.
|
||||
|
||||
Legend:
|
||||
| Arg name | Description |
|
||||
|-|-|
|
||||
| `<jail type>` | The template dir-name in `$JLMKR_PATH/templates` (it will load the `config` file within it) |
|
||||
| `<jail name>`\* | The name of the jail that will be created and destroyed during the testing. <br/> If not supplied, the default is `<jail type>-test` |
|
||||
| `<template/path>` | relative or absolute path to a config template. (`<jail name>` must be supplied) |
|
||||
|
||||
> PRE-REQUISITE:
|
||||
> \* WARNING: If `<jail name>` exists, it will be removed.
|
||||
|
||||
Environment variables control the test behavior:
|
||||
| Variable name | Default |Description |
|
||||
|-|:-:|-|
|
||||
|`JLMKR_PATH`|optional - see description|When unspecified, and `SCALE_POOL_ROOT` is defined, it will check if `$SCALE_POOL_ROOT/jailmaker` contians the `jlmkr.py` script. Otherwise, it will use `${PWD:-.}` instead (the local dir)|
|
||||
|`SCALE_POOL_ROOT`|optional|Used to define `JLMKR_PATH`, it should point to the root of the ZFS dataset hosting the `jailmaker` script dir. |
|
||||
|`STOP`|0| `0` - perform all non-blocking tests<br/>`l` - only list and images, nothing else<br/>`i` - interactive test, includ console-blocking waiting for input tests (edit and shell)|
|
||||
|`FULL_TEST`|0|`0` - perform a single run<br/>`1`* - perform a full-test|
|
||||
|
||||
> \* `FULL_TEST=1` will perform a full run ONLY when passing a `<jail type>` (will not work with `<template/path>`)
|
||||
|
||||
## Type of Run
|
||||
|
||||
In `STOP=0` (the default) all steps, with the interactive steps manipulated to run without prompting.
|
||||
The interactive steps are steps which prompt the user for input. These are `shell` and `edit`.
|
||||
In `STOP=i` all steps will be performed, and interactive steps will wait for user input.
|
||||
|
||||
A special shorthand option exists `STOP=l` which will just run `list` and `images`, this is to test basic invocation of the script regardless of specific jails. This is also the only non-destructive mode.
|
||||
|
||||
## Single Run
|
||||
|
||||
By default, a single run in non-interactive mode will run, it runs from whatever CWD path the shell is in.
|
||||
|
||||
## Full Test
|
||||
|
||||
The Full test, starts with the single run (whether interactive or not), if successful, it continues to perform non-interactive tests with the following parameters:
|
||||
|
||||
| S | F | CWD | Path to template |
|
||||
|:-:|:-:|-|-|
|
||||
|+|+|`$JLMKR_PATH`|`./templates/$JAIL_TYPE/config`|
|
||||
||+|`JLMKR_PATH`|`templates/$JAIL_TYPE/config`|
|
||||
||+|`JLMKR_PATH`|`$SCALE_POOL_ROOT/jailmaker/$JAIL_CONFIG`|
|
||||
||+| ~ |`JLMKR_PATH/$JAIL_CONFIG`|
|
||||
||+| ~ | a _temporary file_, it's contents will be copied form the `<jail type>`'s config file. |
|
||||
|
||||
> S - Single Run | F - Full Test
|
||||
|
||||
### Example invocation:
|
||||
Single run, interactive mode, `nixos` jail type:
|
||||
|
||||
```bash
|
||||
cd ${SCALE_POOL_ROOT:?must define this before running}/jailmaker \
|
||||
&& sudo STOP=i SCALE_POOL_ROOT=$SCALE_POOL_ROOT $SCALE_POOL_ROOT/jailmaker/test/test-jlmkr nixos
|
||||
```
|
||||
|
||||
Full test, default settings, `nixos` jail type:
|
||||
|
||||
```bash
|
||||
cd ${JLMKR_PATH:?must define this before running} \
|
||||
&& sudo FULL_TEST=1 ./test/test-jlmkr nixos
|
||||
```
|
||||
|
||||
### What to expect
|
||||
A full non-interactive test run with `nixos` (which is rather lean, yet tests both pre hook and init scripts) ran for ~260 seconds.
|
||||
|
||||
The report summary outputs what type of run it was with CWD and path to config file and for each step, a green checkmark indicating ✅ Success, a red X symbol following the exit code❌(`<error code>`) for erros. Both follow the command executed.
|
||||
In the case a step wasn't performed, a blank checkbox 🔳 followed by the name of the step will be listed.
|
||||
The report is always listed in alphabetic order (although the steps are performed in an order that allows testing as much as possible while taking into account dependent steps.
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
# Example invokation
|
||||
# sudo SCALE_POOL_ROOT=$SCALE_POOL_ROOT $SCALE_POOL_ROOT/jailmaker/test/test-jlmkr docker
|
||||
|
||||
set -e
|
||||
|
||||
#### Global variables
|
||||
|
||||
if [[ -z "$JLMKR_PATH" && -n "$SCALE_POOL_ROOT" ]]; then
|
||||
SCALE_POOL_ROOT=${SCALE_POOL_ROOT:?must be exported before you can run test suite}
|
||||
JLMKR_PATH=${SCALE_POOL_ROOT}/jailmaker
|
||||
elif [[ -z "$JLMKR_PATH" ]]; then
|
||||
JLMKR_PATH=${PWD:-.}
|
||||
fi
|
||||
if [[ ! -r "$JLMKR_PATH/jlmkr.py" ]]; then
|
||||
>&2 printf "%s\n" \
|
||||
"couldn't find jlmkr.py. Are you running from the jailmaker directory?" \
|
||||
"If not, setup either JLMKR_PATH or SCALE_POOL_ROOT" \
|
||||
""
|
||||
|
||||
>&2 printf "\tPWD: %s\n\tJLMKR_PATH: %s\n\tSCALE_POOL_ROOT: %s\n" \
|
||||
"$PWD" "$JLMKR_PATH" "$SCALE_POOL_ROOT"
|
||||
|
||||
>&2 printf "\n"
|
||||
exit 2
|
||||
fi
|
||||
JAIL_TYPE="${1}"
|
||||
if [ -n "${JAIL_TYPE}" ]; then
|
||||
if [ -r "${JAIL_TYPE}" ]; then
|
||||
JAIL_CONFIG="$JAIL_TYPE"
|
||||
JAIL_TYPE=
|
||||
# Can't perform full test with config path
|
||||
FULL_TEST=0
|
||||
else
|
||||
JAIL_CONFIG="$JLMKR_PATH/templates/$JAIL_TYPE/config"
|
||||
# Full test is an option when using JAIL_TYPE
|
||||
FULL_TEST=${FULL_TEST:-0}
|
||||
fi
|
||||
if [ ! -r "${JAIL_CONFIG}" ]; then
|
||||
>&2 printf "Must supply a valid jail type or config path\n"
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2034 # JAIL is used inside perform_test_suite
|
||||
JAIL="${2:-${JAIL_TYPE:-default}-test}"
|
||||
|
||||
# STOP=0 - (default) perform all tests, in non-blocking mode
|
||||
# STOP=l - only list and images, nothing else
|
||||
# STOP=i - interactive test, includ console-blocking waiting for input tests (edit and shell)
|
||||
STOP=${STOP:-0}
|
||||
|
||||
REPORT=(create edit exec images list log remove restart shell start status stop)
|
||||
|
||||
WAIT_FOR_JAIL=${WAIT_FOR_JAIL:-4s}
|
||||
|
||||
#### Functions
|
||||
jlmkr () {
|
||||
/usr/bin/env python3 "$JLMKR_PATH/jlmkr.py" "${@:---help}"
|
||||
}
|
||||
|
||||
iterate () {
|
||||
# shellcheck disable=SC2206 # $1 will pass multiple values, we want splitting here
|
||||
local SET=($1) DO=("${@:2}")
|
||||
local x j _x x_STATUS
|
||||
|
||||
for j in "${SET[@]}"; do
|
||||
for _x in "${DO[@]}"; do
|
||||
x="${_x//\(Jv)/JLMKR_"$j"}"
|
||||
# echo "$x" >&2
|
||||
${NO_EVAL:+:} eval "echo \$JLMKR_${j} \"$x\""
|
||||
x_STATUS=✅
|
||||
${NO_EVAL:+:} eval "JLMKR_$j=$x_STATUS"
|
||||
${NO_EVAL:+:} eval "JLMKR_${j}_x=\"${x//\"/\'}\""
|
||||
|
||||
if [[ -n "$DELAY" ]]; then
|
||||
printf "Waiting %s seconds before test...\n" "${DELAY}"
|
||||
sleep "${DELAY}"
|
||||
fi
|
||||
|
||||
set +e
|
||||
eval "$x" || x_STATUS=$?
|
||||
set -e
|
||||
if [[ "$x_STATUS" != "✅" ]]; then
|
||||
${NO_EVAL:+:} eval "JLMKR_${j}_x=\"($x_STATUS) ${x//\"/\'}\""
|
||||
${NO_EVAL:+:} eval "JLMKR_$j=❌"
|
||||
STOP=E:$x_STATUS
|
||||
return
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
perform_test_suite() {
|
||||
# shellcheck disable=SC2016 # function relies heavily on single quotes preventing expansion
|
||||
|
||||
if [[ "$STOP" =~ ^(0|l|i)$ ]]; then
|
||||
# Initialize REPORT with empty checkboxes - NO_EVAL=1 is important here, otherwise Status will be evaluated
|
||||
NO_EVAL=1 iterate "${REPORT[*]}" '(Jv)="🔳"'
|
||||
|
||||
TESTS=(list images)
|
||||
iterate "${TESTS[*]}" 'jlmkr $j'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(create) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j ${JAIL_CONFIG:+--config} $JAIL_CONFIG $JAIL'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(start) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j $JAIL'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(restart) \
|
||||
&& DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL'
|
||||
|
||||
# If this is an interactive test, edit and shell will wait for input
|
||||
[[ "$STOP" == "i" ]] && TESTS=(edit shell) \
|
||||
&& DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL'
|
||||
|
||||
# This is the non-interactive test for edit
|
||||
[[ "$STOP" == "0" ]] && TESTS=(edit) \
|
||||
&& DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'EDITOR=cat jlmkr $j $JAIL'
|
||||
|
||||
# This is the non-interactive test for shell
|
||||
[[ "$STOP" == "0" ]] && TESTS=(shell) \
|
||||
&& DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL /bin/sh -c "echo shell called successful"'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(exec) \
|
||||
&& DELAY=$WAIT_FOR_JAIL iterate "${TESTS[*]}" 'jlmkr $j $JAIL /bin/sh -c "echo exec called successful"'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(status) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j $JAIL --no-pager'
|
||||
|
||||
[[ "$STOP" =~ ^(0|i)$ ]] && TESTS=(log) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j $JAIL -n 10'
|
||||
|
||||
[[ "$STOP" =~ ^(0|l|i)$ ]] || >&2 printf "Had an Error. Cleanup up and stopping.\n"
|
||||
|
||||
# Always perform these cleanup steps, even if something failed
|
||||
[[ "$STOP" != l ]] \
|
||||
&& TESTS=(stop) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j $JAIL' \
|
||||
|
||||
[[ "$STOP" != l ]] \
|
||||
&& TESTS=(remove) \
|
||||
&& iterate "${TESTS[*]}" 'jlmkr $j $JAIL <<<"$JAIL" '
|
||||
|
||||
fi
|
||||
printf '\n\nReport for:\n\tCWD: %s\t\tJAIL_CONFIG: %s\n\n' "$(pwd)" "${JAIL_CONFIG}"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
NO_EVAL=1 iterate "${REPORT[*]}" 'echo "$(Jv) ${(Jv)_x:-$j}"'
|
||||
}
|
||||
|
||||
#### Execution starts here
|
||||
|
||||
perform_test_suite
|
||||
|
||||
if [[ "$STOP" =~ ^(0|i)$ ]]; then
|
||||
[[ "$FULL_TEST" == 1 ]] || STOP="Single Test"
|
||||
fi
|
||||
|
||||
if [[ "$STOP" =~ ^(0|i)$ ]]; then
|
||||
pushd "$JLMKR_PATH" > /dev/null || STOP=pushd
|
||||
STOP=0 # The following test suite should only be run non-interactivley
|
||||
|
||||
JAIL_CONFIG="./templates/$JAIL_TYPE/config"
|
||||
perform_test_suite
|
||||
|
||||
JAIL_CONFIG="templates/$JAIL_TYPE/config"
|
||||
perform_test_suite
|
||||
|
||||
JAIL_CONFIG="$JLMKR_PATH/$JAIL_CONFIG"
|
||||
perform_test_suite
|
||||
|
||||
cd ~
|
||||
perform_test_suite
|
||||
|
||||
TMP_JAIL_CFG=$(mktemp) \
|
||||
&& cp "$JAIL_CONFIG" "$TMP_JAIL_CFG" \
|
||||
&& JAIL_CONFIG="${TMP_JAIL_CFG}" perform_test_suite \
|
||||
&& rm "$TMP_JAIL_CFG" \
|
||||
|| STOP=E:temp_file
|
||||
|
||||
popd > /dev/null || STOP=popd
|
||||
fi
|
||||
|
||||
if [[ "$STOP" = 0 ]]; then
|
||||
printf "All tests completed\n"
|
||||
else
|
||||
printf "Stopped: %s\n" "$STOP"
|
||||
[[ "$STOP" = "Single Test" ]] || exit 1
|
||||
fi
|
||||
|
||||
30
test/test.sh
30
test/test.sh
|
|
@ -1,36 +1,12 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
uname -r
|
||||
cat /etc/os-release
|
||||
python3 --version
|
||||
|
||||
apt-get update -qq && apt-get install -qq -y systemd-container
|
||||
|
||||
# # TODO: create zpool with virtual disks, create jailmaker dataset and test jlmkr.py from there
|
||||
# # https://medium.com/@abaddonsd/zfs-usage-with-virtual-disks-62898064a29b
|
||||
# apt-get install -y -qq zfsutils-linux
|
||||
# modinfo zfs | grep version
|
||||
# zfs --version
|
||||
# zpool --version
|
||||
|
||||
# TODO: create a path and/or zfs pool with a space in it to test if jlmkr.py still works properly when ran from inside
|
||||
# mkdir -p "/tmp/path with space/jailmaker"
|
||||
|
||||
chown 0:0 jlmkr.py
|
||||
chmod +x jlmkr.py
|
||||
|
||||
# Setup NAT to give the jail access to the outside network
|
||||
# https://wiki.archlinux.org/title/systemd-nspawn#Use_a_virtual_Ethernet_link
|
||||
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
|
||||
iptables -I DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
||||
iptables -I DOCKER-USER -i ve-+ -o eth0 -j ACCEPT
|
||||
iptables -A INPUT -i ve-+ -p udp -m udp --dport 67 -j ACCEPT
|
||||
# TODO: many more test cases and checking if actual output (text, files on disk etc.) is correct instead of just a 0 exit code
|
||||
|
||||
# TODO: test jlmkr.py from inside another working directory, with a relative path to a config file to test if it uses the config file (and doesn't look for it relative to the jlmkr.py file itself)
|
||||
./jlmkr.py create --start --config=./templates/docker/config test --network-veth --system-call-filter='add_key' --system-call-filter='bpf' --system-call-filter='keyctl'
|
||||
./jlmkr.py exec test docker run hello-world
|
||||
|
||||
# TODO: many more test cases and checking if actual output (text, files on disk etc.) is correct instead of just a 0 exit code
|
||||
./jlmkr.py create --start --config=./templates/docker/config test
|
||||
./jlmkr.py exec test docker run hello-world
|
||||
|
|
|
|||
Loading…
Reference in New Issue