Source code for flux_batch.service.usernetes.service

import json
import os
import shlex
import uuid

import flux_batch.utils as utils
from flux_batch.service.service import Service

# The usernetes module drives a usernetes checkout (hetchy branch) that
# provides the hardened per-node scripts under <usernetes_root>/service/.
# flux-batch only ships thin launchers (script/launch-*.sh) that orchestrate
# startup order, readiness, and teardown around those scripts:
#
#   cluster mode (default): rank 0 starts the control plane, every other rank
#   starts a worker that waits for the join command. User jobs are gated on
#   the whole cluster being Ready.
#
#   worker mode (bursting): EVERY rank starts a worker that joins an existing
#   cluster, identified by cluster_name. The control plane of the original
#   job reconciles the new nodes (sync-external-ip, calico-node restarts) and
#   deletes them again when the burst job ends.
#
# State is shared through <shared_root>/<cluster_name> on a shared filesystem.

START_CLUSTER_TEMPLATE = """
from flux.modprobe import task
import flux.subprocess as subprocess

@task(
    "start-{{service_name}}",
    ranks="0",
    needs_config=["{{service_name}}"],
    after=["resource", "job-list"],
)
def start_{{service_func}}(context):
    subprocess.rexec_bg(
        handle=context.handle,
        command=["/bin/bash", "-c", {{control_plane_command}}],
        label="{{service_name}}",
        nodeid=0,
    )

@task(
    "start-{{service_name}}-worker",
    ranks=">0",
    needs_config=["{{service_name}}"],
    after=["resource", "job-list"],
)
def start_{{service_func}}_worker(context):
    subprocess.rexec_bg(
        handle=context.handle,
        command=["/bin/bash", "-c", {{worker_command}}],
        label="{{service_name}}-worker",
    )
"""

# Burst job: every rank (rank 0 included) is a worker.
START_WORKERS_TEMPLATE = """
from flux.modprobe import task
import flux.subprocess as subprocess

@task(
    "start-{{service_name}}",
    ranks="0",
    needs_config=["{{service_name}}"],
    after=["resource", "job-list"],
)
def start_{{service_func}}(context):
    subprocess.rexec_bg(
        handle=context.handle,
        command=["/bin/bash", "-c", {{worker_command}}],
        label="{{service_name}}",
        nodeid=0,
    )

@task(
    "start-{{service_name}}-worker",
    ranks=">0",
    needs_config=["{{service_name}}"],
    after=["resource", "job-list"],
)
def start_{{service_func}}_worker(context):
    subprocess.rexec_bg(
        handle=context.handle,
        command=["/bin/bash", "-c", {{worker_command}}],
        label="{{service_name}}-worker",
    )
"""

# SIGTERM (15) so the launcher traps fire: workers leave a departure marker
# (the control plane deletes their node objects) and both run 'make down-v'.
STOP_MODULE_TEMPLATE = """
from flux.modprobe import task
import flux.subprocess as subprocess

@task(
    "stop-{{service_name}}",
    ranks="0",
    needs_config=["{{service_name}}"],
    before=["resource", "job-list"],
)
def stop_{{service_func}}(context):
    subprocess.kill(context.handle, signum=15, label="{{service_name}}").get()
    try:
        status = subprocess.wait(context.handle, label="{{service_name}}").get()["status"]
        print(status)
    except Exception:
        pass

@task(
    "stop-{{service_name}}-worker",
    ranks=">0",
    needs_config=["{{service_name}}"],
    before=["resource", "job-list"],
)
def stop_{{service_func}}_worker(context):
    subprocess.kill(context.handle, signum=15, label="{{service_name}}-worker").get()
    try:
        status = subprocess.wait(context.handle, label="{{service_name}}-worker").get()["status"]
        print(status)
    except Exception:
        pass
"""


[docs] class UsernetesService(Service): """ Deploy user space Kubernetes (usernetes, hetchy branch) under a batch job. Attributes are set on the jobspec attributes or passed to add_module: usernetes_root Usernetes checkout whose service/ scripts do the per-node work (default: $USERNETES_ROOT in the submission environment, then /usr/workspace/usernetes/usernetes-wip). mode "cluster" (default) or "worker" (burst: all ranks join an existing cluster). cluster_name Name of the cluster; a burst job must use the name of the cluster it joins (default: "default"). shared_root Shared-filesystem directory holding per-cluster state (default: ${HOME}/.usernetes/clusters). wait_timeout Seconds to wait for each phase (default: 3600). container_tech Rootless engine (default: podman). logs_dir Directory for launcher/service logs on each node (default: /tmp/$USER). use_ssh Launch through 'ssh $(hostname) ...' so the node has a systemd user session, which the usernetes preflight check requires and tasks started by the broker lack (default: True). Needs passwordless ssh between nodes. usernetes_env Dict of extra environment passed through to the usernetes service scripts (USERNETES_STORAGE_ROOT, USERNETES_CNI, CALICO_VXLAN_IIFNAME). """ # The working copy, podman state and node container are per user and per # host: two usernetes jobs on one node tear each other down. exclusive_nodes = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # One id per submission: the control plane writes it to the cluster # directory after clearing stale state, and same-job workers refuse # to read a join command until they see it. A burst job does not use # it (it trusts the existing, ready cluster instead). self.launch_id = str(uuid.uuid4()) # Attribute accessors with defaults @property def mode(self): mode = (self.attributes.get("mode") or "cluster").lower() if mode == "burst": mode = "worker" if mode not in ("cluster", "worker"): raise ValueError(f"Unknown usernetes mode '{mode}': use 'cluster' or 'worker'") return mode @property def usernetes_root(self): return ( self.attributes.get("usernetes_root") or os.environ.get("USERNETES_ROOT") or "/usr/workspace/usernetes/usernetes-wip" ) @property def cluster_name(self): return self.attributes.get("cluster_name") or "default" @property def shared_root(self): # Rendered into bash, so ${HOME} expands on the compute node. return self.attributes.get("shared_root") or "${HOME}/.usernetes/clusters" @property def shared_dir(self): return f"{self.shared_root}/{self.cluster_name}" @property def wait_timeout(self): return int(self.attributes.get("wait_timeout") or 3600) @property def script_dir(self): return os.path.join(os.path.dirname(__file__), "script") # Launcher command construction def launcher_environment(self): """ Environment rendered into the launcher invocations. Values the user did not set are left to the launchers' own defaults. """ env = { "USERNETES_ROOT": self.usernetes_root, "USERNETES_CLUSTER_NAME": self.cluster_name, "USERNETES_MODE": self.mode, "USERNETES_LAUNCH_ID": self.launch_id, "USERNETES_WAIT_TIMEOUT": str(self.wait_timeout), } if self.attributes.get("shared_root"): env["USERNETES_SHARED_ROOT"] = self.attributes["shared_root"] if self.attributes.get("nodes"): env["USERNETES_EXPECTED_NODES"] = str(self.attributes["nodes"]) if self.attributes.get("container_tech"): env["USERNETES_CONTAINER_TECH"] = self.attributes["container_tech"] if self.attributes.get("logs_dir"): env["USERNETES_LOG_DIR"] = str(self.attributes["logs_dir"]) # Anything else the usernetes service scripts understand env.update(self.attributes.get("usernetes_env") or {}) return env
[docs] def launcher_command(self, script): """ One shell command string running a launcher with its environment. """ env = self.launcher_environment() assignments = " ".join(f"{key}={shlex.quote(str(value))}" for key, value in env.items()) script_path = os.path.join(self.script_dir, script) command = f"{assignments} exec /bin/bash {shlex.quote(script_path)}" if self.attributes.get("use_ssh", True): # -tt: when Flux kills the ssh client, the remote pty hangs up and # the launcher gets SIGHUP, so its teardown still runs. command = f"ssh -tt -o BatchMode=yes $(hostname) {shlex.quote(command)}" return command
@property def module_template_args(self): # json.dumps gives valid Python string literals for the modprobe file. return { "control_plane_command": json.dumps(self.launcher_command("launch-control-plane.sh")), "worker_command": json.dumps(self.launcher_command("launch-worker.sh")), } # Hooks used by Service.setup_module / the jobspec @property def module_start_template(self): template = START_CLUSTER_TEMPLATE if self.mode == "cluster" else START_WORKERS_TEMPLATE return utils.undefined_template(template).render(**self.module_template_args) @property def module_shutdown_template(self): return utils.undefined_template(STOP_MODULE_TEMPLATE).render(**self.module_template_args) def generate_batch_prolog(self): """ Lines placed before user jobs in the batch wrapper: gate the workload on the cluster (and every node of this job) being Ready. """ wait_script = os.path.join(self.script_dir, "wait-for-ready.sh") lines = [ "", "# usernetes: wait until the cluster and every node of this job are Ready", f'/bin/bash {shlex.quote(wait_script)} "{self.shared_dir}" {self.wait_timeout}', f'export USERNETES_SHARED_DIR="{self.shared_dir}"', ] if self.mode == "cluster": # Rank 0 of a cluster-mode batch job is the control plane node. lines.append('export KUBECONFIG="/tmp/$(whoami)/usernetes/kubeconfig"') lines.append("") return lines def setup(self, base_dir=None): """ Validate what we can at submit time, then write the modprobe scripts. """ root = self.usernetes_root marker = os.path.join(root, "service", "usernetes-start-control-plane.sh") if not os.path.exists(marker): print( f"WARNING: {marker} not found from the submission host. " "usernetes_root must be a usernetes checkout (hetchy branch) " "visible on every compute node; the launchers will fail fast " "if it is missing there." ) super().setup(base_dir=base_dir)