"""Run a FireSim workload on the F2 FPGAs of a farm, one job per FPGA at a time.
:meth:`FireSimManagerNode.run_workload` runs in the calling process: it sends
each job to an FPGA as a :meth:`FireSimManagerNode.run_job` task, and takes down
the FPGAs with no job left through their farm. ``run_job`` runs in the FireSim
container on an F2 instance (``chia.firesim.specs.F2_SIM``). The manager's run
farm is the instance itself (FireSim's ``ExternallyProvisioned`` mode), and
``_MANAGER`` runs FireSim's commands for it on the instance, with no ssh.
"""
from __future__ import annotations
import logging
import os
import socket
import subprocess
import time
from dataclasses import replace
from urllib.request import Request, urlopen
import ray
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
from chia.aws.manager import Farm
from chia.base.ChiaFunction import ChiaFunction, get
from chia.chipyard.state_def import FireMarshalArtifact
from chia.firesim.fs_bitstream import FSBitstream
from chia.firesim.render import RUN_FARM_USER, render_hwdb, render_runtime_config, stage_workload
from chia.firesim.specs import FPGA_RESOURCE
from chia.firesim.state_def import RunConfig, SimJobResult
from chia.trace.live_status_monitor import report, watch
FIRESIM_DIR = "/home/ray/firesim"
DEPLOY_DIR = f"{FIRESIM_DIR}/deploy"
# `firesim` exits unless sourceme-manager.sh has run: it sets FIRESIM_SOURCED,
# which check_env() requires.
_RUN = r"""set -e
cd "$1"
task="$2"
source sourceme-manager.sh --skip-ssh-setup
cd deploy
python - "$task"
"""
# FireSim's manager with its run farm host commands run on this instance, not
# over ssh: run() runs on the instance through nsenter, as $USER in a login
# shell in its home, as over ssh; put/get/rsync_project copy in the instance's
# home, which the container mounts at the same path.
_MANAGER = r"""
import os, runpy, shlex, sys
import fabric.api, fabric.contrib.project
from fabric.api import abort, env, local, settings
from fabric.operations import _prefix_commands, _prefix_env_vars
from fabric.state import output
def run_on_host(command, **kw):
command = _prefix_env_vars(_prefix_commands(command, "remote"))
with settings(command_prefixes=[], warn_only=True):
out = local(f"sudo nsenter -t 1 -a -- sudo -H -u {os.environ['USER']} /bin/bash -l -c "
+ shlex.quote(f"cd && {command}"), shell="/bin/bash", capture=True)
if output.stdout and out:
print(out, flush=True)
if output.stderr and out.stderr:
print(out.stderr, file=sys.stderr, flush=True)
if out.failed and not env.warn_only:
abort(f"run() failed (rc={out.return_code}): {command}")
return out
def put_in_home(local_path, remote_path, **kw):
return local(f"cp -r {local_path} {remote_path}")
def get_from_home(remote_path, local_path, **kw):
return local(f"cp -r {remote_path} {local_path}")
def rsync_in_home(remote_dir, local_dir, exclude=(), delete=False, extra_opts="",
capture=False, upload=True, default_opts="-pthrvz", **kw):
exclude = [exclude] if isinstance(exclude, str) else exclude
options = ("--delete" if delete else "") + "".join(f' --exclude "{e}"' for e in exclude)
src, dst = (local_dir, remote_dir) if upload else (remote_dir, local_dir)
return local(f"rsync {options} {default_opts} {extra_opts} {src} {dst}", capture=capture)
fabric.api.run = run_on_host
fabric.api.put = put_in_home
fabric.api.get = get_from_home
fabric.contrib.project.rsync_project = rsync_in_home
sys.argv = [os.path.abspath("firesim"), sys.argv[1]]
runpy.run_path(sys.argv[0], run_name="__main__")
"""
[docs]
class FireSimManagerNode:
"""Runs ``firesim infrasetup`` and ``firesim runworkload`` on the local FPGA."""
def __init__(self, timeout_seconds: int | None = None):
"""
Args:
timeout_seconds: Wall-clock limit per manager step; None for no limit.
"""
self.timeout_seconds = timeout_seconds
self.logger = logging.getLogger("FireSimManagerNode")
watch("FPGA") # the process that creates the node prints the FPGA table
[docs]
def run_workload(self, farm: Farm, workload: FireMarshalArtifact,
bitstream: FSBitstream, config: RunConfig | None = None,
teardown: bool = True) -> list[SimJobResult]:
"""Run each job of ``workload`` on an FPGA of ``farm``, one job at a time per
FPGA, and return the results in the order they finish. Runs in the calling
process.
With ``teardown`` and a farm that has its manager, an FPGA with no job left
goes down at once, and at the end so does the whole farm.
"""
release = teardown and farm.manager is not None
pending, results = workload.split(), []
try:
running = [self._start(pending.pop(0), bitstream, config)
for _ in farm.ips[:len(pending)]]
while running:
[ref], running = ray.wait(running)
results.append(result := get(ref))
if pending: # the next job goes to the FPGA that just finished
running.append(self._start(pending.pop(0), bitstream, config,
result.node_id))
elif release:
replace(farm, ips=[result.ip]).teardown()
finally:
if release:
farm.teardown()
return results
def _start(self, job: FireMarshalArtifact, bitstream: FSBitstream,
config: RunConfig | None, node_id: str | None = None):
"""Start ``run_job`` as a task; with ``node_id``, on that Ray node."""
strategy = NodeAffinitySchedulingStrategy(node_id, soft=False) if node_id else None
return self.run_job.options(scheduling_strategy=strategy).chia_remote(
self, job=job, bitstream=bitstream, config=config)
[docs]
@ChiaFunction(resources={FPGA_RESOURCE: 1})
def run_job(self, job: FireMarshalArtifact, bitstream: FSBitstream,
config: RunConfig | None = None) -> SimJobResult:
"""Run the single-job workload ``job`` with ``bitstream`` on this worker's
FPGA, and collect its results.
``config`` holds the FireSim runtime settings (``config_runtime.yaml``).
Infrasetup and runworkload are one task so both use the same FPGA. Each
phase is reported on the ``FPGA`` live status monitor.
"""
host = socket.gethostname()
report("FPGA", host, job=os.path.splitext(job.json_name)[0], state="staging")
try:
result = self._run(host, job, bitstream, config)
except Exception:
report("FPGA", host, state="failed")
raise
report("FPGA", host, state="done" if result.success else "failed")
return replace(result, node_id=ray.get_runtime_context().get_node_id(), ip=_public_ip())
def _run(self, host: str, workload: FireMarshalArtifact, bitstream: FSBitstream,
config: RunConfig | None) -> SimJobResult:
t0 = time.monotonic()
name = stage_workload(DEPLOY_DIR, workload)
render_runtime_config(DEPLOY_DIR, name, config)
render_hwdb(DEPLOY_DIR, bitstream)
log = ""
for task in ("infrasetup", "runworkload"):
report("FPGA", host, job=name, state=task)
self.logger.info(f"firesim {task} for {name}")
rc, out = self._firesim(task)
log += f"=== {task} (rc={rc}) ===\n{out[-4000:]}\n"
if rc != 0:
report("FPGA", host, detail=f"{task} rc={rc}")
return SimJobResult(name, success=False, log=log,
duration_seconds=time.monotonic() - t0)
outputs = self._collect(name)
uartlog = next((v.decode(errors="replace") for k, v in outputs.items()
if os.path.basename(k) == "uartlog"), "")
return SimJobResult(name, success=True, uartlog=uartlog, outputs=outputs,
duration_seconds=time.monotonic() - t0, log=log)
def _firesim(self, task: str) -> tuple[int, str]:
"""Run one manager task; rc=-1 on timeout."""
# FireSim names the run farm host's paths after $USER (/home/$USER/aws-fpga),
# as if the manager ran as that host's user.
env = {**os.environ, "USER": RUN_FARM_USER}
try:
p = subprocess.run(["bash", "-c", _RUN, "_", FIRESIM_DIR, task], env=env,
input=_MANAGER, capture_output=True, text=True,
timeout=self.timeout_seconds)
return p.returncode, p.stdout + p.stderr
except subprocess.TimeoutExpired:
return -1, f"timeout after {self.timeout_seconds}s"
@staticmethod
def _collect(name: str) -> dict[str, bytes]:
"""Files of the newest ``results-workload`` run of ``name``."""
root = os.path.join(DEPLOY_DIR, "results-workload")
runs = sorted(d for d in os.listdir(root) if d.endswith(name))
if not runs:
return {}
run_dir = os.path.join(root, runs[-1])
outputs = {}
for dirpath, _, files in os.walk(run_dir):
for f in files:
path = os.path.join(dirpath, f)
with open(path, "rb") as fh:
outputs[os.path.relpath(path, run_dir)] = fh.read()
return outputs
def _public_ip() -> str:
"""This machine's public IP, from EC2's instance metadata (IMDSv2)."""
imds = "http://169.254.169.254/latest"
token = urlopen(Request(f"{imds}/api/token", method="PUT", headers={
"X-aws-ec2-metadata-token-ttl-seconds": "60"})).read().decode()
return urlopen(Request(f"{imds}/meta-data/public-ipv4", headers={
"X-aws-ec2-metadata-token": token})).read().decode()