diff --git a/asap-tools/experiments/experiment_utils/providers/base.py b/asap-tools/experiments/experiment_utils/providers/base.py index 13a0f4e0..7f3f3e8f 100644 --- a/asap-tools/experiments/experiment_utils/providers/base.py +++ b/asap-tools/experiments/experiment_utils/providers/base.py @@ -56,6 +56,7 @@ def execute_command_parallel( popen: bool = True, redirect: bool = False, wait: bool = True, + ignore_errors: bool = False, ) -> List[subprocess.Popen]: """ Execute a command on multiple nodes in parallel. @@ -68,6 +69,7 @@ def execute_command_parallel( popen: Must be True for parallel execution redirect: Whether to redirect output to /dev/null wait: Whether to wait for all commands to complete + ignore_errors: Whether to ignore command execution errors Returns: List of Popen objects for each node diff --git a/asap-tools/experiments/experiment_utils/providers/cloudlab.py b/asap-tools/experiments/experiment_utils/providers/cloudlab.py index 3845c8ea..08a8a586 100644 --- a/asap-tools/experiments/experiment_utils/providers/cloudlab.py +++ b/asap-tools/experiments/experiment_utils/providers/cloudlab.py @@ -78,6 +78,7 @@ def execute_command_parallel( popen: bool = True, redirect: bool = False, wait: bool = True, + ignore_errors: bool = False, ) -> List[subprocess.Popen]: """ Execute a command on multiple CloudLab nodes in parallel via SSH. diff --git a/asap-tools/experiments/experiment_utils/providers/cloudlab_local.py b/asap-tools/experiments/experiment_utils/providers/cloudlab_local.py index 2e7ed0e5..f775c5b0 100644 --- a/asap-tools/experiments/experiment_utils/providers/cloudlab_local.py +++ b/asap-tools/experiments/experiment_utils/providers/cloudlab_local.py @@ -81,14 +81,20 @@ def execute_command( # Execute locally if popen: - return subprocess.Popen( - cmd, - shell=True, - cwd=cmd_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) + try: + return subprocess.Popen( + cmd, + shell=True, + cwd=cmd_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except OSError as e: + if ignore_errors: + print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}") + return None + raise else: try: result = subprocess.run( @@ -100,7 +106,7 @@ def execute_command( check=not ignore_errors, ) return result - except subprocess.CalledProcessError as e: + except (subprocess.CalledProcessError, OSError) as e: if ignore_errors: return e raise @@ -114,6 +120,7 @@ def execute_command_parallel( popen: bool = True, redirect: bool = False, wait: bool = True, + ignore_errors: bool = False, ) -> List[subprocess.Popen]: """ Execute a command in parallel locally. @@ -129,6 +136,7 @@ def execute_command_parallel( popen: Must be True for parallel execution redirect: Whether to redirect output to /dev/null wait: Whether to wait for all commands to complete + ignore_errors: Whether to ignore command execution errors Returns: List containing single Popen object @@ -144,6 +152,7 @@ def execute_command_parallel( cmd_dir=cmd_dir, nohup=nohup, popen=True, # Always return Popen for parallel + ignore_errors=ignore_errors, ) processes = [process] diff --git a/asap-tools/experiments/experiment_utils/providers/local.py b/asap-tools/experiments/experiment_utils/providers/local.py index 15bfe2b9..af19aaeb 100644 --- a/asap-tools/experiments/experiment_utils/providers/local.py +++ b/asap-tools/experiments/experiment_utils/providers/local.py @@ -64,13 +64,10 @@ def execute_command( text=True, ) except OSError as e: - # execute_command_parallel has no ignore_errors knob (see - # follow-up issue), so this can't be conditional yet. Mirrors - # CloudLab's SSH transport: launching `ssh` locally never - # fails even if the remote cmd_dir is missing — the failure - # is invisible unless something later checks output/health. - print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}") - return None + if ignore_errors: + print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}") + return None + raise try: return subprocess.run( @@ -101,6 +98,7 @@ def execute_command_parallel( popen: bool = True, redirect: bool = False, wait: bool = True, + ignore_errors: bool = False, ) -> List[subprocess.Popen]: """Execute a command once locally (there is only one local node).""" if redirect: @@ -112,6 +110,7 @@ def execute_command_parallel( cmd_dir=cmd_dir, nohup=nohup, popen=True, + ignore_errors=ignore_errors, ) processes = [process] if wait: diff --git a/asap-tools/experiments/experiment_utils/services/misc.py b/asap-tools/experiments/experiment_utils/services/misc.py index bae507a8..790e2862 100644 --- a/asap-tools/experiments/experiment_utils/services/misc.py +++ b/asap-tools/experiments/experiment_utils/services/misc.py @@ -79,6 +79,7 @@ def stop(self, **kwargs) -> None: nohup=False, popen=True, wait=True, + ignore_errors=True, ) def run_workload(