feat(tools): added support in asap-tools for running benchmarks experiments locally (instead of Cloudlab)

asap-tools/experiments/experiment_run_clickhouse.py

@@ -233,35 +233,41 @@ def main(cfg: DictConfig) -> None:
         cfg,
         required_params=[
             ("experiment.name", "Human-readable experiment name"),
-            ("cloudlab.num_nodes", "Number of CloudLab nodes to use"),
-            ("cloudlab.username", "Your CloudLab username"),
-            ("cloudlab.hostname_suffix", "CloudLab experiment hostname suffix"),
-        ],
+        ]
+        + config.required_cloudlab_params(cfg),
         script_name="experiment_run_clickhouse",
     )
     config.validate_experiment_config(cfg.experiment_params)
 
     provider = create_provider(cfg)
 
     experiment_name = cfg.experiment.name
-    node_offset = cfg.cloudlab.node_offset
+    num_nodes, node_offset = config.get_node_params(cfg)
     no_teardown = cfg.flow.no_teardown
     use_container = cfg.use_container.prometheus_client
     parallel = cfg.prometheus_client.parallel
 
-    local_experiment_root_dir = os.path.join(
-        constants.LOCAL_EXPERIMENT_DIR, experiment_name
-    )
+    if provider.is_remote():
+        local_experiment_root_dir = os.path.join(
+            constants.LOCAL_EXPERIMENT_DIR, experiment_name
+        )
+        experiment_root_output_dir = os.path.join(
+            provider.get_home_dir(), "experiment_outputs", experiment_name
+        )
+    else:
+        # Local mode: "remote" and "local" roots are the same filesystem path,
+        # so every rsync/scp call downstream becomes a no-op.
+        local_experiment_root_dir = os.path.join(
+            provider.get_home_dir(), "experiment_outputs", experiment_name
+        )
+        experiment_root_output_dir = local_experiment_root_dir
     os.makedirs(local_experiment_root_dir, exist_ok=True)
 
     with open(os.path.join(local_experiment_root_dir, "hydra_config.yaml"), "w") as f:
         OmegaConf.save(cfg, f)
     with open(os.path.join(local_experiment_root_dir, "cmdline_args.txt"), "w") as f:
         json.dump({"experiment_name": experiment_name, "node_offset": node_offset}, f)
 
-    experiment_root_output_dir = (
-        f"{constants.CLOUDLAB_HOME_DIR}/experiment_outputs/{experiment_name}"
-    )
     provider.execute_command(
         node_idx=node_offset,
         cmd=f"mkdir -p {experiment_root_output_dir}",
@@ -314,7 +320,7 @@ def main(cfg: DictConfig) -> None:
 
     # --- start ClickHouse (persists across all modes) ---
     clickhouse_service = ClickHouseService(
-        provider, num_nodes=cfg.cloudlab.num_nodes, node_offset=node_offset
+        provider, num_nodes=num_nodes, node_offset=node_offset
     )
     clickhouse_service.start(
         experiment_output_dir=experiment_root_output_dir,
@@ -326,7 +332,7 @@ def main(cfg: DictConfig) -> None:
     # --- load data once before the mode loop (DROP + reload) ---
     data_loader = ClickHouseDataLoaderService(
         provider,
-        num_nodes=cfg.cloudlab.num_nodes,
+        num_nodes=num_nodes,
         node_offset=node_offset,
         clickhouse_http_port=clickhouse_http_port,
     )

asap-tools/experiments/experiment_run_e2e.py

@@ -59,15 +59,35 @@ def main(cfg: DictConfig):
     config.validate_config(cfg)
     # Validate experiment configuration
     config.validate_experiment_config(cfg.experiment_params)
-    # Convert config to args-like object for backward compatibility
-    args = config.Args(cfg)
 
     # Create infrastructure provider
     provider = create_provider(cfg)
 
-    local_experiment_root_dir = os.path.join(
-        constants.LOCAL_EXPERIMENT_DIR, args.experiment_name
+    # Route the remote-write IP through the provider (127.0.0.1 for local mode,
+    # CloudLab's 10.10.1.x scheme otherwise) before anything resolves
+    # cfg.streaming.remote_write.ip (Args does, below).
+    OmegaConf.register_new_resolver(
+        "remote_write_ip", provider.get_node_ip, replace=True
     )
+
+    # Convert config to args-like object for backward compatibility
+    args = config.Args(cfg)
+
+    if provider.is_remote():
+        local_experiment_root_dir = os.path.join(
+            constants.LOCAL_EXPERIMENT_DIR, args.experiment_name
+        )
+        experiment_root_output_dir = os.path.join(
+            provider.get_home_dir(), "experiment_outputs", args.experiment_name
+        )
+    else:
+        # Local mode: the "remote" and "local" roots are the same filesystem
+        # path by construction, so every rsync/scp call downstream becomes a
+        # no-op rather than something to opportunistically skip.
+        local_experiment_root_dir = os.path.join(
+            provider.get_home_dir(), "experiment_outputs", args.experiment_name
+        )
+        experiment_root_output_dir = local_experiment_root_dir
     os.makedirs(local_experiment_root_dir, exist_ok=True)
 
     # dump config to a file
@@ -78,10 +98,6 @@ def main(cfg: DictConfig):
     with open(os.path.join(local_experiment_root_dir, "cmdline_args.txt"), "w") as f:
         json.dump(vars(args), f)
 
-    experiment_root_output_dir = (
-        f"{constants.CLOUDLAB_HOME_DIR}/experiment_outputs/{args.experiment_name}"
-    )
-
     global CONTROLLER_REMOTE_OUTPUT_DIR, CONTROLLER_LOCAL_OUTPUT_DIR
     CONTROLLER_LOCAL_OUTPUT_DIR = os.path.join(
         local_experiment_root_dir, "controller_output"
@@ -93,7 +109,7 @@ def main(cfg: DictConfig):
     provider.execute_command(
         node_idx=args.get_coordinator_node(),
         cmd="mkdir -p {} {}".format(
-            os.path.dirname(constants.CLOUDLAB_QUERY_LOG_FILE),
+            os.path.dirname(provider.get_query_log_file()),
             experiment_root_output_dir,
         ),
         cmd_dir="",

asap-tools/experiments/experiment_utils/config.py

@@ -55,6 +55,26 @@ def validate_basic_config(
         raise ValueError(error_msg)
 
 
+def get_node_params(cfg: DictConfig) -> Tuple[int, int]:
+    """Return (num_nodes, node_offset). Local mode is a single node and never
+    touches cfg.cloudlab (whose username/hostname_suffix are mandatory-missing
+    markers that raise on mere attribute access if not provided)."""
+    if hasattr(cfg, "local"):
+        return 0, 0
+    return cfg.cloudlab.num_nodes, cfg.cloudlab.node_offset
+
+
+def required_cloudlab_params(cfg: DictConfig) -> List[Tuple[str, str]]:
+    """Return cloudlab.* required-param tuples, or [] when running in local mode."""
+    if hasattr(cfg, "local"):
+        return []
+    return [
+        ("cloudlab.num_nodes", "Number of CloudLab nodes to use"),
+        ("cloudlab.username", "Your CloudLab username"),
+        ("cloudlab.hostname_suffix", "CloudLab experiment hostname suffix"),
+    ]
+
+
 def _is_clickhouse_experiment(experiment_params: DictConfig) -> bool:
     """Return True if experiment_params describes a ClickHouse (SQL) experiment."""
     return "dataset" in experiment_params
@@ -469,11 +489,14 @@ def __init__(self, cfg: DictConfig):
         # Experiment configuration
         self.experiment_name = cfg.experiment.name
 
-        # CloudLab configuration
-        self.num_nodes = cfg.cloudlab.num_nodes
-        self.node_offset = cfg.cloudlab.node_offset
-        self.cloudlab_username = cfg.cloudlab.username
-        self.hostname_suffix = cfg.cloudlab.hostname_suffix
+        # CloudLab configuration (or local-mode equivalents)
+        self.num_nodes, self.node_offset = get_node_params(cfg)
+        if hasattr(cfg, "local"):
+            self.cloudlab_username = None
+            self.hostname_suffix = None
+        else:
+            self.cloudlab_username = cfg.cloudlab.username
+            self.hostname_suffix = cfg.cloudlab.hostname_suffix
 
         # Logging and debugging
         self.log_level = cfg.logging.level
@@ -577,10 +600,7 @@ def validate_config(cfg: DictConfig, script_name: str = "experiment_run_e2e"):
     # Check for required parameters that must be provided via command line
     required_params = [
         ("experiment.name", "Human-readable experiment name"),
-        ("cloudlab.num_nodes", "Number of CloudLab nodes to use"),
-        ("cloudlab.username", "Your CloudLab username"),
-        ("cloudlab.hostname_suffix", "CloudLab experiment hostname suffix"),
-    ]
+    ] + required_cloudlab_params(cfg)
 
     # Use the existing validate_basic_config function
     validate_basic_config(cfg, required_params, script_name)

asap-tools/experiments/experiment_utils/providers/base.py

@@ -132,6 +132,19 @@ def get_provider_type(self) -> str:
         """
         return self.__class__.__name__.replace("Provider", "").lower()
 
+    def is_remote(self) -> bool:
+        """
+        Whether this provider executes on a separate machine reachable over SSH.
+
+        Local providers override this to False so that callers can skip
+        rsync/scp file-transfer steps entirely (source and destination are
+        the same filesystem).
+
+        Returns:
+            True by default (e.g. CloudLab); False for local execution.
+        """
+        return True
+
     def __str__(self) -> str:
         """String representation of the provider."""
         return f"{self.__class__.__name__}()"

asap-tools/experiments/experiment_utils/providers/cloudlab_local.py

@@ -154,6 +154,10 @@ def execute_command_parallel(
 
         return processes
 
+    def is_remote(self) -> bool:
+        """Commands run locally on this node (no SSH), even though paths/usernames are CloudLab-shaped."""
+        return False
+
     def get_node_address(self, node_idx: int) -> str:
         """
         Get the network address for the local node.

asap-tools/experiments/experiment_utils/providers/factory.py

@@ -9,6 +9,7 @@
 
 from .base import InfrastructureProvider
 from .cloudlab import CloudLabProvider
+from .local import LocalProvider
 
 
 def create_provider(cfg: DictConfig) -> InfrastructureProvider:
@@ -28,8 +29,13 @@ def create_provider(cfg: DictConfig) -> InfrastructureProvider:
         ValueError: If the configuration doesn't contain required parameters
                    or specifies an unsupported provider type
     """
-    # Phase 1: Always return CloudLab provider for backward compatibility
-    # Future phases will add detection logic for other providers
+    # Local mode (e.g. `+local.home_dir=...`) takes priority and must be checked
+    # before any `cfg.cloudlab.*` access — those fields are Hydra `???` mandatory-
+    # missing markers, so merely reading them raises if not provided via CLI.
+    if hasattr(cfg, "local"):
+        if not hasattr(cfg.local, "home_dir") or not cfg.local.home_dir:
+            raise ValueError("Missing 'local.home_dir' configuration parameter")
+        return LocalProvider(home_dir=cfg.local.home_dir)
 
     # Validate that we have the required CloudLab parameters
     if not hasattr(cfg, "cloudlab"):
@@ -62,11 +68,13 @@ def detect_provider_type(cfg: DictConfig) -> str:
     Returns:
         String identifier for the provider type ('cloudlab', 'aws', 'local', etc.)
     """
-    # Phase 1: Always detect as CloudLab
+    if hasattr(cfg, "local"):
+        return "local"
+
     # Future phases will add logic to detect other provider types
     # based on configuration parameters like:
     # - cfg.infrastructure.provider
-    # - presence of aws/kubernetes/local configuration sections
+    # - presence of aws/kubernetes configuration sections
     # - environment variables
 
     if (
@@ -83,10 +91,9 @@ def detect_provider_type(cfg: DictConfig) -> str:
     #     return "aws"
     # if hasattr(cfg, "kubernetes"):
     #     return "kubernetes"
-    # if cfg.get("local", False):
-    #     return "local"
 
     raise ValueError(
         "Unable to detect infrastructure provider type from configuration. "
-        "Currently supported: CloudLab (requires cloudlab.username and cloudlab.hostname_suffix)"
+        "Currently supported: CloudLab (requires cloudlab.username and cloudlab.hostname_suffix) "
+        "or Local (requires local.home_dir)"
     )