diff --git a/asap-tools/experiments/experiment_only_ingest_path.py b/asap-tools/experiments/experiment_only_ingest_path.py index 1026d82..dc3cbf6 100644 --- a/asap-tools/experiments/experiment_only_ingest_path.py +++ b/asap-tools/experiments/experiment_only_ingest_path.py @@ -227,7 +227,9 @@ def main(cfg: DictConfig): node_offset=args.node_offset, ) - prometheus_scrape_interval = config.get_prometheus_scrape_interval(cfg.prometheus) + prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( + cfg.prometheus + ) # Start fake exporter if configured if config.check_exporter_and_queries_exist("fake_exporter", cfg.experiment_params): @@ -288,16 +290,16 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() - label_discovery_wait = prometheus_scrape_interval * 2 + label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 print( - f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data " + f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." ) - time.sleep(label_discovery_wait) + time.sleep(label_discovery_wait_ms / 1000) controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 4a098db..3b6bb70 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -324,7 +324,7 @@ def main(cfg: DictConfig): prometheus_config_output_dir, node_offset=args.node_offset, ) - prometheus_scrape_interval = config.get_prometheus_scrape_interval( + prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( cfg.prometheus ) @@ -352,12 +352,12 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() # Wait for two scrape intervals so Prometheus has series to return. - label_discovery_wait = prometheus_scrape_interval * 2 + label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 print( - f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data " + f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." ) - time.sleep(label_discovery_wait) + time.sleep(label_discovery_wait_ms / 1000) prometheus_url = ( f"http://localhost:{prometheus_service.get_query_endpoint_port()}" @@ -372,7 +372,7 @@ def main(cfg: DictConfig): url=prometheus_url, database=None, ), - prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, ) sync.rsync_controller_config_remote_to_local( provider, @@ -516,7 +516,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_run_grafana_demo.py b/asap-tools/experiments/experiment_run_grafana_demo.py index c3bb106..6eff1a2 100644 --- a/asap-tools/experiments/experiment_run_grafana_demo.py +++ b/asap-tools/experiments/experiment_run_grafana_demo.py @@ -273,7 +273,9 @@ def main(cfg: DictConfig): prometheus_config_output_dir, node_offset=args.node_offset, ) - prometheus_scrape_interval = config.get_prometheus_scrape_interval(cfg.prometheus) + prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( + cfg.prometheus + ) if config.check_exporter_and_queries_exist("fake_exporter", cfg.experiment_params): # this DOES NOT block @@ -341,16 +343,16 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() - label_discovery_wait = prometheus_scrape_interval * 2 + label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 print( - f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data " + f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." ) - time.sleep(label_discovery_wait) + time.sleep(label_discovery_wait_ms / 1000) controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, @@ -467,7 +469,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index 2c447e0..0797a01 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -478,20 +478,18 @@ def read_workloads_config(experiment_params: DictConfig): return workloads_config -def get_prometheus_scrape_interval(prometheus_config): - """Extract scrape interval from Prometheus configuration.""" - prometheus_scrape_interval_string = prometheus_config.scrape_interval - # convert to seconds - if prometheus_scrape_interval_string.endswith("s"): - prometheus_scrape_interval = int(prometheus_scrape_interval_string[:-1]) - elif prometheus_scrape_interval_string.endswith("m"): - prometheus_scrape_interval = int(prometheus_scrape_interval_string[:-1]) * 60 +def get_prometheus_scrape_interval_ms(prometheus_config): + """Extract scrape interval from Prometheus configuration, returned in milliseconds.""" + s = prometheus_config.scrape_interval + # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute + if s.endswith("ms"): + return int(s[:-2]) + elif s.endswith("s"): + return int(s[:-1]) * 1000 + elif s.endswith("m"): + return int(s[:-1]) * 60 * 1000 else: - raise ValueError( - f"Invalid scrape interval string: {prometheus_scrape_interval_string}" - ) - - return prometheus_scrape_interval + raise ValueError(f"Invalid scrape interval string: {s}") class Args: