diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a0cf5ddf..54c84d01 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -203,6 +203,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -e asap-common/dependencies/py/promql_utilities/ - pip install pytest pyyaml jinja2 + pip install pytest pyyaml jinja2 loguru requests - name: Run pytest run: python -m pytest asap-summary-ingest/tests/ -v diff --git a/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py index 81b710dd..d4374905 100644 --- a/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py +++ b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py @@ -35,7 +35,6 @@ def get_statistics_to_compute( # template_config.tumblingWindowSize = self.t_repeat elif query_pattern_type == QueryPatternType.ONLY_SPATIAL: statistic_to_compute = query_pattern_match.tokens["aggregation"]["op"] - # template_config.tumblingWindowSize = self.prometheus_scrape_interval else: raise ValueError("Invalid query pattern type") @@ -55,7 +54,7 @@ def get_spatial_aggregation_output_labels( # Fixing issue https://github.com/ProjectASAP/asap-internal/issues/24 if aggregation_modifier is None: return KeyByLabelNames([]) - + if aggregation_modifier.type == aggregation_modifier.type.By: aggregation_modifier_labels = KeyByLabelNames(aggregation_modifier.labels) elif aggregation_modifier.type == aggregation_modifier.type.Without: diff --git a/asap-planner-rs/docker-compose.yml.j2 b/asap-planner-rs/docker-compose.yml.j2 index 9978b670..301b5c0a 100644 --- a/asap-planner-rs/docker-compose.yml.j2 +++ b/asap-planner-rs/docker-compose.yml.j2 @@ -8,12 +8,11 @@ services: command: [ "--input_config", "/app/input/config.yaml", "--output_dir", "/app/output", - "--prometheus_scrape_interval_ms", "{{ prometheus_scrape_interval_ms }}", + "--data-ingestion-interval-ms", "{{ data_ingestion_interval_ms }}", "--streaming_engine", "{{ streaming_engine }}", "--prometheus-url", "{{ prometheus_url }}", "--query-language", "{{ query_language }}"{% if punting %}, - "--enable-punting"{% endif %}{% if data_ingestion_interval_ms is not none %}, - "--data-ingestion-interval-ms", "{{ data_ingestion_interval_ms }}"{% endif %} + "--enable-punting"{% endif %} ] network_mode: "host" restart: no diff --git a/asap-planner-rs/src/bin/optimizer_cli.rs b/asap-planner-rs/src/bin/optimizer_cli.rs index 70d2352f..2ce62d41 100644 --- a/asap-planner-rs/src/bin/optimizer_cli.rs +++ b/asap-planner-rs/src/bin/optimizer_cli.rs @@ -20,8 +20,8 @@ struct Args { #[arg(long = "input_config")] input_config: PathBuf, - #[arg(long = "prometheus_scrape_interval_ms")] - prometheus_scrape_interval_ms: u64, + #[arg(long = "data-ingestion-interval-ms")] + data_ingestion_interval_ms: u64, /// Placeholder arrival rate (items/sec) applied uniformly to every candidate's /// IngestCost. Real per-config rates aren't wired up yet — see the open TODOs @@ -56,12 +56,8 @@ fn main() -> anyhow::Result<()> { let config: ControllerConfig = serde_yaml::from_str(&yaml_str)?; let schema = config.schema_from_hints(); - let (streaming, inference) = run_greedy_pipeline( - &config, - &schema, - args.prometheus_scrape_interval_ms, - args.rho, - ); + let (streaming, inference) = + run_greedy_pipeline(&config, &schema, args.data_ingestion_interval_ms, args.rho); let deployed = streaming.get_all_aggregation_configs(); println!("=== Deployed streaming configs: {} ===", deployed.len()); diff --git a/asap-planner-rs/src/lib.rs b/asap-planner-rs/src/lib.rs index 82fc8c13..9f83c515 100644 --- a/asap-planner-rs/src/lib.rs +++ b/asap-planner-rs/src/lib.rs @@ -35,7 +35,7 @@ pub enum StreamingEngine { #[derive(Debug, Clone)] pub struct RuntimeOptions { - pub prometheus_scrape_interval_ms: u64, + pub data_ingestion_interval_ms: u64, pub streaming_engine: StreamingEngine, pub enable_punting: bool, pub range_duration_ms: u64, diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index f9c325eb..bf99c419 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -20,9 +20,6 @@ struct Args { #[arg(long = "output_dir")] output_dir: PathBuf, - #[arg(long = "prometheus_scrape_interval_ms", required = false)] - prometheus_scrape_interval_ms: Option, - /// Base URL of the Prometheus instance used to auto-infer metric label sets. /// Optional: when provided, the planner queries Prometheus for label discovery. /// When absent, labels are taken from the `metrics` hint in the config file. @@ -86,11 +83,11 @@ fn main() -> anyhow::Result<()> { match args.query_language { QueryLanguage::promql => { - let scrape_interval_ms = args.prometheus_scrape_interval_ms.ok_or_else(|| { - anyhow::anyhow!("--prometheus_scrape_interval_ms is required for PromQL mode") + let scrape_interval_ms = args.data_ingestion_interval_ms.ok_or_else(|| { + anyhow::anyhow!("--data-ingestion-interval-ms is required for PromQL mode") })?; let opts = RuntimeOptions { - prometheus_scrape_interval_ms: scrape_interval_ms, + data_ingestion_interval_ms: scrape_interval_ms, streaming_engine: engine, enable_punting: args.enable_punting, range_duration_ms: args.range_duration_ms, diff --git a/asap-planner-rs/src/planner/promql.rs b/asap-planner-rs/src/planner/promql.rs index aa9470c8..5127968a 100644 --- a/asap-planner-rs/src/planner/promql.rs +++ b/asap-planner-rs/src/planner/promql.rs @@ -72,7 +72,7 @@ pub(crate) fn parse_binary_arms(query: &str) -> Option<(BinaryArm, BinaryArm)> { pub struct SingleQueryProcessor { query: String, t_repeat_ms: u64, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, metric_schema: PromQLSchema, #[allow(dead_code)] streaming_engine: StreamingEngine, @@ -87,7 +87,7 @@ impl SingleQueryProcessor { pub fn new( query: String, t_repeat_ms: u64, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, metric_schema: PromQLSchema, streaming_engine: StreamingEngine, sketch_parameters: Option, @@ -98,7 +98,7 @@ impl SingleQueryProcessor { Self { query, t_repeat_ms, - prometheus_scrape_interval_ms, + data_ingestion_interval_ms, metric_schema, streaming_engine, sketch_parameters, @@ -159,7 +159,7 @@ impl SingleQueryProcessor { SingleQueryProcessor::new( arm_query, self.t_repeat_ms, - self.prometheus_scrape_interval_ms, + self.data_ingestion_interval_ms, self.metric_schema.clone(), self.streaming_engine, self.sketch_parameters.clone(), @@ -199,7 +199,7 @@ impl SingleQueryProcessor { | PromQLFunction::QuantileOverTime) ) { let num_data_points = - self.t_repeat_ms as f64 / self.prometheus_scrape_interval_ms as f64; + self.t_repeat_ms as f64 / self.data_ingestion_interval_ms as f64; if num_data_points < 60.0 { return false; } @@ -248,7 +248,7 @@ impl SingleQueryProcessor { set_window_parameters( pattern_type, self.t_repeat_ms, - self.prometheus_scrape_interval_ms, + self.data_ingestion_interval_ms, "any", // aggregation_type doesn't matter (sliding always false) self.step_ms, &mut window_cfg, diff --git a/asap-planner-rs/src/planner/window.rs b/asap-planner-rs/src/planner/window.rs index 2d1b6a0e..b89a599c 100644 --- a/asap-planner-rs/src/planner/window.rs +++ b/asap-planner-rs/src/planner/window.rs @@ -20,7 +20,7 @@ pub fn should_use_sliding_window( pub fn set_window_parameters( query_pattern_type: QueryPatternType, t_repeat_ms: u64, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, aggregation_type: &str, step_ms: u64, config: &mut IntermediateWindowConfig, @@ -31,7 +31,7 @@ pub fn set_window_parameters( set_tumbling_window_parameters( query_pattern_type, effective_repeat, - prometheus_scrape_interval_ms, + data_ingestion_interval_ms, config, ); } @@ -39,7 +39,7 @@ pub fn set_window_parameters( fn set_tumbling_window_parameters( query_pattern_type: QueryPatternType, effective_repeat: u64, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, config: &mut IntermediateWindowConfig, ) { match query_pattern_type { @@ -49,8 +49,8 @@ fn set_tumbling_window_parameters( config.window_type = WindowType::Tumbling; } QueryPatternType::OnlySpatial => { - config.window_size_ms = prometheus_scrape_interval_ms; - config.slide_interval_ms = prometheus_scrape_interval_ms; + config.window_size_ms = data_ingestion_interval_ms; + config.slide_interval_ms = data_ingestion_interval_ms; config.window_type = WindowType::Tumbling; } } diff --git a/asap-planner-rs/src/promql/controller.rs b/asap-planner-rs/src/promql/controller.rs index 30dea006..6e85f7f6 100644 --- a/asap-planner-rs/src/promql/controller.rs +++ b/asap-planner-rs/src/promql/controller.rs @@ -114,7 +114,7 @@ impl Controller { ) -> Result { let entries = query_log::parse_log_file(log_path)?; let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval_ms); + query_log::infer_queries(&entries, opts.data_ingestion_interval_ms); let config = query_log::to_controller_config(instants, ranges); let all_queries: Vec = config .query_groups @@ -139,7 +139,7 @@ impl Controller { ) -> Result { let entries = query_log::parse_log_file(log_path)?; let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval_ms); + query_log::infer_queries(&entries, opts.data_ingestion_interval_ms); let config = query_log::to_controller_config(instants, ranges); Ok(Self { config, diff --git a/asap-planner-rs/src/promql/generator.rs b/asap-planner-rs/src/promql/generator.rs index 0f450a66..b211f665 100644 --- a/asap-planner-rs/src/promql/generator.rs +++ b/asap-planner-rs/src/promql/generator.rs @@ -60,7 +60,7 @@ pub fn generate_plan( let processor = SingleQueryProcessor::new( query_string.clone(), qg.repetition_delay_ms, - opts.prometheus_scrape_interval_ms, + opts.data_ingestion_interval_ms, metric_schema.clone(), opts.streaming_engine, controller_config.sketch_parameters.clone(), diff --git a/asap-planner-rs/tests/integration.rs b/asap-planner-rs/tests/integration.rs index 8501580d..18b0f098 100644 --- a/asap-planner-rs/tests/integration.rs +++ b/asap-planner-rs/tests/integration.rs @@ -6,7 +6,7 @@ use std::path::Path; fn arroyo_opts() -> RuntimeOptions { RuntimeOptions { - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, streaming_engine: StreamingEngine::Arroyo, enable_punting: false, range_duration_ms: 0, @@ -598,7 +598,7 @@ fn sub_second_scrape_interval_window_size_equals_scrape_interval() { // OnlySpatial query: window size = scrape interval (planner/window.rs). // 100ms would be indistinguishable from 0 under the old seconds-only model. let opts = RuntimeOptions { - prometheus_scrape_interval_ms: 100, + data_ingestion_interval_ms: 100, ..arroyo_opts() }; let c = Controller::from_file_with_schema( @@ -617,7 +617,7 @@ fn sub_second_scrape_interval_round_trips_through_generated_yaml() { // under the renamed wire key (windowSizeMs, not windowSize) — not silently // rounded or truncated. let opts = RuntimeOptions { - prometheus_scrape_interval_ms: 100, + data_ingestion_interval_ms: 100, ..arroyo_opts() }; let c = Controller::from_file_with_schema( diff --git a/asap-query-engine/examples/engine_config.yaml b/asap-query-engine/examples/engine_config.yaml index 8dc31486..ec934c25 100644 --- a/asap-query-engine/examples/engine_config.yaml +++ b/asap-query-engine/examples/engine_config.yaml @@ -12,8 +12,8 @@ output_dir: "./output" log_level: "INFO" # DEBUG | INFO | WARN | ERROR (also respects RUST_LOG) -# Prometheus scrape interval in milliseconds. Used by the query tracker and planner. -prometheus_scrape_interval_ms: 15000 +# Data ingestion interval in milliseconds. Used by the query tracker and planner. +data_ingestion_interval_ms: 15000 # Processing backend. Must be consistent with the ingest source chosen below: # arroyo → requires ingest.type=kafka diff --git a/asap-query-engine/src/engine_config.rs b/asap-query-engine/src/engine_config.rs index 8385ab58..8ef1ef65 100644 --- a/asap-query-engine/src/engine_config.rs +++ b/asap-query-engine/src/engine_config.rs @@ -31,8 +31,8 @@ pub fn check_config(config: &EngineConfig) -> Result<(), String> { return Err("ingest.ts_step_ms is required when ingest.timestamp_col is not set".into()); } - if config.prometheus_scrape_interval_ms == 0 { - return Err("prometheus_scrape_interval_ms must be greater than 0".into()); + if config.data_ingestion_interval_ms == 0 { + return Err("data_ingestion_interval_ms must be greater than 0".into()); } if config.query_tracker.enabled && !matches!(config.backend, BackendConfig::Prometheus { .. }) { @@ -47,7 +47,7 @@ pub fn check_config(config: &EngineConfig) -> Result<(), String> { pub struct EngineConfig { pub output_dir: String, pub log_level: String, - pub prometheus_scrape_interval_ms: u64, + pub data_ingestion_interval_ms: u64, pub streaming_engine: StreamingEngine, pub http_server: HttpServerSettings, pub backend: BackendConfig, @@ -65,7 +65,7 @@ impl Default for EngineConfig { Self { output_dir: "./output".to_string(), log_level: "INFO".to_string(), - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, streaming_engine: StreamingEngine::Precompute, http_server: HttpServerSettings::default(), backend: BackendConfig::default(), @@ -522,7 +522,7 @@ streaming_engine: "precompute" ingest: type: "http_remote_write" port: 9090 -prometheus_scrape_interval_ms: 0 +data_ingestion_interval_ms: 0 output_dir: "./output" "#; let config: EngineConfig = Figment::new().merge(Yaml::string(yaml)).extract().unwrap(); diff --git a/asap-query-engine/src/engines/simple_engine/mod.rs b/asap-query-engine/src/engines/simple_engine/mod.rs index d734d0e4..4fe429a8 100644 --- a/asap-query-engine/src/engines/simple_engine/mod.rs +++ b/asap-query-engine/src/engines/simple_engine/mod.rs @@ -136,7 +136,7 @@ pub struct SimpleEngine { /// Updated at runtime via update_streaming_config(). Readers briefly lock to /// clone the Arc pointer, then use without holding the lock. streaming_config: RwLock>, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, controller_patterns: HashMap>, query_language: QueryLanguage, } @@ -147,7 +147,7 @@ impl SimpleEngine { // promsketch_store: Option>, inference_config: InferenceConfig, streaming_config: Arc, - prometheus_scrape_interval_ms: u64, + data_ingestion_interval_ms: u64, query_language: QueryLanguage, ) -> Self { // Create temporal pattern blocks @@ -287,7 +287,7 @@ impl SimpleEngine { // promsketch_store, inference_config: RwLock::new(inference_config), streaming_config: RwLock::new(streaming_config), - prometheus_scrape_interval_ms, + data_ingestion_interval_ms, controller_patterns, query_language, } @@ -332,13 +332,13 @@ impl SimpleEngine { mut end_timestamp: u64, query_pattern_type: QueryPatternType, ) -> u64 { - let interval_ms = self.prometheus_scrape_interval_ms; + let interval_ms = self.data_ingestion_interval_ms; if !end_timestamp.is_multiple_of(interval_ms) { warn!( - "Query end timestamp {} is not aligned with Prometheus scrape interval of {} ms. \ + "Query end timestamp {} is not aligned with data ingestion interval of {} ms. \ This may lead to inaccurate results.", - end_timestamp, self.prometheus_scrape_interval_ms + end_timestamp, self.data_ingestion_interval_ms ); } @@ -348,8 +348,8 @@ impl SimpleEngine { { let aligned_end_timestamp = (end_timestamp / interval_ms) * interval_ms; debug!( - "OnlySpatial query: Aligning end_timestamp from {} to {} using scrape interval of {} ms", - end_timestamp, aligned_end_timestamp, self.prometheus_scrape_interval_ms + "OnlySpatial query: Aligning end_timestamp from {} to {} using data ingestion interval of {} ms", + end_timestamp, aligned_end_timestamp, self.data_ingestion_interval_ms ); end_timestamp = aligned_end_timestamp; } diff --git a/asap-query-engine/src/engines/simple_engine/promql.rs b/asap-query-engine/src/engines/simple_engine/promql.rs index 88a2cf2c..e8502d82 100644 --- a/asap-query-engine/src/engines/simple_engine/promql.rs +++ b/asap-query-engine/src/engines/simple_engine/promql.rs @@ -43,7 +43,7 @@ impl SimpleEngine { .num_milliseconds() as u64; end_timestamp - range_ms } - QueryPatternType::OnlySpatial => end_timestamp - self.prometheus_scrape_interval_ms, + QueryPatternType::OnlySpatial => end_timestamp - self.data_ingestion_interval_ms, } } diff --git a/asap-query-engine/src/engines/simple_engine/sql.rs b/asap-query-engine/src/engines/simple_engine/sql.rs index eb687c90..fc6b4883 100644 --- a/asap-query-engine/src/engines/simple_engine/sql.rs +++ b/asap-query-engine/src/engines/simple_engine/sql.rs @@ -197,7 +197,7 @@ impl SimpleEngine { .get_duration() as u64; end_timestamp - (duration_secs * 1000) } - QueryPatternType::OnlySpatial => end_timestamp - self.prometheus_scrape_interval_ms, + QueryPatternType::OnlySpatial => end_timestamp - self.data_ingestion_interval_ms, } } @@ -398,10 +398,10 @@ impl SimpleEngine { } }; - // SQLPatternMatcher (sql_utilities, out of scope for the ms rename) divides a + // SQLPatternMatcher (sql_utilities, out of scope for this rename) divides a // seconds-denominated SQL query duration by this value — convert back to seconds. let matcher = - SQLPatternMatcher::new(schema, self.prometheus_scrape_interval_ms as f64 / 1000.0); + SQLPatternMatcher::new(schema, self.data_ingestion_interval_ms as f64 / 1000.0); let match_result = matcher.query_info_to_pattern(&query_data); debug!("Match result: {:?}", match_result); diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index b808c6d7..f00a1135 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -117,7 +117,7 @@ async fn main() -> Result<()> { store.clone(), inference_config, streaming_config.clone(), - config.prometheus_scrape_interval_ms, + config.data_ingestion_interval_ms, query_language, )); @@ -395,14 +395,14 @@ async fn main() -> Result<()> { let tracker_config = QueryTrackerConfig { observation_window_secs: config.query_tracker.observation_window_secs, - prometheus_scrape_interval_ms: config.prometheus_scrape_interval_ms, + data_ingestion_interval_ms: config.data_ingestion_interval_ms, }; let runtime_options = asap_planner::RuntimeOptions { - prometheus_scrape_interval_ms: config.prometheus_scrape_interval_ms, + data_ingestion_interval_ms: config.data_ingestion_interval_ms, streaming_engine: asap_planner::StreamingEngine::Precompute, enable_punting: false, range_duration_ms: 300_000, - step_ms: config.prometheus_scrape_interval_ms, + step_ms: config.data_ingestion_interval_ms, }; let prometheus_url = match &config.backend { BackendConfig::Prometheus { server, .. } => server.clone(), diff --git a/asap-query-engine/src/planner_client.rs b/asap-query-engine/src/planner_client.rs index 87616928..925e15e5 100644 --- a/asap-query-engine/src/planner_client.rs +++ b/asap-query-engine/src/planner_client.rs @@ -124,7 +124,7 @@ mod tests { fn sample_runtime_options() -> RuntimeOptions { RuntimeOptions { - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, streaming_engine: StreamingEngine::Precompute, enable_punting: false, range_duration_ms: 300_000, diff --git a/asap-query-engine/src/query_tracker/tracker.rs b/asap-query-engine/src/query_tracker/tracker.rs index ab2124d2..5f45e08d 100644 --- a/asap-query-engine/src/query_tracker/tracker.rs +++ b/asap-query-engine/src/query_tracker/tracker.rs @@ -16,8 +16,8 @@ use crate::planner_client::{PlannerClient, PlannerResult}; pub struct QueryTrackerConfig { /// How often to evaluate and trigger planning (default: 600s = 10 min). pub observation_window_secs: u64, - /// Prometheus scrape interval (ms), passed through to `infer_queries`. - pub prometheus_scrape_interval_ms: u64, + /// Data ingestion interval (ms), passed through to `infer_queries`. + pub data_ingestion_interval_ms: u64, } pub struct QueryTracker { @@ -121,7 +121,7 @@ impl QueryTracker { entries.len() ); - let scrape_interval_ms = tracker.config.prometheus_scrape_interval_ms; + let scrape_interval_ms = tracker.config.data_ingestion_interval_ms; let (instants, ranges) = infer_queries(&entries, scrape_interval_ms); info!( @@ -228,7 +228,7 @@ mod tests { fn record_instant_appends_entry() { let tracker = make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, }); tracker.record_instant("rate(http_requests_total[5m])", 1700000000.0); tracker.record_instant("rate(http_requests_total[5m])", 1700000060.0); @@ -242,7 +242,7 @@ mod tests { fn record_range_appends_entry() { let tracker = make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, }); tracker.record_range( "rate(http_requests_total[5m])", @@ -260,7 +260,7 @@ mod tests { async fn evaluate_calls_planner_with_entries() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, })); // Record enough entries for infer_queries to produce results (need >=2 per query). @@ -289,7 +289,7 @@ mod tests { async fn evaluate_skips_when_no_entries() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, })); let mock_client = Arc::new(MockPlannerClient::new()); @@ -308,7 +308,7 @@ mod tests { async fn evaluate_sends_plan_only_once() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, })); // Record enough for infer_queries to produce results. @@ -400,7 +400,7 @@ mod tests { let tracker = Arc::new(QueryTracker::new( QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval_ms: 15_000, + data_ingestion_interval_ms: 15_000, }, sc, ic, diff --git a/asap-query-engine/src/tests/query_equivalence_tests.rs b/asap-query-engine/src/tests/query_equivalence_tests.rs index b5231e68..da1141fd 100644 --- a/asap-query-engine/src/tests/query_equivalence_tests.rs +++ b/asap-query-engine/src/tests/query_equivalence_tests.rs @@ -196,7 +196,7 @@ mod tests { let (_, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(vec!["L1", "L2", "L3", "L4"]) - .with_scrape_interval_ms(scrape_interval_ms) // USED: propagated to SimpleEngine::prometheus_scrape_interval_ms + .with_scrape_interval_ms(scrape_interval_ms) // USED: propagated to SimpleEngine::data_ingestion_interval_ms .add_temporal_query( "sum_over_time(cpu_usage[150s])", // NOT USED: TestConfigBuilder requires a PromQL string; timestamp calculation is SQL-only sql_query, // USED: parsed to extract the 150s duration diff --git a/asap-quickstart/docker-compose.dev.yml b/asap-quickstart/docker-compose.dev.yml index 44612013..d0f964ca 100644 --- a/asap-quickstart/docker-compose.dev.yml +++ b/asap-quickstart/docker-compose.dev.yml @@ -18,7 +18,7 @@ services: command: - "--input_config=/config/controller-config.yaml" - "--output_dir=/asap-planner-output" - - "--prometheus_scrape_interval_ms=1000" + - "--data-ingestion-interval-ms=1000" - "--streaming_engine=precompute" - "--range-duration-ms=300000" - "--step-ms=10000" diff --git a/asap-tools/experiments/experiment_only_ingest_path.py b/asap-tools/experiments/experiment_only_ingest_path.py index 41af9d97..497c6bbb 100644 --- a/asap-tools/experiments/experiment_only_ingest_path.py +++ b/asap-tools/experiments/experiment_only_ingest_path.py @@ -228,7 +228,7 @@ def main(cfg: DictConfig): node_offset=args.node_offset, ) - prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( + data_ingestion_interval_ms = config.get_prometheus_data_ingestion_interval_ms( cfg.prometheus ) @@ -291,7 +291,7 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() - label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 + label_discovery_wait_ms = data_ingestion_interval_ms * 2 print( f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." @@ -300,7 +300,7 @@ def main(cfg: DictConfig): controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, diff --git a/asap-tools/experiments/experiment_run_clickhouse.py b/asap-tools/experiments/experiment_run_clickhouse.py index 0e5bcc0a..3ca71375 100644 --- a/asap-tools/experiments/experiment_run_clickhouse.py +++ b/asap-tools/experiments/experiment_run_clickhouse.py @@ -437,7 +437,7 @@ def main(cfg: DictConfig) -> None: experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format="json", - prometheus_scrape_interval_ms=data_ingestion_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, log_level="INFO", profile_query_engine=False, manual=False, diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 3b6bb708..74b22577 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -324,7 +324,7 @@ def main(cfg: DictConfig): prometheus_config_output_dir, node_offset=args.node_offset, ) - prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( + data_ingestion_interval_ms = config.get_prometheus_data_ingestion_interval_ms( cfg.prometheus ) @@ -352,7 +352,7 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() # Wait for two scrape intervals so Prometheus has series to return. - label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 + label_discovery_wait_ms = data_ingestion_interval_ms * 2 print( f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." @@ -372,7 +372,7 @@ def main(cfg: DictConfig): url=prometheus_url, database=None, ), - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, ) sync.rsync_controller_config_remote_to_local( provider, @@ -516,7 +516,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_run_grafana_demo.py b/asap-tools/experiments/experiment_run_grafana_demo.py index 589fe821..52377c88 100644 --- a/asap-tools/experiments/experiment_run_grafana_demo.py +++ b/asap-tools/experiments/experiment_run_grafana_demo.py @@ -274,7 +274,7 @@ def main(cfg: DictConfig): prometheus_config_output_dir, node_offset=args.node_offset, ) - prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms( + data_ingestion_interval_ms = config.get_prometheus_data_ingestion_interval_ms( cfg.prometheus ) @@ -344,7 +344,7 @@ def main(cfg: DictConfig): prometheus_service.wait_until_ready() - label_discovery_wait_ms = prometheus_scrape_interval_ms * 2 + label_discovery_wait_ms = data_ingestion_interval_ms * 2 print( f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data " f"before running controller label inference..." @@ -353,7 +353,7 @@ def main(cfg: DictConfig): controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, @@ -474,7 +474,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index a08a9a50..c4655e8d 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -466,7 +466,7 @@ def read_workloads_config(experiment_params: DictConfig): return workloads_config -def get_prometheus_scrape_interval_ms(prometheus_config): +def get_prometheus_data_ingestion_interval_ms(prometheus_config): """Extract scrape interval from Prometheus configuration, returned in milliseconds.""" s = prometheus_config.scrape_interval # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute diff --git a/asap-tools/experiments/experiment_utils/services/misc.py b/asap-tools/experiments/experiment_utils/services/misc.py index 790e2862..fdb398eb 100644 --- a/asap-tools/experiments/experiment_utils/services/misc.py +++ b/asap-tools/experiments/experiment_utils/services/misc.py @@ -206,9 +206,8 @@ def start( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval_ms: Optional[int] = None, + data_ingestion_interval_ms: int, query_language: str = "promql", - data_ingestion_interval_ms: Optional[int] = None, **kwargs, ) -> None: """ @@ -222,10 +221,8 @@ def start( discovery_backend: Backend used for label/column auto-discovery. PromQL mode: DiscoveryBackend(type="prometheus", url=, database=None) SQL mode: DiscoveryBackend(type="clickhouse", url=, database=) - prometheus_scrape_interval_ms: Required for PromQL mode (milliseconds; passed as - --prometheus_scrape_interval_ms / --prometheus-scrape-interval-ms) + data_ingestion_interval_ms: Data ingestion interval in milliseconds (required for all modes) query_language: 'promql' (default) or 'sql' - data_ingestion_interval_ms: Required for SQL mode (milliseconds) **kwargs: Additional configuration """ if self.use_container: @@ -235,9 +232,8 @@ def start( controller_remote_output_dir, punting, discovery_backend, - prometheus_scrape_interval_ms, - query_language, data_ingestion_interval_ms, + query_language, ) else: return self._start_bare_metal( @@ -246,9 +242,8 @@ def start( controller_remote_output_dir, punting, discovery_backend, - prometheus_scrape_interval_ms, - query_language, data_ingestion_interval_ms, + query_language, ) def _start_bare_metal( @@ -258,9 +253,8 @@ def _start_bare_metal( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval_ms: Optional[int], + data_ingestion_interval_ms: int, query_language: str, - data_ingestion_interval_ms: Optional[int], ) -> None: controller_log = os.path.join(controller_remote_output_dir, "controller.log") cmd = ( @@ -270,10 +264,7 @@ def _start_bare_metal( f" --streaming_engine {streaming_engine}" f" --query-language {query_language}" ) - if prometheus_scrape_interval_ms is not None: - cmd += f" --prometheus_scrape_interval_ms {prometheus_scrape_interval_ms}" - if data_ingestion_interval_ms is not None: - cmd += f" --data-ingestion-interval-ms {data_ingestion_interval_ms}" + cmd += f" --data-ingestion-interval-ms {data_ingestion_interval_ms}" if discovery_backend.type == "prometheus": cmd += f" --prometheus-url {discovery_backend.url}" elif discovery_backend.type == "clickhouse": @@ -301,9 +292,8 @@ def _start_containerized( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval_ms: Optional[int], + data_ingestion_interval_ms: int, query_language: str, - data_ingestion_interval_ms: Optional[int], ): controller_dir = os.path.join( self.provider.get_home_dir(), "code", "asap-planner-rs" @@ -331,14 +321,7 @@ def _start_containerized( generate_cmd += f" --controller-output-dir {controller_remote_output_dir}" generate_cmd += f" --streaming-engine {streaming_engine}" generate_cmd += f" --query-language {query_language}" - if prometheus_scrape_interval_ms is not None: - generate_cmd += ( - f" --prometheus-scrape-interval-ms {prometheus_scrape_interval_ms}" - ) - if data_ingestion_interval_ms is not None: - generate_cmd += ( - f" --data-ingestion-interval-ms {data_ingestion_interval_ms}" - ) + generate_cmd += f" --data-ingestion-interval-ms {data_ingestion_interval_ms}" if discovery_backend.type == "prometheus": generate_cmd += f" --prometheus-url {discovery_backend.url}" elif discovery_backend.type == "clickhouse": diff --git a/asap-tools/experiments/experiment_utils/services/query_engine.py b/asap-tools/experiments/experiment_utils/services/query_engine.py index 19a6a884..092a6ea4 100644 --- a/asap-tools/experiments/experiment_utils/services/query_engine.py +++ b/asap-tools/experiments/experiment_utils/services/query_engine.py @@ -73,7 +73,7 @@ def _build_engine_config( self, output_dir: str, flink_output_format: str, - prometheus_scrape_interval_ms: int, + data_ingestion_interval_ms: int, log_level: str, streaming_engine: str, controller_config_dir: str, @@ -93,7 +93,7 @@ def _build_engine_config( Args: output_dir: Output directory path (remote host path or container-internal path) flink_output_format: Kafka input_format when streaming_engine=arroyo - prometheus_scrape_interval_ms: Prometheus scraping interval in milliseconds + data_ingestion_interval_ms: Prometheus scraping interval in milliseconds log_level: Logging level streaming_engine: 'arroyo' (Kafka ingest) or 'precompute' (HTTP remote write) controller_config_dir: Directory containing inference_config.yaml and streaming_config.yaml @@ -140,7 +140,7 @@ def _build_engine_config( return { "output_dir": output_dir, "log_level": log_level, - "prometheus_scrape_interval_ms": prometheus_scrape_interval_ms, + "data_ingestion_interval_ms": data_ingestion_interval_ms, "streaming_engine": streaming_engine, "http_server": {"port": http_port}, "backend": backend, # already fully resolved by caller @@ -205,7 +205,7 @@ def start( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval_ms: int, + data_ingestion_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -228,7 +228,7 @@ def start( config YAML locally before rsyncing to remote) flink_output_format: Format of data from Flink (used as Kafka input_format when streaming_engine=arroyo) - prometheus_scrape_interval_ms: Prometheus scraping interval in milliseconds + data_ingestion_interval_ms: Prometheus scraping interval in milliseconds log_level: Logging level profile_query_engine: Whether to enable profiling manual: Whether to run in manual mode @@ -250,7 +250,7 @@ def start( experiment_output_dir, local_experiment_dir, flink_output_format, - prometheus_scrape_interval_ms, + data_ingestion_interval_ms, log_level, profile_query_engine, manual, @@ -269,7 +269,7 @@ def start( experiment_output_dir, local_experiment_dir, flink_output_format, - prometheus_scrape_interval_ms, + data_ingestion_interval_ms, log_level, profile_query_engine, manual, @@ -289,7 +289,7 @@ def _start_bare_metal( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval_ms: int, + data_ingestion_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -310,7 +310,7 @@ def _start_bare_metal( config = self._build_engine_config( output_dir=output_dir, flink_output_format=flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, log_level=log_level, streaming_engine=streaming_engine, controller_config_dir=controller_remote_output_dir, @@ -353,7 +353,7 @@ def _start_containerized( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval_ms: int, + data_ingestion_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -380,7 +380,7 @@ def _start_containerized( config = self._build_engine_config( output_dir=container_output_dir, flink_output_format=flink_output_format, - prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, + data_ingestion_interval_ms=data_ingestion_interval_ms, log_level=log_level, streaming_engine=streaming_engine, controller_config_dir=container_controller_dir, diff --git a/asap-tools/experiments/generate_controller_compose.py b/asap-tools/experiments/generate_controller_compose.py index 15c643dc..f7eac6bd 100644 --- a/asap-tools/experiments/generate_controller_compose.py +++ b/asap-tools/experiments/generate_controller_compose.py @@ -5,7 +5,6 @@ import argparse import os import sys -from typing import Optional from jinja2 import Template @@ -16,12 +15,11 @@ def generate_compose_file( container_name: str, input_config_path: str, output_dir: str, - prometheus_scrape_interval_ms: int, + data_ingestion_interval_ms: int, streaming_engine: str, punting: bool, prometheus_url: str, query_language: str, - data_ingestion_interval_ms: Optional[int], ): """Generate docker-compose.yml from template with provided variables.""" @@ -42,12 +40,11 @@ def generate_compose_file( "container_name": container_name, "input_config_path": input_config_path, "output_dir": output_dir, - "prometheus_scrape_interval_ms": prometheus_scrape_interval_ms, + "data_ingestion_interval_ms": data_ingestion_interval_ms, "streaming_engine": streaming_engine, "punting": punting, "prometheus_url": prometheus_url, "query_language": query_language, - "data_ingestion_interval_ms": data_ingestion_interval_ms, } # Render the template @@ -107,12 +104,6 @@ def main(): required=True, help="Output directory for generated configs", ) - parser.add_argument( - "--prometheus-scrape-interval-ms", - type=int, - required=True, - help="Prometheus scrape interval in milliseconds", - ) parser.add_argument( "--streaming-engine", required=True, @@ -138,17 +129,12 @@ def main(): parser.add_argument( "--data-ingestion-interval-ms", type=int, - default=None, - help="Data ingestion interval in milliseconds (SQL mode only)", + required=True, + help="Data ingestion interval in milliseconds", ) args = parser.parse_args() - if args.data_ingestion_interval_ms is not None and args.query_language != "sql": - parser.error( - "--data-ingestion-interval-ms is only valid when --query-language is 'sql'" - ) - generate_compose_file( template_path=args.template_path, output_path=args.compose_output_path, @@ -156,12 +142,11 @@ def main(): container_name=args.container_name, input_config_path=args.input_config_path, output_dir=args.controller_output_dir, - prometheus_scrape_interval_ms=args.prometheus_scrape_interval_ms, + data_ingestion_interval_ms=args.data_ingestion_interval_ms, streaming_engine=args.streaming_engine, punting=args.punting, prometheus_url=args.prometheus_url, query_language=args.query_language, - data_ingestion_interval_ms=args.data_ingestion_interval_ms, ) diff --git a/benchmarks/config/engine_config.yaml b/benchmarks/config/engine_config.yaml index 1618ff72..57abe29a 100644 --- a/benchmarks/config/engine_config.yaml +++ b/benchmarks/config/engine_config.yaml @@ -1,6 +1,6 @@ output_dir: "/app/outputs" log_level: "INFO" -prometheus_scrape_interval_ms: 1000 +data_ingestion_interval_ms: 1000 streaming_engine: "precompute" http_server: diff --git a/benchmarks/docker-compose.yml b/benchmarks/docker-compose.yml index 00e68896..a331a1fc 100644 --- a/benchmarks/docker-compose.yml +++ b/benchmarks/docker-compose.yml @@ -15,7 +15,7 @@ services: command: - "--input_config=/config/controller-config.yaml" - "--output_dir=/asap-planner-output" - - "--prometheus_scrape_interval_ms=1000" + - "--data-ingestion-interval-ms=1000" - "--streaming_engine=precompute" - "--range-duration-ms=300000" - "--step-ms=10000"