diff --git a/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py index 843b16c1..6934df9e 100644 --- a/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py +++ b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py @@ -26,8 +26,8 @@ class StreamingAggregationConfig: aggregationType: str aggregationSubType: str - windowSize: int # Window size in seconds (e.g., 900s for 15m) - slideInterval: int # Slide/hop interval in seconds (e.g., 30s) + windowSizeMs: int # Window size in milliseconds (e.g., 900_000ms for 15m) + slideIntervalMs: int # Slide/hop interval in milliseconds (e.g., 30_000ms) windowType: str # "tumbling" or "sliding" spatialFilter: str @@ -61,9 +61,9 @@ def from_dict(aggregation_config: dict) -> "StreamingAggregationConfig": aggregation.aggregationSubType = aggregation_config["aggregationSubType"] aggregation.windowType = aggregation_config.get("windowType", "tumbling") - aggregation.windowSize = aggregation_config["windowSize"] - aggregation.slideInterval = aggregation_config.get( - "slideInterval", aggregation.windowSize + aggregation.windowSizeMs = aggregation_config["windowSizeMs"] + aggregation.slideIntervalMs = aggregation_config.get( + "slideIntervalMs", aggregation.windowSizeMs ) aggregation.spatialFilter = aggregation_config["spatialFilter"] @@ -139,8 +139,8 @@ def get_identifying_key(self) -> Tuple: self.aggregationType, self.aggregationSubType, self.windowType, - self.windowSize, - self.slideInterval, + self.windowSizeMs, + self.slideIntervalMs, self.spatialFilter, self.metric, self.table_name, # SQL mode: table name diff --git a/asap-common/dependencies/rs/asap_types/src/aggregation_config.rs b/asap-common/dependencies/rs/asap_types/src/aggregation_config.rs index aae18a7a..93516104 100644 --- a/asap-common/dependencies/rs/asap_types/src/aggregation_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/aggregation_config.rs @@ -20,8 +20,8 @@ pub struct AggregationConfig { pub rollup_labels: KeyByLabelNames, pub original_yaml: String, - pub window_size: u64, // Window size in seconds (e.g., 900s for 15m) - pub slide_interval: u64, // Slide/hop interval in seconds (e.g., 30s) + pub window_size_ms: u64, // Window size in milliseconds (e.g., 900_000ms for 15m) + pub slide_interval_ms: u64, // Slide/hop interval in milliseconds (e.g., 30_000ms) pub window_type: WindowType, // Tumbling or Sliding pub spatial_filter: String, @@ -59,8 +59,8 @@ impl AggregationConfig { aggregated_labels: KeyByLabelNames, rollup_labels: KeyByLabelNames, original_yaml: String, - window_size: u64, - slide_interval: u64, + window_size_ms: u64, + slide_interval_ms: u64, window_type: WindowType, spatial_filter: String, metric: String, @@ -82,8 +82,8 @@ impl AggregationConfig { aggregated_labels, rollup_labels, original_yaml, - window_size, - slide_interval, + window_size_ms, + slide_interval_ms, window_type, spatial_filter, spatial_filter_normalized, @@ -144,7 +144,9 @@ impl AggregationConfig { let aggregated_labels = KeyByLabelNames::deserialize_from_json(&data["aggregatedLabels"])?; let rollup_labels = KeyByLabelNames::deserialize_from_json(&data["rollupLabels"])?; - let window_size = data["windowSize"].as_u64().ok_or("Missing windowSize")?; + let window_size_ms = data["windowSizeMs"] + .as_u64() + .ok_or("Missing windowSizeMs")?; let window_type = data .get("windowType") @@ -153,10 +155,10 @@ impl AggregationConfig { .parse::() .unwrap_or_default(); - let slide_interval = data - .get("slideInterval") + let slide_interval_ms = data + .get("slideIntervalMs") .and_then(|v| v.as_u64()) - .unwrap_or(window_size); + .unwrap_or(window_size_ms); let spatial_filter = data["spatialFilter"].as_str().unwrap_or("").to_string(); @@ -184,8 +186,8 @@ impl AggregationConfig { aggregated_labels, rollup_labels, original_yaml, - window_size, - slide_interval, + window_size_ms, + slide_interval_ms, window_type, spatial_filter, metric, @@ -266,9 +268,9 @@ impl AggregationConfig { }) .collect(); - let window_size = aggregation_data["windowSize"] + let window_size_ms = aggregation_data["windowSizeMs"] .as_u64() - .ok_or_else(|| anyhow::anyhow!("Missing windowSize"))?; + .ok_or_else(|| anyhow::anyhow!("Missing windowSizeMs"))?; let window_type = aggregation_data .get("windowType") @@ -277,10 +279,10 @@ impl AggregationConfig { .parse::() .unwrap_or_default(); - let slide_interval = aggregation_data - .get("slideInterval") + let slide_interval_ms = aggregation_data + .get("slideIntervalMs") .and_then(|v| v.as_u64()) - .unwrap_or(window_size); + .unwrap_or(window_size_ms); let spatial_filter = aggregation_data["spatialFilter"] .as_str() @@ -330,8 +332,8 @@ impl AggregationConfig { aggregated_labels, rollup_labels, String::new(), // original_yaml - empty as in Python - window_size, - slide_interval, + window_size_ms, + slide_interval_ms, window_type, spatial_filter, metric, @@ -351,8 +353,8 @@ impl SerializableToSink for AggregationConfig { "aggregationSubType": self.aggregation_sub_type, "parameters": self.parameters, "originalYaml": self.original_yaml, - "windowSize": self.window_size, - "slideInterval": self.slide_interval, + "windowSizeMs": self.window_size_ms, + "slideIntervalMs": self.slide_interval_ms, "windowType": self.window_type.to_string(), "spatialFilter": self.spatial_filter, "metric": self.metric, diff --git a/asap-common/dependencies/rs/asap_types/src/capability_matching.rs b/asap-common/dependencies/rs/asap_types/src/capability_matching.rs index 4763c47d..a1240b93 100644 --- a/asap-common/dependencies/rs/asap_types/src/capability_matching.rs +++ b/asap-common/dependencies/rs/asap_types/src/capability_matching.rs @@ -70,7 +70,7 @@ pub fn window_compatible(config: &AggregationConfig, data_range_ms: Option) let Some(range) = data_range_ms else { return true; }; - let window_ms = config.window_size * 1000; + let window_ms = config.window_size_ms; if window_ms == 0 || range == 0 { return false; } @@ -134,10 +134,10 @@ pub fn topk_weighting_compatible( } } -/// Aggregation priority comparator: prefer larger `window_size` (descending). +/// Aggregation priority comparator: prefer larger `window_size_ms` (descending). /// This is a separate function so callers can swap the policy without touching matching logic. pub fn aggregation_priority(a: &AggregationConfig, b: &AggregationConfig) -> Ordering { - b.window_size.cmp(&a.window_size) + b.window_size_ms.cmp(&a.window_size_ms) } // --------------------------------------------------------------------------- @@ -152,7 +152,7 @@ pub fn aggregation_priority(a: &AggregationConfig, b: &AggregationConfig) -> Ord /// Algorithm: /// 1. For each statistic, collect and sort compatible candidates. /// 2. For multi-statistic requirements (e.g. avg = [Sum, Count]), all must be -/// served by configs sharing the same `window_size` and `grouping_labels`. +/// served by configs sharing the same `window_size_ms` and `grouping_labels`. /// 3. If the selected value aggregation type is multi-population, also find a /// paired key aggregation (`SetAggregator` / `DeltaSetAggregator`) on the same metric. pub fn find_compatible_aggregation( @@ -197,7 +197,7 @@ pub fn find_compatible_aggregation( agg_id = c.aggregation_id, agg_type = %c.aggregation_type, metric = %c.metric, - window_size_s = c.window_size, + window_size_ms = c.window_size_ms, "capability matching: rejected config for {:?}", stat, ); @@ -222,7 +222,7 @@ pub fn find_compatible_aggregation( num_candidates = candidates.len(), chosen_agg_id = candidates[0].aggregation_id, chosen_agg_type = %candidates[0].aggregation_type, - chosen_window_size_s = candidates[0].window_size, + chosen_window_size_ms = candidates[0].window_size_ms, "capability matching: found candidates, chose best", ); @@ -233,16 +233,17 @@ pub fn find_compatible_aggregation( let value_agg = per_stat_candidates[0][0]; // For multi-statistic requirements, the remaining statistics must be served by a - // config that agrees on window_size and grouping_labels with the chosen value agg. + // config that agrees on window_size_ms and grouping_labels with the chosen value agg. for (i, candidates) in per_stat_candidates.iter().enumerate().skip(1) { let found = candidates.iter().any(|c| { - c.window_size == value_agg.window_size && c.grouping_labels == value_agg.grouping_labels + c.window_size_ms == value_agg.window_size_ms + && c.grouping_labels == value_agg.grouping_labels }); if !found { warn!( metric = %requirements.metric, statistic = ?requirements.statistics[i], - required_window_size_s = value_agg.window_size, + required_window_size_ms = value_agg.window_size_ms, "capability matching: no matching window/labels for multi-statistic requirement", ); return None; @@ -305,7 +306,7 @@ mod tests { metric: &str, agg_type: &str, sub_type: &str, - window_size_s: u64, + window_size_ms: u64, window_type: &str, grouping: &[&str], spatial_filter: &str, @@ -322,8 +323,8 @@ mod tests { aggregated_labels: KeyByLabelNames::new(vec![]), rollup_labels: KeyByLabelNames::new(vec![]), original_yaml: String::new(), - window_size: window_size_s, - slide_interval: window_size_s, + window_size_ms, + slide_interval_ms: window_size_ms, window_type: window_type.parse::().unwrap_or_default(), spatial_filter: spatial_filter.to_string(), spatial_filter_normalized, @@ -362,7 +363,16 @@ mod tests { #[test] fn basic_sum_match() { - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(300_000), &[], ""), @@ -378,7 +388,7 @@ mod tests { "lat", "DatasketchesKLL", "", - 300, + 300_000, "tumbling", &[], "", @@ -403,7 +413,7 @@ mod tests { "lat", "HydraKLL", "", - 300, + 300_000, "tumbling", &[], "", @@ -417,7 +427,16 @@ mod tests { #[test] fn no_match_wrong_metric() { - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("mem", &[Statistic::Sum], Some(300_000), &[], ""), @@ -432,7 +451,7 @@ mod tests { "cpu", "DatasketchesKLL", "", - 300, + 300_000, "tumbling", &[], "", @@ -448,7 +467,16 @@ mod tests { #[test] fn window_tumbling_exact() { - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(300_000), &[], ""), @@ -458,8 +486,17 @@ mod tests { #[test] fn window_tumbling_divisible() { - // 900_000 ms / 300 s = 3 buckets — valid merge - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], "")); + // 900_000 ms / 300_000 ms = 3 buckets — valid merge + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(900_000), &[], ""), @@ -469,8 +506,17 @@ mod tests { #[test] fn window_tumbling_not_divisible() { - // 600_000 ms / 900 s is not a whole number - let configs = single_config(make_config(1, "cpu", "Sum", "", 900, "tumbling", &[], "")); + // 600_000 ms / 900_000 ms is not a whole number + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 900_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(600_000), &[], ""), @@ -480,7 +526,16 @@ mod tests { #[test] fn window_sliding_exact() { - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "sliding", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "sliding", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(300_000), &[], ""), @@ -490,8 +545,17 @@ mod tests { #[test] fn window_sliding_too_large() { - // Query range 600 s but sliding window only covers 300 s - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "sliding", &[], "")); + // Query range 600_000 ms but sliding window only covers 300_000 ms + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "sliding", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(600_000), &[], ""), @@ -504,13 +568,13 @@ mod tests { let mut configs = HashMap::new(); configs.insert( 1, - make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], ""), + make_config(1, "cpu", "Sum", "", 300_000, "tumbling", &[], ""), ); configs.insert( 2, - make_config(2, "cpu", "Sum", "", 900, "tumbling", &[], ""), + make_config(2, "cpu", "Sum", "", 900_000, "tumbling", &[], ""), ); - // 900_000 ms is divisible by both 300 s and 900 s — prefer 900 s + // 900_000 ms is divisible by both 300_000 ms and 900_000 ms — prefer 900_000 ms let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(900_000), &[], ""), @@ -521,7 +585,16 @@ mod tests { #[test] fn spatial_only_no_range() { // data_range_ms = None → any window size is compatible - let configs = single_config(make_config(1, "cpu", "Sum", "", 900, "tumbling", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 900_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation(&configs, &req("cpu", &[Statistic::Sum], None, &[], "")); assert!(result.is_some()); @@ -536,7 +609,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &["job"], "", @@ -556,7 +629,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &["job", "instance"], "", @@ -575,7 +648,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &["region"], "", @@ -591,7 +664,16 @@ mod tests { #[test] fn spatial_filter_empty_both() { - let configs = single_config(make_config(1, "cpu", "Sum", "", 300, "tumbling", &[], "")); + let configs = single_config(make_config( + 1, + "cpu", + "Sum", + "", + 300_000, + "tumbling", + &[], + "", + )); let result = find_compatible_aggregation( &configs, &req("cpu", &[Statistic::Sum], Some(300_000), &[], ""), @@ -607,7 +689,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &[], "env=prod", @@ -626,7 +708,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &[], "env=prod", @@ -645,7 +727,7 @@ mod tests { "cpu", "Sum", "", - 300, + 300_000, "tumbling", &[], "env=prod", @@ -666,7 +748,7 @@ mod tests { "cpu", "MinMax", "min", - 300, + 300_000, "tumbling", &[], "", @@ -686,7 +768,7 @@ mod tests { "cpu", "MinMax", "min", - 300, + 300_000, "tumbling", &[], "", @@ -710,7 +792,7 @@ mod tests { "req", "CountMinSketchWithHeap", "", - 300, + 300_000, "tumbling", &[], "", @@ -723,7 +805,7 @@ mod tests { "req", "DeltaSetAggregator", "", - 300, + 300_000, "tumbling", &[], "", @@ -746,7 +828,7 @@ mod tests { "req", "CountMinSketchWithHeap", "", - 300, + 300_000, "tumbling", &[], "", @@ -765,7 +847,7 @@ mod tests { let mut configs = HashMap::new(); configs.insert( 1, - make_config(1, "cpu", "Sum", "", 300, "tumbling", &["job"], ""), + make_config(1, "cpu", "Sum", "", 300_000, "tumbling", &["job"], ""), ); configs.insert( 2, @@ -774,7 +856,7 @@ mod tests { "cpu", "CountMinSketch", "", - 300, + 300_000, "tumbling", &["job"], "", @@ -806,7 +888,7 @@ mod tests { "peers", "HLL", "", - 1, + 1_000, "tumbling", &["srcip"], "", @@ -850,9 +932,9 @@ mod tests { let mut configs = HashMap::new(); configs.insert( 1, - make_config(1, "cpu", "Sum", "", 300, "tumbling", &["job"], ""), + make_config(1, "cpu", "Sum", "", 300_000, "tumbling", &["job"], ""), ); - // Count config has different window_size — must be rejected + // Count config has different window_size_ms — must be rejected configs.insert( 2, make_config( @@ -860,7 +942,7 @@ mod tests { "cpu", "CountMinSketch", "", - 900, + 900_000, "tumbling", &["job"], "", @@ -889,7 +971,16 @@ mod tests { /// Paired key aggregation required by the multi-population fallback. fn make_key_agg(id: u64, metric: &str) -> AggregationConfig { - make_config(id, metric, "DeltaSetAggregator", "", 1, "tumbling", &[], "") + make_config( + id, + metric, + "DeltaSetAggregator", + "", + 1_000, + "tumbling", + &[], + "", + ) } /// `CountMinSketchWithHeap` config with an explicit `count_events` parameter. @@ -899,7 +990,7 @@ mod tests { metric, "CountMinSketchWithHeap", "", - 1, + 1_000, "tumbling", &[], "", @@ -971,7 +1062,7 @@ mod tests { "netflow_table", "CountMinSketchWithHeap", "", - 1, + 1_000, "tumbling", &[], "", diff --git a/asap-common/dependencies/rs/asap_types/src/query_requirements.rs b/asap-common/dependencies/rs/asap_types/src/query_requirements.rs index fbb9fc87..876ef2c7 100644 --- a/asap-common/dependencies/rs/asap_types/src/query_requirements.rs +++ b/asap-common/dependencies/rs/asap_types/src/query_requirements.rs @@ -64,9 +64,11 @@ pub fn build_query_requirements_promql( let data_range_ms = match pattern_type { QueryPatternType::OnlySpatial => None, + // promql-parser supports a literal `ms` duration suffix (e.g. `[500ms]`), + // so .num_seconds() would truncate sub-second ranges to 0. _ => match_result .get_range_duration() - .map(|d| d.num_seconds() as u64 * 1000), + .map(|d| d.num_milliseconds() as u64), }; let all_labels = metric_schema diff --git a/asap-planner-rs/docker-compose.yml.j2 b/asap-planner-rs/docker-compose.yml.j2 index 4211a612..19d797d7 100644 --- a/asap-planner-rs/docker-compose.yml.j2 +++ b/asap-planner-rs/docker-compose.yml.j2 @@ -8,7 +8,7 @@ services: command: [ "--input_config", "/app/input/config.yaml", "--output_dir", "/app/output", - "--prometheus_scrape_interval", "{{ prometheus_scrape_interval }}", + "--prometheus_scrape_interval_ms", "{{ prometheus_scrape_interval_ms }}", "--streaming_engine", "{{ streaming_engine }}", "--prometheus-url", "{{ prometheus_url }}", "--query-language", "{{ query_language }}"{% if punting %}, diff --git a/asap-planner-rs/src/bin/optimizer_cli.rs b/asap-planner-rs/src/bin/optimizer_cli.rs index 02eec53c..70d2352f 100644 --- a/asap-planner-rs/src/bin/optimizer_cli.rs +++ b/asap-planner-rs/src/bin/optimizer_cli.rs @@ -20,8 +20,8 @@ struct Args { #[arg(long = "input_config")] input_config: PathBuf, - #[arg(long = "prometheus_scrape_interval")] - prometheus_scrape_interval: u64, + #[arg(long = "prometheus_scrape_interval_ms")] + prometheus_scrape_interval_ms: u64, /// Placeholder arrival rate (items/sec) applied uniformly to every candidate's /// IngestCost. Real per-config rates aren't wired up yet — see the open TODOs @@ -56,18 +56,22 @@ fn main() -> anyhow::Result<()> { let config: ControllerConfig = serde_yaml::from_str(&yaml_str)?; let schema = config.schema_from_hints(); - let (streaming, inference) = - run_greedy_pipeline(&config, &schema, args.prometheus_scrape_interval, args.rho); + let (streaming, inference) = run_greedy_pipeline( + &config, + &schema, + args.prometheus_scrape_interval_ms, + args.rho, + ); let deployed = streaming.get_all_aggregation_configs(); println!("=== Deployed streaming configs: {} ===", deployed.len()); for (id, cfg) in deployed { println!( - " [{id}] {} sub_type={:?} window={}s slide={}s type={:?} metric={} params={:?}", + " [{id}] {} sub_type={:?} window={}ms slide={}ms type={:?} metric={} params={:?}", cfg.aggregation_type, cfg.aggregation_sub_type, - cfg.window_size, - cfg.slide_interval, + cfg.window_size_ms, + cfg.slide_interval_ms, cfg.window_type, cfg.metric, cfg.parameters, diff --git a/asap-planner-rs/src/config/input.rs b/asap-planner-rs/src/config/input.rs index 6143493a..6752ca33 100644 --- a/asap-planner-rs/src/config/input.rs +++ b/asap-planner-rs/src/config/input.rs @@ -48,15 +48,15 @@ impl ControllerConfig { pub struct QueryGroup { pub id: Option, pub queries: Vec, - pub repetition_delay: u64, + pub repetition_delay_ms: u64, #[serde(default)] pub controller_options: ControllerOptions, - /// Per-group step override (seconds). Falls back to `RuntimeOptions::step` when None. + /// Per-group step override (ms). Falls back to `RuntimeOptions::step_ms` when None. #[serde(default)] - pub step: Option, - /// Per-group range_duration override (seconds). Falls back to `RuntimeOptions::range_duration` when None. + pub step_ms: Option, + /// Per-group range_duration override (ms). Falls back to `RuntimeOptions::range_duration_ms` when None. #[serde(default)] - pub range_duration: Option, + pub range_duration_ms: Option, } #[derive(Debug, Clone, Deserialize, Default)] diff --git a/asap-planner-rs/src/generator.rs b/asap-planner-rs/src/generator.rs index 62bc29d2..3193cd21 100644 --- a/asap-planner-rs/src/generator.rs +++ b/asap-planner-rs/src/generator.rs @@ -28,14 +28,14 @@ pub(crate) const KEY_PARAMETERS: &str = "parameters"; pub(crate) const KEY_QUERIES: &str = "queries"; pub(crate) const KEY_QUERY: &str = "query"; pub(crate) const KEY_READ_COUNT_THRESHOLD: &str = "read_count_threshold"; -pub(crate) const KEY_SLIDE_INTERVAL: &str = "slideInterval"; +pub(crate) const KEY_SLIDE_INTERVAL_MS: &str = "slideIntervalMs"; pub(crate) const KEY_SPATIAL_FILTER: &str = "spatialFilter"; pub(crate) const KEY_TABLE_NAME: &str = "table_name"; pub(crate) const KEY_TABLES: &str = "tables"; pub(crate) const KEY_TIME_COLUMN: &str = "time_column"; pub(crate) const KEY_VALUE_COLUMN: &str = "value_column"; pub(crate) const KEY_VALUE_COLUMNS: &str = "value_columns"; -pub(crate) const KEY_WINDOW_SIZE: &str = "windowSize"; +pub(crate) const KEY_WINDOW_SIZE_MS: &str = "windowSizeMs"; pub(crate) const KEY_WINDOW_TYPE: &str = "windowType"; pub fn key_by_labels_to_yaml(labels: &KeyByLabelNames) -> YamlValue { @@ -90,8 +90,8 @@ pub fn build_aggregation_entry(id: u32, cfg: &IntermediateAggConfig) -> YamlValu params_to_yaml(&cfg.parameters), ); map.insert( - YamlValue::String(KEY_SLIDE_INTERVAL.to_string()), - YamlValue::Number(cfg.slide_interval.into()), + YamlValue::String(KEY_SLIDE_INTERVAL_MS.to_string()), + YamlValue::Number(cfg.slide_interval_ms.into()), ); map.insert( YamlValue::String(KEY_SPATIAL_FILTER.to_string()), @@ -112,8 +112,8 @@ pub fn build_aggregation_entry(id: u32, cfg: &IntermediateAggConfig) -> YamlValu }, ); map.insert( - YamlValue::String(KEY_WINDOW_SIZE.to_string()), - YamlValue::Number(cfg.window_size.into()), + YamlValue::String(KEY_WINDOW_SIZE_MS.to_string()), + YamlValue::Number(cfg.window_size_ms.into()), ); map.insert( YamlValue::String(KEY_WINDOW_TYPE.to_string()), diff --git a/asap-planner-rs/src/lib.rs b/asap-planner-rs/src/lib.rs index f6d691ea..82fc8c13 100644 --- a/asap-planner-rs/src/lib.rs +++ b/asap-planner-rs/src/lib.rs @@ -35,9 +35,9 @@ pub enum StreamingEngine { #[derive(Debug, Clone)] pub struct RuntimeOptions { - pub prometheus_scrape_interval: u64, + pub prometheus_scrape_interval_ms: u64, pub streaming_engine: StreamingEngine, pub enable_punting: bool, - pub range_duration: u64, - pub step: u64, + pub range_duration_ms: u64, + pub step_ms: u64, } diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index ccb96003..28f796a1 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -20,8 +20,8 @@ struct Args { #[arg(long = "output_dir")] output_dir: PathBuf, - #[arg(long = "prometheus_scrape_interval", required = false)] - prometheus_scrape_interval: Option, + #[arg(long = "prometheus_scrape_interval_ms", required = false)] + prometheus_scrape_interval_ms: Option, /// Base URL of the Prometheus instance used to auto-infer metric label sets. /// Optional: when provided, the planner queries Prometheus for label discovery. @@ -36,11 +36,11 @@ struct Args { #[arg(long = "enable-punting", default_value = "false")] enable_punting: bool, - #[arg(long = "range-duration", default_value = "0")] - range_duration: u64, + #[arg(long = "range-duration-ms", default_value = "0")] + range_duration_ms: u64, - #[arg(long = "step", default_value = "0")] - step: u64, + #[arg(long = "step-ms", default_value = "0")] + step_ms: u64, #[arg(long = "query-language", value_enum, default_value = "promql")] query_language: QueryLanguage, @@ -86,15 +86,15 @@ fn main() -> anyhow::Result<()> { match args.query_language { QueryLanguage::promql => { - let scrape_interval = args.prometheus_scrape_interval.ok_or_else(|| { - anyhow::anyhow!("--prometheus_scrape_interval is required for PromQL mode") + let scrape_interval_ms = args.prometheus_scrape_interval_ms.ok_or_else(|| { + anyhow::anyhow!("--prometheus_scrape_interval_ms is required for PromQL mode") })?; let opts = RuntimeOptions { - prometheus_scrape_interval: scrape_interval, + prometheus_scrape_interval_ms: scrape_interval_ms, streaming_engine: engine, enable_punting: args.enable_punting, - range_duration: args.range_duration, - step: args.step, + range_duration_ms: args.range_duration_ms, + step_ms: args.step_ms, }; let controller = match (args.input_config, args.query_log, args.prometheus_url) { (Some(config_path), None, Some(url)) => { diff --git a/asap-planner-rs/src/optimizer/aqe_extractor.rs b/asap-planner-rs/src/optimizer/aqe_extractor.rs index ec84f87c..8d714e91 100644 --- a/asap-planner-rs/src/optimizer/aqe_extractor.rs +++ b/asap-planner-rs/src/optimizer/aqe_extractor.rs @@ -16,7 +16,7 @@ use super::solution::AQE; #[derive(Debug, Clone)] pub struct RQE { pub query_string: String, - pub t_repeat_secs: u64, + pub t_repeat_ms: u64, } /// Stable deduplication key for an AQE. @@ -54,9 +54,9 @@ impl AQEKey { /// /// Three frequency-related values are computed per AQE: /// - `query_frequency_hz`: Σ 1/T_r — total query load for the MIP objective. -/// - `min_t_repeat_secs`: min(T_r) — freshness bound on window size W ≤ min_t. -/// - `t_repeat_gcd_secs`: GCD(T_r) — natural slide interval S for candidate -/// generation (windows completing every GCD secs align with all dashboards). +/// - `min_t_repeat_ms`: min(T_r) — freshness bound on window size W ≤ min_t. +/// - `t_repeat_gcd_ms`: GCD(T_r) — natural slide interval S for candidate +/// generation (windows completing every GCD ms align with all dashboards). /// /// Leaf queries that do not match any supported pattern (e.g. unsupported /// functions, parse errors) are skipped with a warning. @@ -65,10 +65,10 @@ pub fn extract_aqes(rqes: &[RQE], metric_schema: &PromQLSchema) -> Vec { let mut acc: HashMap, f64, u64, u64)> = HashMap::new(); for rqe in rqes { - if rqe.t_repeat_secs == 0 { + if rqe.t_repeat_ms == 0 { warn!( query = %rqe.query_string, - "aqe_extractor: skipping RQE with repetition_delay=0 \ + "aqe_extractor: skipping RQE with repetition_delay_ms=0 \ (would produce infinite query frequency and corrupt GCD)" ); continue; @@ -86,12 +86,14 @@ pub fn extract_aqes(rqes: &[RQE], metric_schema: &PromQLSchema) -> Vec { if !entry.1.contains(&leaf) { entry.1.push(leaf); } - entry.2 += 1.0 / rqe.t_repeat_secs as f64; - entry.3 = entry.3.min(rqe.t_repeat_secs); + // query_frequency_hz must stay in Hz (queries per real second) + // regardless of t_repeat_ms's internal unit — 1000.0 / ms, not 1.0 / ms. + entry.2 += 1000.0 / rqe.t_repeat_ms as f64; + entry.3 = entry.3.min(rqe.t_repeat_ms); entry.4 = if entry.4 == 0 { - rqe.t_repeat_secs + rqe.t_repeat_ms } else { - gcd(entry.4, rqe.t_repeat_secs) + gcd(entry.4, rqe.t_repeat_ms) }; } None => { @@ -110,14 +112,14 @@ pub fn extract_aqes(rqes: &[RQE], metric_schema: &PromQLSchema) -> Vec { requirements, query_strings, query_frequency_hz, - min_t_repeat_secs, - t_repeat_gcd_secs, + min_t_repeat_ms, + t_repeat_gcd_ms, )| AQE { requirements, query_strings, query_frequency_hz, - min_t_repeat_secs, - t_repeat_gcd_secs, + min_t_repeat_ms, + t_repeat_gcd_ms, }, ) .collect() @@ -180,16 +182,16 @@ mod tests { PromQLSchema::new() } - fn rqe(query: &str, t: u64) -> RQE { + fn rqe(query: &str, t_ms: u64) -> RQE { RQE { query_string: query.to_string(), - t_repeat_secs: t, + t_repeat_ms: t_ms, } } #[test] fn single_temporal_query() { - let rqes = vec![rqe("sum_over_time(metric[5m])", 60)]; + let rqes = vec![rqe("sum_over_time(metric[5m])", 60_000)]; let aqes = extract_aqes(&rqes, &empty_schema()); assert_eq!(aqes.len(), 1); assert!((aqes[0].query_frequency_hz - 1.0 / 60.0).abs() < 1e-9); @@ -201,7 +203,7 @@ mod tests { fn binary_query_produces_two_aqes() { let rqes = vec![rqe( "sum_over_time(metric_a[5m]) / sum_over_time(metric_b[5m])", - 60, + 60_000, )]; let aqes = extract_aqes(&rqes, &empty_schema()); assert_eq!(aqes.len(), 2); @@ -209,7 +211,7 @@ mod tests { #[test] fn binary_with_scalar_produces_one_aqe() { - let rqes = vec![rqe("sum_over_time(metric[5m]) * 100", 60)]; + let rqes = vec![rqe("sum_over_time(metric[5m]) * 100", 60_000)]; let aqes = extract_aqes(&rqes, &empty_schema()); assert_eq!(aqes.len(), 1); } @@ -217,8 +219,8 @@ mod tests { #[test] fn same_aqe_in_two_rqes_deduplicates_and_sums_frequency() { let rqes = vec![ - rqe("sum_over_time(metric[5m])", 60), - rqe("sum_over_time(metric[5m])", 30), + rqe("sum_over_time(metric[5m])", 60_000), + rqe("sum_over_time(metric[5m])", 30_000), ]; let aqes = extract_aqes(&rqes, &empty_schema()); assert_eq!(aqes.len(), 1); @@ -226,14 +228,14 @@ mod tests { let expected_freq = 1.0 / 60.0 + 1.0 / 30.0; assert!((aqes[0].query_frequency_hz - expected_freq).abs() < 1e-9); // min_t and gcd_t used for windowing constraints - assert_eq!(aqes[0].min_t_repeat_secs, 30); - assert_eq!(aqes[0].t_repeat_gcd_secs, 30); // gcd(60, 30) = 30 + assert_eq!(aqes[0].min_t_repeat_ms, 30_000); + assert_eq!(aqes[0].t_repeat_gcd_ms, 30_000); // gcd(60_000, 30_000) = 30_000 assert_eq!(aqes[0].query_strings.len(), 1); // same string, deduplicated } #[test] fn unsupported_query_is_skipped() { - let rqes = vec![rqe("not_a_real_function(metric[5m])", 60)]; + let rqes = vec![rqe("not_a_real_function(metric[5m])", 60_000)]; let aqes = extract_aqes(&rqes, &empty_schema()); assert_eq!(aqes.len(), 0); } diff --git a/asap-planner-rs/src/optimizer/candidate_gen.rs b/asap-planner-rs/src/optimizer/candidate_gen.rs index c294841d..ba878862 100644 --- a/asap-planner-rs/src/optimizer/candidate_gen.rs +++ b/asap-planner-rs/src/optimizer/candidate_gen.rs @@ -31,7 +31,7 @@ pub struct CandidateConfig { /// /// Multi-statistic AQEs (e.g. avg = [Sum, Count]) return only EXACT — a single /// sketch family cannot serve two incompatible statistics simultaneously. -pub fn enumerate_candidates(aqe: &AQE, scrape_interval_secs: u64) -> Vec { +pub fn enumerate_candidates(aqe: &AQE, scrape_interval_ms: u64) -> Vec { let mut candidates = Vec::new(); if aqe.requirements.statistics.len() != 1 { @@ -41,7 +41,7 @@ pub fn enumerate_candidates(aqe: &AQE, scrape_interval_secs: u64) -> Vec Vec CandidateConfig { } } -/// Window candidates: (WindowType, W_secs, slide_interval_secs, n_windows). +/// Window candidates: (WindowType, W_ms, slide_interval_ms, n_windows). /// /// For spatial-only queries (range = None): one tumbling window of scrape_interval. /// For range-based queries: @@ -99,39 +99,39 @@ fn exact_candidate() -> CandidateConfig { /// ingest CPU/mem but gives fresher results. Doubling from scrape_interval up to /// W keeps the grid small while covering that tradeoff. fn window_candidates( - range_a_secs: Option, - min_t_repeat_secs: u64, - scrape_interval_secs: u64, + range_a_ms: Option, + min_t_repeat_ms: u64, + scrape_interval_ms: u64, ) -> Vec<(WindowType, u64, u64, u64)> { - let Some(range_a) = range_a_secs else { + let Some(range_a) = range_a_ms else { // Spatial-only: any window works (window_compatible returns true for None range). return vec![( WindowType::Tumbling, - scrape_interval_secs, - scrape_interval_secs, + scrape_interval_ms, + scrape_interval_ms, 1, )]; }; - if range_a == 0 || scrape_interval_secs == 0 { + if range_a == 0 || scrape_interval_ms == 0 { return vec![]; } - let max_w = range_a.min(min_t_repeat_secs); + let max_w = range_a.min(min_t_repeat_ms); let mut out = Vec::new(); // Tumbling: W divides range_a, is a multiple of scrape_interval, and ≤ max_w. - let mut w = scrape_interval_secs; + let mut w = scrape_interval_ms; while w <= max_w { if range_a % w == 0 { let n = range_a / w; out.push((WindowType::Tumbling, w, w, n)); } - w += scrape_interval_secs; + w += scrape_interval_ms; } // Sliding: W = range_a exactly; S doubles from scrape_interval up to W. - if range_a <= min_t_repeat_secs { - let mut s = scrape_interval_secs; + if range_a <= min_t_repeat_ms { + let mut s = scrape_interval_ms; while s <= range_a { out.push((WindowType::Sliding, range_a, s, 1)); s *= 2; @@ -314,15 +314,15 @@ mod tests { }, query_strings: vec!["test_query".into()], query_frequency_hz: 1.0 / 60.0, - min_t_repeat_secs: min_t, - t_repeat_gcd_secs: min_t, + min_t_repeat_ms: min_t, + t_repeat_gcd_ms: min_t, } } #[test] fn always_includes_exact_fallback() { - let aqe = make_aqe(Statistic::Sum, Some(300_000), 60); - let candidates = enumerate_candidates(&aqe, 15); + let aqe = make_aqe(Statistic::Sum, Some(300_000), 60_000); + let candidates = enumerate_candidates(&aqe, 15_000); assert!(candidates .iter() .any(|c| c.config.is_none() && c.query_method == QueryMethod::Exact)); @@ -330,8 +330,8 @@ mod tests { #[test] fn spatial_only_produces_neither_candidates() { - let aqe = make_aqe(Statistic::Sum, None, 60); - let candidates = enumerate_candidates(&aqe, 15); + let aqe = make_aqe(Statistic::Sum, None, 60_000); + let candidates = enumerate_candidates(&aqe, 15_000); // All non-EXACT candidates should be Direct (no temporal range to cover). for c in candidates.iter().filter(|c| c.config.is_some()) { assert_eq!(c.query_method, QueryMethod::Direct); @@ -340,9 +340,9 @@ mod tests { #[test] fn tumbling_w_equals_range_produces_neither() { - // range_a = 60s, scrape = 60s → only W=60, n=1 → Direct - let aqe = make_aqe(Statistic::Sum, Some(60_000), 60); - let candidates = enumerate_candidates(&aqe, 60); + // range_a = 60_000ms, scrape = 60_000ms → only W=60_000, n=1 → Direct + let aqe = make_aqe(Statistic::Sum, Some(60_000), 60_000); + let candidates = enumerate_candidates(&aqe, 60_000); for c in candidates.iter().filter(|c| c.config.is_some()) { assert_eq!(c.query_method, QueryMethod::Direct); } @@ -350,10 +350,10 @@ mod tests { #[test] fn mergeable_sketch_with_multiple_windows_produces_merge() { - // Min → MinMax (mergeable, not subtractable): range_a=300s, scrape=60s, min_t=300s - // → W=60 → n=5, Merge{5}. (Sum would prefer Subtract since it's also subtractable.) - let aqe = make_aqe(Statistic::Min, Some(300_000), 300); - let candidates = enumerate_candidates(&aqe, 60); + // Min → MinMax (mergeable, not subtractable): range_a=300_000ms, scrape=60_000ms, min_t=300_000ms + // → W=60_000 → n=5, Merge{5}. (Sum would prefer Subtract since it's also subtractable.) + let aqe = make_aqe(Statistic::Min, Some(300_000), 300_000); + let candidates = enumerate_candidates(&aqe, 60_000); let merge_candidates: Vec<_> = candidates .iter() .filter(|c| matches!(c.query_method, QueryMethod::Merge { .. })) @@ -367,8 +367,8 @@ mod tests { #[test] fn cms_with_heap_only_neither_no_merge() { // CMS+Heap is neither mergeable nor subtractable → only n=1 (Direct) valid. - let aqe = make_aqe(Statistic::Topk, Some(300_000), 300); - let candidates = enumerate_candidates(&aqe, 60); + let aqe = make_aqe(Statistic::Topk, Some(300_000), 300_000); + let candidates = enumerate_candidates(&aqe, 60_000); for c in candidates.iter().filter(|c| c.config.is_some()) { assert_eq!( c.query_method, diff --git a/asap-planner-rs/src/optimizer/cost_model.rs b/asap-planner-rs/src/optimizer/cost_model.rs index 00ba0c97..9cb9360c 100644 --- a/asap-planner-rs/src/optimizer/cost_model.rs +++ b/asap-planner-rs/src/optimizer/cost_model.rs @@ -81,12 +81,12 @@ pub fn ingest_cost( // branches collapse to the same value until that lands. let subpopulation_count = 1.0_f64; - // Defensive floor: slide_interval is a plain u64 on a widely-shared struct; + // Defensive floor: slide_interval_ms is a plain u64 on a widely-shared struct; // guard against div-by-zero producing `inf` and poisoning cost comparisons. let n_concurrent = match agg_config.window_type { WindowType::Tumbling => 1.0, WindowType::Sliding => { - (agg_config.window_size as f64 / agg_config.slide_interval.max(1) as f64).ceil() + (agg_config.window_size_ms as f64 / agg_config.slide_interval_ms.max(1) as f64).ceil() } }; @@ -184,8 +184,8 @@ mod tests { }, query_strings: vec!["test_query".into()], query_frequency_hz: 1.0 / 60.0, - min_t_repeat_secs: min_t, - t_repeat_gcd_secs: min_t, + min_t_repeat_ms: min_t, + t_repeat_gcd_ms: min_t, } } @@ -198,7 +198,7 @@ mod tests { }; let costs = AtomicCosts::default(); let weights = CostWeights::default(); - let a = make_aqe(Statistic::Sum, Some(300_000), 300); + let a = make_aqe(Statistic::Sum, Some(300_000), 300_000); assert_eq!(ingest_cost(&candidate, 1.0, &costs, &weights), 0.0); assert!(query_cost(&a, &candidate, &costs, &weights) > 0.0); } @@ -208,8 +208,8 @@ mod tests { // Calibration-independent: Subtract is O(1) (one subtract + one read) while // Merge is O(n) (n-1 merges + one read), so for the same n and same // underlying config, Subtract must cost less regardless of weight tuning. - let a = make_aqe(Statistic::Sum, Some(300_000), 300); - let candidates = enumerate_candidates(&a, 60); + let a = make_aqe(Statistic::Sum, Some(300_000), 300_000); + let candidates = enumerate_candidates(&a, 60_000); let template = candidates .iter() .find_map(|c| c.config.clone()) diff --git a/asap-planner-rs/src/optimizer/greedy.rs b/asap-planner-rs/src/optimizer/greedy.rs index cd5a9688..719c73a5 100644 --- a/asap-planner-rs/src/optimizer/greedy.rs +++ b/asap-planner-rs/src/optimizer/greedy.rs @@ -18,7 +18,7 @@ use super::solution::{AQEAssignment, OptimizerSolution, AQE}; /// which isn't wired up yet — a single placeholder value is applied uniformly. pub fn greedy_assign( aqes: Vec, - scrape_interval_secs: u64, + scrape_interval_ms: u64, arrival_rate_hz: f64, costs: &AtomicCosts, weights: &CostWeights, @@ -30,7 +30,7 @@ pub fn greedy_assign( let mut next_id: u64 = 1; for aqe in aqes { - let candidates = enumerate_candidates(&aqe, scrape_interval_secs); + let candidates = enumerate_candidates(&aqe, scrape_interval_ms); let best = candidates .into_iter() @@ -106,20 +106,20 @@ mod tests { }, query_strings: vec!["test_query".into()], query_frequency_hz: freq_hz, - min_t_repeat_secs: min_t, - t_repeat_gcd_secs: min_t, + min_t_repeat_ms: min_t, + t_repeat_gcd_ms: min_t, } } #[test] fn assigns_unique_ids_to_each_deployed_config() { let aqes = vec![ - make_aqe(Statistic::Min, Some(300_000), 300, 1.0 / 60.0), - make_aqe(Statistic::Max, Some(300_000), 300, 1.0 / 60.0), + make_aqe(Statistic::Min, Some(300_000), 300_000, 1.0 / 60.0), + make_aqe(Statistic::Max, Some(300_000), 300_000, 1.0 / 60.0), ]; let solution = greedy_assign( aqes, - 60, + 60_000, 1.0, &AtomicCosts::default(), &CostWeights::default(), @@ -148,12 +148,12 @@ mod tests { }, query_strings: vec!["avg_query".into()], query_frequency_hz: 1.0 / 60.0, - min_t_repeat_secs: 60, - t_repeat_gcd_secs: 60, + min_t_repeat_ms: 60_000, + t_repeat_gcd_ms: 60_000, }; let solution = greedy_assign( vec![aqe], - 60, + 60_000, 1.0, &AtomicCosts::default(), &CostWeights::default(), diff --git a/asap-planner-rs/src/optimizer/pipeline.rs b/asap-planner-rs/src/optimizer/pipeline.rs index 854bb5ff..cb093af5 100644 --- a/asap-planner-rs/src/optimizer/pipeline.rs +++ b/asap-planner-rs/src/optimizer/pipeline.rs @@ -63,13 +63,13 @@ pub fn run_all_exact_pipeline( pub fn run_greedy_pipeline( config: &ControllerConfig, schema: &PromQLSchema, - scrape_interval_secs: u64, + scrape_interval_ms: u64, arrival_rate_hz: f64, ) -> (StreamingConfig, InferenceConfig) { run_pipeline(config, schema, "greedy", |aqes| { greedy_assign( aqes, - scrape_interval_secs, + scrape_interval_ms, arrival_rate_hz, &AtomicCosts::default(), &CostWeights::default(), @@ -86,7 +86,7 @@ fn config_to_rqes(config: &ControllerConfig) -> Vec { .flat_map(|qg| { qg.queries.iter().map(|q| RQE { query_string: q.clone(), - t_repeat_secs: qg.repetition_delay, + t_repeat_ms: qg.repetition_delay_ms, }) }) .collect() @@ -104,10 +104,10 @@ mod tests { .map(|(q, t)| QueryGroup { id: None, queries: vec![q.to_string()], - repetition_delay: *t, + repetition_delay_ms: *t, controller_options: Default::default(), - step: None, - range_duration: None, + step_ms: None, + range_duration_ms: None, }) .collect(); @@ -123,7 +123,10 @@ mod tests { #[test] fn all_exact_pipeline_produces_empty_streaming_config() { - let config = make_config(&[("sum_over_time(metric[5m])", 60), ("sum(other_metric)", 30)]); + let config = make_config(&[ + ("sum_over_time(metric[5m])", 60_000), + ("sum(other_metric)", 30_000), + ]); let schema = PromQLSchema::new(); let (streaming, _inference) = run_all_exact_pipeline(&config, &schema); // All-EXACT: no streaming configs deployed. @@ -132,19 +135,22 @@ mod tests { #[test] fn greedy_pipeline_deploys_a_config_for_a_mergeable_aqe() { - let config = make_config(&[("min_over_time(metric[5m])", 60)]); + let config = make_config(&[("min_over_time(metric[5m])", 60_000)]); let schema = PromQLSchema::new(); - let (streaming, inference) = run_greedy_pipeline(&config, &schema, 60, 1.0); + let (streaming, inference) = run_greedy_pipeline(&config, &schema, 60_000, 1.0); assert!(!streaming.get_all_aggregation_configs().is_empty()); assert!(!inference.query_configs.is_empty()); } #[test] fn config_to_rqes_flattens_groups() { - let config = make_config(&[("sum_over_time(a[5m])", 60), ("sum_over_time(b[5m])", 30)]); + let config = make_config(&[ + ("sum_over_time(a[5m])", 60_000), + ("sum_over_time(b[5m])", 30_000), + ]); let rqes = config_to_rqes(&config); assert_eq!(rqes.len(), 2); - assert_eq!(rqes[0].t_repeat_secs, 60); - assert_eq!(rqes[1].t_repeat_secs, 30); + assert_eq!(rqes[0].t_repeat_ms, 60_000); + assert_eq!(rqes[1].t_repeat_ms, 30_000); } } diff --git a/asap-planner-rs/src/optimizer/solution.rs b/asap-planner-rs/src/optimizer/solution.rs index 5ca974c7..bbd60ba9 100644 --- a/asap-planner-rs/src/optimizer/solution.rs +++ b/asap-planner-rs/src/optimizer/solution.rs @@ -21,18 +21,18 @@ pub struct AQE { /// hitting the sketch. pub query_frequency_hz: f64, - /// Minimum repeat interval across all RQEs that reference this AQE (secs). + /// Minimum repeat interval across all RQEs that reference this AQE (ms). /// Determines the freshness constraint on the window size: W ≤ min_t_repeat /// ensures a completed window is available for every dashboard's cycle. /// When multiple RQEs share this AQE, the fastest dashboard is the binding /// constraint. - pub min_t_repeat_secs: u64, + pub min_t_repeat_ms: u64, - /// GCD of all repeat intervals across RQEs that reference this AQE (secs). + /// GCD of all repeat intervals across RQEs that reference this AQE (ms). /// The natural candidate for the slide interval S: windows that complete - /// every GCD seconds align harmonically with all dashboard refresh cycles, + /// every GCD ms align harmonically with all dashboard refresh cycles, /// ensuring every dashboard can always be served a fresh result on-cycle. - pub t_repeat_gcd_secs: u64, + pub t_repeat_gcd_ms: u64, } /// How an AQE is answered from its assigned streaming config. diff --git a/asap-planner-rs/src/planner/agg_config.rs b/asap-planner-rs/src/planner/agg_config.rs index af891f22..9e0b4a8c 100644 --- a/asap-planner-rs/src/planner/agg_config.rs +++ b/asap-planner-rs/src/planner/agg_config.rs @@ -14,8 +14,8 @@ pub struct IntermediateAggConfig { pub aggregation_type: AggregationType, pub aggregation_sub_type: String, pub window_type: WindowType, - pub window_size: u64, - pub slide_interval: u64, + pub window_size_ms: u64, + pub slide_interval_ms: u64, pub spatial_filter: String, pub metric: String, pub table_name: Option, @@ -56,8 +56,8 @@ impl IntermediateAggConfig { self.aggregation_type, self.aggregation_sub_type, self.window_type, - self.window_size, - self.slide_interval, + self.window_size_ms, + self.slide_interval_ms, self.spatial_filter, self.metric, self.table_name, @@ -112,8 +112,8 @@ pub fn build_agg_configs_for_statistics( aggregation_type: AggregationType::DeltaSetAggregator, aggregation_sub_type: String::new(), window_type: window_cfg.window_type, - window_size: window_cfg.window_size, - slide_interval: window_cfg.slide_interval, + window_size_ms: window_cfg.window_size_ms, + slide_interval_ms: window_cfg.slide_interval_ms, spatial_filter: spatial_filter.to_string(), metric: metric.to_string(), table_name: table_name.map(str::to_string), @@ -130,8 +130,8 @@ pub fn build_agg_configs_for_statistics( aggregation_type: agg_type, aggregation_sub_type: agg_sub_type, window_type: window_cfg.window_type, - window_size: window_cfg.window_size, - slide_interval: window_cfg.slide_interval, + window_size_ms: window_cfg.window_size_ms, + slide_interval_ms: window_cfg.slide_interval_ms, spatial_filter: spatial_filter.to_string(), metric: metric.to_string(), table_name: table_name.map(str::to_string), @@ -157,8 +157,8 @@ mod tests { aggregation_type: AggregationType::MultipleIncrease, aggregation_sub_type: "rate".to_string(), window_type: WindowType::Tumbling, - window_size: 300, - slide_interval: 300, + window_size_ms: 300_000, + slide_interval_ms: 300_000, spatial_filter: String::new(), metric: "http_requests_total".to_string(), table_name: None, @@ -196,7 +196,7 @@ mod tests { fn different_window_size_produces_different_key() { let cfg1 = base_config(); let mut cfg2 = base_config(); - cfg2.window_size = 60; + cfg2.window_size_ms = 60_000; assert_ne!(cfg1.identifying_key(), cfg2.identifying_key()); } diff --git a/asap-planner-rs/src/planner/cleanup.rs b/asap-planner-rs/src/planner/cleanup.rs index 23730967..ff0182d9 100644 --- a/asap-planner-rs/src/planner/cleanup.rs +++ b/asap-planner-rs/src/planner/cleanup.rs @@ -8,33 +8,37 @@ pub fn get_cleanup_param( cleanup_policy: CleanupPolicy, query_pattern_type: QueryPatternType, match_result: &PromQLMatchResult, - t_repeat: u64, + t_repeat_ms: u64, window_type: WindowType, - range_duration: u64, - step: u64, + range_duration_ms: u64, + step_ms: u64, ) -> Result { // Validation - if (range_duration == 0) != (step == 0) { + if (range_duration_ms == 0) != (step_ms == 0) { return Err(format!( - "range_duration and step must both be 0 or both > 0. Got range_duration={}, step={}", - range_duration, step + "range_duration_ms and step_ms must both be 0 or both > 0. Got range_duration_ms={}, step_ms={}", + range_duration_ms, step_ms )); } - let is_range_query = step > 0; + let is_range_query = step_ms > 0; let t_lookback: u64 = if query_pattern_type == QueryPatternType::OnlySpatial { - t_repeat + t_repeat_ms } else { match_result .get_range_duration() - .map(|d| d.num_seconds() as u64) + .map(|d| { + let ms = d.num_milliseconds(); + debug_assert!(ms >= 0, "PromQL range duration should never be negative"); + ms as u64 + }) .ok_or_else(|| "No range_vector token found".to_string())? }; if window_type == WindowType::Sliding { let result = if is_range_query { - range_duration / step + 1 + range_duration_ms / step_ms + 1 } else { 1 }; @@ -42,19 +46,19 @@ pub fn get_cleanup_param( } // Tumbling - let effective_repeat = get_effective_repeat(t_repeat, step); + let effective_repeat = get_effective_repeat(t_repeat_ms, step_ms); let result = match cleanup_policy { CleanupPolicy::CircularBuffer => { - // ceil((t_lookback + range_duration) / effective_repeat) - let numerator = t_lookback + range_duration; + // ceil((t_lookback + range_duration_ms) / effective_repeat) + let numerator = t_lookback + range_duration_ms; numerator.div_ceil(effective_repeat) } CleanupPolicy::ReadBased => { - // ceil(t_lookback / effective_repeat) * (range_duration / step + 1) + // ceil(t_lookback / effective_repeat) * (range_duration_ms / step_ms + 1) let lookback_buckets = t_lookback.div_ceil(effective_repeat); let num_steps = if is_range_query { - range_duration / step + 1 + range_duration_ms / step_ms + 1 } else { 1 }; @@ -115,14 +119,14 @@ mod tests { fn cleanup_param_circular_buffer_spatial_instant_query() { let (pt, mr) = match_query("sum(some_metric)"); assert_eq!(pt, QueryPatternType::OnlySpatial); - // t_lookback = t_repeat = 300 (OnlySpatial path) - // effective_repeat = 300 (step=0) - // ceil((300 + 0) / 300) = 1 + // t_lookback = t_repeat_ms = 300_000 (OnlySpatial path) + // effective_repeat = 300_000 (step_ms=0) + // ceil((300_000 + 0) / 300_000) = 1 let result = get_cleanup_param( CleanupPolicy::CircularBuffer, pt, &mr, - 300, + 300_000, WindowType::Tumbling, 0, 0, @@ -134,16 +138,16 @@ mod tests { #[test] fn cleanup_param_circular_buffer_spatial_range_query() { let (pt, mr) = match_query("sum(some_metric)"); - // t_lookback = t_repeat = 300, effective_repeat = min(300, 30) = 30 - // ceil((300 + 3600) / 30) = ceil(130) = 130 + // t_lookback = t_repeat_ms = 300_000, effective_repeat = min(300_000, 30_000) = 30_000 + // ceil((300_000 + 3_600_000) / 30_000) = ceil(130) = 130 let result = get_cleanup_param( CleanupPolicy::CircularBuffer, pt, &mr, - 300, + 300_000, WindowType::Tumbling, - 3600, - 30, + 3_600_000, + 30_000, ) .unwrap(); assert_eq!(result, 130); @@ -152,12 +156,12 @@ mod tests { #[test] fn cleanup_param_read_based_spatial_instant_query() { let (pt, mr) = match_query("sum(some_metric)"); - // lookback_buckets = ceil(300/300) = 1, num_steps = 1 → result = 1 + // lookback_buckets = ceil(300_000/300_000) = 1, num_steps = 1 → result = 1 let result = get_cleanup_param( CleanupPolicy::ReadBased, pt, &mr, - 300, + 300_000, WindowType::Tumbling, 0, 0, @@ -169,16 +173,16 @@ mod tests { #[test] fn cleanup_param_read_based_spatial_range_query() { let (pt, mr) = match_query("sum(some_metric)"); - // lookback_buckets = ceil(300/30) = 10, num_steps = 3600/30 + 1 = 121 + // lookback_buckets = ceil(300_000/30_000) = 10, num_steps = 3_600_000/30_000 + 1 = 121 // result = 10 * 121 = 1210 let result = get_cleanup_param( CleanupPolicy::ReadBased, pt, &mr, - 300, + 300_000, WindowType::Tumbling, - 3600, - 30, + 3_600_000, + 30_000, ) .unwrap(); assert_eq!(result, 1210); @@ -188,13 +192,13 @@ mod tests { fn cleanup_param_circular_buffer_temporal_instant_query() { let (pt, mr) = match_query("rate(some_metric[5m])"); assert_eq!(pt, QueryPatternType::OnlyTemporal); - // t_lookback = 5m = 300s (from [5m] range vector), range_duration=0, step=0 - // effective_repeat = 60, ceil((300 + 0) / 60) = 5 + // t_lookback = 5m = 300_000ms (from [5m] range vector), range_duration_ms=0, step_ms=0 + // effective_repeat = 60_000, ceil((300_000 + 0) / 60_000) = 5 let result = get_cleanup_param( CleanupPolicy::CircularBuffer, pt, &mr, - 60, + 60_000, WindowType::Tumbling, 0, 0, @@ -210,7 +214,7 @@ mod tests { CleanupPolicy::NoCleanup, pt, &mr, - 300, + 300_000, WindowType::Tumbling, 0, 0, @@ -221,14 +225,14 @@ mod tests { #[test] fn cleanup_param_mismatched_range_and_step_returns_error() { let (pt, mr) = match_query("sum(some_metric)"); - // range_duration > 0 but step == 0 is invalid + // range_duration_ms > 0 but step_ms == 0 is invalid let result = get_cleanup_param( CleanupPolicy::CircularBuffer, pt, &mr, - 300, + 300_000, WindowType::Tumbling, - 3600, + 3_600_000, 0, ); assert!(result.is_err()); diff --git a/asap-planner-rs/src/planner/elastic_dsl.rs b/asap-planner-rs/src/planner/elastic_dsl.rs index 713218c7..6e6b9483 100644 --- a/asap-planner-rs/src/planner/elastic_dsl.rs +++ b/asap-planner-rs/src/planner/elastic_dsl.rs @@ -65,10 +65,14 @@ impl ElasticSingleQueryProcessor { // Get aggregation type and statistics let (treatment_type, statistics) = get_elastic_statistics(&query_info.aggregation)?; - // Build window config (always tumbling for Elasticsearch queries) + // Build window config (always tumbling for Elasticsearch queries). + // self.t_repeat is seconds (Elastic DSL is out of scope for the ms-precision + // rename, see issue #427) — IntermediateWindowConfig is now ms-typed throughout, + // so convert at this boundary, same as planner::sql::compute_sql_window. + let t_repeat_ms = self.t_repeat * 1000; let window_cfg = IntermediateWindowConfig { - window_size: self.t_repeat, - slide_interval: self.t_repeat, + window_size_ms: t_repeat_ms, + slide_interval_ms: t_repeat_ms, window_type: WindowType::Tumbling, }; @@ -131,8 +135,8 @@ impl ElasticSingleQueryProcessor { }) .and_then(|p| range_query_to_time_range(p, 0)); let t_lookback = match time_range { - Some(tr) => tr.duration_ms().unwrap_or(self.t_repeat), - None => self.t_repeat, // Default to repetition delay if no time range found + Some(tr) => tr.duration_ms().unwrap_or(t_repeat_ms), + None => t_repeat_ms, }; // Calculate cleanup param based on query's time window @@ -140,7 +144,7 @@ impl ElasticSingleQueryProcessor { None } else { Some( - get_sql_cleanup_param(self.cleanup_policy, t_lookback, self.t_repeat) + get_sql_cleanup_param(self.cleanup_policy, t_lookback, t_repeat_ms) .map_err(ControllerError::PlannerError)?, ) }; diff --git a/asap-planner-rs/src/planner/promql.rs b/asap-planner-rs/src/planner/promql.rs index a0e952aa..aa9470c8 100644 --- a/asap-planner-rs/src/planner/promql.rs +++ b/asap-planner-rs/src/planner/promql.rs @@ -71,14 +71,14 @@ pub(crate) fn parse_binary_arms(query: &str) -> Option<(BinaryArm, BinaryArm)> { pub struct SingleQueryProcessor { query: String, - t_repeat: u64, - prometheus_scrape_interval: u64, + t_repeat_ms: u64, + prometheus_scrape_interval_ms: u64, metric_schema: PromQLSchema, #[allow(dead_code)] streaming_engine: StreamingEngine, sketch_parameters: Option, - range_duration: u64, - step: u64, + range_duration_ms: u64, + step_ms: u64, cleanup_policy: CleanupPolicy, } @@ -86,24 +86,24 @@ impl SingleQueryProcessor { #[allow(clippy::too_many_arguments)] pub fn new( query: String, - t_repeat: u64, - prometheus_scrape_interval: u64, + t_repeat_ms: u64, + prometheus_scrape_interval_ms: u64, metric_schema: PromQLSchema, streaming_engine: StreamingEngine, sketch_parameters: Option, - range_duration: u64, - step: u64, + range_duration_ms: u64, + step_ms: u64, cleanup_policy: CleanupPolicy, ) -> Self { Self { query, - t_repeat, - prometheus_scrape_interval, + t_repeat_ms, + prometheus_scrape_interval_ms, metric_schema, streaming_engine, sketch_parameters, - range_duration, - step, + range_duration_ms, + step_ms, cleanup_policy, } } @@ -158,13 +158,13 @@ impl SingleQueryProcessor { pub fn make_arm_processor(&self, arm_query: String) -> Self { SingleQueryProcessor::new( arm_query, - self.t_repeat, - self.prometheus_scrape_interval, + self.t_repeat_ms, + self.prometheus_scrape_interval_ms, self.metric_schema.clone(), self.streaming_engine, self.sketch_parameters.clone(), - self.range_duration, - self.step, + self.range_duration_ms, + self.step_ms, self.cleanup_policy, ) } @@ -198,14 +198,15 @@ impl SingleQueryProcessor { | PromQLFunction::Increase | PromQLFunction::QuantileOverTime) ) { - let num_data_points = self.t_repeat as f64 / self.prometheus_scrape_interval as f64; + let num_data_points = + self.t_repeat_ms as f64 / self.prometheus_scrape_interval_ms as f64; if num_data_points < 60.0 { return false; } if parsed_fn == Ok(PromQLFunction::QuantileOverTime) { if let Some(range_dur) = match_result.get_range_duration() { - let range_secs = range_dur.num_seconds() as f64; - if range_secs / self.t_repeat as f64 > 15.0 { + let range_ms = range_dur.num_milliseconds() as f64; + if range_ms / self.t_repeat_ms as f64 > 15.0 { return false; } } @@ -246,10 +247,10 @@ impl SingleQueryProcessor { let mut window_cfg = IntermediateWindowConfig::default(); set_window_parameters( pattern_type, - self.t_repeat, - self.prometheus_scrape_interval, + self.t_repeat_ms, + self.prometheus_scrape_interval_ms, "any", // aggregation_type doesn't matter (sliding always false) - self.step, + self.step_ms, &mut window_cfg, ); @@ -286,10 +287,10 @@ impl SingleQueryProcessor { self.cleanup_policy, pattern_type, &match_result, - self.t_repeat, + self.t_repeat_ms, window_cfg.window_type, - self.range_duration, - self.step, + self.range_duration_ms, + self.step_ms, ) .map_err(ControllerError::PlannerError)?, ) diff --git a/asap-planner-rs/src/planner/sql.rs b/asap-planner-rs/src/planner/sql.rs index 523203ff..168ce39d 100644 --- a/asap-planner-rs/src/planner/sql.rs +++ b/asap-planner-rs/src/planner/sql.rs @@ -218,13 +218,17 @@ fn compute_sql_window( data_ingestion_interval: u64, t_repeat: u64, ) -> IntermediateWindowConfig { - let window_size = match query_type { - QueryType::Spatial => data_ingestion_interval, - _ => t_repeat, + // data_ingestion_interval and t_repeat are seconds (SQL mode is out of scope for the + // ms-precision rename, see issue #427) — IntermediateWindowConfig is now ms-typed + // throughout, so convert at this boundary, same as the pre-existing pattern in + // asap-query-engine/src/engines/simple_engine/sql.rs. + let window_size_ms = match query_type { + QueryType::Spatial => data_ingestion_interval * 1000, + _ => t_repeat * 1000, }; IntermediateWindowConfig { - window_size, - slide_interval: window_size, + window_size_ms, + slide_interval_ms: window_size_ms, window_type: WindowType::Tumbling, } } diff --git a/asap-planner-rs/src/planner/window.rs b/asap-planner-rs/src/planner/window.rs index 171b09af..2d1b6a0e 100644 --- a/asap-planner-rs/src/planner/window.rs +++ b/asap-planner-rs/src/planner/window.rs @@ -1,11 +1,11 @@ use asap_types::enums::WindowType; use promql_utilities::query_logics::enums::QueryPatternType; -pub fn get_effective_repeat(t_repeat: u64, step: u64) -> u64 { - if step > 0 { - t_repeat.min(step) +pub fn get_effective_repeat(t_repeat_ms: u64, step_ms: u64) -> u64 { + if step_ms > 0 { + t_repeat_ms.min(step_ms) } else { - t_repeat + t_repeat_ms } } @@ -19,19 +19,19 @@ pub fn should_use_sliding_window( pub fn set_window_parameters( query_pattern_type: QueryPatternType, - t_repeat: u64, - prometheus_scrape_interval: u64, + t_repeat_ms: u64, + prometheus_scrape_interval_ms: u64, aggregation_type: &str, - step: u64, + step_ms: u64, config: &mut IntermediateWindowConfig, ) { - let effective_repeat = get_effective_repeat(t_repeat, step); + let effective_repeat = get_effective_repeat(t_repeat_ms, step_ms); let _use_sliding = should_use_sliding_window(query_pattern_type, aggregation_type); // use_sliding is always false, so always tumbling set_tumbling_window_parameters( query_pattern_type, effective_repeat, - prometheus_scrape_interval, + prometheus_scrape_interval_ms, config, ); } @@ -39,18 +39,18 @@ pub fn set_window_parameters( fn set_tumbling_window_parameters( query_pattern_type: QueryPatternType, effective_repeat: u64, - prometheus_scrape_interval: u64, + prometheus_scrape_interval_ms: u64, config: &mut IntermediateWindowConfig, ) { match query_pattern_type { QueryPatternType::OnlyTemporal | QueryPatternType::OneTemporalOneSpatial => { - config.window_size = effective_repeat; - config.slide_interval = effective_repeat; + config.window_size_ms = effective_repeat; + config.slide_interval_ms = effective_repeat; config.window_type = WindowType::Tumbling; } QueryPatternType::OnlySpatial => { - config.window_size = prometheus_scrape_interval; - config.slide_interval = prometheus_scrape_interval; + config.window_size_ms = prometheus_scrape_interval_ms; + config.slide_interval_ms = prometheus_scrape_interval_ms; config.window_type = WindowType::Tumbling; } } @@ -59,8 +59,8 @@ fn set_tumbling_window_parameters( /// A mutable window config holder used during planning #[derive(Debug, Clone, Default)] pub struct IntermediateWindowConfig { - pub window_size: u64, - pub slide_interval: u64, + pub window_size_ms: u64, + pub slide_interval_ms: u64, pub window_type: WindowType, } @@ -70,16 +70,16 @@ mod tests { #[test] fn effective_repeat_no_step() { - assert_eq!(get_effective_repeat(300, 0), 300); + assert_eq!(get_effective_repeat(300_000, 0), 300_000); } #[test] fn effective_repeat_step_smaller_than_t_repeat() { - assert_eq!(get_effective_repeat(300, 30), 30); + assert_eq!(get_effective_repeat(300_000, 30_000), 30_000); } #[test] fn effective_repeat_step_larger_than_t_repeat() { - assert_eq!(get_effective_repeat(30, 300), 30); + assert_eq!(get_effective_repeat(30_000, 300_000), 30_000); } } diff --git a/asap-planner-rs/src/planner_output.rs b/asap-planner-rs/src/planner_output.rs index ee739aae..036a8c79 100644 --- a/asap-planner-rs/src/planner_output.rs +++ b/asap-planner-rs/src/planner_output.rs @@ -7,7 +7,7 @@ use asap_types::streaming_config::StreamingConfig; use crate::generator::{ GeneratorOutput, PuntedQuery, KEY_AGGREGATIONS, KEY_AGG_SUB_TYPE, KEY_AGG_TYPE, KEY_LABELS, KEY_NUM_AGG_TO_RETAIN, KEY_PARAMETERS, KEY_QUERIES, KEY_QUERY, KEY_READ_COUNT_THRESHOLD, - KEY_TABLE_NAME, KEY_VALUE_COLUMN, KEY_WINDOW_SIZE, + KEY_TABLE_NAME, KEY_VALUE_COLUMN, KEY_WINDOW_SIZE_MS, }; /// Output of the planning process — contains the two YAML configs @@ -83,7 +83,7 @@ impl PlannerOutput { .map(|aggs| { aggs.iter().all(|agg| { if let YamlValue::Mapping(m) = agg { - if let Some(val) = m.get(KEY_WINDOW_SIZE) { + if let Some(val) = m.get(KEY_WINDOW_SIZE_MS) { let size = match val { YamlValue::Number(n) => n.as_u64().unwrap_or(0), _ => 0, diff --git a/asap-planner-rs/src/promql/controller.rs b/asap-planner-rs/src/promql/controller.rs index 24b4e6ba..88c404f8 100644 --- a/asap-planner-rs/src/promql/controller.rs +++ b/asap-planner-rs/src/promql/controller.rs @@ -111,7 +111,7 @@ impl Controller { ) -> Result { let entries = query_log::parse_log_file(log_path)?; let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval); + query_log::infer_queries(&entries, opts.prometheus_scrape_interval_ms); let config = query_log::to_controller_config(instants, ranges); let all_queries: Vec = config .query_groups @@ -136,7 +136,7 @@ impl Controller { ) -> Result { let entries = query_log::parse_log_file(log_path)?; let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval); + query_log::infer_queries(&entries, opts.prometheus_scrape_interval_ms); let config = query_log::to_controller_config(instants, ranges); Ok(Self { config, diff --git a/asap-planner-rs/src/promql/generator.rs b/asap-planner-rs/src/promql/generator.rs index 63761492..0f450a66 100644 --- a/asap-planner-rs/src/promql/generator.rs +++ b/asap-planner-rs/src/promql/generator.rs @@ -59,13 +59,13 @@ pub fn generate_plan( for query_string in &qg.queries { let processor = SingleQueryProcessor::new( query_string.clone(), - qg.repetition_delay, - opts.prometheus_scrape_interval, + qg.repetition_delay_ms, + opts.prometheus_scrape_interval_ms, metric_schema.clone(), opts.streaming_engine, controller_config.sketch_parameters.clone(), - qg.range_duration.unwrap_or(opts.range_duration), - qg.step.unwrap_or(opts.step), + qg.range_duration_ms.unwrap_or(opts.range_duration_ms), + qg.step_ms.unwrap_or(opts.step_ms), cleanup_policy, ); diff --git a/asap-planner-rs/src/query_log/converter.rs b/asap-planner-rs/src/query_log/converter.rs index 44dda288..9a9d70e3 100644 --- a/asap-planner-rs/src/query_log/converter.rs +++ b/asap-planner-rs/src/query_log/converter.rs @@ -17,10 +17,10 @@ pub fn to_controller_config( query_groups.push(QueryGroup { id: None, queries: vec![info.query], - repetition_delay: info.repetition_delay, + repetition_delay_ms: info.repetition_delay_ms, controller_options: Default::default(), - step: None, - range_duration: None, + step_ms: None, + range_duration_ms: None, }); } @@ -28,10 +28,10 @@ pub fn to_controller_config( query_groups.push(QueryGroup { id: None, queries: vec![info.query], - repetition_delay: info.repetition_delay, + repetition_delay_ms: info.repetition_delay_ms, controller_options: Default::default(), - step: Some(info.step), - range_duration: Some(info.range_duration), + step_ms: Some(info.step_ms), + range_duration_ms: Some(info.range_duration_ms), }); } diff --git a/asap-planner-rs/src/query_log/frequency.rs b/asap-planner-rs/src/query_log/frequency.rs index 975f7520..5ec6e3b1 100644 --- a/asap-planner-rs/src/query_log/frequency.rs +++ b/asap-planner-rs/src/query_log/frequency.rs @@ -9,19 +9,19 @@ type QueryVariant<'a> = ((String, u64), Vec<&'a LogEntry>); #[derive(Debug, Clone, PartialEq)] pub struct InstantQueryInfo { pub query: String, - /// Median inter-arrival time rounded to nearest scrape interval (seconds). - pub repetition_delay: u64, + /// Median inter-arrival time rounded to nearest scrape interval (ms). + pub repetition_delay_ms: u64, } #[derive(Debug, Clone, PartialEq)] pub struct RangeQueryInfo { pub query: String, - /// Median inter-arrival time rounded to nearest scrape interval (seconds). - pub repetition_delay: u64, - /// Step from params.step (seconds). - pub step: u64, - /// Median of (end − start) across occurrences (seconds). - pub range_duration: u64, + /// Median inter-arrival time rounded to nearest scrape interval (ms). + pub repetition_delay_ms: u64, + /// Step from params.step, converted from the log's native seconds to ms. + pub step_ms: u64, + /// Median of (end − start) across occurrences (ms). + pub range_duration_ms: u64, } /// Infer query repetition delays from a slice of parsed log entries. @@ -34,12 +34,12 @@ pub struct RangeQueryInfo { /// - If the same query string appears under multiple step values, keep the /// variant with the most occurrences and warn about discarded variants. /// - Groups with only 1 occurrence are skipped with a warning. -/// - `repetition_delay` = median inter-arrival time, rounded to nearest -/// `scrape_interval` second. -/// - If `|raw_median − rounded| / scrape_interval ≥ 0.1` a warning is emitted. +/// - `repetition_delay_ms` = median inter-arrival time, rounded to nearest +/// `scrape_interval_ms`. +/// - If `|raw_median − rounded| / scrape_interval_ms ≥ 0.1` a warning is emitted. pub fn infer_queries( entries: &[LogEntry], - scrape_interval: u64, + scrape_interval_ms: u64, ) -> (Vec, Vec) { // Group by (query_string, step) let mut groups: HashMap<(String, u64), Vec<&LogEntry>> = HashMap::new(); @@ -82,20 +82,21 @@ pub fn infer_queries( continue; } - let repetition_delay = infer_repetition_delay(&variant_entries, scrape_interval, &query); + let repetition_delay_ms = + infer_repetition_delay(&variant_entries, scrape_interval_ms, &query); if step == 0 { instant_results.push(InstantQueryInfo { query, - repetition_delay, + repetition_delay_ms, }); } else { - let range_duration = median_range_duration(&variant_entries); + let range_duration_ms = median_range_duration(&variant_entries); range_results.push(RangeQueryInfo { query, - repetition_delay, - step, - range_duration, + repetition_delay_ms, + step_ms: step * 1000, // LogEntry.step is seconds (Prometheus log format) + range_duration_ms, }); } } @@ -104,24 +105,24 @@ pub fn infer_queries( } /// Compute median inter-arrival time from timestamps and round to nearest scrape interval. -fn infer_repetition_delay(entries: &[&LogEntry], scrape_interval: u64, query: &str) -> u64 { +fn infer_repetition_delay(entries: &[&LogEntry], scrape_interval_ms: u64, query: &str) -> u64 { let mut timestamps: Vec> = entries.iter().map(|e| e.ts).collect(); timestamps.sort(); let deltas: Vec = timestamps .windows(2) - .map(|w| (w[1] - w[0]).num_seconds() as f64) + .map(|w| (w[1] - w[0]).num_milliseconds() as f64) .collect(); let raw_median = median_f64(&deltas); - let rounded = round_to_nearest(raw_median, scrape_interval); + let rounded = round_to_nearest(raw_median, scrape_interval_ms); - let misalignment = (raw_median - rounded as f64).abs() / scrape_interval as f64; + let misalignment = (raw_median - rounded as f64).abs() / scrape_interval_ms as f64; if misalignment >= 0.1 { tracing::warn!( %query, - raw_median_secs = raw_median, - rounded_secs = rounded, + raw_median_ms = raw_median, + rounded_ms = rounded, misalignment_pct = misalignment * 100.0, "inferred repetition_delay is poorly aligned with scrape_interval; result may be inaccurate" ); @@ -130,11 +131,11 @@ fn infer_repetition_delay(entries: &[&LogEntry], scrape_interval: u64, query: &s rounded } -/// Median of (end − start) durations in seconds. +/// Median of (end − start) durations in ms. fn median_range_duration(entries: &[&LogEntry]) -> u64 { let mut durations: Vec = entries .iter() - .map(|e| (e.end - e.start).num_seconds() as f64) + .map(|e| (e.end - e.start).num_milliseconds() as f64) .collect(); durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); median_f64(&durations) as u64 @@ -211,55 +212,55 @@ mod tests { #[test] fn single_occurrence_skipped() { let entries = make_instant_entries(base_ts(), 60, 1); - let (instants, ranges) = infer_queries(&entries, 15); + let (instants, ranges) = infer_queries(&entries, 15_000); assert!(instants.is_empty()); assert!(ranges.is_empty()); } #[test] fn median_inter_arrival_odd_count() { - // 5 entries at exactly 60s apart → 4 deltas all 60s → median=60 → rounded=60 + // 5 entries at exactly 60s apart → 4 deltas all 60_000ms → median=60_000 → rounded=60_000 let entries = make_instant_entries(base_ts(), 60, 5); - let (instants, _) = infer_queries(&entries, 15); + let (instants, _) = infer_queries(&entries, 15_000); assert_eq!(instants.len(), 1); - assert_eq!(instants[0].repetition_delay, 60); + assert_eq!(instants[0].repetition_delay_ms, 60_000); } #[test] fn median_inter_arrival_even_count() { - // 4 entries at 60s apart → 3 deltas all 60s → median=60 → rounded=60 + // 4 entries at 60s apart → 3 deltas all 60_000ms → median=60_000 → rounded=60_000 let entries = make_instant_entries(base_ts(), 60, 4); - let (instants, _) = infer_queries(&entries, 15); + let (instants, _) = infer_queries(&entries, 15_000); assert_eq!(instants.len(), 1); - assert_eq!(instants[0].repetition_delay, 60); + assert_eq!(instants[0].repetition_delay_ms, 60_000); } #[test] fn round_down_to_nearest_scrape() { - // raw=16s, scrape=15 → nearest=15 (|16-15|/15=6.7% < 10%, no warn) - assert_eq!(round_to_nearest(16.0, 15), 15); + // raw=16_000ms, scrape=15_000ms → nearest=15_000 (|16-15|/15=6.7% < 10%, no warn) + assert_eq!(round_to_nearest(16_000.0, 15_000), 15_000); } #[test] fn round_up_to_nearest_scrape() { - // raw=23s, scrape=15 → nearest=30 (23 is closer to 30 than 15) - assert_eq!(round_to_nearest(23.0, 15), 30); + // raw=23_000ms, scrape=15_000ms → nearest=30_000 (23 is closer to 30 than 15) + assert_eq!(round_to_nearest(23_000.0, 15_000), 30_000); } #[test] fn misaligned_still_returns_result() { - // raw=22s, scrape=15 → rounds to 15, but |22-15|/15=46.7% ≥ 10% → warn emitted + // raw=22_000ms, scrape=15_000ms → rounds to 15_000, but |22-15|/15=46.7% ≥ 10% → warn emitted // We only verify the value is returned (warning is logged, not returned) - assert_eq!(round_to_nearest(22.0, 15), 15); + assert_eq!(round_to_nearest(22_000.0, 15_000), 15_000); } #[test] fn range_duration_from_start_end() { let entries = make_range_entries(base_ts(), 60, 3600, 30, 5); - let (_, ranges) = infer_queries(&entries, 15); + let (_, ranges) = infer_queries(&entries, 15_000); assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0].range_duration, 3600); - assert_eq!(ranges[0].step, 30); + assert_eq!(ranges[0].range_duration_ms, 3_600_000); + assert_eq!(ranges[0].step_ms, 30_000); } #[test] @@ -268,7 +269,7 @@ mod tests { let range_entries = make_range_entries(base_ts(), 60, 3600, 30, 2); // step=30, 2 occurrences entries.extend(range_entries); - let (instants, ranges) = infer_queries(&entries, 15); + let (instants, ranges) = infer_queries(&entries, 15_000); // step=0 variant has more occurrences → kept as instant assert_eq!(instants.len(), 1); // step=30 variant discarded diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml index c6b81011..30d55df0 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(errors_total[5m]) / rate(requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml index 4dc2eabe..18b8fd4d 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml @@ -3,7 +3,7 @@ query_groups: queries: - "rate(errors_total[5m]) / rate(requests_total[5m])" - "rate(errors_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml index f5abf8a3..879d918a 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "(rate(a_total[5m]) + rate(b_total[5m])) / rate(c_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml index 6f53e4cf..2542bb7f 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "foo(errors_total[5m]) / rate(requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml index 35e7dd32..63d2e22b 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(errors_total[5m]) * 100" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml index 40d6c1f0..6e53ddac 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml index 2fcd7996..96637d51 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml b/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml index 4fe88975..0e8017b0 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml @@ -4,14 +4,14 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 - id: 2 queries: - "increase(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml b/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml index d7235571..38d23e94 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "increase(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml b/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml index c4cac069..79d11262 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml @@ -3,7 +3,7 @@ query_groups: queries: - "rate(http_requests_total[5m])" - "sum by (job) (rate(http_requests_total[5m]))" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml b/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml index 0d37afb6..a5f4c1b9 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "quantile_over_time(0.99, http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml b/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml index 2fcd7996..96637d51 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml b/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml index 2fcd7996..96637d51 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml b/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml index 91b9ca99..4e2ba500 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "quantile by (job) (0.99, http_requests_total)" - repetition_delay: 60 + repetition_delay_ms: 60000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml index 7c68d86e..ad8aaf50 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "sum by (job, method) (http_requests_total)" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml index 1324c781..9a9b57bd 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "sum by (job, method) (http_requests_total)" - repetition_delay: 60 + repetition_delay_ms: 60000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml index c560bddc..8e0eee95 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "sum_over_time(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml b/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml index 4e74cb84..23017f48 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml @@ -5,7 +5,7 @@ query_groups: - "increase(http_requests_total[5m])" - "sum_over_time(http_requests_total[5m])" - "quantile_over_time(0.99, http_requests_total[5m])" - repetition_delay: 60 + repetition_delay_ms: 60000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml b/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml index 3d996d8c..7ab4b3cb 100644 --- a/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml +++ b/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml @@ -2,7 +2,7 @@ query_groups: - id: 1 queries: - "topk(5, http_requests_total)" - repetition_delay: 60 + repetition_delay_ms: 60000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 diff --git a/asap-planner-rs/tests/elastic_dsl_integration.rs b/asap-planner-rs/tests/elastic_dsl_integration.rs index d46cdf49..7d248853 100644 --- a/asap-planner-rs/tests/elastic_dsl_integration.rs +++ b/asap-planner-rs/tests/elastic_dsl_integration.rs @@ -149,7 +149,7 @@ fn elastic_sum_produces_basic_plan_and_schema() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics".to_string()) @@ -201,7 +201,7 @@ fn elastic_avg_produces_three_configs() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("MultipleSum")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("MultipleSum"), Some("metrics".to_string()) @@ -254,7 +254,7 @@ fn elastic_min_produces_exact_plan() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("MultipleMinMax")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("MultipleMinMax"), Some("metrics".to_string()) @@ -307,7 +307,7 @@ fn elastic_percentiles_produce_kll_plan() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics".to_string()) @@ -420,7 +420,7 @@ aggregate_cleanup: assert_eq!(out.inference_query_count(), 2); assert!(out.has_aggregation_type("MultipleSum")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); let inference_config = out .to_inference_config(QueryLanguage::elastic_querydsl) diff --git a/asap-planner-rs/tests/integration.rs b/asap-planner-rs/tests/integration.rs index 5426ac76..8501580d 100644 --- a/asap-planner-rs/tests/integration.rs +++ b/asap-planner-rs/tests/integration.rs @@ -6,11 +6,11 @@ use std::path::Path; fn arroyo_opts() -> RuntimeOptions { RuntimeOptions { - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, streaming_engine: StreamingEngine::Arroyo, enable_punting: false, - range_duration: 0, - step: 0, + range_duration_ms: 0, + step_ms: 0, } } @@ -81,8 +81,8 @@ fn query_log_range_produces_valid_configs() { ) .unwrap(); let out = c.generate().unwrap(); - // range_only.log has step=30, so effective window size must be ≤ 30 - assert!(out.all_tumbling_window_sizes_leq(30)); + // range_only.log has step=30 (seconds, native log format) → step_ms=30_000 + assert!(out.all_tumbling_window_sizes_leq(30_000)); } #[test] @@ -162,7 +162,7 @@ fn only_spatial_window_equals_scrape_interval() { ) .unwrap(); let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(15)); + assert!(out.all_tumbling_window_sizes_eq(15_000)); } #[test] @@ -193,8 +193,8 @@ fn topk_produces_count_min_sketch_with_heap() { #[test] fn range_query_uses_effective_repeat() { let opts = RuntimeOptions { - range_duration: 3600, - step: 30, + range_duration_ms: 3_600_000, + step_ms: 30_000, ..arroyo_opts() }; let c = Controller::from_file_with_schema( @@ -204,7 +204,7 @@ fn range_query_uses_effective_repeat() { ) .unwrap(); let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_leq(30)); + assert!(out.all_tumbling_window_sizes_leq(30_000)); } #[test] @@ -223,11 +223,11 @@ fn output_files_written_to_dir() { #[test] fn rate_tumbling_window_size_equals_effective_repeat() { - // For range queries, effective_repeat = min(t_repeat=300, step=30) = 30 + // For range queries, effective_repeat = min(t_repeat=300_000, step=30_000) = 30_000 // Tumbling window size must equal effective_repeat (sliding is always disabled) let opts = RuntimeOptions { - range_duration: 3600, - step: 30, + range_duration_ms: 3_600_000, + step_ms: 30_000, ..arroyo_opts() }; let c = Controller::from_file_with_schema( @@ -237,15 +237,15 @@ fn rate_tumbling_window_size_equals_effective_repeat() { ) .unwrap(); let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(30)); + assert!(out.all_tumbling_window_sizes_eq(30_000)); } #[test] fn increase_tumbling_window_size_equals_effective_repeat() { - // effective_repeat = min(t_repeat=300, step=30) = 30 + // effective_repeat = min(t_repeat=300_000, step=30_000) = 30_000 let opts = RuntimeOptions { - range_duration: 3600, - step: 30, + range_duration_ms: 3_600_000, + step_ms: 30_000, ..arroyo_opts() }; let c = Controller::from_file_with_schema( @@ -256,15 +256,15 @@ fn increase_tumbling_window_size_equals_effective_repeat() { .unwrap(); let out = c.generate().unwrap(); assert!(out.has_aggregation_type("MultipleIncrease")); - assert!(out.all_tumbling_window_sizes_eq(30)); + assert!(out.all_tumbling_window_sizes_eq(30_000)); } #[test] fn quantile_over_time_tumbling_window_size_equals_effective_repeat() { - // effective_repeat = min(t_repeat=300, step=30) = 30 + // effective_repeat = min(t_repeat=300_000, step=30_000) = 30_000 let opts = RuntimeOptions { - range_duration: 3600, - step: 30, + range_duration_ms: 3_600_000, + step_ms: 30_000, ..arroyo_opts() }; let c = Controller::from_file_with_schema( @@ -275,7 +275,7 @@ fn quantile_over_time_tumbling_window_size_equals_effective_repeat() { .unwrap(); let out = c.generate().unwrap(); assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(out.all_tumbling_window_sizes_eq(30)); + assert!(out.all_tumbling_window_sizes_eq(30_000)); } #[test] @@ -332,7 +332,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 @@ -355,7 +355,7 @@ query_groups: queries: - "rate(http_requests_total[5m])" - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 @@ -374,14 +374,14 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 - id: 2 queries: - "rate(http_requests_total[5m])" - repetition_delay: 60 + repetition_delay_ms: 60000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 @@ -401,7 +401,7 @@ query_groups: - id: 1 queries: - "rate(unknown_metric[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 controller_options: accuracy_sla: 0.99 latency_sla: 1.0 @@ -429,7 +429,7 @@ query_groups: - id: 1 queries: - "rate(http_requests_total[5m])" - repetition_delay: 300 + repetition_delay_ms: 300000 metrics: - metric: "http_requests_total" labels: ["instance"] @@ -445,7 +445,7 @@ metrics: #[test] fn temporal_overlapping_window_size_equals_t_repeat() { - // [5m] range repeated every 60s → windowSize = 60, not 300 + // [5m] range repeated every 60_000ms → windowSizeMs = 60_000, not 300_000 let c = Controller::from_file_with_schema( Path::new("tests/comparison/test_data/configs/temporal_overlapping.yaml"), http_requests_schema(), @@ -453,7 +453,7 @@ fn temporal_overlapping_window_size_equals_t_repeat() { ) .unwrap(); let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(60)); + assert!(out.all_tumbling_window_sizes_eq(60_000)); } #[test] @@ -587,3 +587,49 @@ fn temporal_overlapping_rate_increase_deduped() { assert_eq!(out.inference_query_count(), 4); assert_eq!(out.streaming_aggregation_count(), 4); // not 5 } + +// --- Sub-second scrape interval (issue #398) --- +// This is the actual capability the issue asks for: a 100ms scrape interval +// rounds to 0 under the old whole-seconds representation. These tests would +// have failed before the ms-precision rename and must pass after it. + +#[test] +fn sub_second_scrape_interval_window_size_equals_scrape_interval() { + // OnlySpatial query: window size = scrape interval (planner/window.rs). + // 100ms would be indistinguishable from 0 under the old seconds-only model. + let opts = RuntimeOptions { + prometheus_scrape_interval_ms: 100, + ..arroyo_opts() + }; + let c = Controller::from_file_with_schema( + Path::new("tests/comparison/test_data/configs/spatial_quantile.yaml"), + http_requests_schema(), + opts, + ) + .unwrap(); + let out = c.generate().unwrap(); + assert!(out.all_tumbling_window_sizes_eq(100)); +} + +#[test] +fn sub_second_scrape_interval_round_trips_through_generated_yaml() { + // The generated streaming_config.yaml must carry the sub-second value through + // under the renamed wire key (windowSizeMs, not windowSize) — not silently + // rounded or truncated. + let opts = RuntimeOptions { + prometheus_scrape_interval_ms: 100, + ..arroyo_opts() + }; + let c = Controller::from_file_with_schema( + Path::new("tests/comparison/test_data/configs/spatial_quantile.yaml"), + http_requests_schema(), + opts, + ) + .unwrap(); + let out = c.generate().unwrap(); + let yaml = out.to_streaming_yaml_string().unwrap(); + assert!( + yaml.contains("windowSizeMs: 100"), + "expected a sub-second windowSizeMs in the generated YAML, got:\n{yaml}" + ); +} diff --git a/asap-planner-rs/tests/sql_integration.rs b/asap-planner-rs/tests/sql_integration.rs index 1b210740..5c281547 100644 --- a/asap-planner-rs/tests/sql_integration.rs +++ b/asap-planner-rs/tests/sql_integration.rs @@ -77,7 +77,7 @@ fn temporal_sum() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics_table".to_string()) @@ -114,7 +114,7 @@ fn temporal_count() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics_table".to_string()) @@ -151,7 +151,7 @@ fn temporal_min() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("MultipleMinMax")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("MultipleMinMax"), Some("metrics_table".to_string()) @@ -188,7 +188,7 @@ fn temporal_max() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("MultipleMinMax")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("MultipleMinMax"), Some("metrics_table".to_string()) @@ -225,7 +225,7 @@ fn temporal_avg() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics_table".to_string()) @@ -262,7 +262,7 @@ fn temporal_quantile() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics_table".to_string()) @@ -308,7 +308,7 @@ fn temporal_quantile_half_open() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics_table".to_string()) @@ -369,7 +369,7 @@ fn temporal_quantile_percentile_syntax() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics_table".to_string()) @@ -406,7 +406,7 @@ fn temporal_quantile_quoted_dateadd_unit() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics_table".to_string()) @@ -443,7 +443,7 @@ fn temporal_quantile_cast_datetime_bounds() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("DatasketchesKLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); + assert!(out.all_tumbling_window_sizes_eq(300_000)); assert_eq!( out.aggregation_table_name("DatasketchesKLL"), Some("metrics_table".to_string()) @@ -512,7 +512,7 @@ fn spatial_count_distinct_hll() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("HLL")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(1)); + assert!(out.all_tumbling_window_sizes_eq(1_000)); assert_eq!( out.aggregation_table_name("HLL"), Some("netflow_table".to_string()) @@ -554,7 +554,7 @@ fn temporal_sum_t30() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(30)); + assert!(out.all_tumbling_window_sizes_eq(30_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics_table".to_string()) @@ -592,7 +592,7 @@ fn temporal_sum_t600() { assert_eq!(out.inference_query_count(), 1); assert!(out.has_aggregation_type("CountMinSketch")); assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(600)); + assert!(out.all_tumbling_window_sizes_eq(600_000)); assert_eq!( out.aggregation_table_name("CountMinSketch"), Some("metrics_table".to_string()) @@ -913,7 +913,7 @@ fn spatial_count_topk_heap() { assert!(out.has_aggregation_type_and_sub_type("CountMinSketchWithHeap", "topk")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); assert!(!out.has_aggregation_type("CountMinSketch")); - assert!(out.all_tumbling_window_sizes_eq(1)); + assert!(out.all_tumbling_window_sizes_eq(1_000)); assert_eq!( out.aggregation_labels("CountMinSketchWithHeap", "grouping"), Vec::::new() @@ -954,7 +954,7 @@ fn spatial_sum_topk_heap() { assert!(out.has_aggregation_type_and_sub_type("CountMinSketchWithHeap", "topk")); assert!(!out.has_aggregation_type("DeltaSetAggregator")); assert!(!out.has_aggregation_type("CountMinSketch")); - assert!(out.all_tumbling_window_sizes_eq(1)); + assert!(out.all_tumbling_window_sizes_eq(1_000)); assert_eq!( out.aggregation_labels("CountMinSketchWithHeap", "grouping"), Vec::::new() diff --git a/asap-query-engine/benches/simple_store_bench.rs b/asap-query-engine/benches/simple_store_bench.rs index ef49abc4..831f019d 100644 --- a/asap-query-engine/benches/simple_store_bench.rs +++ b/asap-query-engine/benches/simple_store_bench.rs @@ -135,8 +135,8 @@ fn make_agg_config( KeyByLabelNames::empty(), KeyByLabelNames::empty(), "".to_string(), - 60, // window_size (seconds) - 60, // slide_interval (seconds) + 60_000, // window_size_ms + 60_000, // slide_interval_ms WindowType::Tumbling, // window_type "".to_string(), // spatial_filter metric.to_string(), diff --git a/asap-query-engine/examples/engine_config.yaml b/asap-query-engine/examples/engine_config.yaml index 753ccfdf..8dc31486 100644 --- a/asap-query-engine/examples/engine_config.yaml +++ b/asap-query-engine/examples/engine_config.yaml @@ -12,16 +12,14 @@ output_dir: "./output" log_level: "INFO" # DEBUG | INFO | WARN | ERROR (also respects RUST_LOG) -# Prometheus scrape interval in seconds. Used by the query tracker and planner. -prometheus_scrape_interval: 15 +# Prometheus scrape interval in milliseconds. Used by the query tracker and planner. +prometheus_scrape_interval_ms: 15000 # Processing backend. Must be consistent with the ingest source chosen below: # arroyo → requires ingest.type=kafka # precompute → requires ingest.type=http_remote_write or csv streaming_engine: "precompute" # arroyo | precompute -do_profiling: false - # --------------------------------------------------------------------------- # HTTP query server # --------------------------------------------------------------------------- diff --git a/asap-query-engine/examples/promql/streaming_config.yaml b/asap-query-engine/examples/promql/streaming_config.yaml index 03001c95..73241995 100644 --- a/asap-query-engine/examples/promql/streaming_config.yaml +++ b/asap-query-engine/examples/promql/streaming_config.yaml @@ -9,7 +9,7 @@ aggregations: metric: fake_metric parameters: K: 20 - windowSize: 1 + windowSizeMs: 1000 windowType: tumbling spatialFilter: '' metrics: diff --git a/asap-query-engine/examples/sql/streaming_config.yaml b/asap-query-engine/examples/sql/streaming_config.yaml index dc5265e4..8dee664d 100644 --- a/asap-query-engine/examples/sql/streaming_config.yaml +++ b/asap-query-engine/examples/sql/streaming_config.yaml @@ -15,6 +15,6 @@ aggregations: value_column: cpu_usage parameters: K: 20 - windowSize: 1 + windowSizeMs: 1000 windowType: tumbling spatialFilter: '' diff --git a/asap-query-engine/src/bin/bench_precompute_sketch.rs b/asap-query-engine/src/bin/bench_precompute_sketch.rs index f2646956..0dd909ae 100644 --- a/asap-query-engine/src/bin/bench_precompute_sketch.rs +++ b/asap-query-engine/src/bin/bench_precompute_sketch.rs @@ -121,7 +121,7 @@ fn make_timeseries(metric: &str, label_0: &str, samples: Vec) -> TimeSer fn make_kll_streaming_config( aggregation_id: u64, - window_size_secs: u64, + window_size_ms: u64, k: u16, ) -> Arc { let mut params = HashMap::new(); @@ -136,8 +136,8 @@ fn make_kll_streaming_config( promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - window_size_secs, - window_size_secs, + window_size_ms, + window_size_ms, WindowType::Tumbling, "bench_metric".to_string(), "bench_metric".to_string(), @@ -274,7 +274,7 @@ async fn run_latency_benchmark( let aggregation_id = 101; let metric = "bench_metric"; let window_size_ms = (args.window_size_secs * 1000) as i64; - let streaming_config = make_kll_streaming_config(aggregation_id, args.window_size_secs, args.k); + let streaming_config = make_kll_streaming_config(aggregation_id, window_size_ms as u64, args.k); let store = make_store(streaming_config.clone()); let sink = Arc::new(TrackingStoreSink::new(store.clone())); @@ -327,7 +327,7 @@ async fn run_latency_benchmark( metric, aggregation_id, 0, - ((args.latency_repetitions as u64) + 10) * args.window_size_secs * 1000, + ((args.latency_repetitions as u64) + 10) * window_size_ms as u64, )?; let stored_windows: usize = latency_store_results .values() @@ -370,7 +370,7 @@ async fn run_throughput_benchmark( let total_samples = (args.num_requests * args.num_series * args.samples_per_series) as u64; let expected_outputs = (args.num_requests * args.num_series) as u64; - let streaming_config = make_kll_streaming_config(aggregation_id, args.window_size_secs, args.k); + let streaming_config = make_kll_streaming_config(aggregation_id, window_size_ms as u64, args.k); let store = make_store(streaming_config.clone()); let sink = Arc::new(TrackingStoreSink::new(store.clone())); @@ -431,7 +431,7 @@ async fn run_throughput_benchmark( metric, aggregation_id, 0, - ((args.num_requests as u64) + 2) * args.window_size_secs * 1000, + ((args.num_requests as u64) + 2) * window_size_ms as u64, )?; let stored_windows: usize = store_results.values().map(|buckets| buckets.len()).sum(); diff --git a/asap-query-engine/src/bin/e2e_quickstart_resource_test.rs b/asap-query-engine/src/bin/e2e_quickstart_resource_test.rs index cc34b502..7be32efd 100644 --- a/asap-query-engine/src/bin/e2e_quickstart_resource_test.rs +++ b/asap-query-engine/src/bin/e2e_quickstart_resource_test.rs @@ -149,8 +149,8 @@ fn make_kll_streaming_config() -> Arc { rollup, aggregated, String::new(), - 10, // window size = 10s (matching quickstart range-duration/step) - 10, // tumbling + 10_000, // window size = 10s (matching quickstart range-duration/step) + 10_000, // tumbling WindowType::Tumbling, "sensor_reading".to_string(), "sensor_reading".to_string(), diff --git a/asap-query-engine/src/bin/precompute_engine.rs b/asap-query-engine/src/bin/precompute_engine.rs index 8c479980..8f32d080 100644 --- a/asap-query-engine/src/bin/precompute_engine.rs +++ b/asap-query-engine/src/bin/precompute_engine.rs @@ -144,7 +144,7 @@ async fn main() -> Result<(), Box> { store.clone(), inference_config, streaming_config.clone(), - 15, // default prometheus scrape interval + 15_000, // default prometheus scrape interval (ms) QueryLanguage::promql, )); let http_config = HttpServerConfig { diff --git a/asap-query-engine/src/bin/test_e2e_precompute.rs b/asap-query-engine/src/bin/test_e2e_precompute.rs index 085aa9c3..6f542bfa 100644 --- a/asap-query-engine/src/bin/test_e2e_precompute.rs +++ b/asap-query-engine/src/bin/test_e2e_precompute.rs @@ -33,7 +33,7 @@ use std::sync::Arc; const INGEST_PORT: u16 = 19090; const QUERY_PORT: u16 = 18080; const RAW_INGEST_PORT: u16 = 19091; -const SCRAPE_INTERVAL: u64 = 1; // 1 second to match tumblingWindowSize +const SCRAPE_INTERVAL_MS: u64 = 1000; // 1s to match windowSizeMs in examples/promql/streaming_config.yaml fn build_remote_write_body(timeseries: Vec) -> Vec { let write_req = WriteRequest { timeseries }; @@ -122,7 +122,7 @@ async fn main() -> Result<(), Box> { store.clone(), inference_config, streaming_config.clone(), - SCRAPE_INTERVAL, + SCRAPE_INTERVAL_MS, QueryLanguage::promql, )); let http_config = HttpServerConfig { @@ -601,10 +601,10 @@ struct BenchRunConfig { /// Build an AggregationConfig for Sum with specified window parameters. fn make_sum_agg_config( agg_id: u64, - window_size_secs: u64, - slide_interval_secs: u64, + window_size_ms: u64, + slide_interval_ms: u64, ) -> AggregationConfig { - let window_type = if slide_interval_secs == 0 || slide_interval_secs == window_size_secs { + let window_type = if slide_interval_ms == 0 || slide_interval_ms == window_size_ms { WindowType::Tumbling } else { WindowType::Sliding @@ -618,8 +618,8 @@ fn make_sum_agg_config( promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - window_size_secs, - slide_interval_secs, + window_size_ms, + slide_interval_ms, window_type, "bench_metric".to_string(), "bench_metric".to_string(), @@ -779,10 +779,10 @@ async fn run_windowed_benchmarks( let num_series = 50u64; let configs: Vec<(&str, u16, u64, u64)> = vec![ - // (label, port, window_size_secs, slide_interval_secs) - ("Tumbling 10s Sum", 19100, 10, 0), - ("Sliding 30s/10s Sum", 19101, 30, 10), - ("Sliding 60s/10s Sum (W=6)", 19102, 60, 10), + // (label, port, window_size_ms, slide_interval_ms) + ("Tumbling 10s Sum", 19100, 10_000, 0), + ("Sliding 30s/10s Sum", 19101, 30_000, 10_000), + ("Sliding 60s/10s Sum (W=6)", 19102, 60_000, 10_000), ]; println!("\n=== Windowed aggregation benchmarks ({num_requests} req × {samples_per_request} samples, {num_series} series) ==="); @@ -833,7 +833,7 @@ async fn run_scalability_benchmark( let port = base_port + i as u16; let label = format!("{num_workers}"); - let agg_config = make_sum_agg_config(200 + i as u64, 30, 10); + let agg_config = make_sum_agg_config(200 + i as u64, 30_000, 10_000); let mut agg_map = HashMap::new(); agg_map.insert(200 + i as u64, agg_config); let sc = Arc::new(StreamingConfig::new(agg_map)); diff --git a/asap-query-engine/src/engine_config.rs b/asap-query-engine/src/engine_config.rs index 1bd78fbd..8385ab58 100644 --- a/asap-query-engine/src/engine_config.rs +++ b/asap-query-engine/src/engine_config.rs @@ -31,8 +31,8 @@ pub fn check_config(config: &EngineConfig) -> Result<(), String> { return Err("ingest.ts_step_ms is required when ingest.timestamp_col is not set".into()); } - if config.prometheus_scrape_interval == 0 { - return Err("prometheus_scrape_interval must be greater than 0".into()); + if config.prometheus_scrape_interval_ms == 0 { + return Err("prometheus_scrape_interval_ms must be greater than 0".into()); } if config.query_tracker.enabled && !matches!(config.backend, BackendConfig::Prometheus { .. }) { @@ -43,11 +43,11 @@ pub fn check_config(config: &EngineConfig) -> Result<(), String> { } #[derive(Debug, serde::Deserialize)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct EngineConfig { pub output_dir: String, pub log_level: String, - pub prometheus_scrape_interval: u64, + pub prometheus_scrape_interval_ms: u64, pub streaming_engine: StreamingEngine, pub http_server: HttpServerSettings, pub backend: BackendConfig, @@ -65,7 +65,7 @@ impl Default for EngineConfig { Self { output_dir: "./output".to_string(), log_level: "INFO".to_string(), - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, streaming_engine: StreamingEngine::Precompute, http_server: HttpServerSettings::default(), backend: BackendConfig::default(), @@ -522,7 +522,7 @@ streaming_engine: "precompute" ingest: type: "http_remote_write" port: 9090 -prometheus_scrape_interval: 0 +prometheus_scrape_interval_ms: 0 output_dir: "./output" "#; let config: EngineConfig = Figment::new().merge(Yaml::string(yaml)).extract().unwrap(); diff --git a/asap-query-engine/src/engines/simple_engine/mod.rs b/asap-query-engine/src/engines/simple_engine/mod.rs index 3cd069f1..d734d0e4 100644 --- a/asap-query-engine/src/engines/simple_engine/mod.rs +++ b/asap-query-engine/src/engines/simple_engine/mod.rs @@ -46,7 +46,9 @@ pub struct QueryMetadata { pub struct StoreQueryParams { pub metric: String, pub aggregation_id: u64, + /// Milliseconds since epoch. pub start_timestamp: u64, + /// Milliseconds since epoch. pub end_timestamp: u64, /// true for sliding windows (exact match), false for tumbling (range) pub is_exact_query: bool, @@ -63,7 +65,9 @@ pub struct StoreQueryPlan { /// Timestamps for query execution #[derive(Debug, Clone)] pub struct QueryTimestamps { + /// Milliseconds since epoch. pub start_timestamp: u64, + /// Milliseconds since epoch. pub end_timestamp: u64, } @@ -132,7 +136,7 @@ pub struct SimpleEngine { /// Updated at runtime via update_streaming_config(). Readers briefly lock to /// clone the Arc pointer, then use without holding the lock. streaming_config: RwLock>, - prometheus_scrape_interval: u64, + prometheus_scrape_interval_ms: u64, controller_patterns: HashMap>, query_language: QueryLanguage, } @@ -143,7 +147,7 @@ impl SimpleEngine { // promsketch_store: Option>, inference_config: InferenceConfig, streaming_config: Arc, - prometheus_scrape_interval: u64, + prometheus_scrape_interval_ms: u64, query_language: QueryLanguage, ) -> Self { // Create temporal pattern blocks @@ -283,7 +287,7 @@ impl SimpleEngine { // promsketch_store, inference_config: RwLock::new(inference_config), streaming_config: RwLock::new(streaming_config), - prometheus_scrape_interval, + prometheus_scrape_interval_ms, controller_patterns, query_language, } @@ -328,13 +332,13 @@ impl SimpleEngine { mut end_timestamp: u64, query_pattern_type: QueryPatternType, ) -> u64 { - let interval_ms = self.prometheus_scrape_interval * 1000; + let interval_ms = self.prometheus_scrape_interval_ms; if !end_timestamp.is_multiple_of(interval_ms) { warn!( - "Query end timestamp {} is not aligned with Prometheus scrape interval of {} seconds. \ + "Query end timestamp {} is not aligned with Prometheus scrape interval of {} ms. \ This may lead to inaccurate results.", - end_timestamp, self.prometheus_scrape_interval + end_timestamp, self.prometheus_scrape_interval_ms ); } @@ -344,8 +348,8 @@ impl SimpleEngine { { let aligned_end_timestamp = (end_timestamp / interval_ms) * interval_ms; debug!( - "OnlySpatial query: Aligning end_timestamp from {} to {} using scrape interval of {} seconds", - end_timestamp, aligned_end_timestamp, self.prometheus_scrape_interval + "OnlySpatial query: Aligning end_timestamp from {} to {} using scrape interval of {} ms", + end_timestamp, aligned_end_timestamp, self.prometheus_scrape_interval_ms ); end_timestamp = aligned_end_timestamp; } @@ -372,7 +376,7 @@ impl SimpleEngine { .read() .unwrap() .get_aggregation_config(agg_info.aggregation_id_for_key) - .map(|config| config.window_size * 1000) + .map(|config| config.window_size_ms) .ok_or_else(|| { format!( "Failed to get window size for aggregation {}", @@ -420,7 +424,7 @@ impl SimpleEngine { let (values_start, values_end) = if is_exact_query { // Sliding window: exact window match let exact_start = - timestamps.end_timestamp - (aggregation_config_for_value.window_size * 1000); + timestamps.end_timestamp - aggregation_config_for_value.window_size_ms; (exact_start, timestamps.end_timestamp) } else { // Tumbling window: range query diff --git a/asap-query-engine/src/engines/simple_engine/promql.rs b/asap-query-engine/src/engines/simple_engine/promql.rs index 3426b1c6..88a2cf2c 100644 --- a/asap-query-engine/src/engines/simple_engine/promql.rs +++ b/asap-query-engine/src/engines/simple_engine/promql.rs @@ -35,12 +35,15 @@ impl SimpleEngine { ) -> u64 { match query_pattern_type { QueryPatternType::OnlyTemporal | QueryPatternType::OneTemporalOneSpatial => { - let range_seconds = match_result.get_range_duration().unwrap().num_seconds() as u64; - end_timestamp - (range_seconds * 1000) - } - QueryPatternType::OnlySpatial => { - end_timestamp - (self.prometheus_scrape_interval * 1000) + // promql-parser supports a literal `ms` duration suffix (e.g. `[500ms]`), + // so .num_seconds() would truncate genuinely sub-second range vectors to 0. + let range_ms = match_result + .get_range_duration() + .unwrap() + .num_milliseconds() as u64; + end_timestamp - range_ms } + QueryPatternType::OnlySpatial => end_timestamp - self.prometheus_scrape_interval_ms, } } @@ -486,7 +489,7 @@ impl SimpleEngine { .read() .unwrap() .get_aggregation_config(base_context.agg_info.aggregation_id_for_value) - .map(|c| c.window_size * 1000)?; + .map(|c| c.window_size_ms)?; self.validate_range_query_params(start_ms, end_ms, step_ms, tumbling_window_ms) .map_err(|e| { @@ -1093,7 +1096,7 @@ impl SimpleEngine { .read() .unwrap() .get_aggregation_config(base_context.agg_info.aggregation_id_for_value) - .map(|config| config.window_size * 1000)?; + .map(|config| config.window_size_ms)?; // Validate parameters self.validate_range_query_params(start_ms, end_ms, step_ms, tumbling_window_ms) @@ -1288,8 +1291,8 @@ mod topk_pipeline_tests { aggregated_labels: KeyByLabelNames::new(vec!["srcip".to_string()]), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -1315,7 +1318,7 @@ mod topk_pipeline_tests { store.clone(), inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ); (engine, store) diff --git a/asap-query-engine/src/engines/simple_engine/sql.rs b/asap-query-engine/src/engines/simple_engine/sql.rs index 0e20fe58..eb687c90 100644 --- a/asap-query-engine/src/engines/simple_engine/sql.rs +++ b/asap-query-engine/src/engines/simple_engine/sql.rs @@ -197,9 +197,7 @@ impl SimpleEngine { .get_duration() as u64; end_timestamp - (duration_secs * 1000) } - QueryPatternType::OnlySpatial => { - end_timestamp - (self.prometheus_scrape_interval * 1000) - } + QueryPatternType::OnlySpatial => end_timestamp - self.prometheus_scrape_interval_ms, } } @@ -400,7 +398,10 @@ impl SimpleEngine { } }; - let matcher = SQLPatternMatcher::new(schema, self.prometheus_scrape_interval as f64); + // SQLPatternMatcher (sql_utilities, out of scope for the ms rename) divides a + // seconds-denominated SQL query duration by this value — convert back to seconds. + let matcher = + SQLPatternMatcher::new(schema, self.prometheus_scrape_interval_ms as f64 / 1000.0); let match_result = matcher.query_info_to_pattern(&query_data); debug!("Match result: {:?}", match_result); @@ -1239,8 +1240,8 @@ mod topk_pipeline_tests { aggregated_labels: KeyByLabelNames::new(vec!["srcip".to_string()]), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -1266,7 +1267,7 @@ mod topk_pipeline_tests { store.clone(), inference_config, streaming_config, - 1, // 1s scrape interval ⇒ the 1s window classifies as OnlySpatial + 1000, // 1s scrape interval ⇒ the 1s window classifies as OnlySpatial QueryLanguage::sql, ); (engine, store) @@ -1329,8 +1330,8 @@ mod topk_pipeline_tests { aggregated_labels: KeyByLabelNames::new(vec!["srcip".to_string()]), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -1354,7 +1355,7 @@ mod topk_pipeline_tests { store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::sql, ) } @@ -1391,8 +1392,8 @@ mod topk_pipeline_tests { aggregated_labels: KeyByLabelNames::new(vec!["srcip".to_string()]), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -1414,8 +1415,8 @@ mod topk_pipeline_tests { aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -1452,7 +1453,7 @@ mod topk_pipeline_tests { store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::sql, ) } diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index 7581ab23..b808c6d7 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -117,7 +117,7 @@ async fn main() -> Result<()> { store.clone(), inference_config, streaming_config.clone(), - config.prometheus_scrape_interval, + config.prometheus_scrape_interval_ms, query_language, )); @@ -395,14 +395,14 @@ async fn main() -> Result<()> { let tracker_config = QueryTrackerConfig { observation_window_secs: config.query_tracker.observation_window_secs, - prometheus_scrape_interval: config.prometheus_scrape_interval, + prometheus_scrape_interval_ms: config.prometheus_scrape_interval_ms, }; let runtime_options = asap_planner::RuntimeOptions { - prometheus_scrape_interval: config.prometheus_scrape_interval, + prometheus_scrape_interval_ms: config.prometheus_scrape_interval_ms, streaming_engine: asap_planner::StreamingEngine::Precompute, enable_punting: false, - range_duration: 300, - step: config.prometheus_scrape_interval, + range_duration_ms: 300_000, + step_ms: config.prometheus_scrape_interval_ms, }; let prometheus_url = match &config.backend { BackendConfig::Prometheus { server, .. } => server.clone(), diff --git a/asap-query-engine/src/planner_client.rs b/asap-query-engine/src/planner_client.rs index 42f4a85a..87616928 100644 --- a/asap-query-engine/src/planner_client.rs +++ b/asap-query-engine/src/planner_client.rs @@ -106,10 +106,10 @@ mod tests { query_groups: vec![QueryGroup { id: Some(1), queries: vec!["sum(rate(http_requests_total[5m]))".to_string()], - repetition_delay: 60, + repetition_delay_ms: 60_000, controller_options: ControllerOptions::default(), - step: None, - range_duration: None, + step_ms: None, + range_duration_ms: None, }], metrics: Some(vec![MetricDefinition { metric: "http_requests_total".to_string(), @@ -124,11 +124,11 @@ mod tests { fn sample_runtime_options() -> RuntimeOptions { RuntimeOptions { - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, streaming_engine: StreamingEngine::Precompute, enable_punting: false, - range_duration: 300, - step: 15, + range_duration_ms: 300_000, + step_ms: 15_000, } } diff --git a/asap-query-engine/src/precompute_engine/accumulator_factory.rs b/asap-query-engine/src/precompute_engine/accumulator_factory.rs index ed25f0ad..cc02854d 100644 --- a/asap-query-engine/src/precompute_engine/accumulator_factory.rs +++ b/asap-query-engine/src/precompute_engine/accumulator_factory.rs @@ -981,7 +981,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1102,7 +1102,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1157,7 +1157,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1190,7 +1190,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1224,7 +1224,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1262,7 +1262,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1293,7 +1293,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1324,7 +1324,7 @@ mod tests { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 60, + 60_000, 0, WindowType::Tumbling, "m".to_string(), @@ -1378,7 +1378,7 @@ mod tests { ]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - 1, + 1_000, 0, WindowType::Tumbling, "netflow_table".to_string(), diff --git a/asap-query-engine/src/precompute_engine/window_manager.rs b/asap-query-engine/src/precompute_engine/window_manager.rs index b6d513b9..3b04b124 100644 --- a/asap-query-engine/src/precompute_engine/window_manager.rs +++ b/asap-query-engine/src/precompute_engine/window_manager.rs @@ -13,14 +13,13 @@ pub struct WindowManager { impl WindowManager { /// Create a new WindowManager. /// - /// `window_size_secs` and `slide_interval_secs` come from `AggregationConfig` - /// (which stores them in seconds). They are converted to milliseconds internally. - pub fn new(window_size_secs: u64, slide_interval_secs: u64) -> Self { - let window_size_ms = (window_size_secs * 1000) as i64; - let slide_interval_ms = if slide_interval_secs == 0 { - window_size_ms // tumbling window - } else { - (slide_interval_secs * 1000) as i64 + /// `window_size_ms` and `slide_interval_ms` come straight from `AggregationConfig`, + /// which is ms-typed already — no conversion needed here. + pub fn new(window_size_ms: u64, slide_interval_ms: u64) -> Self { + let window_size_ms = window_size_ms as i64; + let slide_interval_ms = match slide_interval_ms { + 0 => window_size_ms, // tumbling window + ms => ms as i64, }; Self { window_size_ms, @@ -123,7 +122,7 @@ mod tests { #[test] fn test_tumbling_window_start() { // 60-second (60000ms) tumbling windows - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); assert_eq!(wm.window_start_for(0), 0); assert_eq!(wm.window_start_for(59_999), 0); @@ -134,7 +133,7 @@ mod tests { #[test] fn test_no_closed_windows_on_first_sample() { - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); let closed = wm.closed_windows(i64::MIN, 30_000); assert!(closed.is_empty()); } @@ -142,7 +141,7 @@ mod tests { #[test] fn test_tumbling_window_close() { // 60s tumbling windows - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); // Watermark advances from 30_000 to 70_000 // Window [0, 60_000) closes when wm >= 60_000 @@ -153,7 +152,7 @@ mod tests { #[test] fn test_multiple_window_closes() { // 10s (10000ms) tumbling windows - let wm = WindowManager::new(10, 0); + let wm = WindowManager::new(10_000, 0); // Watermark jumps from 5_000 to 35_000 — closes windows 0, 10_000, 20_000 let closed = wm.closed_windows(5_000, 35_000); @@ -162,14 +161,14 @@ mod tests { #[test] fn test_no_close_when_watermark_stagnant() { - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); let closed = wm.closed_windows(30_000, 30_000); assert!(closed.is_empty()); } #[test] fn test_window_bounds() { - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); assert_eq!(wm.window_bounds(0), (0, 60_000)); assert_eq!(wm.window_bounds(60_000), (60_000, 120_000)); } @@ -177,7 +176,7 @@ mod tests { #[test] fn test_sliding_window() { // 30s window, 10s slide - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); assert_eq!(wm.window_start_for(0), 0); assert_eq!(wm.window_start_for(9_999), 0); @@ -192,7 +191,7 @@ mod tests { #[test] fn test_window_starts_containing_tumbling() { // 60s tumbling windows — each sample belongs to exactly one window - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); let mut starts = wm.window_starts_containing(15_000); starts.sort(); assert_eq!(starts, vec![0]); @@ -205,7 +204,7 @@ mod tests { #[test] fn test_window_starts_containing_sliding() { // 30s window, 10s slide — each sample belongs to 3 windows - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); // t=15_000 belongs to [0, 30_000), [10_000, 40_000) // and [-10_000, 20_000) which starts negative — still returned @@ -224,7 +223,7 @@ mod tests { #[test] fn test_pane_start_for_equals_window_start_for() { // Pane start and window start use the same slide-aligned grid - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); for ts in [0, 5_000, 9_999, 10_000, 15_000, 25_000, 30_000] { assert_eq!(wm.pane_start_for(ts), wm.window_start_for(ts)); } @@ -233,7 +232,7 @@ mod tests { #[test] fn test_panes_for_window_sliding() { // 30s window, 10s slide → 3 panes per window - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); assert_eq!(wm.panes_for_window(0), vec![0, 10_000, 20_000]); assert_eq!(wm.panes_for_window(10_000), vec![10_000, 20_000, 30_000]); @@ -243,7 +242,7 @@ mod tests { #[test] fn test_panes_for_window_tumbling_degeneration() { // 60s tumbling window → 1 pane per window (no merges needed) - let wm = WindowManager::new(60, 0); + let wm = WindowManager::new(60_000, 0); assert_eq!(wm.panes_for_window(0), vec![0]); assert_eq!(wm.panes_for_window(60_000), vec![60_000]); @@ -251,30 +250,30 @@ mod tests { #[test] fn test_slide_interval_ms_accessor() { - let wm_tumbling = WindowManager::new(60, 0); + let wm_tumbling = WindowManager::new(60_000, 0); assert_eq!(wm_tumbling.slide_interval_ms(), 60_000); - let wm_sliding = WindowManager::new(30, 10); + let wm_sliding = WindowManager::new(30_000, 10_000); assert_eq!(wm_sliding.slide_interval_ms(), 10_000); } #[test] fn test_panes_for_window_count() { // W = window_size / slide_interval - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); assert_eq!(wm.panes_for_window(0).len(), 3); // 30/10 = 3 - let wm2 = WindowManager::new(50, 10); + let wm2 = WindowManager::new(50_000, 10_000); assert_eq!(wm2.panes_for_window(0).len(), 5); // 50/10 = 5 - let wm3 = WindowManager::new(60, 0); + let wm3 = WindowManager::new(60_000, 0); assert_eq!(wm3.panes_for_window(0).len(), 1); // tumbling = 1 } #[test] fn test_consecutive_windows_share_panes() { // 30s window, 10s slide — consecutive windows share W-1 = 2 panes - let wm = WindowManager::new(30, 10); + let wm = WindowManager::new(30_000, 10_000); let panes_a = wm.panes_for_window(0); // [0, 10_000, 20_000] let panes_b = wm.panes_for_window(10_000); // [10_000, 20_000, 30_000] diff --git a/asap-query-engine/src/precompute_engine/worker.rs b/asap-query-engine/src/precompute_engine/worker.rs index 1db00f7a..d408c412 100644 --- a/asap-query-engine/src/precompute_engine/worker.rs +++ b/asap-query-engine/src/precompute_engine/worker.rs @@ -348,7 +348,7 @@ impl Worker { // Creation path: requires a config, and allocates the interned key once. let config = Arc::clone(self.agg_configs.get(&agg_id)?); let gs = GroupState { - window_manager: WindowManager::new(config.window_size, config.slide_interval), + window_manager: WindowManager::new(config.window_size_ms, config.slide_interval_ms), config, active_panes: BTreeMap::new(), previous_watermark_ms: i64::MIN, @@ -1133,8 +1133,8 @@ mod tests { "netflow_table", AggregationType::HLL, "", - 1, - 1, + 1000, + 1000, vec!["srcip"], ); config.value_column = Some("dstip".to_string()); @@ -1219,8 +1219,8 @@ mod tests { metric: &str, agg_type: AggregationType, agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, + window_size_ms: u64, + slide_interval_ms: u64, grouping: Vec<&str>, ) -> AggregationConfig { make_agg_config_full( @@ -1228,8 +1228,8 @@ mod tests { metric, agg_type, agg_sub_type, - window_secs, - slide_secs, + window_size_ms, + slide_interval_ms, grouping, vec![], ) @@ -1241,12 +1241,12 @@ mod tests { metric: &str, agg_type: AggregationType, agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, + window_size_ms: u64, + slide_interval_ms: u64, grouping: Vec<&str>, aggregated: Vec<&str>, ) -> AggregationConfig { - let window_type = if slide_secs == 0 || slide_secs == window_secs { + let window_type = if slide_interval_ms == 0 || slide_interval_ms == window_size_ms { WindowType::Tumbling } else { WindowType::Sliding @@ -1264,8 +1264,8 @@ mod tests { ), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - window_secs, - slide_secs, + window_size_ms, + slide_interval_ms, window_type, metric.to_string(), metric.to_string(), @@ -1362,7 +1362,7 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -1428,7 +1428,7 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -1493,7 +1493,7 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec!["pattern"], ); @@ -1566,7 +1566,7 @@ mod tests { "latency", AggregationType::DatasketchesKLL, "", - 10, + 10_000, 0, vec!["pattern"], ); @@ -1650,8 +1650,8 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 30, - 10, + 30_000, + 10_000, vec![], ); let mut agg_configs = HashMap::new(); @@ -1717,7 +1717,7 @@ mod tests { "cpu", AggregationType::MultipleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], // grouping: empty — one output group vec!["host"], // aggregated: host is the key INSIDE the sketch @@ -1802,7 +1802,7 @@ mod tests { "netflow_table", AggregationType::MultipleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], // grouping: empty — one output group vec!["dstip"], // aggregated: dstip is the key INSIDE the sketch @@ -1891,7 +1891,7 @@ mod tests { "cpu", AggregationType::MultipleSum, "sum", - 10, + 10_000, 0, vec!["host"], ); @@ -1967,7 +1967,7 @@ mod tests { "latency", AggregationType::DatasketchesKLL, "", - 10, + 10_000, 0, vec![], ); @@ -2067,7 +2067,7 @@ mod tests { "cpu", AggregationType::MultipleSum, "sum", - 10, + 10_000, 0, vec![], vec!["host"], @@ -2168,7 +2168,7 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2221,7 +2221,7 @@ mod tests { "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2300,9 +2300,9 @@ aggregations: metric: requests_total parameters: {} tumblingWindowSize: 10 - windowSize: 10 + windowSizeMs: 10000 windowType: tumbling - slideInterval: 0 + slideIntervalMs: 0 spatialFilter: '' "#; @@ -2408,7 +2408,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2553,7 +2553,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2607,7 +2607,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, // 10s tumbling window + 10_000, // 10s tumbling window 0, vec![], ); @@ -2738,14 +2738,14 @@ aggregations: // Realistic event time: ~2023-11-14T22:13:20Z in ms. With the old // i64::MAX code this is what made closed_windows blow up. let base_ms: i64 = 1_700_000_000_000; - let window_secs = 10u64; + let window_size_ms = 10_000u64; let config = make_agg_config( 1, "cpu", AggregationType::SingleSubpopulation, "Sum", - window_secs, + window_size_ms, 0, vec![], ); @@ -2802,7 +2802,7 @@ aggregations: "removing the agg_id must force-close the single open window" ); let (output, acc) = &captured[0]; - let window_ms = window_secs as i64 * 1_000; + let window_ms = window_size_ms as i64; let expected_start = base_ms - (base_ms % window_ms); assert_eq!(output.start_timestamp as i64, expected_start); assert_eq!(output.end_timestamp as i64, expected_start + window_ms); @@ -2865,7 +2865,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2932,7 +2932,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); @@ -2979,7 +2979,7 @@ aggregations: "cpu", AggregationType::SingleSubpopulation, "Sum", - 10, + 10_000, 0, vec![], ); diff --git a/asap-query-engine/src/query_tracker/tracker.rs b/asap-query-engine/src/query_tracker/tracker.rs index 08223092..ab2124d2 100644 --- a/asap-query-engine/src/query_tracker/tracker.rs +++ b/asap-query-engine/src/query_tracker/tracker.rs @@ -16,8 +16,8 @@ use crate::planner_client::{PlannerClient, PlannerResult}; pub struct QueryTrackerConfig { /// How often to evaluate and trigger planning (default: 600s = 10 min). pub observation_window_secs: u64, - /// Prometheus scrape interval, passed through to `infer_queries`. - pub prometheus_scrape_interval: u64, + /// Prometheus scrape interval (ms), passed through to `infer_queries`. + pub prometheus_scrape_interval_ms: u64, } pub struct QueryTracker { @@ -121,8 +121,8 @@ impl QueryTracker { entries.len() ); - let scrape_interval = tracker.config.prometheus_scrape_interval; - let (instants, ranges) = infer_queries(&entries, scrape_interval); + let scrape_interval_ms = tracker.config.prometheus_scrape_interval_ms; + let (instants, ranges) = infer_queries(&entries, scrape_interval_ms); info!( "query_tracker: inferred {} instant queries, {} range queries", @@ -228,7 +228,7 @@ mod tests { fn record_instant_appends_entry() { let tracker = make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, }); tracker.record_instant("rate(http_requests_total[5m])", 1700000000.0); tracker.record_instant("rate(http_requests_total[5m])", 1700000060.0); @@ -242,7 +242,7 @@ mod tests { fn record_range_appends_entry() { let tracker = make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, }); tracker.record_range( "rate(http_requests_total[5m])", @@ -260,7 +260,7 @@ mod tests { async fn evaluate_calls_planner_with_entries() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, })); // Record enough entries for infer_queries to produce results (need >=2 per query). @@ -289,7 +289,7 @@ mod tests { async fn evaluate_skips_when_no_entries() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, })); let mock_client = Arc::new(MockPlannerClient::new()); @@ -308,7 +308,7 @@ mod tests { async fn evaluate_sends_plan_only_once() { let tracker = Arc::new(make_tracker(QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, })); // Record enough for infer_queries to produce results. @@ -400,7 +400,7 @@ mod tests { let tracker = Arc::new(QueryTracker::new( QueryTrackerConfig { observation_window_secs: 600, - prometheus_scrape_interval: 15, + prometheus_scrape_interval_ms: 15_000, }, sc, ic, diff --git a/asap-query-engine/src/stores/simple_map_store/mod.rs b/asap-query-engine/src/stores/simple_map_store/mod.rs index 17e6bace..7ac1fd60 100644 --- a/asap-query-engine/src/stores/simple_map_store/mod.rs +++ b/asap-query-engine/src/stores/simple_map_store/mod.rs @@ -169,7 +169,7 @@ mod tests { KeyByLabelNames::new(vec![]), KeyByLabelNames::new(vec![]), String::new(), - 10, + 10_000, 0, WindowType::Tumbling, metric.to_string(), diff --git a/asap-query-engine/src/tests/capability_matching_tests.rs b/asap-query-engine/src/tests/capability_matching_tests.rs index 9cc8d3fb..9311b45e 100644 --- a/asap-query-engine/src/tests/capability_matching_tests.rs +++ b/asap-query-engine/src/tests/capability_matching_tests.rs @@ -27,7 +27,7 @@ fn make_agg_config( id: u64, metric: &str, agg_type: AggregationType, - window_size_s: u64, + window_size_ms: u64, window_type: WindowType, grouping: &[&str], ) -> AggregationConfig { @@ -40,8 +40,8 @@ fn make_agg_config( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: window_size_s, - slide_interval: window_size_s, + window_size_ms, + slide_interval_ms: window_size_ms, window_type, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -75,7 +75,7 @@ fn engine_no_query_configs( // Insert a data point for each aggregation so queries can actually execute. let ts = 1_000_000_u64; for c in &agg_configs { - let window_ms = c.window_size * 1000; + let window_ms = c.window_size_ms; let output = PrecomputedOutput::new(ts - window_ms, ts, None, c.aggregation_id); let acc: Box = match c.aggregation_type.as_str() { "DatasketchesKLL" => { @@ -102,7 +102,7 @@ fn engine_no_query_configs( store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -126,7 +126,7 @@ fn engine_with_query_config( )); let ts = 1_000_000_u64; - let window_ms = agg_config.window_size * 1000; + let window_ms = agg_config.window_size_ms; let output = PrecomputedOutput::new(ts - window_ms, ts, None, agg_id); store .insert_precomputed_output(output, Box::new(SumAccumulator::with_sum(99.0))) @@ -149,7 +149,7 @@ fn engine_with_query_config( store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -166,7 +166,7 @@ fn capability_fallback_fires_when_no_config() { 1, "cpu", AggregationType::Sum, - 300, + 300_000, WindowType::Tumbling, &[], ); @@ -190,7 +190,7 @@ fn config_path_takes_priority_over_capability_matching() { 42, "cpu", AggregationType::Sum, - 300, + 300_000, WindowType::Tumbling, &[], ); @@ -212,7 +212,7 @@ fn quantile_different_values_resolve_to_same_aggregation() { 7, "latency", AggregationType::DatasketchesKLL, - 300, + 300_000, WindowType::Tumbling, &[], ); @@ -250,7 +250,7 @@ fn no_match_returns_none() { 1, "cpu", AggregationType::DatasketchesKLL, - 300, + 300_000, WindowType::Tumbling, &[], ); @@ -271,7 +271,7 @@ fn priority_largest_window_wins() { 1, "cpu", AggregationType::Sum, - 300, + 300_000, WindowType::Tumbling, &[], ); @@ -279,7 +279,7 @@ fn priority_largest_window_wins() { 2, "cpu", AggregationType::Sum, - 900, + 900_000, WindowType::Tumbling, &[], ); diff --git a/asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs b/asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs index 0cdaf6bc..3908c309 100644 --- a/asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs +++ b/asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs @@ -51,7 +51,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -81,7 +81,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -118,7 +118,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -155,7 +155,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -193,7 +193,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -226,7 +226,7 @@ mod tests { vec!["host"], vec![], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -251,7 +251,7 @@ mod tests { Box::new(SumAccumulator::with_sum(1.0)), )], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -275,7 +275,7 @@ mod tests { Box::new(SumAccumulator::with_sum(1.0)), )], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -324,7 +324,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -376,7 +376,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -415,7 +415,7 @@ mod tests { Box::new(SumAccumulator::with_sum(1.0)), )], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -443,7 +443,7 @@ mod tests { Box::new(SumAccumulator::with_sum(1.0)), )], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -467,7 +467,7 @@ mod tests { vec!["host"], vec![], query, - 5, + 5_000, WindowType::Tumbling, ); @@ -503,7 +503,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); @@ -533,7 +533,7 @@ mod tests { vec!["host"], data, query, - 5, + 5_000, WindowType::Tumbling, ); diff --git a/asap-query-engine/src/tests/query_equivalence_tests.rs b/asap-query-engine/src/tests/query_equivalence_tests.rs index b9ca4ca8..b5231e68 100644 --- a/asap-query-engine/src/tests/query_equivalence_tests.rs +++ b/asap-query-engine/src/tests/query_equivalence_tests.rs @@ -80,21 +80,21 @@ mod tests { #[test] fn test_temporal_sum_equivalence() { - let scrape_interval = 1; + let scrape_interval_ms = 1000; let promql_query = "sum_over_time(cpu_usage[10s])"; let sql_query = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"; let grouping_labels = vec!["L1", "L2", "L3", "L4"]; - let window_seconds = 10; + let window_size_ms = 10_000; // Setup test configuration let (promql_config, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(grouping_labels) - .with_scrape_interval(scrape_interval) + .with_scrape_interval_ms(scrape_interval_ms) .add_temporal_query( promql_query, sql_query, 1, - window_seconds, + window_size_ms, WindowType::Tumbling, ) .build_both(); @@ -105,7 +105,7 @@ mod tests { // None, promql_config, streaming_config.clone(), - scrape_interval, + scrape_interval_ms, QueryLanguage::promql, ); @@ -114,7 +114,7 @@ mod tests { // None, sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -135,7 +135,7 @@ mod tests { #[test] fn test_spatial_sum_equivalence() { - let scrape_interval = 1; + let scrape_interval_ms = 1000; let promql_query = "sum(cpu_usage) by (L1, L2)"; let sql_query = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY L1, L2"; let grouping_labels = vec!["L1", "L2"]; @@ -145,7 +145,7 @@ mod tests { let (promql_config, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(grouping_labels) .with_rollup_labels(rollup_labels) - .with_scrape_interval(scrape_interval) + .with_scrape_interval_ms(scrape_interval_ms) .add_spatial_query(promql_query, sql_query, 2) .build_both(); @@ -155,7 +155,7 @@ mod tests { // None, promql_config, streaming_config.clone(), - scrape_interval, + scrape_interval_ms, QueryLanguage::promql, ); @@ -164,7 +164,7 @@ mod tests { // None, sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -188,20 +188,20 @@ mod tests { /// Bug: start = end - (150 * 15 * 1000) = end - 2_250_000ms (15× too wide). #[test] fn test_bug_sql_start_timestamp_multiplied_by_scrape_interval() { - let scrape_interval = 15u64; - let window_seconds = 150u64; + let scrape_interval_ms = 15_000u64; + let window_size_ms = 150_000u64; let sql_query = "SELECT SUM(value) FROM cpu_usage \ WHERE time BETWEEN DATEADD(s, -150, '2025-10-01 00:00:00') AND '2025-10-01 00:00:00' \ GROUP BY L1, L2, L3, L4"; let (_, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(vec!["L1", "L2", "L3", "L4"]) - .with_scrape_interval(scrape_interval) // USED: propagated to SimpleEngine::prometheus_scrape_interval + .with_scrape_interval_ms(scrape_interval_ms) // USED: propagated to SimpleEngine::prometheus_scrape_interval_ms .add_temporal_query( "sum_over_time(cpu_usage[150s])", // NOT USED: TestConfigBuilder requires a PromQL string; timestamp calculation is SQL-only sql_query, // USED: parsed to extract the 150s duration 1, // NOT USED: agg_id just satisfies the builder; calculate_start_timestamp_sql never consults it - window_seconds, // NOT USED: stored in AggregationConfig, not read by timestamp calculation + window_size_ms, // NOT USED: stored in AggregationConfig, not read by timestamp calculation WindowType::Tumbling, // NOT USED: same as above ) .build_both(); @@ -210,7 +210,7 @@ mod tests { Arc::new(NoOpStore), sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -222,7 +222,7 @@ mod tests { let start_ms = sql_context.store_plan.values_query.start_timestamp; let end_ms = sql_context.store_plan.values_query.end_timestamp; let actual_window_ms = end_ms - start_ms; - let expected_window_ms = window_seconds * 1000; + let expected_window_ms = window_size_ms; // already ms — this field is ms-typed end to end now assert_eq!( actual_window_ms, expected_window_ms, @@ -239,7 +239,7 @@ mod tests { /// query forms answer the identical question. #[test] fn test_half_open_equivalent_to_between() { - let scrape_interval = 1; + let scrape_interval_ms = 1000; let promql_query = "sum_over_time(cpu_usage[10s])"; let between_query = "SELECT SUM(value) FROM cpu_usage \ WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:00') AND '2025-10-01 00:00:00' \ @@ -252,15 +252,15 @@ mod tests { // half-open incoming query resolves against it (duration-based match). let (_, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(vec!["L1", "L2", "L3", "L4"]) - .with_scrape_interval(scrape_interval) - .add_temporal_query(promql_query, between_query, 1, 10, WindowType::Tumbling) + .with_scrape_interval_ms(scrape_interval_ms) + .add_temporal_query(promql_query, between_query, 1, 10_000, WindowType::Tumbling) .build_both(); let sql_engine = SimpleEngine::new( Arc::new(NoOpStore), sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -285,7 +285,7 @@ mod tests { /// half-open window that wouldn't match the ClickHouse baseline). #[test] fn test_gt_lte_combination_not_matched() { - let scrape_interval = 1; + let scrape_interval_ms = 1000; let promql_query = "sum_over_time(cpu_usage[10s])"; let between_query = "SELECT SUM(value) FROM cpu_usage \ WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:00') AND '2025-10-01 00:00:00' \ @@ -296,15 +296,15 @@ mod tests { let (_, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(vec!["L1", "L2", "L3", "L4"]) - .with_scrape_interval(scrape_interval) - .add_temporal_query(promql_query, between_query, 1, 10, WindowType::Tumbling) + .with_scrape_interval_ms(scrape_interval_ms) + .add_temporal_query(promql_query, between_query, 1, 10_000, WindowType::Tumbling) .build_both(); let sql_engine = SimpleEngine::new( Arc::new(NoOpStore), sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -318,21 +318,21 @@ mod tests { #[test] fn test_spatial_of_temporal_sum_equivalence() { - let scrape_interval = 1; + let scrape_interval_ms = 1000; let promql_query = "sum(sum_over_time(cpu_usage[10s])) by (L1)"; let sql_query = "SELECT SUM(result) FROM (SELECT SUM(value) AS result FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4) GROUP BY L1"; // let all_labels = vec!["L1", "L2", "L3", "L4"]; let grouping_labels = vec!["L1"]; let rollup_labels = vec!["L2", "L3", "L4"]; - let window_seconds = 10; + let window_size_ms = 10_000; // Setup test configuration // Using SUM of SUM which is collapsable (spatial="sum", temporal="sum_over_time") let (promql_config, sql_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(grouping_labels) .with_rollup_labels(rollup_labels) - .with_scrape_interval(scrape_interval) - .add_spatial_of_temporal_query(promql_query, sql_query, 3, window_seconds) + .with_scrape_interval_ms(scrape_interval_ms) + .add_spatial_of_temporal_query(promql_query, sql_query, 3, window_size_ms) .build_both(); // Create engines @@ -341,7 +341,7 @@ mod tests { // None, promql_config, streaming_config.clone(), - scrape_interval, + scrape_interval_ms, QueryLanguage::promql, ); @@ -350,7 +350,7 @@ mod tests { // None, sql_config, streaming_config, - scrape_interval, + scrape_interval_ms, QueryLanguage::sql, ); @@ -372,4 +372,50 @@ mod tests { "spatial_of_temporal_sum", ); } + + // --- Sub-second range vector (issue #398) --- + // This is the actual capability the issue asks for: a `[500ms]` PromQL range + // vector must produce a 500ms execution window, not round to 0 under the old + // whole-seconds representation. This test would have failed before the + // ms-precision rename (the engine's `.num_seconds()` truncation bug) and + // must pass after it. + #[test] + fn sub_second_range_vector_produces_sub_second_window() { + let scrape_interval_ms = 100; + let promql_query = "sum_over_time(cpu_usage[500ms])"; + let window_size_ms = 500; + + let (promql_config, _, streaming_config) = TestConfigBuilder::new("cpu_usage") + .with_grouping_labels(vec!["L1"]) + .with_scrape_interval_ms(scrape_interval_ms) + .add_temporal_query( + promql_query, + promql_query, + 1, + window_size_ms, + WindowType::Tumbling, + ) + .build_both(); + + let promql_engine = SimpleEngine::new( + Arc::new(NoOpStore), + promql_config, + streaming_config, + scrape_interval_ms, + QueryLanguage::promql, + ); + + let query_time_sec: f64 = 1000.0; + let context = promql_engine + .build_query_execution_context_promql(promql_query.to_string(), query_time_sec) + .expect("Failed to build PromQL context for sub-second range vector"); + + let start_ms = context.store_plan.values_query.start_timestamp; + let end_ms = context.store_plan.values_query.end_timestamp; + assert_eq!( + end_ms - start_ms, + 500, + "[500ms] range vector must produce a 500ms window, got [{start_ms}, {end_ms})" + ); + } } diff --git a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs index 9db601dc..4c9f579c 100644 --- a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs +++ b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs @@ -19,10 +19,10 @@ mod tests { /// Build a minimal SQL SimpleEngine with one template query config. /// - /// * `template_sql` — the NOW()-based query stored in inference_config - /// * `agg_id` — aggregation id - /// * `window_secs` — window size in seconds - fn build_sql_engine(template_sql: &str, agg_id: u64, window_secs: u64) -> SimpleEngine { + /// * `template_sql` — the NOW()-based query stored in inference_config + /// * `agg_id` — aggregation id + /// * `window_size_ms` — window size in ms + fn build_sql_engine(template_sql: &str, agg_id: u64, window_size_ms: u64) -> SimpleEngine { // Schema: cpu_usage table let labels: HashSet = ["L1", "L2", "L3", "L4"] .iter() @@ -62,8 +62,8 @@ mod tests { aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: window_secs, - slide_interval: window_secs, + window_size_ms, + slide_interval_ms: window_size_ms, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -89,7 +89,7 @@ mod tests { store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::sql, ) } @@ -98,7 +98,7 @@ mod tests { fn test_temporal_query_matches_now_template() { // Template in inference_config uses NOW() let template = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); // Incoming query uses absolute timestamps for the same 10s window let incoming = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4"; @@ -116,7 +116,7 @@ mod tests { fn test_spatiotemporal_query_matches_now_template() { // SpatioTemporal: same metric, spans multiple intervals, GROUP BY subset of labels let template = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); let incoming = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1"; let query_time = 1727740810.0_f64; @@ -133,7 +133,7 @@ mod tests { // Spatial: window equals the scrape interval (1s), GROUP BY all labels let template = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"; // scrape_interval=1, window=1 → classified as Spatial by the matcher - let engine = build_sql_engine(template, 1, 1); + let engine = build_sql_engine(template, 1, 1000); let incoming = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -1, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4"; let query_time = 1727740810.0_f64; @@ -149,7 +149,7 @@ mod tests { fn test_temporal_quantile_query_matches_now_template() { // TemporalQuantile: QUANTILE aggregation, window > scrape interval, GROUP BY all labels let template = "SELECT QUANTILE(0.95, value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); let incoming = "SELECT QUANTILE(0.95, value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4"; let query_time = 1727740810.0_f64; @@ -165,7 +165,7 @@ mod tests { fn test_spatial_of_temporal_subquery_matches_now_template() { // Spatial-of-temporal: outer GROUP BY L1 (subset), inner GROUP BY all labels let template = "SELECT SUM(result) FROM (SELECT SUM(value) AS result FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4) GROUP BY L1"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); let incoming = "SELECT SUM(result) FROM (SELECT SUM(value) AS result FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4) GROUP BY L1"; let query_time = 1727740810.0_f64; @@ -182,7 +182,7 @@ mod tests { // Outer ORDER BY + LIMIT on a spatial-over-temporal query must not be routed // through CountMinSketchWithHeap; the inner temporal SUM is not a flat top-k. let template = "SELECT L1, SUM(result) FROM (SELECT SUM(value) AS result FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4) sub GROUP BY L1"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); let incoming = "SELECT L1, SUM(result) AS rollup FROM (SELECT SUM(value) AS result FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4) sub GROUP BY L1 ORDER BY rollup DESC LIMIT 10"; let query_time = 1727740810.0_f64; @@ -201,7 +201,7 @@ mod tests { fn test_no_match_returns_none() { // Engine has a SUM template; incoming uses AVG — should never match let template = "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"; - let engine = build_sql_engine(template, 1, 10); + let engine = build_sql_engine(template, 1, 10_000); let incoming = "SELECT AVG(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, '2025-10-01 00:00:10') AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4"; let query_time = 1727740810.0_f64; diff --git a/asap-query-engine/src/tests/store_correctness_tests.rs b/asap-query-engine/src/tests/store_correctness_tests.rs index b15ce460..46485056 100644 --- a/asap-query-engine/src/tests/store_correctness_tests.rs +++ b/asap-query-engine/src/tests/store_correctness_tests.rs @@ -60,8 +60,8 @@ fn make_agg_config( KeyByLabelNames::empty(), KeyByLabelNames::empty(), "".to_string(), - 60, // window_size (seconds) - 60, // slide_interval (seconds) + 60_000, // window_size_ms + 60_000, // slide_interval_ms WindowType::Tumbling, // window_type "".to_string(), // spatial_filter "cpu_usage".to_string(), diff --git a/asap-query-engine/src/tests/test_utilities/config_builders.rs b/asap-query-engine/src/tests/test_utilities/config_builders.rs index 4bd68ade..d432c204 100644 --- a/asap-query-engine/src/tests/test_utilities/config_builders.rs +++ b/asap-query-engine/src/tests/test_utilities/config_builders.rs @@ -19,7 +19,7 @@ pub struct TestConfigBuilder { value_col: String, grouping_labels: Vec, rollup_labels: Vec, - scrape_interval: u64, + scrape_interval_ms: u64, query_configs: Vec, streaming_configs: HashMap, } @@ -33,7 +33,7 @@ impl TestConfigBuilder { value_col: "value".to_string(), grouping_labels: Vec::new(), rollup_labels: Vec::new(), - scrape_interval: 1, + scrape_interval_ms: 1000, query_configs: Vec::new(), streaming_configs: HashMap::new(), } @@ -62,9 +62,9 @@ impl TestConfigBuilder { self } - /// Set the scrape interval in seconds (default: 1) - pub fn with_scrape_interval(mut self, interval: u64) -> Self { - self.scrape_interval = interval; + /// Set the scrape interval in ms (default: 1000) + pub fn with_scrape_interval_ms(mut self, interval_ms: u64) -> Self { + self.scrape_interval_ms = interval_ms; self } @@ -74,7 +74,7 @@ impl TestConfigBuilder { promql: &str, sql: &str, agg_id: u64, - window_seconds: u64, + window_size_ms: u64, window_type: WindowType, ) -> Self { // Add PromQL query config @@ -97,8 +97,8 @@ impl TestConfigBuilder { aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::new(self.rollup_labels.clone()), original_yaml: String::new(), - window_size: window_seconds, - slide_interval: window_seconds, + window_size_ms, + slide_interval_ms: window_size_ms, window_type, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -134,8 +134,8 @@ impl TestConfigBuilder { aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::new(self.rollup_labels.clone()), original_yaml: String::new(), - window_size: self.scrape_interval, - slide_interval: self.scrape_interval, + window_size_ms: self.scrape_interval_ms, + slide_interval_ms: self.scrape_interval_ms, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -156,7 +156,7 @@ impl TestConfigBuilder { promql: &str, sql: &str, agg_id: u64, - window_seconds: u64, + window_size_ms: u64, ) -> Self { // Add PromQL query config let promql_config = QueryConfig::new(promql.to_string()) @@ -177,8 +177,8 @@ impl TestConfigBuilder { aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::new(self.rollup_labels.clone()), original_yaml: String::new(), - window_size: window_seconds, - slide_interval: window_seconds, + window_size_ms, + slide_interval_ms: window_size_ms, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -290,12 +290,12 @@ mod tests { fn test_builder_creates_valid_configs() { let (inference_config, streaming_config) = TestConfigBuilder::new("cpu_usage") .with_grouping_labels(vec!["L1", "L2", "L3", "L4"]) - .with_scrape_interval(1) + .with_scrape_interval_ms(1000) .add_temporal_query( "sum_over_time(cpu_usage[10s])", "SELECT SUM(value) FROM cpu_usage WHERE time BETWEEN NOW() AND DATEADD(s, -10, NOW()) GROUP BY L1, L2, L3, L4", 1, - 10, + 10_000, WindowType::Tumbling, ) .build(); @@ -316,7 +316,7 @@ mod tests { // Verify streaming config assert!(streaming_config.get_aggregation_config(1).is_some()); let agg_config = streaming_config.get_aggregation_config(1).unwrap(); - assert_eq!(agg_config.window_size, 10); + assert_eq!(agg_config.window_size_ms, 10_000); assert_eq!(agg_config.window_type, WindowType::Tumbling); } diff --git a/asap-query-engine/src/tests/test_utilities/engine_factories.rs b/asap-query-engine/src/tests/test_utilities/engine_factories.rs index 53848f5f..ab44a9b0 100644 --- a/asap-query-engine/src/tests/test_utilities/engine_factories.rs +++ b/asap-query-engine/src/tests/test_utilities/engine_factories.rs @@ -80,8 +80,8 @@ pub fn create_engine_single_pop_with_aggregated( aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -128,7 +128,7 @@ pub fn create_engine_single_pop_with_aggregated( // None, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -177,8 +177,8 @@ pub fn create_engine_dual_input( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -200,8 +200,8 @@ pub fn create_engine_dual_input( aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -256,7 +256,7 @@ pub fn create_engine_dual_input( // None, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -293,8 +293,8 @@ pub fn create_engine_two_metrics( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -315,8 +315,8 @@ pub fn create_engine_two_metrics( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -369,7 +369,7 @@ pub fn create_engine_two_metrics( store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -418,8 +418,8 @@ pub fn create_engine_three_metrics( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -472,7 +472,7 @@ pub fn create_engine_three_metrics( store, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -499,8 +499,8 @@ pub fn create_engine_multi_timestamp( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size: 1, - slide_interval: 1, + window_size_ms: 1000, + slide_interval_ms: 1000, window_type: WindowType::Tumbling, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -546,7 +546,7 @@ pub fn create_engine_multi_timestamp( // None, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } @@ -563,7 +563,7 @@ pub fn create_engine_multi_timestamp_with_window( grouping_labels: Vec<&str>, data: Vec<(u64, Option>, Box)>, promql_query: &str, - window_size: u64, + window_size_ms: u64, window_type: WindowType, ) -> SimpleEngine { let grouping_label_strings: Vec = @@ -579,8 +579,8 @@ pub fn create_engine_multi_timestamp_with_window( aggregated_labels: KeyByLabelNames::empty(), rollup_labels: KeyByLabelNames::empty(), original_yaml: String::new(), - window_size, - slide_interval: 1, + window_size_ms, + slide_interval_ms: 1000, window_type, spatial_filter: String::new(), spatial_filter_normalized: String::new(), @@ -626,7 +626,7 @@ pub fn create_engine_multi_timestamp_with_window( // None, inference_config, streaming_config, - 1, + 1000, QueryLanguage::promql, ) } diff --git a/asap-query-engine/src/utils/file_io.rs b/asap-query-engine/src/utils/file_io.rs index 6fb950bb..4866088a 100644 --- a/asap-query-engine/src/utils/file_io.rs +++ b/asap-query-engine/src/utils/file_io.rs @@ -62,7 +62,7 @@ aggregations: parameters: K: 200 spatialFilter: '' - windowSize: 10 + windowSizeMs: 10000 metrics: fake_metric_total: - instance diff --git a/asap-query-engine/tests/e2e_netflow_single_second.rs b/asap-query-engine/tests/e2e_netflow_single_second.rs index 8c99c200..b4dd93ec 100644 --- a/asap-query-engine/tests/e2e_netflow_single_second.rs +++ b/asap-query-engine/tests/e2e_netflow_single_second.rs @@ -27,7 +27,7 @@ use query_engine_rust::precompute_engine::{ }; use query_engine_rust::precompute_operators::sum_accumulator::SumAccumulator; -fn netflow_agg_config(metric: &str, window_secs: u64) -> AggregationConfig { +fn netflow_agg_config(metric: &str, window_size_ms: u64) -> AggregationConfig { AggregationConfig::new( 1, AggregationType::SingleSubpopulation, @@ -37,7 +37,7 @@ fn netflow_agg_config(metric: &str, window_secs: u64) -> AggregationConfig { promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - window_secs, + window_size_ms, 0, WindowType::Tumbling, metric.to_string(), @@ -97,7 +97,7 @@ async fn netflow_single_second_batch_is_not_lost_on_shutdown() { // --- Wire up the real engine: 1s tumbling Sum over `bytes`. let metric = "netflow_bytes"; let mut agg_map = HashMap::new(); - agg_map.insert(1u64, netflow_agg_config(metric, 1)); + agg_map.insert(1u64, netflow_agg_config(metric, 1_000)); let streaming_config = Arc::new(StreamingConfig::new(agg_map)); let json_cfg = JsonFileIngestConfig { diff --git a/asap-query-engine/tests/e2e_precompute_equivalence.rs b/asap-query-engine/tests/e2e_precompute_equivalence.rs index 1c613077..c1a76589 100644 --- a/asap-query-engine/tests/e2e_precompute_equivalence.rs +++ b/asap-query-engine/tests/e2e_precompute_equivalence.rs @@ -36,8 +36,8 @@ fn make_agg_config( metric: &str, agg_type: AggregationType, agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, + window_size_ms: u64, + slide_interval_ms: u64, grouping: Vec<&str>, ) -> AggregationConfig { make_agg_config_full( @@ -45,8 +45,8 @@ fn make_agg_config( metric, agg_type, agg_sub_type, - window_secs, - slide_secs, + window_size_ms, + slide_interval_ms, grouping, vec![], ) @@ -58,12 +58,12 @@ fn make_agg_config_full( metric: &str, agg_type: AggregationType, agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, + window_size_ms: u64, + slide_interval_ms: u64, grouping: Vec<&str>, aggregated: Vec<&str>, ) -> AggregationConfig { - let window_type = if slide_secs == 0 || slide_secs == window_secs { + let window_type = if slide_interval_ms == 0 || slide_interval_ms == window_size_ms { WindowType::Tumbling } else { WindowType::Sliding @@ -81,8 +81,8 @@ fn make_agg_config_full( ), promql_utilities::data_model::key_by_label_names::KeyByLabelNames::new(vec![]), String::new(), - window_secs, - slide_secs, + window_size_ms, + slide_interval_ms, window_type, metric.to_string(), metric.to_string(), @@ -174,7 +174,7 @@ fn gzip_hex(bytes: &[u8]) -> String { async fn e2e_kll_output_matches_arroyo() { let port = 19400u16; let agg_id = 1u64; - let window_secs = 10u64; + let window_size_ms = 10_000u64; let k = 20u16; let mut kll_config = make_agg_config( @@ -182,7 +182,7 @@ async fn e2e_kll_output_matches_arroyo() { "latency", AggregationType::DatasketchesKLL, "", - window_secs, + window_size_ms, 0, vec![], ); @@ -264,7 +264,7 @@ async fn e2e_kll_output_matches_arroyo() { // Window metadata assert_eq!(handcrafted_output.aggregation_id, agg_id); assert_eq!(handcrafted_output.start_timestamp, 0); - assert_eq!(handcrafted_output.end_timestamp, window_secs * 1_000); + assert_eq!(handcrafted_output.end_timestamp, window_size_ms); // Sketch contents assert_eq!( @@ -294,14 +294,14 @@ async fn e2e_kll_output_matches_arroyo() { async fn e2e_multiple_sum_output_matches_arroyo() { let port = 19401u16; let agg_id = 2u64; - let window_secs = 10u64; + let window_size_ms = 10_000u64; let config = make_agg_config_full( agg_id, "cpu", AggregationType::MultipleSum, "sum", - window_secs, + window_size_ms, 0, vec![], // grouping: none vec!["host"], // aggregated: host is the key INSIDE the sketch @@ -377,7 +377,7 @@ async fn e2e_multiple_sum_output_matches_arroyo() { // Window metadata assert_eq!(handcrafted_output.aggregation_id, agg_id); assert_eq!(handcrafted_output.start_timestamp, 0); - assert_eq!(handcrafted_output.end_timestamp, window_secs * 1_000); + assert_eq!(handcrafted_output.end_timestamp, window_size_ms); // Accumulator contents assert_eq!( diff --git a/asap-quickstart/docker-compose.dev.yml b/asap-quickstart/docker-compose.dev.yml index 10531c8d..44612013 100644 --- a/asap-quickstart/docker-compose.dev.yml +++ b/asap-quickstart/docker-compose.dev.yml @@ -13,6 +13,16 @@ services: build: context: .. dockerfile: asap-planner-rs/Dockerfile + volumes: + - ./config/controller-config.dev.yaml:/config/controller-config.yaml:ro + command: + - "--input_config=/config/controller-config.yaml" + - "--output_dir=/asap-planner-output" + - "--prometheus_scrape_interval_ms=1000" + - "--streaming_engine=precompute" + - "--range-duration-ms=300000" + - "--step-ms=10000" + - "-v" queryengine: build: diff --git a/asap-summary-ingest/examples/configs/streaming_config.yaml b/asap-summary-ingest/examples/configs/streaming_config.yaml index c5c1f10b..d76ff8c5 100644 --- a/asap-summary-ingest/examples/configs/streaming_config.yaml +++ b/asap-summary-ingest/examples/configs/streaming_config.yaml @@ -16,7 +16,7 @@ aggregations: metric: fake_metric_total parameters: {} spatialFilter: '' - windowSize: 10 + windowSizeMs: 10000 # - aggregationId: 2 # aggregationSubType: '' # aggregationType: MultipleIncrease @@ -32,7 +32,7 @@ aggregations: # metric: fake_metric_total # parameters: {} # spatialFilter: '' -# windowSize: 10 +# windowSizeMs: 10000 # - aggregationId: 3 # aggregationSubType: sum # aggregationType: MultipleSum @@ -48,7 +48,7 @@ aggregations: # metric: fake_metric_total # parameters: {} # spatialFilter: '' -# windowSize: 10 +# windowSizeMs: 10000 - aggregationId: 4 aggregationSubType: sum aggregationType: MultipleSum @@ -64,7 +64,7 @@ aggregations: metric: fake_metric_total parameters: {} spatialFilter: '' - windowSize: 10 + windowSizeMs: 10000 metrics: fake_metric_total: - instance diff --git a/asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml b/asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml index 36483ca5..6118742c 100644 --- a/asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml +++ b/asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml @@ -21,7 +21,7 @@ aggregations: metric: fake_metric_total parameters: K: 20 - slideInterval: 60 + slideIntervalMs: 60000 spatialFilter: '' - windowSize: 60 + windowSizeMs: 60000 windowType: tumbling diff --git a/asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml b/asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml index 670f2f4c..4015818c 100644 --- a/asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml +++ b/asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml @@ -21,7 +21,7 @@ aggregations: - hostname parameters: K: 20 - slideInterval: 1 + slideIntervalMs: 1000 spatialFilter: '' - windowSize: 1 + windowSizeMs: 1000 windowType: tumbling diff --git a/asap-summary-ingest/run_arroyosketch.py b/asap-summary-ingest/run_arroyosketch.py index d15b0cdc..0ef524aa 100644 --- a/asap-summary-ingest/run_arroyosketch.py +++ b/asap-summary-ingest/run_arroyosketch.py @@ -607,9 +607,11 @@ def get_sql_query( ) -> Tuple[str, str, dict]: window_type = streaming_aggregation_config.windowType - window_size = "{} seconds".format(streaming_aggregation_config.windowSize) + window_size = "{} milliseconds".format(streaming_aggregation_config.windowSizeMs) window_interval = window_size - slide_interval = "{} seconds".format(streaming_aggregation_config.slideInterval) + slide_interval = "{} milliseconds".format( + streaming_aggregation_config.slideIntervalMs + ) logger.info( f"Preparing SQL query for aggregation {streaming_aggregation_config.aggregationId}: " diff --git a/asap-summary-ingest/tests/test_integration.py b/asap-summary-ingest/tests/test_integration.py index 834133b4..c3bd6160 100644 --- a/asap-summary-ingest/tests/test_integration.py +++ b/asap-summary-ingest/tests/test_integration.py @@ -193,7 +193,7 @@ def sql_agg_config(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } ) @@ -277,7 +277,7 @@ def promql_agg_config(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } ) @@ -367,7 +367,7 @@ def test_parse_sql_config_file(self): rollup: [] parameters: {} spatialFilter: '' - windowSize: 10 + windowSizeMs: 10000 """ config = yaml.safe_load(config_content) @@ -403,7 +403,7 @@ def test_parse_promql_config_file(self): metric: fake_metric_total parameters: {} spatialFilter: '' - windowSize: 10 + windowSizeMs: 10000 metrics: fake_metric_total: diff --git a/asap-summary-ingest/tests/test_sql_schema.py b/asap-summary-ingest/tests/test_sql_schema.py index 0af03bef..9b842e7e 100644 --- a/asap-summary-ingest/tests/test_sql_schema.py +++ b/asap-summary-ingest/tests/test_sql_schema.py @@ -153,7 +153,7 @@ def test_parse_sql_aggregation_config(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -176,7 +176,7 @@ def test_default_value_column(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -197,7 +197,7 @@ def test_get_source_identifier_sql(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -230,7 +230,7 @@ def test_validate_sql_mode_success(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -255,7 +255,7 @@ def test_validate_sql_mode_missing_table(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -290,7 +290,7 @@ def test_validate_sql_mode_invalid_value_column(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -325,7 +325,7 @@ def test_validate_sql_mode_mismatched_labels(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -351,7 +351,7 @@ def test_parse_promql_aggregation_config(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -373,7 +373,7 @@ def test_get_source_identifier_promql(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -396,7 +396,7 @@ def test_validate_promql_mode_success(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) @@ -420,7 +420,7 @@ def test_validate_promql_mode_default(self): }, "parameters": {}, "spatialFilter": "", - "windowSize": 10, + "windowSizeMs": 10000, } agg_config = StreamingAggregationConfig.from_dict(config_dict) diff --git a/asap-tools/execution-utilities/asap_benchmark_pipeline/README.md b/asap-tools/execution-utilities/asap_benchmark_pipeline/README.md index d3d5b55c..6c5fcef7 100644 --- a/asap-tools/execution-utilities/asap_benchmark_pipeline/README.md +++ b/asap-tools/execution-utilities/asap_benchmark_pipeline/README.md @@ -81,7 +81,7 @@ python3 plot_latency.py To benchmark with a different tumbling window size (e.g., 120s): -1. Set `tumblingWindowSize` and `windowSize` in `streaming_config.yaml` +1. Set `windowSizeMs` in `streaming_config.yaml` (e.g., `120000` for 120s; `tumblingWindowSize` is a legacy key, no longer read) 2. Set the DATEADD offset in `inference_config.yaml` to match (e.g., `-120`) 3. Regenerate query files with matching window boundaries diff --git a/asap-tools/execution-utilities/asap_benchmark_pipeline/streaming_config.yaml b/asap-tools/execution-utilities/asap_benchmark_pipeline/streaming_config.yaml index 4883e33b..bb5f4f84 100644 --- a/asap-tools/execution-utilities/asap_benchmark_pipeline/streaming_config.yaml +++ b/asap-tools/execution-utilities/asap_benchmark_pipeline/streaming_config.yaml @@ -16,7 +16,6 @@ aggregations: value_column: v1 parameters: K: 200 - tumblingWindowSize: 10 - windowSize: 10 + windowSizeMs: 10000 windowType: tumbling spatialFilter: '' diff --git a/asap-tools/execution-utilities/asap_query_latency/streaming_config.yaml b/asap-tools/execution-utilities/asap_query_latency/streaming_config.yaml index 33079903..ad923a15 100644 --- a/asap-tools/execution-utilities/asap_query_latency/streaming_config.yaml +++ b/asap-tools/execution-utilities/asap_query_latency/streaming_config.yaml @@ -99,7 +99,7 @@ aggregations: value_column: ResolutionWidth parameters: K: 200 - windowSize: 10 + windowSizeMs: 10000 windowType: tumbling spatialFilter: '' diff --git a/asap-tools/execution-utilities/elastic-asap-benchmarking/streaming_config.yaml b/asap-tools/execution-utilities/elastic-asap-benchmarking/streaming_config.yaml index 3e2a59a2..07b89736 100644 --- a/asap-tools/execution-utilities/elastic-asap-benchmarking/streaming_config.yaml +++ b/asap-tools/execution-utilities/elastic-asap-benchmarking/streaming_config.yaml @@ -18,7 +18,6 @@ aggregations: value_column: v3 parameters: K: 200 - tumblingWindowSize: 10 - windowSize: 10 + windowSizeMs: 10000 windowType: tumbling spatialFilter: '' diff --git a/asap-tools/experiments/experiment_only_ingest_path.py b/asap-tools/experiments/experiment_only_ingest_path.py index d2654dc0..1026d821 100644 --- a/asap-tools/experiments/experiment_only_ingest_path.py +++ b/asap-tools/experiments/experiment_only_ingest_path.py @@ -297,7 +297,7 @@ def main(cfg: DictConfig): controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, diff --git a/asap-tools/experiments/experiment_run_clickhouse.py b/asap-tools/experiments/experiment_run_clickhouse.py index d452e062..e20dff84 100644 --- a/asap-tools/experiments/experiment_run_clickhouse.py +++ b/asap-tools/experiments/experiment_run_clickhouse.py @@ -437,7 +437,7 @@ def main(cfg: DictConfig) -> None: experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format="json", - prometheus_scrape_interval=15, + prometheus_scrape_interval_ms=15000, log_level="INFO", profile_query_engine=False, manual=False, diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 4e841d97..4a098db8 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -372,7 +372,7 @@ def main(cfg: DictConfig): url=prometheus_url, database=None, ), - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, ) sync.rsync_controller_config_remote_to_local( provider, @@ -516,7 +516,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_run_grafana_demo.py b/asap-tools/experiments/experiment_run_grafana_demo.py index 5f9456c8..c3bb1064 100644 --- a/asap-tools/experiments/experiment_run_grafana_demo.py +++ b/asap-tools/experiments/experiment_run_grafana_demo.py @@ -350,7 +350,7 @@ def main(cfg: DictConfig): controller_service.start( controller_input_file=controller_input_config, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, streaming_engine=args.streaming_engine, controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, punting=args.controller_punting, @@ -467,7 +467,7 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, flink_output_format=args.flink_output_format, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000, log_level=args.log_level, profile_query_engine=args.profile_query_engine, manual=args.manual_query_engine, diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index 89cc8e9e..2c447e06 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -369,6 +369,18 @@ def generate_controller_client_configs( controller_only_config = { k: v for k, v in full_config.items() if k in CONTROLLER_ALLOWED_KEYS } + # asap-planner-rs renamed QueryGroup.repetition_delay -> repetition_delay_ms + # (issue #398). full_config keeps the original seconds value for + # prometheus_client (a separate tool, unaffected by that rename) — + # only the controller-bound copy needs the renamed/scaled key. + if "query_groups" in controller_only_config: + controller_only_config["query_groups"] = [ + { + **{k: v for k, v in qg.items() if k != "repetition_delay"}, + "repetition_delay_ms": qg["repetition_delay"] * 1000, + } + for qg in controller_only_config["query_groups"] + ] with open( os.path.join( output_dir, "{}_controller_input.yaml".format(experiment_mode["mode"]) diff --git a/asap-tools/experiments/experiment_utils/services/misc.py b/asap-tools/experiments/experiment_utils/services/misc.py index 01d28f2c..5ca942ec 100644 --- a/asap-tools/experiments/experiment_utils/services/misc.py +++ b/asap-tools/experiments/experiment_utils/services/misc.py @@ -205,7 +205,7 @@ def start( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval: Optional[int] = None, + prometheus_scrape_interval_ms: Optional[int] = None, query_language: str = "promql", data_ingestion_interval: Optional[int] = None, **kwargs, @@ -221,7 +221,8 @@ def start( discovery_backend: Backend used for label/column auto-discovery. PromQL mode: DiscoveryBackend(type="prometheus", url=, database=None) SQL mode: DiscoveryBackend(type="clickhouse", url=, database=) - prometheus_scrape_interval: Required for PromQL mode + prometheus_scrape_interval_ms: Required for PromQL mode (milliseconds; passed as + --prometheus_scrape_interval_ms / --prometheus-scrape-interval-ms) query_language: 'promql' (default) or 'sql' data_ingestion_interval: Required for SQL mode (seconds) **kwargs: Additional configuration @@ -233,7 +234,7 @@ def start( controller_remote_output_dir, punting, discovery_backend, - prometheus_scrape_interval, + prometheus_scrape_interval_ms, query_language, data_ingestion_interval, ) @@ -244,7 +245,7 @@ def start( controller_remote_output_dir, punting, discovery_backend, - prometheus_scrape_interval, + prometheus_scrape_interval_ms, query_language, data_ingestion_interval, ) @@ -256,7 +257,7 @@ def _start_bare_metal( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval: Optional[int], + prometheus_scrape_interval_ms: Optional[int], query_language: str, data_ingestion_interval: Optional[int], ) -> None: @@ -268,8 +269,8 @@ def _start_bare_metal( f" --streaming_engine {streaming_engine}" f" --query-language {query_language}" ) - if prometheus_scrape_interval is not None: - cmd += f" --prometheus_scrape_interval {prometheus_scrape_interval}" + if prometheus_scrape_interval_ms is not None: + cmd += f" --prometheus_scrape_interval_ms {prometheus_scrape_interval_ms}" if data_ingestion_interval is not None: cmd += f" --data-ingestion-interval {data_ingestion_interval}" if discovery_backend.type == "prometheus": @@ -299,7 +300,7 @@ def _start_containerized( controller_remote_output_dir: str, punting: bool, discovery_backend: DiscoveryBackend, - prometheus_scrape_interval: Optional[int], + prometheus_scrape_interval_ms: Optional[int], query_language: str, data_ingestion_interval: Optional[int], ): @@ -329,9 +330,9 @@ def _start_containerized( generate_cmd += f" --controller-output-dir {controller_remote_output_dir}" generate_cmd += f" --streaming-engine {streaming_engine}" generate_cmd += f" --query-language {query_language}" - if prometheus_scrape_interval is not None: + if prometheus_scrape_interval_ms is not None: generate_cmd += ( - f" --prometheus-scrape-interval {prometheus_scrape_interval}" + f" --prometheus-scrape-interval-ms {prometheus_scrape_interval_ms}" ) if data_ingestion_interval is not None: generate_cmd += f" --data-ingestion-interval {data_ingestion_interval}" diff --git a/asap-tools/experiments/experiment_utils/services/query_engine.py b/asap-tools/experiments/experiment_utils/services/query_engine.py index 001cc52b..19a6a884 100644 --- a/asap-tools/experiments/experiment_utils/services/query_engine.py +++ b/asap-tools/experiments/experiment_utils/services/query_engine.py @@ -73,7 +73,7 @@ def _build_engine_config( self, output_dir: str, flink_output_format: str, - prometheus_scrape_interval: int, + prometheus_scrape_interval_ms: int, log_level: str, streaming_engine: str, controller_config_dir: str, @@ -93,7 +93,7 @@ def _build_engine_config( Args: output_dir: Output directory path (remote host path or container-internal path) flink_output_format: Kafka input_format when streaming_engine=arroyo - prometheus_scrape_interval: Prometheus scraping interval in seconds + prometheus_scrape_interval_ms: Prometheus scraping interval in milliseconds log_level: Logging level streaming_engine: 'arroyo' (Kafka ingest) or 'precompute' (HTTP remote write) controller_config_dir: Directory containing inference_config.yaml and streaming_config.yaml @@ -140,7 +140,7 @@ def _build_engine_config( return { "output_dir": output_dir, "log_level": log_level, - "prometheus_scrape_interval": prometheus_scrape_interval, + "prometheus_scrape_interval_ms": prometheus_scrape_interval_ms, "streaming_engine": streaming_engine, "http_server": {"port": http_port}, "backend": backend, # already fully resolved by caller @@ -205,7 +205,7 @@ def start( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval: int, + prometheus_scrape_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -228,7 +228,7 @@ def start( config YAML locally before rsyncing to remote) flink_output_format: Format of data from Flink (used as Kafka input_format when streaming_engine=arroyo) - prometheus_scrape_interval: Prometheus scraping interval + prometheus_scrape_interval_ms: Prometheus scraping interval in milliseconds log_level: Logging level profile_query_engine: Whether to enable profiling manual: Whether to run in manual mode @@ -250,7 +250,7 @@ def start( experiment_output_dir, local_experiment_dir, flink_output_format, - prometheus_scrape_interval, + prometheus_scrape_interval_ms, log_level, profile_query_engine, manual, @@ -269,7 +269,7 @@ def start( experiment_output_dir, local_experiment_dir, flink_output_format, - prometheus_scrape_interval, + prometheus_scrape_interval_ms, log_level, profile_query_engine, manual, @@ -289,7 +289,7 @@ def _start_bare_metal( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval: int, + prometheus_scrape_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -310,7 +310,7 @@ def _start_bare_metal( config = self._build_engine_config( output_dir=output_dir, flink_output_format=flink_output_format, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, log_level=log_level, streaming_engine=streaming_engine, controller_config_dir=controller_remote_output_dir, @@ -353,7 +353,7 @@ def _start_containerized( experiment_output_dir: str, local_experiment_dir: str, flink_output_format: str, - prometheus_scrape_interval: int, + prometheus_scrape_interval_ms: int, log_level: str, profile_query_engine: bool, manual: bool, @@ -380,7 +380,7 @@ def _start_containerized( config = self._build_engine_config( output_dir=container_output_dir, flink_output_format=flink_output_format, - prometheus_scrape_interval=prometheus_scrape_interval, + prometheus_scrape_interval_ms=prometheus_scrape_interval_ms, log_level=log_level, streaming_engine=streaming_engine, controller_config_dir=container_controller_dir, diff --git a/asap-tools/experiments/generate_controller_compose.py b/asap-tools/experiments/generate_controller_compose.py index b41838f9..79311d66 100644 --- a/asap-tools/experiments/generate_controller_compose.py +++ b/asap-tools/experiments/generate_controller_compose.py @@ -16,7 +16,7 @@ def generate_compose_file( container_name: str, input_config_path: str, output_dir: str, - prometheus_scrape_interval: int, + prometheus_scrape_interval_ms: int, streaming_engine: str, punting: bool, prometheus_url: str, @@ -42,7 +42,7 @@ def generate_compose_file( "container_name": container_name, "input_config_path": input_config_path, "output_dir": output_dir, - "prometheus_scrape_interval": prometheus_scrape_interval, + "prometheus_scrape_interval_ms": prometheus_scrape_interval_ms, "streaming_engine": streaming_engine, "punting": punting, "prometheus_url": prometheus_url, @@ -108,10 +108,10 @@ def main(): help="Output directory for generated configs", ) parser.add_argument( - "--prometheus-scrape-interval", + "--prometheus-scrape-interval-ms", type=int, required=True, - help="Prometheus scrape interval in seconds", + help="Prometheus scrape interval in milliseconds", ) parser.add_argument( "--streaming-engine", @@ -156,7 +156,7 @@ def main(): container_name=args.container_name, input_config_path=args.input_config_path, output_dir=args.controller_output_dir, - prometheus_scrape_interval=args.prometheus_scrape_interval, + prometheus_scrape_interval_ms=args.prometheus_scrape_interval_ms, streaming_engine=args.streaming_engine, punting=args.punting, prometheus_url=args.prometheus_url, diff --git a/benchmarks/config/controller-config.yaml b/benchmarks/config/controller-config.yaml new file mode 100644 index 00000000..444be132 --- /dev/null +++ b/benchmarks/config/controller-config.yaml @@ -0,0 +1,28 @@ +# ASAP Configuration for Pattern-based Demo (CI benchmarks) +# Uses deterministic time-based patterns for visual comparison + +query_groups: + - id: 1 + queries: + # Quantile queries grouped by pattern - shows each pattern type + - "quantile by (pattern) (0.99, sensor_reading)" + - "quantile by (pattern) (0.95, sensor_reading)" + - "quantile by (pattern) (0.90, sensor_reading)" + - "quantile by (pattern) (0.50, sensor_reading)" + repetition_delay_ms: 10000 + controller_options: + accuracy_sla: 0.99 + latency_sla: 1 + +# Metric metadata - includes the 'pattern' label +metrics: + - metric: "sensor_reading" + labels: ["instance", "job", "pattern", "region", "service", "host"] + +# Cleanup policy for aggregates +aggregate_cleanup: + policy: "read_based" + +sketch_parameters: + DatasketchesKLL: + K: 200 diff --git a/benchmarks/config/engine_config.yaml b/benchmarks/config/engine_config.yaml index 63c6feb0..1618ff72 100644 --- a/benchmarks/config/engine_config.yaml +++ b/benchmarks/config/engine_config.yaml @@ -1,8 +1,7 @@ output_dir: "/app/outputs" log_level: "INFO" -prometheus_scrape_interval: 1 +prometheus_scrape_interval_ms: 1000 streaming_engine: "precompute" -do_profiling: false http_server: port: 8088 diff --git a/benchmarks/docker-compose.yml b/benchmarks/docker-compose.yml index e6106eb3..00e68896 100644 --- a/benchmarks/docker-compose.yml +++ b/benchmarks/docker-compose.yml @@ -12,6 +12,16 @@ services: asap-planner-rs: image: ghcr.io/projectasap/asap-planner-rs:${ASAP_IMAGE_TAG} + command: + - "--input_config=/config/controller-config.yaml" + - "--output_dir=/asap-planner-output" + - "--prometheus_scrape_interval_ms=1000" + - "--streaming_engine=precompute" + - "--range-duration-ms=300000" + - "--step-ms=10000" + - "-v" + volumes: + - ../benchmarks/config/controller-config.yaml:/config/controller-config.yaml:ro queryengine: image: ghcr.io/projectasap/asap-query-engine:${ASAP_IMAGE_TAG} diff --git a/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md index cad67df1..2f1c950e 100644 --- a/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md +++ b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md @@ -27,7 +27,7 @@ asap-planner \ --query-log /var/log/prometheus/query.log \ --prometheus-url http://localhost:9090 \ --output_dir ./configs \ - --prometheus_scrape_interval 15 \ + --prometheus_scrape_interval_ms 15000 \ --streaming_engine arroyo ``` diff --git a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md index 4bb29608..14278f29 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md @@ -76,7 +76,7 @@ aggregations: parameters: K: 20 tumblingWindowSize: 1 - windowSize: 1 + windowSizeMs: 1000 windowType: tumbling spatialFilter: '' tables: diff --git a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md index c09bf44d..736d4b16 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md @@ -58,7 +58,7 @@ aggregations: parameters: K: 20 tumblingWindowSize: 1 - windowSize: 1 + windowSizeMs: 1000 windowType: tumbling spatialFilter: '' metrics: diff --git a/docs/03-how-to-guides/operations/try-asap-planner-promql.md b/docs/03-how-to-guides/operations/try-asap-planner-promql.md index 9e2ff062..01a57c9c 100644 --- a/docs/03-how-to-guides/operations/try-asap-planner-promql.md +++ b/docs/03-how-to-guides/operations/try-asap-planner-promql.md @@ -20,7 +20,7 @@ metrics: labels: [instance, job, method, status] query_groups: - id: 1 - repetition_delay: 300 # seconds between repeats of this query + repetition_delay_ms: 300000 # ms between repeats of this query controller_options: accuracy_sla: 0.95 latency_sla: 100.0 @@ -46,16 +46,16 @@ auto-infer label sets per metric. `accuracy_sla` and `latency_sla` are required by the config schema but not currently used by the planner's decision logic — any numeric values are fine (e.g. the placeholders above). -### Choosing `repetition_delay` and `--prometheus_scrape_interval` +### Choosing `repetition_delay_ms` and `--prometheus_scrape_interval_ms` -- `--prometheus_scrape_interval` is your actual Prometheus scrape interval — a fact about your +- `--prometheus_scrape_interval_ms` is your actual Prometheus scrape interval — a fact about your existing setup, not something to tune. -- `repetition_delay` is how often this specific query actually re-runs — e.g. `300` for a +- `repetition_delay_ms` is how often this specific query actually re-runs — e.g. `300000` for a dashboard panel refreshing every 5 minutes, or an alert rule's `evaluation_interval`. -- Unlike SQL mode, there's no hard error if `repetition_delay` isn't a multiple of the scrape +- Unlike SQL mode, there's no hard error if `repetition_delay_ms` isn't a multiple of the scrape interval, but `rate`/`increase`/`quantile_over_time` queries need at least 60 scraped data points per repeat window to be considered worth accelerating when `--enable-punting` is set — - i.e. `repetition_delay >= 60 * prometheus_scrape_interval`. + i.e. `repetition_delay_ms >= 60 * prometheus_scrape_interval_ms`. ## 3. Run the planner @@ -63,7 +63,7 @@ the planner's decision logic — any numeric values are fine (e.g. the placehold asap-planner \ --input_config promql_workload.yaml \ --output_dir ./out \ - --prometheus_scrape_interval 15 \ + --prometheus_scrape_interval_ms 15000 \ --streaming_engine precompute \ -v ```