diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml index b79ef571..4e6f6ab3 100644 --- a/.github/workflows/correctness.yml +++ b/.github/workflows/correctness.yml @@ -140,31 +140,3 @@ jobs: - name: Compare serialised patterns working-directory: asap-common/tests/compare_patterns run: python compare_serialized_patterns.py - - # ── 3. Rust pattern-matching unit tests ──────────────────────────────────── - rust-pattern-matching: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false - name: Rust pattern matching unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Install protoc - run: | - sudo apt-get update -qq - sudo apt-get install -y protobuf-compiler - - - name: Run sccache - uses: mozilla-actions/sccache-action@v0.0.4 - - - name: Cache cargo - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 50fc271e..a0cf5ddf 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -8,6 +8,7 @@ on: - 'asap-tools/**' - 'asap-tools/data-sources/prometheus-exporters/**' - 'asap-tools/execution-utilities/**' + - 'asap-summary-ingest/**' - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' pull_request: @@ -18,6 +19,7 @@ on: - 'asap-tools/**' - 'asap-tools/data-sources/prometheus-exporters/**' - 'asap-tools/execution-utilities/**' + - 'asap-summary-ingest/**' - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' workflow_dispatch: @@ -39,6 +41,7 @@ jobs: utilities: ${{ steps.filter.outputs.utilities }} prometheus_exporters: ${{ steps.filter.outputs.prometheus_exporters }} execution_utilities: ${{ steps.filter.outputs.execution_utilities }} + summary_ingest: ${{ steps.filter.outputs.summary_ingest }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v2 @@ -57,6 +60,9 @@ jobs: execution_utilities: - 'asap-tools/execution-utilities/**' - 'asap-common/dependencies/py/**' + summary_ingest: + - 'asap-summary-ingest/**' + - 'asap-common/dependencies/py/**' test-prometheus-client: needs: detect-changes @@ -182,3 +188,21 @@ jobs: flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics + + test-summary-ingest: + needs: detect-changes + if: needs.detect-changes.outputs.summary_ingest == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e asap-common/dependencies/py/promql_utilities/ + pip install pytest pyyaml jinja2 + - name: Run pytest + run: python -m pytest asap-summary-ingest/tests/ -v diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 42bfdcfe..01049098 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -5,6 +5,7 @@ on: branches: [ main ] paths: - 'asap-query-engine/**' + - 'asap-planner-rs/**' - 'asap-common/dependencies/rs/**' - 'asap-common/tests/**' - 'asap-common/sketch-core/**' @@ -16,6 +17,7 @@ on: branches: [ main ] paths: - 'asap-query-engine/**' + - 'asap-planner-rs/**' - 'asap-common/dependencies/rs/**' - 'asap-common/tests/**' - 'asap-common/sketch-core/**' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b573ca40..869d836f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,3 +100,11 @@ repos: language: system files: \.rs$ pass_filenames: false + + - id: pytest-summary-ingest + name: pytest (asap-summary-ingest) + description: Run asap-summary-ingest unit tests against vendored promql_utilities. + entry: bash -c 'pip install -q -e asap-common/dependencies/py/promql_utilities/ && python3 -m pytest asap-summary-ingest/tests/' + language: system + files: ^(asap-summary-ingest|asap-common/dependencies/py)/ + pass_filenames: false diff --git a/asap-tools/experiments/experiment_utils/test_local_provider.py b/asap-tools/experiments/experiment_utils/test_local_provider.py deleted file mode 100644 index 90ce600e..00000000 --- a/asap-tools/experiments/experiment_utils/test_local_provider.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Self-check for local-vs-CloudLab provider dispatch (issue #424). - -Runnable standalone (`python3 test_local_provider.py`) or via pytest. -""" - -import os -import sys - -_EXPERIMENTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, _EXPERIMENTS_DIR) - -from omegaconf import OmegaConf # noqa: E402 -import omegaconf.errors as oerr # noqa: E402 - -import config # noqa: E402 -from providers.factory import create_provider, detect_provider_type # noqa: E402 -from providers.local import LocalProvider # noqa: E402 -from providers.cloudlab import CloudLabProvider # noqa: E402 - - -def _base_cfg(**overrides): - base = OmegaConf.load(os.path.join(_EXPERIMENTS_DIR, "config", "config.yaml")) - del base["defaults"] # not relevant to provider dispatch - return OmegaConf.merge(base, {"experiment": {"name": "smoke"}, **overrides}) - - -def test_local_mode_never_touches_cloudlab_mandatory_fields(): - cfg = _base_cfg(local={"home_dir": "/tmp/sketchdb_home"}) - provider = create_provider(cfg) - assert isinstance(provider, LocalProvider) - assert provider.is_remote() is False - assert provider.get_node_ip(0) == "127.0.0.1" - assert provider.get_home_dir() == "/tmp/sketchdb_home" - assert detect_provider_type(cfg) == "local" - assert config.get_node_params(cfg) == (0, 0) - assert config.required_cloudlab_params(cfg) == [] - - try: - cfg.cloudlab.username - raise AssertionError("cfg.cloudlab.username should still be mandatory-missing") - except oerr.MissingMandatoryValue: - pass - - -def test_local_args_resolve_remote_write_ip_through_provider(): - cfg = _base_cfg(local={"home_dir": "/tmp/sketchdb_home"}) - provider = create_provider(cfg) - OmegaConf.register_new_resolver( - "remote_write_ip", provider.get_node_ip, replace=True - ) - args = config.Args(cfg) - assert args.num_nodes == 0 - assert args.node_offset == 0 - assert args.cloudlab_username is None - assert args.hostname_suffix is None - assert args.remote_write_ip == "127.0.0.1" - assert args.get_node_range(include_coordinator=True) == [0] - - -def test_cloudlab_mode_unchanged(): - cfg = _base_cfg( - cloudlab={ - "num_nodes": 2, - "username": "milindsr", - "hostname_suffix": "exp.cloudlab.us", - } - ) - provider = create_provider(cfg) - assert isinstance(provider, CloudLabProvider) - assert provider.is_remote() is True - assert provider.get_home_dir() == "/scratch/sketch_db_for_prometheus" - assert detect_provider_type(cfg) == "cloudlab" - assert config.get_node_params(cfg) == (2, 0) - assert len(config.required_cloudlab_params(cfg)) == 3 - - OmegaConf.register_new_resolver( - "remote_write_ip", provider.get_node_ip, replace=True - ) - args = config.Args(cfg) - assert args.cloudlab_username == "milindsr" - assert args.remote_write_ip == "10.10.1.1" - - -if __name__ == "__main__": - test_local_mode_never_touches_cloudlab_mandatory_fields() - test_local_args_resolve_remote_write_ip_through_provider() - test_cloudlab_mode_unchanged() - print("OK")