From 4c611092286cb57565ac3f60781d756d517ea92e Mon Sep 17 00:00:00 2001 From: nk Date: Fri, 12 Jun 2026 19:26:04 +0900 Subject: [PATCH] fix(dlt): surface underlying error and hint when attaching to pipeline fails When `sqlmesh init -t dlt --dlt-path ` could not attach to the pipeline, the underlying dlt error was swallowed and replaced with an uninformative message. `--dlt-path` points to dlt's pipelines working directory (by default ~/.dlt/pipelines), not the directory containing the pipeline scripts, which is a common source of confusion. Now the error includes the original dlt exception, the directory that was searched, and a hint to omit `--dlt-path` when the pipeline exists in the default working directory. Also clarifies the `--dlt-path` help text and docstring accordingly. Fixes #5660. Co-Authored-By: Claude Fable 5 --- docs/reference/cli.md | 3 ++- sqlmesh/cli/main.py | 4 ++-- sqlmesh/integrations/dlt.py | 20 +++++++++++++++++--- tests/cli/test_cli.py | 8 +++++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a6d4fa9514..b65f8256ac 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -279,7 +279,8 @@ Options: empty. --dlt-pipeline TEXT DLT pipeline for which to generate a SQLMesh project. Use alongside template: dlt - --dlt-path TEXT The directory where the DLT pipeline resides. Use + --dlt-path TEXT The DLT pipelines working directory, where DLT stores + pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt --help Show this message and exit. ``` diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py index 940a9ebcf4..5b45d72153 100644 --- a/sqlmesh/cli/main.py +++ b/sqlmesh/cli/main.py @@ -165,7 +165,7 @@ def cli( @click.option( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides. Use alongside template: dlt", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt", ) @click.pass_context @error_handler @@ -1151,7 +1151,7 @@ def table_name( @click.option( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides.", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines).", ) @click.pass_context @error_handler diff --git a/sqlmesh/integrations/dlt.py b/sqlmesh/integrations/dlt.py index 2d601a0e22..a2202bea02 100644 --- a/sqlmesh/integrations/dlt.py +++ b/sqlmesh/integrations/dlt.py @@ -22,7 +22,8 @@ def generate_dlt_models_and_settings( pipeline_name: The name of the DLT pipeline to attach to. dialect: The SQL dialect to use for generating SQLMesh models. tables: A list of table names to include. - dlt_path: The path to the directory containing the DLT pipelines. + dlt_path: The path to the DLT pipelines working directory, where DLT stores + pipeline state (by default ~/.dlt/pipelines). Returns: A tuple containing a set of the SQLMesh model definitions, the connection config and the start date. @@ -34,8 +35,21 @@ def generate_dlt_models_and_settings( try: pipeline = dlt.attach(pipeline_name=pipeline_name, pipelines_dir=dlt_path or "") - except CannotRestorePipelineException: - raise click.ClickException(f"Could not attach to pipeline {pipeline_name}") + except CannotRestorePipelineException as e: + from pathlib import Path + from dlt.common.pipeline import get_dlt_pipelines_dir + + searched_dir = dlt_path or get_dlt_pipelines_dir() + msg = f"Could not attach to pipeline {pipeline_name}.\nSearched in: {searched_dir}\n{e}" + if dlt_path and (Path(get_dlt_pipelines_dir()) / pipeline_name).exists(): + msg += ( + f"\nHint: A pipeline named '{pipeline_name}' exists in the default pipelines " + f"working directory '{get_dlt_pipelines_dir()}'. Note that --dlt-path must " + "point to the directory where DLT stores pipeline working state (by default " + "~/.dlt/pipelines), not the directory containing your pipeline scripts. " + "Try omitting --dlt-path." + ) + raise click.ClickException(msg) schema = pipeline.default_schema dataset = pipeline.dataset_name diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 5e0737e1b6..bf3af9eebd 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -995,7 +995,7 @@ def test_dlt_pipeline(runner, tmp_path): exec(file.read()) # This should fail since it won't be able to locate the pipeline in this path - with pytest.raises(ClickException, match=r".*Could not attach to pipeline*"): + with pytest.raises(ClickException, match=r".*Could not attach to pipeline*") as excinfo: init_example_project( tmp_path, "duckdb", @@ -1004,6 +1004,12 @@ def test_dlt_pipeline(runner, tmp_path): dlt_path="./dlt2/pipelines", ) + # The error should surface where the pipeline was searched for and, since the + # pipeline exists in the default working directory, a hint about --dlt-path + error_message = str(excinfo.value) + assert "Searched in: ./dlt2/pipelines" in error_message + assert "Try omitting --dlt-path" in error_message + # By setting the pipelines path where the pipeline directory is located, it should work dlt_path = get_dlt_pipelines_dir() init_example_project(