From b5e310ef2ffbb35d8a19172729afe92d21068677 Mon Sep 17 00:00:00 2001 From: cookieark Date: Sat, 21 Nov 2026 22:53:26 +0530 Subject: [PATCH 1/2] feat(cli): add --use-project-index to sqlmesh test With test or model file selectors, sqlmesh test now loads only the models those tests need plus their upstream dependencies, using the persistent project index. Errors in unrelated models no longer block the run, and a selected test whose model is not found fails instead of being skipped. Without selectors, or with --select-model, the whole project is still loaded and the index is refreshed. --local and --use-project-index stay independent. Closes #6024 Signed-off-by: cookieark --- docs/concepts/tests.md | 12 +++ docs/reference/cli.md | 6 ++ sqlmesh/cli/main.py | 19 ++++ sqlmesh/core/context.py | 73 ++++++++++++++- sqlmesh/core/loader.py | 78 ++++++++++++++-- tests/cli/test_cli.py | 99 ++++++++++++++++++++ tests/core/test_test.py | 195 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 474 insertions(+), 8 deletions(-) diff --git a/docs/concepts/tests.md b/docs/concepts/tests.md index 8bf0fe0cbf..9119433a36 100644 --- a/docs/concepts/tests.md +++ b/docs/concepts/tests.md @@ -494,6 +494,18 @@ In multi-repository setups, or when running tests for only a subset of projects, So a passing exit code alone does not mean every test you expected actually ran. Watch the output for these warnings, and keep in mind that a hook using `--local` will not fail on them. +By default, `sqlmesh test` loads every model in the project before it runs the selected tests. In a large project, or a commit hook that only tests a few changed files, you can pass `--use-project-index` to load only the models the selected tests need, together with their upstream dependencies: + +``` +$ sqlmesh test --use-project-index models/full_model.sql +``` + +This uses the persistent [project index](../guides/configuration.md#project-index). Model files are looked up in the index, and test files through the `model` of each test. Errors in unrelated models are not reported, the same as with `sqlmesh lint --use-project-index`. If the index is missing or stale, SQLMesh falls back to a full load and rebuilds it. + +When the load is scoped this way, a selected test whose model can't be found is an error rather than a warning, so the run can't pass without testing anything. + +Without test or model file arguments, or together with `--select-model`, every model is still loaded and the index is refreshed. `--use-project-index` and `--local` are independent and can be combined. + ### Testing using notebooks You can execute tests on demand using the `%run_test` notebook magic as follows: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 460b6e520b..bfd284b81b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -639,6 +639,12 @@ Options: --local Run tests using only locally loaded project files without loading state. Tests whose model is not loaded are skipped with a warning rather than failing. + --use-project-index Use the persistent project index. With TESTS, only the + models they test and their upstream dependencies are + loaded, so errors in unrelated models are not reported, + and a selected test whose model is not found fails. + Without TESTS, or with --select-model, every model is + still loaded. --help Show this message and exit. ``` diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py index 896b47abd7..4d24404e62 100644 --- a/sqlmesh/cli/main.py +++ b/sqlmesh/cli/main.py @@ -150,6 +150,9 @@ def cli( # Unlike the other commands above, lint can scope its own load for multi-project contexts. if ctx.invoked_subcommand == "lint": load = False + # test scopes its own load only when it's asked to use the project index. + if ctx.invoked_subcommand == "test" and "--use-project-index" in ctx.meta["subcommand_args"]: + load = False configs = load_configs(config, Context.CONFIG_TYPE, paths, dotenv_path=dotenv) log_limit = list(configs.values())[0].log_limit @@ -823,6 +826,12 @@ def create_test( expose_value=False, help="Run tests using only locally loaded project files without loading state. Tests whose model is not loaded are skipped with a warning rather than failing.", ) +@click.option( + "--use-project-index", + is_flag=True, + default=False, + help="Use the persistent project index. With TESTS, only the models they test and their upstream dependencies are loaded, so errors in unrelated models are not reported, and a selected test whose model is not found fails. Without TESTS, or with --select-model, every model is still loaded.", +) @click.argument("tests", nargs=-1) @click.pass_obj @error_handler @@ -833,6 +842,7 @@ def test( verbose: int, preserve_fixtures: bool, select_model: t.List[str], + use_project_index: bool, tests: t.List[str], ) -> None: """Run model unit tests. @@ -841,6 +851,9 @@ def test( tests for those models are run. They are unioned, and a test selected more than once still only runs once. """ + if use_project_index and select_model: + # Model selections are expanded against the whole project, so it has to be loaded. + obj.load(use_project_index=True) model_names = ( obj._new_selector().expand_model_selections(select_model) if select_model else None ) @@ -851,7 +864,13 @@ def test( preserve_fixtures=preserve_fixtures, model_names=model_names, raise_on_unknown_paths=True, + use_project_index=use_project_index, ) + # The group callback skips its own empty-project check when test does the loading. + if use_project_index and not obj.models: + raise click.ClickException( + f"`{obj.path}` doesn't seem to have any models... cd into the proper directory or specify the path(s) with -p." + ) if not result.wasSuccessful(): exit(1) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index fd7f8ad071..a02c2c7a90 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -2434,13 +2434,30 @@ def test( stream: t.Optional[t.TextIO] = None, model_names: t.Optional[t.Collection[str]] = None, raise_on_unknown_paths: bool = False, + use_project_index: bool = False, ) -> ModelTextTestResult: - """Discover and run model tests""" + """Discover and run model tests. + + Args: + use_project_index: Whether to use the persistent project index when the context + isn't loaded yet. With test or model file selectors, only the models they need + and their upstream dependencies are loaded, and a selected test whose model + isn't found is an error. Without selectors, or together with ``model_names``, + the whole project is loaded. + """ if verbosity >= Verbosity.VERBOSE: import pandas as pd pd.set_option("display.max_columns", None) + target_fqns: t.Optional[t.Set[str]] = None + # A context that is already loaded is left as is, so callers that go on to use it + # don't end up with only part of the project. + if use_project_index and not self._loaded: + if tests and model_names is None: + target_fqns = self._model_fqns_for_test_selectors(tests) + self.load(model_fqns=target_fqns, use_project_index=True) + baseline_meta = self.select_tests( tests=tests, patterns=match_patterns, @@ -2459,6 +2476,9 @@ def test( test_meta = baseline_meta tests_skipped = 0 + if target_fqns is not None: + self._raise_for_tests_without_models(test_meta) + result = run_tests( model_test_metadata=test_meta, models=self._models, @@ -3725,6 +3745,57 @@ def _select_tests_by_test_path(self, selector: str) -> t.Optional[t.List[ModelTe return None + def _model_fqns_for_test_selectors(self, selectors: t.Iterable[str]) -> t.Optional[t.Set[str]]: + """Resolves test selectors to model FQNs without loading the project's models. + + Returns None if any selector can't be resolved, so that a full load runs and the + usual selector errors are reported against the whole project. + """ + model_fqns: t.Set[str] = set() + for selector in selectors: + path, _, test_name = ( + selector.rpartition("::") if "::" in selector else (selector, "", "") + ) + absolute_path = Path(os.path.abspath(path)) + for loader in self._loaders: + model_names = loader.model_names_for_test_selector(absolute_path, test_name or None) + if model_names is not None: + break + else: + return None + + model_fqns.update( + normalize_model_name( + model_name, default_catalog=self.default_catalog, dialect=self.default_dialect + ) + for model_name in model_names + ) + + return model_fqns or None + + def _raise_for_tests_without_models(self, test_meta: t.Iterable[ModelTestMetadata]) -> None: + """Fails a scoped test run when a selected test's model wasn't loaded. + + A full run warns and skips such a test, but a scoped run that does so can pass while + testing nothing. + """ + errors = [] + for metadata in test_meta: + if not metadata.model_name: + continue + fqn = normalize_model_name( + metadata.model_name, + default_catalog=self.default_catalog, + dialect=self.default_dialect, + ) + if fqn not in self._models: + errors.append( + f"Model '{fqn}' targeted by test '{metadata.fully_qualified_test_name}' was not found." + ) + + if errors: + raise SQLMeshError("\n".join(errors)) + def _unknown_test_selector_error(self, selector: str) -> str: """Explains why a selector matched nothing. diff --git a/sqlmesh/core/loader.py b/sqlmesh/core/loader.py index 5a1ce8b17d..4344f3e204 100644 --- a/sqlmesh/core/loader.py +++ b/sqlmesh/core/loader.py @@ -449,6 +449,16 @@ def load_model_tests(self) -> t.List[ModelTestMetadata]: """Loads YAML-based model tests""" return [] + def model_names_for_test_selector( + self, path: Path, test_name: t.Optional[str] = None + ) -> t.Optional[t.Set[str]]: + """Resolves a `sqlmesh test` selector to the models it needs, before any model is loaded. + + Returns None if the selector can't be resolved this way, which makes the caller fall + back to a full load. + """ + return None + def _glob_paths( self, path: Path, @@ -633,9 +643,8 @@ def _model_paths(self) -> t.Set[Path]: ) return paths - def _selected_model_paths( - self, model_fqns: t.Set[str] - ) -> t.Optional[t.Tuple[t.Set[Path], t.Set[str]]]: + def _read_model_index(self) -> t.Optional[t.Tuple[t.Any, t.Dict[str, t.Dict[str, t.Any]]]]: + """Returns the stored index ID and files, or None if the index is missing or malformed.""" try: index = json.loads(self._model_index_path.read_text(encoding="utf-8")) except (OSError, ValueError): @@ -644,10 +653,6 @@ def _selected_model_paths( if not isinstance(index, dict): return None - if index.get("index_id") != self._model_index_id(): - return None - - current_paths = self._model_paths() indexed_files = index.get("files") if not isinstance(indexed_files, dict) or not all( isinstance(relative_path, str) and isinstance(file_models, dict) @@ -655,6 +660,65 @@ def _selected_model_paths( ): return None + return index.get("index_id"), indexed_files + + def model_names_for_test_selector( + self, path: Path, test_name: t.Optional[str] = None + ) -> t.Optional[t.Set[str]]: + """Resolves a `sqlmesh test` selector to the models it needs, before any model is loaded. + + A test file resolves through the `model:` of its tests. A model file resolves through + the project index, which can't be validated until the project is loaded, so its result + is only a hint: `Context.load` falls back to a full load if the hint turns out stale. + + Args: + path: The absolute path of a test file or a model file. + test_name: The test to select from a test file, if the selector named one. + + Returns: + The model names, or None if the selector can't be resolved this way. + """ + try: + relative_path = path.relative_to(os.path.abspath(self.config_path)) + except ValueError: + return None + + if ( + relative_path.parts[:1] == (c.TESTS,) + and path.name.startswith("test") + and path.suffix in (".yaml", ".yml") + and path.is_file() + ): + # Same rules as load_model_tests, so an ignored file isn't treated as a test file. + if any(path.match(pattern) for pattern in self.config.ignore_patterns or []): + return None + tests = self._load_model_test_file(path) + if test_name is not None: + tests = {test_name: tests[test_name]} if test_name in tests else {} + model_names = { + metadata.model_name for metadata in tests.values() if metadata.model_name + } + return model_names or None + + if test_name is None and relative_path.parts[:1] == (c.MODELS,): + model_index = self._read_model_index() + file_models = model_index[1].get(str(relative_path)) if model_index else None + return set(file_models) if file_models else None + + return None + + def _selected_model_paths( + self, model_fqns: t.Set[str] + ) -> t.Optional[t.Tuple[t.Set[Path], t.Set[str]]]: + model_index = self._read_model_index() + if model_index is None: + return None + + index_id, indexed_files = model_index + if index_id != self._model_index_id(): + return None + + current_paths = self._model_paths() indexed_paths = {self.config_path / relative_path for relative_path in indexed_files} if current_paths != indexed_paths: return None diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 1602dc3b11..25859dd90c 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -21,6 +21,11 @@ from sqlmesh.utils import major_minor from sqlmesh.utils.date import now_ds, time_like_to_str, timedelta, to_datetime, yesterday_ds from sqlmesh.core.config.connection import DIALECT_TO_TYPE +from tests.core.test_test import ( + break_unrelated_model, + create_indexed_test_project, + warm_project_index, +) FREEZE_TIME = "2023-01-01 00:00:00 UTC" @@ -2976,3 +2981,97 @@ def execute(context: ExecutionContext, **kwargs: t.Any) -> pd.DataFrame: assert result.exit_code != 0 assert "is not a known model or test file" in result.output mock.assert_not_called() + + +def test_test_use_project_index_skips_unrelated_models(runner: CliRunner, tmp_path: Path) -> None: + """The pre-commit case from #6024: a broken model elsewhere doesn't block a scoped test run.""" + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + break_unrelated_model(tmp_path) + model_path = str(tmp_path / "models" / "a.sql") + + # Without the index the whole project is loaded, so the unrelated model fails the run. + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", model_path]) + assert result.exit_code != 0 + + result = runner.invoke( + cli, ["--paths", str(tmp_path), "test", "--use-project-index", model_path] + ) + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + assert "Successfully Ran 1 tests" in " ".join(result.output.split()) + + +@pytest.mark.parametrize( + "flags, expected_load, expected_load_state", + [ + (["--use-project-index"], False, True), + (["--local"], True, False), + (["--local", "--use-project-index"], False, False), + ], +) +def test_test_use_project_index_and_local_are_independent( + runner: CliRunner, + tmp_path: Path, + mocker, + flags: list[str], + expected_load: bool, + expected_load_state: bool, +) -> None: + """`--use-project-index` defers the load to `test`, `--local` skips state; neither implies the other.""" + create_indexed_test_project(tmp_path) + init_spy = mocker.spy(Context, "__init__") + + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", *flags]) + + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + # With no selectors, every test still runs. + assert "Successfully Ran 2 tests" in " ".join(result.output.split()) + assert init_spy.call_count == 1 + assert init_spy.call_args.kwargs["load"] is expected_load + assert init_spy.call_args.kwargs["load_state"] is expected_load_state + + +def test_test_use_project_index_with_select_model(runner: CliRunner, tmp_path: Path) -> None: + create_indexed_test_project(tmp_path) + + result = runner.invoke( + cli, + ["--paths", str(tmp_path), "test", "--use-project-index", "--select-model", "demo.a"], + ) + + assert result.exit_code == 0, f"Test failed: {result.output}\nException: {result.exception}" + assert "Successfully Ran 1 tests" in " ".join(result.output.split()) + + +def test_test_use_project_index_errors_when_test_model_is_missing( + runner: CliRunner, tmp_path: Path +) -> None: + create_indexed_test_project(tmp_path) + test_path = tmp_path / "tests" / "test_missing.yaml" + test_path.write_text( + """test_missing: + model: demo.nope + outputs: + query: + rows: + - id: 1 +""", + encoding="utf-8", + ) + warm_project_index(tmp_path) + + result = runner.invoke( + cli, ["--paths", str(tmp_path), "test", "--use-project-index", str(test_path)] + ) + + assert result.exit_code != 0 + assert "was not found" in " ".join(result.output.split()) + + +def test_test_use_project_index_no_models(runner: CliRunner, tmp_path: Path) -> None: + with open(tmp_path / "config.yaml", "w", encoding="utf-8") as f: + f.write("model_defaults:\n dialect: duckdb\n") + + result = runner.invoke(cli, ["--paths", str(tmp_path), "test", "--use-project-index"]) + assert result.exit_code == 1 + assert "doesn't seem to have any models" in result.output diff --git a/tests/core/test_test.py b/tests/core/test_test.py index 9be1237b58..07dfb02aaf 100644 --- a/tests/core/test_test.py +++ b/tests/core/test_test.py @@ -3809,3 +3809,198 @@ def test_filter_tests_by_model_names(): dialect="duckdb", ) assert [t.test_name for t in filtered_short] == ["t1"] + + +INDEXED_PROJECT_FQNS = { + name: f'"memory"."demo"."{name}"' for name in ("src", "a", "b", "unrelated") +} + + +def create_indexed_test_project(path: Path) -> None: + """A project where `a` reads from `src` and `b` and `unrelated` stand alone.""" + (path / "models").mkdir(parents=True, exist_ok=True) + (path / "tests").mkdir(parents=True, exist_ok=True) + (path / "config.yaml").write_text( + "gateways:\n" + " local:\n" + " connection:\n" + " type: duckdb\n" + "default_gateway: local\n" + "model_defaults:\n" + " dialect: duckdb\n", + encoding="utf-8", + ) + + models = { + "src": "SELECT 1 AS id, 'x' AS name", + # `SELECT *` only resolves to columns if `src` is loaded alongside `a`. + "a": "SELECT * FROM demo.src", + "b": "SELECT 2 AS id", + "unrelated": "SELECT 3 AS id", + } + for name, query in models.items(): + (path / "models" / f"{name}.sql").write_text( + f"MODEL(name demo.{name}, kind FULL); {query}", encoding="utf-8" + ) + + (path / "tests" / "test_a.yaml").write_text( + """test_a: + model: demo.a + inputs: + demo.src: + rows: + - id: 1 + name: x + outputs: + query: + rows: + - id: 1 + name: x +""", + encoding="utf-8", + ) + (path / "tests" / "test_b.yaml").write_text( + """test_b: + model: demo.b + outputs: + query: + rows: + - id: 2 +""", + encoding="utf-8", + ) + + +def warm_project_index(path: Path) -> None: + Context(paths=path, load=False).load(use_project_index=True) + + +def break_unrelated_model(path: Path) -> None: + """Make `unrelated` fail to load without changing the set of model files. + + The index only tracks which files exist, so it stays valid and a scoped load never + opens this file. A full load does, and fails. + """ + (path / "models" / "unrelated.sql").write_text( + "MODEL(name demo.unrelated, kind NOT_A_KIND); SELECT 3 AS id", encoding="utf-8" + ) + + +@pytest.mark.parametrize( + "selector", + ["models/a.sql", "tests/test_a.yaml", "tests/test_a.yaml::test_a"], +) +def test_use_project_index_loads_only_selected_models( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, selector: str +) -> None: + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + break_unrelated_model(tmp_path) + monkeypatch.chdir(tmp_path) + + context = Context(paths=tmp_path, load=False) + result = context.test(tests=[selector], use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 1 + # The model under test and its upstream are loaded, nothing else. + assert set(context.models) == {INDEXED_PROJECT_FQNS["src"], INDEXED_PROJECT_FQNS["a"]} + # The upstream model is there for column resolution of `SELECT *`. + assert set(context.get_model("demo.a", raise_if_missing=True).columns_to_types or {}) == { + "id", + "name", + } + + +def test_use_project_index_unions_model_and_test_paths( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + break_unrelated_model(tmp_path) + monkeypatch.chdir(tmp_path) + + context = Context(paths=tmp_path, load=False) + result = context.test(tests=["models/b.sql", "tests/test_a.yaml"], use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 2 + assert set(context.models) == { + INDEXED_PROJECT_FQNS["src"], + INDEXED_PROJECT_FQNS["a"], + INDEXED_PROJECT_FQNS["b"], + } + + +def test_use_project_index_errors_when_test_model_is_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + create_indexed_test_project(tmp_path) + (tmp_path / "tests" / "test_missing.yaml").write_text( + """test_missing: + model: demo.nope + outputs: + query: + rows: + - id: 1 +""", + encoding="utf-8", + ) + warm_project_index(tmp_path) + monkeypatch.chdir(tmp_path) + + # Without the index the test is skipped with a warning, which is unchanged. + assert Context(paths=tmp_path).test(tests=["tests/test_missing.yaml"]).wasSuccessful() + + # A scoped run must not pass with nothing tested. + context = Context(paths=tmp_path, load=False) + with pytest.raises(SQLMeshError, match=r"demo.*nope.*test_missing.*was not found"): + context.test(tests=["tests/test_missing.yaml"], use_project_index=True) + + +def test_use_project_index_without_selectors_runs_full_suite(tmp_path: Path) -> None: + create_indexed_test_project(tmp_path) + + context = Context(paths=tmp_path, load=False) + index_path = context._loaders[0]._model_index_path # type: ignore[attr-defined] + assert not index_path.exists() + + result = context.test(use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 2 + assert set(context.models) == set(INDEXED_PROJECT_FQNS.values()) + # A full run refreshes the index, the same as `sqlmesh lint --use-project-index`. + assert index_path.exists() + + +def test_use_project_index_with_cold_index_falls_back_to_full_load( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + create_indexed_test_project(tmp_path) + monkeypatch.chdir(tmp_path) + + context = Context(paths=tmp_path, load=False) + index_path = context._loaders[0]._model_index_path # type: ignore[attr-defined] + result = context.test(tests=["models/a.sql"], use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 1 + assert set(context.models) == set(INDEXED_PROJECT_FQNS.values()) + assert index_path.exists() + + +def test_use_project_index_keeps_loaded_context_whole( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A context that's already loaded must not shrink to the selected models.""" + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + monkeypatch.chdir(tmp_path) + + context = Context(paths=tmp_path) + result = context.test(tests=["models/a.sql"], use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 1 + assert set(context.models) == set(INDEXED_PROJECT_FQNS.values()) From fc4807f07256e9f1a7e703550f104009cd27cae2 Mon Sep 17 00:00:00 2001 From: cookieark Date: Sat, 21 Nov 2026 23:11:07 +0530 Subject: [PATCH 2/2] fix: reload partial contexts and resolve schemas for tested models A context left partially loaded by a scoped test run is reloaded on the next indexed run instead of silently dropping tests whose models it lacks. Scoped runs now update schemas for the models the selected tests target, which can differ from the index hint after a model is renamed in place. Signed-off-by: cookieark --- sqlmesh/core/context.py | 34 +++++++++++++++++++-------- tests/core/test_test.py | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index a02c2c7a90..1ebd8e2b1a 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -420,6 +420,8 @@ def __init__( self._engine_adapter: t.Optional[EngineAdapter] = None self._linters: t.Dict[str, Linter] = {} self._loaded: bool = False + # Whether the last load used the project index to load only some of the models. + self._partially_loaded: bool = False self._materializations_loaded: bool = False self._load_state: bool = load_state self._uncached_model_names: t.Set[str] = set() @@ -808,6 +810,8 @@ def load( ) self._loaded = True + # Loaders only report indexed models when the index actually scoped the load. + self._partially_loaded = bool(indexed_model_fqns) self._materializations_loaded = True return self @@ -2451,12 +2455,18 @@ def test( pd.set_option("display.max_columns", None) target_fqns: t.Optional[t.Set[str]] = None - # A context that is already loaded is left as is, so callers that go on to use it - # don't end up with only part of the project. - if use_project_index and not self._loaded: + # A fully loaded context is left as is, so callers that go on to use it don't end up + # with only part of the project. A partially loaded one may lack the models needed now. + if use_project_index and (not self._loaded or self._partially_loaded): if tests and model_names is None: target_fqns = self._model_fqns_for_test_selectors(tests) - self.load(model_fqns=target_fqns, use_project_index=True) + # For a scoped load, schemas are updated below for the models the selected tests + # target. Those can differ from the index's hint, e.g. after a model is renamed. + self.load( + model_fqns=target_fqns, + use_project_index=True, + update_schemas=target_fqns is None, + ) baseline_meta = self.select_tests( tests=tests, @@ -2477,7 +2487,7 @@ def test( tests_skipped = 0 if target_fqns is not None: - self._raise_for_tests_without_models(test_meta) + self._update_model_schemas_and_validate(self._scoped_test_model_fqns(test_meta)) result = run_tests( model_test_metadata=test_meta, @@ -3773,12 +3783,13 @@ def _model_fqns_for_test_selectors(self, selectors: t.Iterable[str]) -> t.Option return model_fqns or None - def _raise_for_tests_without_models(self, test_meta: t.Iterable[ModelTestMetadata]) -> None: - """Fails a scoped test run when a selected test's model wasn't loaded. + def _scoped_test_model_fqns(self, test_meta: t.Iterable[ModelTestMetadata]) -> t.Set[str]: + """Returns the FQNs of the models targeted by the tests of a scoped run. - A full run warns and skips such a test, but a scoped run that does so can pass while - testing nothing. + Fails if a selected test's model wasn't loaded. A full run warns and skips such a test, + but a scoped run that does so can pass while testing nothing. """ + fqns = set() errors = [] for metadata in test_meta: if not metadata.model_name: @@ -3788,13 +3799,16 @@ def _raise_for_tests_without_models(self, test_meta: t.Iterable[ModelTestMetadat default_catalog=self.default_catalog, dialect=self.default_dialect, ) - if fqn not in self._models: + if fqn in self._models: + fqns.add(fqn) + else: errors.append( f"Model '{fqn}' targeted by test '{metadata.fully_qualified_test_name}' was not found." ) if errors: raise SQLMeshError("\n".join(errors)) + return fqns def _unknown_test_selector_error(self, selector: str) -> str: """Explains why a selector matched nothing. diff --git a/tests/core/test_test.py b/tests/core/test_test.py index 07dfb02aaf..ae53152d3b 100644 --- a/tests/core/test_test.py +++ b/tests/core/test_test.py @@ -4004,3 +4004,55 @@ def test_use_project_index_keeps_loaded_context_whole( assert result.wasSuccessful(), result.failures + result.errors assert result.testsRun == 1 assert set(context.models) == set(INDEXED_PROJECT_FQNS.values()) + + +def test_use_project_index_reloads_partially_loaded_context( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A context left with only some models by a scoped run must not silently drop tests.""" + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + monkeypatch.chdir(tmp_path) + + context = Context(paths=tmp_path, load=False) + assert context.test(tests=["models/a.sql"], use_project_index=True).testsRun == 1 + + # Case: a second scoped run needs a model the first one didn't load. + result = context.test(tests=["models/b.sql"], use_project_index=True) + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 1 + + # Case: a run without selectors loads everything again. + result = context.test(use_project_index=True) + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 2 + assert set(context.models) == set(INDEXED_PROJECT_FQNS.values()) + + +def test_use_project_index_resolves_schemas_after_stale_hint( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Renaming a model in place leaves the index hint stale, which forces a full load.""" + create_indexed_test_project(tmp_path) + warm_project_index(tmp_path) + monkeypatch.chdir(tmp_path) + + (tmp_path / "models" / "a.sql").write_text( + "MODEL(name demo.renamed, kind FULL); SELECT * FROM demo.src", encoding="utf-8" + ) + test_path = tmp_path / "tests" / "test_a.yaml" + test_path.write_text( + test_path.read_text(encoding="utf-8").replace("model: demo.a", "model: demo.renamed"), + encoding="utf-8", + ) + + context = Context(paths=tmp_path, load=False) + result = context.test(tests=["models/a.sql"], use_project_index=True) + + assert result.wasSuccessful(), result.failures + result.errors + assert result.testsRun == 1 + # The schema is resolved for the model the test targets, not the stale name in the index. + assert set(context.get_model("demo.renamed", raise_if_missing=True).columns_to_types or {}) == { + "id", + "name", + }