From a085782aeacab179f53711332de68d7a103b6573 Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Sat, 21 Nov 2026 10:19:31 +0100 Subject: [PATCH 1/2] fix(redshift): prevent empty-schema CREATE TABLE when late-binding CTAS view metadata is unavailable Unannotated SCD_TYPE_2_BY_TIME and SCD_TYPE_2_BY_COLUMN models on Redshift create their snapshot table via empty CTAS (WHERE FALSE LIMIT 0) through VarcharSizeWorkaroundMixin. When svv_columns returned 0 rows for the temporary late-binding view (e.g. over Redshift Datashares, cross-database relations, or restricted search_path schemas), RedshiftEngineAdapter.columns() returned {}, causing VarcharSizeWorkaroundMixin to emit a truncated column-less CREATE TABLE statement (SQLSTATE 42601: syntax error at end of input). - Fall back in RedshiftEngineAdapter.columns() from svv_columns to pg_get_late_binding_view_cols() and pg_catalog.pg_attribute without aborting active transactions, and respect explicit no_schema_binding in create_view(). - Guard VarcharSizeWorkaroundMixin._build_create_table_exp() so empty or unknown view column metadata falls back to the CTAS query instead of building an empty schema. - Return False from columns_to_types_all_known({}) so create_table() raises SQLMeshError rather than emitting column-less DDL. - Quote unmanaged column identifiers in EngineAdapter._scd_type_2() so reserved keywords and special-character column names render safely. Fixes #5787 Signed-off-by: Vamsi-klu --- sqlmesh/core/engine_adapter/base.py | 8 +- sqlmesh/core/engine_adapter/mixins.py | 30 +- sqlmesh/core/engine_adapter/redshift.py | 69 ++- sqlmesh/utils/__init__.py | 6 +- tests/core/engine_adapter/test_mssql.py | 23 + tests/core/engine_adapter/test_redshift.py | 596 +++++++++++++++++++++ tests/utils/test_helpers.py | 19 +- 7 files changed, 725 insertions(+), 26 deletions(-) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 4242e1ebb0..acc4888f7b 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -1943,8 +1943,10 @@ def remove_managed_columns( f"Column {updated_at_name} not found in {target_table}. Table must contain an `updated_at` timestamp for SCD Type 2" ) time_data_type = target_columns_to_types[valid_from_name] - select_source_columns: t.List[t.Union[str, exp.Alias]] = [ - col for col in unmanaged_columns_to_types if col != updated_at_name + select_source_columns: t.List[exp.Expr] = [ + exp.column(col, quoted=True) + for col in unmanaged_columns_to_types + if col != updated_at_name ] table_columns = [exp.column(c, quoted=True) for c in target_columns_to_types] if updated_at_name: @@ -2245,7 +2247,7 @@ def remove_managed_columns( .with_( "inserted_rows", exp.select( - *unmanaged_columns_to_types, + *(exp.column(col, quoted=True) for col in unmanaged_columns_to_types), insert_valid_from_start.as_(valid_from_col.this), # type: ignore to_time_column(exp.null(), time_data_type, self.dialect, nullable=True).as_( valid_to_col.this diff --git a/sqlmesh/core/engine_adapter/mixins.py b/sqlmesh/core/engine_adapter/mixins.py index 50b8b8752b..0bf241d4dc 100644 --- a/sqlmesh/core/engine_adapter/mixins.py +++ b/sqlmesh/core/engine_adapter/mixins.py @@ -14,6 +14,7 @@ from sqlmesh.core.node import IntervalUnit from sqlmesh.core.dialect import schema_ from sqlmesh.core.schema_diff import TableAlterOperation +from sqlmesh.utils import columns_to_types_all_known from sqlmesh.utils.errors import SQLMeshError if t.TYPE_CHECKING: @@ -281,19 +282,22 @@ def _build_create_table_exp( self.columns(temp_view_name) ) - schema = self._build_schema_exp( - exp.to_table(table_name_or_schema), - columns_to_types_from_view, - ) - statement = super()._build_create_table_exp( - schema, - None, - exists=exists, - replace=replace, - target_columns_to_types=columns_to_types_from_view, - table_description=table_description, - **kwargs, - ) + if columns_to_types_from_view and columns_to_types_all_known( + columns_to_types_from_view + ): + schema = self._build_schema_exp( + exp.to_table(table_name_or_schema), + columns_to_types_from_view, + ) + statement = super()._build_create_table_exp( + schema, + None, + exists=exists, + replace=replace, + target_columns_to_types=columns_to_types_from_view, + table_description=table_description, + **kwargs, + ) finally: self.drop_view(temp_view_name) diff --git a/sqlmesh/core/engine_adapter/redshift.py b/sqlmesh/core/engine_adapter/redshift.py index 8f2f098d08..7e14695f0a 100644 --- a/sqlmesh/core/engine_adapter/redshift.py +++ b/sqlmesh/core/engine_adapter/redshift.py @@ -103,6 +103,59 @@ def columns( sql = sql.where(exp.column("table_schema").eq(table.args["db"].name)) columns_raw = self.fetchall(sql, quote_identifiers=True) + if not columns_raw: + # svv_columns can return 0 rows for late-binding views referencing cross-database / + # datashare objects or schemas without USAGE metadata visibility. Fall back to + # pg_get_late_binding_view_cols() and then pg_catalog.pg_attribute. + late_binding_sql = ( + exp.select("col_name", "col_type") + .from_( + exp.Table( + this=exp.func("pg_get_late_binding_view_cols"), + alias=exp.TableAlias( + this=exp.to_identifier("cols"), + columns=[ + exp.ColumnDef( + this=exp.to_identifier("view_schema"), + kind=exp.DataType.build("NAME", dialect=self.dialect), + ), + exp.ColumnDef( + this=exp.to_identifier("view_name"), + kind=exp.DataType.build("NAME", dialect=self.dialect), + ), + exp.ColumnDef( + this=exp.to_identifier("col_name"), + kind=exp.DataType.build("NAME", dialect=self.dialect), + ), + exp.ColumnDef( + this=exp.to_identifier("col_type"), + kind=exp.DataType.build("VARCHAR", dialect=self.dialect), + ), + exp.ColumnDef( + this=exp.to_identifier("col_num"), + kind=exp.DataType.build("INT", dialect=self.dialect), + ), + ], + ), + ) + ) + .where(exp.column("view_name").eq(table.alias_or_name)) + .order_by("col_num", dialect=self.dialect) + ) + if table.args.get("db"): + late_binding_sql = late_binding_sql.where( + exp.column("view_schema").eq(table.args["db"].name) + ) + late_binding_cols = self.fetchall(late_binding_sql, quote_identifiers=True) + if late_binding_cols: + return { + col_name: exp.DataType.build(col_type, dialect=self.dialect, udt=True) + for col_name, col_type in late_binding_cols + } + try: + return super().columns(table_name, include_pseudo_columns=include_pseudo_columns) + except SQLMeshError: + return {} def build_var_length_col( column_name: str, @@ -228,13 +281,15 @@ def create_view( underlying table without dropping the view first. This is a problem for us since we want to be able to swap tables out from under views. Therefore, we create the view as non-binding. """ - no_schema_binding = True - if isinstance(query_or_df, exp.Expr): - # We can't include NO SCHEMA BINDING if the query has a recursive CTE - has_recursive_cte = any( - w.args.get("recursive", False) for w in query_or_df.find_all(exp.With) - ) - no_schema_binding = not has_recursive_cte + no_schema_binding: t.Optional[bool] = create_kwargs.pop("no_schema_binding", None) + if no_schema_binding is None: + no_schema_binding = True + if isinstance(query_or_df, exp.Expr): + # We can't include NO SCHEMA BINDING if the query has a recursive CTE + has_recursive_cte = any( + w.args.get("recursive", False) for w in query_or_df.find_all(exp.With) + ) + no_schema_binding = not has_recursive_cte return super().create_view( view_name, diff --git a/sqlmesh/utils/__init__.py b/sqlmesh/utils/__init__.py index da8d4f85a4..5f18c566e3 100644 --- a/sqlmesh/utils/__init__.py +++ b/sqlmesh/utils/__init__.py @@ -342,8 +342,10 @@ def type_is_known(d_type: t.Union[exp.DataType, exp.ColumnDef]) -> bool: def columns_to_types_all_known(columns_to_types: t.Dict[str, exp.DataType]) -> bool: - """Checks that all column types are known and not NULL.""" - return all(type_is_known(expression) for expression in columns_to_types.values()) + """Checks that all column types are non-empty, known, and not NULL.""" + return bool(columns_to_types) and all( + type_is_known(expression) for expression in columns_to_types.values() + ) class Verbosity(IntEnum): diff --git a/tests/core/engine_adapter/test_mssql.py b/tests/core/engine_adapter/test_mssql.py index e54a42ff29..2cd6ad0554 100644 --- a/tests/core/engine_adapter/test_mssql.py +++ b/tests/core/engine_adapter/test_mssql.py @@ -1055,3 +1055,26 @@ def test_comments(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture) adapter._build_create_comment_table_exp(table, comment), adapter._build_create_comment_column_exp(table, "a", comment), ] + + +def test_ctas_varchar_workaround_empty_columns_fallback( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(MSSQLEngineAdapter) + mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="test_random_id") + mocker.patch( + "sqlmesh.core.engine_adapter.mssql.MSSQLEngineAdapter.table_exists", return_value=False + ) + mocker.patch("sqlmesh.core.engine_adapter.mssql.MSSQLEngineAdapter.columns", return_value={}) + + adapter.ctas( + table_name="test_schema.test_table", + query_or_df=parse_one("SELECT a, b FROM tbl WHERE FALSE LIMIT 0"), + exists=False, + ) + + assert to_sql_calls(adapter) == [ + "CREATE VIEW [__temp_ctas_test_random_id] AS SELECT [a], [b] FROM [tbl];", + "DROP VIEW IF EXISTS [__temp_ctas_test_random_id];", + "SELECT * INTO [test_schema].[test_table] FROM (SELECT TOP 0 [a] AS [a], [b] AS [b] FROM [tbl] WHERE (1 = 0)) AS temp;", + ] diff --git a/tests/core/engine_adapter/test_redshift.py b/tests/core/engine_adapter/test_redshift.py index a3260ed4cd..1eea84e8a8 100644 --- a/tests/core/engine_adapter/test_redshift.py +++ b/tests/core/engine_adapter/test_redshift.py @@ -14,6 +14,7 @@ from sqlmesh.core.schema_diff import TableAlterDropColumnOperation from sqlmesh.core.model import load_sql_based_model from sqlmesh.core.model.definition import SqlModel +from sqlmesh.core.snapshot.evaluator import SCDType2Strategy from sqlmesh.utils.errors import SQLMeshError from tests.core.engine_adapter import to_sql_calls @@ -956,3 +957,598 @@ def test_redshift_multiple_column_drops_no_clustering_operations( assert len(operations) == 2 assert all(isinstance(op, TableAlterDropColumnOperation) for op in operations) + + +@pytest.mark.parametrize( + "mocked_columns", + [ + {}, + { + "id": exp.DataType.build("UNKNOWN"), + "valid_from": exp.DataType.build("TIMESTAMP"), + }, + ], + ids=["empty_columns", "unknown_column_type"], +) +def test_ctas_varchar_workaround_empty_or_unknown_columns_fallback( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, + mocked_columns: t.Dict[str, exp.DataType], +) -> None: + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter) + + mocker.patch( + "sqlmesh.core.engine_adapter.base.random_id", + return_value="test_random_id", + ) + mocker.patch( + "sqlmesh.core.engine_adapter.redshift.RedshiftEngineAdapter.table_exists", + return_value=False, + ) + mocker.patch( + "sqlmesh.core.engine_adapter.redshift.RedshiftEngineAdapter.columns", + return_value=mocked_columns, + ) + + adapter.ctas( + table_name="sqlmesh_xxx.my_example", + query_or_df=parse_one( + "SELECT id, updated_at, trashed_at, CAST(NULL AS TIMESTAMP) AS valid_from, CAST(NULL AS TIMESTAMP) AS valid_to FROM my_table WHERE FALSE LIMIT 0" + ), + exists=False, + ) + + sql_calls = to_sql_calls(adapter) + assert 'CREATE TABLE "sqlmesh_xxx"."my_example"' not in sql_calls + assert all("UNKNOWN" not in sql for sql in sql_calls) + assert sql_calls == [ + 'CREATE VIEW "__temp_ctas_test_random_id" AS SELECT "id", "updated_at", "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" WITH NO SCHEMA BINDING', + 'DROP VIEW IF EXISTS "__temp_ctas_test_random_id" CASCADE', + 'CREATE TABLE "sqlmesh_xxx"."my_example" AS SELECT "id", "updated_at", "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" WHERE FALSE LIMIT 0', + ] + for sql in sql_calls: + parse_one(sql, read="redshift") + + +def test_columns_late_binding_view_and_pg_catalog_fallback( + make_mocked_engine_adapter: t.Callable, +) -> None: + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter) + + # Subcase 2a: svv_columns returns [] -> fallback to pg_get_late_binding_view_cols() (without schema) + adapter.cursor.fetchall.side_effect = [ + [], + [ + ("id", "character varying(36)"), + ("name", "character varying(256)"), + ("updated_at", "timestamp without time zone"), + ("deleted_at", "timestamp with time zone"), + ], + ] + resp = adapter.columns("__temp_ctas_abc") + assert to_sql_calls(adapter) == [ + """SELECT "column_name", "data_type", "character_maximum_length", "numeric_precision", "numeric_scale" FROM "svv_columns" WHERE "table_name" = '__temp_ctas_abc'""", + """SELECT "col_name", "col_type" FROM PG_GET_LATE_BINDING_VIEW_COLS() AS "cols"("view_schema" NAME, "view_name" NAME, "col_name" NAME, "col_type" VARCHAR, "col_num" INTEGER) WHERE "view_name" = '__temp_ctas_abc' ORDER BY "col_num\"""", + ] + assert list(resp.keys()) == ["id", "name", "updated_at", "deleted_at"] + assert resp == { + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "name": exp.DataType.build("VARCHAR(256)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "deleted_at": exp.DataType.build("TIMESTAMPTZ", dialect="redshift"), + } + assert adapter._default_precision_to_max(resp) == { + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "name": exp.DataType.build("VARCHAR(max)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "deleted_at": exp.DataType.build("TIMESTAMPTZ", dialect="redshift"), + } + + # Subcase 2a (with schema): includes AND "view_schema" = 'my_schema' + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.side_effect = [ + [], + [("id", "character varying(36)")], + ] + resp_schema = adapter.columns("my_schema.my_lbv") + assert to_sql_calls(adapter) == [ + """SELECT "column_name", "data_type", "character_maximum_length", "numeric_precision", "numeric_scale" FROM "svv_columns" WHERE "table_name" = 'my_lbv' AND "table_schema" = 'my_schema'""", + """SELECT "col_name", "col_type" FROM PG_GET_LATE_BINDING_VIEW_COLS() AS "cols"("view_schema" NAME, "view_name" NAME, "col_name" NAME, "col_type" VARCHAR, "col_num" INTEGER) WHERE "view_name" = 'my_lbv' AND "view_schema" = 'my_schema' ORDER BY "col_num\"""", + ] + assert resp_schema == {"id": exp.DataType.build("VARCHAR(36)", dialect="redshift")} + + # Subcase 2b: svv_columns and pg_get_late_binding_view_cols() both return [] -> super().columns() pg_catalog.pg_attribute fallback + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.side_effect = [ + [], + [], + [("id", "integer"), ("ts", "timestamp without time zone")], + ] + resp_pg = adapter.columns("db.local_tbl") + sql_calls_pg = to_sql_calls(adapter) + assert len(sql_calls_pg) == 3 + assert '"pg_catalog"."pg_attribute"' in sql_calls_pg[2] + assert resp_pg == { + "id": exp.DataType.build("INT", dialect="redshift"), + "ts": exp.DataType.build("TIMESTAMP", dialect="redshift"), + } + + # Subcase 2c: all three catalog queries return [] -> catches SQLMeshError and returns {} + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.side_effect = [[], [], []] + assert adapter.columns("db.missing") == {} + + +def test_create_view_explicit_no_schema_binding_override( + make_mocked_engine_adapter: t.Callable, +) -> None: + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter) + + adapter.create_view( + view_name="test_view", + query_or_df=parse_one("SELECT cola FROM tbl"), + no_schema_binding=False, + ) + assert to_sql_calls(adapter) == [ + 'DROP VIEW IF EXISTS "test_view" CASCADE', + 'CREATE VIEW "test_view" AS SELECT "cola" FROM "tbl"', + ] + + adapter.cursor.execute.reset_mock() + adapter.create_view( + view_name="test_view", + query_or_df=parse_one("SELECT cola FROM tbl"), + ) + assert to_sql_calls(adapter) == [ + 'DROP VIEW IF EXISTS "test_view" CASCADE', + 'CREATE VIEW "test_view" AS SELECT "cola" FROM "tbl" WITH NO SCHEMA BINDING', + ] + + adapter.cursor.execute.reset_mock() + adapter.create_view( + view_name="test_view", + query_or_df=parse_one( + "WITH RECURSIVE cte AS (SELECT 1 AS cola) SELECT cola FROM cte", + dialect="redshift", + ), + ) + assert to_sql_calls(adapter) == [ + 'DROP VIEW IF EXISTS "test_view" CASCADE', + 'CREATE VIEW "test_view" AS WITH RECURSIVE "cte" AS (SELECT 1 AS "cola") SELECT "cola" FROM "cte"', + ] + + +def test_scd_type_2_by_time_unannotated_lifecycle_issue_5787( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, +) -> None: + model = load_sql_based_model( + d.parse( + """ + MODEL ( + name sqlmesh_xxx.my_example, + kind SCD_TYPE_2_BY_TIME ( + unique_key id + ) + ); + SELECT + id, + updated_at, + trashed_at + FROM my_table + """ + ), + dialect="redshift", + ) + assert not model.annotated + + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter, default_catalog="dev") + adapter.cursor.fetchone.side_effect = None + adapter.cursor.fetchone.return_value = None + mocker.patch( + "sqlmesh.core.engine_adapter.base.random_id", + return_value="test_random_id", + ) + + strategy = SCDType2Strategy(adapter) + target_table = "sqlmesh__sqlmesh_xxx.sqlmesh_xxx__my_example__1234567890" + truncated_ddl = 'CREATE TABLE "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890"' + + # Step 1a: strategy.create when svv_columns returns [] and pg_get_late_binding_view_cols() resolves columns + adapter.cursor.fetchall.side_effect = [ + [], + [ + ("id", "character varying(36)"), + ("updated_at", "timestamp without time zone"), + ("trashed_at", "timestamp without time zone"), + ("valid_from", "timestamp without time zone"), + ("valid_to", "timestamp without time zone"), + ], + ] + strategy.create( + table_name=target_table, + model=model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + create_calls_lbv = to_sql_calls(adapter) + assert truncated_ddl not in create_calls_lbv + assert ( + 'CREATE TABLE "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890" ("id" VARCHAR(36), "updated_at" TIMESTAMP, "trashed_at" TIMESTAMP, "valid_from" TIMESTAMP, "valid_to" TIMESTAMP)' + in create_calls_lbv + ) + for sql in create_calls_lbv: + parse_one(sql, read="redshift") + + # Step 1b: strategy.create when columns("__temp_ctas_*") returns {} completely (all catalog queries empty) + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.side_effect = [[], [], []] + strategy.create( + table_name=target_table, + model=model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + create_calls_fallback = to_sql_calls(adapter) + assert truncated_ddl not in create_calls_fallback + assert ( + 'CREATE TABLE "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890" AS SELECT "id" AS "id", "updated_at" AS "updated_at", "trashed_at" AS "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" AS "my_table" WHERE FALSE LIMIT 0' + in create_calls_fallback + ) + for sql in create_calls_fallback: + parse_one(sql, read="redshift") + + # Step 2 (maiden load: is_first_insert=True) & Step 3 (incremental load: is_first_insert=False) + adapter.cursor.fetchone.return_value = ("dev",) + mocker.patch.object( + adapter, + "_get_data_objects", + return_value=[ + DataObject( + catalog="dev", + schema="sqlmesh__sqlmesh_xxx", + name="sqlmesh_xxx__my_example__1234567890", + type=DataObjectType.TABLE, + ) + ], + ) + mocker.patch.object( + adapter, + "columns", + return_value={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "trashed_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + + for is_first_insert in (True, False): + adapter.cursor.execute.reset_mock() + strategy.insert( + table_name=target_table, + query_or_df=model.render_query_or_raise(), + model=model, + is_first_insert=is_first_insert, + render_kwargs={}, + execution_time="2024-01-02 00:00:00", + ) + insert_calls = to_sql_calls(adapter) + for sql in insert_calls: + parse_one(sql, read="redshift") + assert "DISTINCT ON" not in sql + assert "TRUNCATE" not in sql + + assert any( + sql.startswith( + 'CREATE TABLE "dev"."sqlmesh__sqlmesh_xxx"."__temp_sqlmesh_xxx__my_example__1234567890_test_random_id" AS SELECT ' + ) + for sql in insert_calls + ) + assert ( + 'DELETE FROM "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890" WHERE TRUE' + in insert_calls + ) + assert ( + 'DROP TABLE IF EXISTS "sqlmesh__sqlmesh_xxx"."__temp_sqlmesh_xxx__my_example__1234567890_test_random_id"' + in insert_calls + ) + + scd_insert_sql = [ + sql + for sql in insert_calls + if sql.startswith( + 'INSERT INTO "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890" ("id", "updated_at", "trashed_at", "valid_from", "valid_to") WITH "source" AS (' + ) + ] + assert len(scd_insert_sql) == 1 + assert ( + 'ROW_NUMBER() OVER (PARTITION BY "id" ORDER BY "id") AS _row_number' + in scd_insert_sql[0] + ) + if is_first_insert: + assert 'WHERE NOT "valid_to" IS NULL LIMIT 0' in scd_insert_sql[0] + assert 'WHERE "valid_to" IS NULL LIMIT 0' in scd_insert_sql[0] + else: + assert "LIMIT 0" not in scd_insert_sql[0] + + +@pytest.mark.parametrize( + "scenario", + [ + "reserved_from_by_time", + "reserved_select_by_time", + "hyphenated_col_by_time", + "reserved_from_by_column", + "reserved_select_by_column", + "hyphenated_col_by_column", + "composite_key_hard_deletes_timestamptz", + "expression_key_no_hard_deletes", + "dataframe_source_by_time", + "by_column_strategy_star_and_explicit", + ], +) +def test_scd_type_2_redshift_edge_cases( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, + scenario: str, +) -> None: + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter, default_catalog="dev") + adapter.cursor.fetchone.return_value = ("dev",) + mocker.patch( + "sqlmesh.core.engine_adapter.base.random_id", + return_value="test_random_id", + ) + mocker.patch.object( + adapter, + "_get_data_objects", + return_value=[ + DataObject( + catalog="dev", + schema="s", + name="t", + type=DataObjectType.TABLE, + ) + ], + ) + + if scenario in ( + "reserved_from_by_time", + "reserved_select_by_time", + "hyphenated_col_by_time", + ): + special_col = { + "reserved_from_by_time": "from", + "reserved_select_by_time": "select", + "hyphenated_col_by_time": "col-name", + }[scenario] + adapter.scd_type_2_by_time( + target_table="s.t", + source_table=parse_one( + f'SELECT id, "{special_col}", updated_at FROM src', + read="redshift", + ), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00", + updated_at_col=exp.column("updated_at", quoted=True), + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + special_col: exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + sql_calls = to_sql_calls(adapter) + insert_sql = [s for s in sql_calls if s.startswith('INSERT INTO "s"."t"')][0] + assert '"col" - "name"' not in insert_sql + assert "AS _col" not in insert_sql + assert f'"inserted_rows" AS (SELECT "id", "{special_col}", "updated_at"' in insert_sql + assert f'SELECT "_exists", "id", "{special_col}", "updated_at" FROM' in insert_sql + + elif scenario in ( + "reserved_from_by_column", + "reserved_select_by_column", + "hyphenated_col_by_column", + ): + special_col = { + "reserved_from_by_column": "from", + "reserved_select_by_column": "select", + "hyphenated_col_by_column": "col-name", + }[scenario] + check_cols: t.Union[exp.Star, t.Sequence[exp.Expr]] = ( + [exp.column(special_col, quoted=True)] + if scenario == "reserved_select_by_column" + else exp.Star() + ) + adapter.scd_type_2_by_column( + target_table="s.t", + source_table=parse_one( + f'SELECT id, "{special_col}" FROM src', + read="redshift", + ), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00", + check_columns=check_cols, + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + special_col: exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + sql_calls = to_sql_calls(adapter) + insert_sql = [s for s in sql_calls if s.startswith('INSERT INTO "s"."t"')][0] + assert '"col" - "name"' not in insert_sql + assert "AS _col" not in insert_sql + assert f'"inserted_rows" AS (SELECT "id", "{special_col}",' in insert_sql + assert f'SELECT "_exists", "id", "{special_col}" FROM' in insert_sql + + elif scenario == "composite_key_hard_deletes_timestamptz": + adapter.scd_type_2_by_time( + target_table="s.t", + source_table=parse_one("SELECT id, sub_id, val, updated_at FROM src", read="redshift"), + unique_key=[exp.column("id"), exp.column("sub_id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00+00:00", + updated_at_col=exp.column("updated_at", quoted=True), + invalidate_hard_deletes=True, + updated_at_as_valid_from=True, + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "sub_id": exp.DataType.build("INT", dialect="redshift"), + "val": exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMPTZ", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMPTZ", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMPTZ", dialect="redshift"), + }, + ) + sql_calls = to_sql_calls(adapter) + insert_sql = [s for s in sql_calls if s.startswith('INSERT INTO "s"."t"')][0] + assert 'PARTITION BY "id", "sub_id" ORDER BY "id", "sub_id"' in insert_sql + assert "TIMESTAMP WITH TIME ZONE" in insert_sql + assert 'WHEN "joined"."_exists" IS NULL THEN' in insert_sql + assert "1970-01-01" not in insert_sql + + elif scenario == "expression_key_no_hard_deletes": + adapter.scd_type_2_by_time( + target_table="s.t", + source_table=parse_one("SELECT id, val, updated_at FROM src", read="redshift"), + unique_key=[parse_one("COALESCE(id, '')", dialect="redshift")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00", + updated_at_col=exp.column("updated_at", quoted=True), + invalidate_hard_deletes=False, + updated_at_as_valid_from=False, + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "val": exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + sql_calls = to_sql_calls(adapter) + insert_sql = [s for s in sql_calls if s.startswith('INSERT INTO "s"."t"')][0] + assert "COALESCE(\"id\", '')" in insert_sql + assert 'WHEN "joined"."_exists" IS NULL THEN' not in insert_sql + assert "CAST('1970-01-01 00:00:00' AS TIMESTAMP)" in insert_sql + + elif scenario == "dataframe_source_by_time": + df = pd.DataFrame( + { + "id": ["a", "b"], + "val": ["x", "y"], + "updated_at": ["2024-01-01 00:00:00", "2024-01-01 01:00:00"], + } + ) + adapter.scd_type_2_by_time( + target_table="s.t", + source_table=df, + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00", + updated_at_col=exp.column("updated_at", quoted=True), + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "val": exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "updated_at": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + sql_calls = to_sql_calls(adapter) + insert_sql = [s for s in sql_calls if s.startswith('INSERT INTO "s"."t"')][0] + assert "UNION ALL SELECT" in insert_sql + assert "VALUES" not in insert_sql + + elif scenario == "by_column_strategy_star_and_explicit": + model_star = load_sql_based_model( + d.parse( + """ + MODEL ( + name s.t, + kind SCD_TYPE_2_BY_COLUMN ( + unique_key id, + columns *, + execution_time_as_valid_from true + ) + ); + SELECT id, val FROM src + """ + ), + dialect="redshift", + ) + assert not model_star.annotated + strategy = SCDType2Strategy(adapter) + + mocker.patch.object(adapter, "table_exists", return_value=False) + mocker.patch.object(adapter, "columns", return_value={}) + strategy.create( + table_name="s.t", + model=model_star, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + create_calls = to_sql_calls(adapter) + assert 'CREATE TABLE "s"."t"' not in create_calls + assert ( + 'CREATE TABLE "s"."t" AS SELECT "id" AS "id", "val" AS "val", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "src" AS "src" WHERE FALSE LIMIT 0' + in create_calls + ) + + mocker.patch.object( + adapter, + "columns", + return_value={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "val": exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + for exec_as_valid_from, check_cols_arg in ( + (True, exp.Star()), + (False, [exp.column("val")]), + ): + adapter.cursor.execute.reset_mock() + adapter.scd_type_2_by_column( + target_table="s.t", + source_table=parse_one("SELECT id, val FROM src", read="redshift"), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + execution_time="2024-01-02 00:00:00", + check_columns=check_cols_arg, + execution_time_as_valid_from=exec_as_valid_from, + truncate=True, + target_columns_to_types={ + "id": exp.DataType.build("VARCHAR(36)", dialect="redshift"), + "val": exp.DataType.build("VARCHAR(100)", dialect="redshift"), + "valid_from": exp.DataType.build("TIMESTAMP", dialect="redshift"), + "valid_to": exp.DataType.build("TIMESTAMP", dialect="redshift"), + }, + ) + sub_calls = to_sql_calls(adapter) + for sql in sub_calls: + parse_one(sql, read="redshift") + sub_insert = [s for s in sub_calls if s.startswith('INSERT INTO "s"."t"')][0] + if exec_as_valid_from: + assert "1970-01-01" not in sub_insert + else: + assert "CAST('1970-01-01 00:00:00' AS TIMESTAMP)" in sub_insert + sql_calls = create_calls + + for sql in sql_calls: + parse_one(sql, read="redshift") diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 20a544512e..3e28fcdad5 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -1,11 +1,13 @@ +import typing as t import pytest from functools import wraps from sqlglot import expressions from sqlglot.optimizer.annotate_types import annotate_types from sqlmesh.core.console import set_console, get_console, TerminalConsole - +from sqlmesh.core.engine_adapter import EngineAdapter, RedshiftEngineAdapter from sqlmesh.utils import columns_to_types_all_known +from sqlmesh.utils.errors import SQLMeshError @pytest.mark.parametrize( @@ -71,12 +73,27 @@ {"a": annotate_types(expressions.DataType.build("VARCHAR(MAX)", dialect="redshift"))}, True, ), + ({}, False), ], ) def test_columns_to_types_all_known(columns_to_types, expected) -> None: assert columns_to_types_all_known(columns_to_types) == expected +@pytest.mark.parametrize("adapter_cls", [EngineAdapter, RedshiftEngineAdapter]) +def test_create_table_empty_columns_raises( + make_mocked_engine_adapter: t.Callable, adapter_cls: t.Type[EngineAdapter] +) -> None: + adapter = make_mocked_engine_adapter(adapter_cls) + with pytest.raises( + SQLMeshError, + match="Cannot create a table without knowing the column types", + ): + adapter.create_table("s.t", {}, exists=False) + + adapter.cursor.execute.assert_not_called() + + def use_terminal_console(func): @wraps(func) def test_wrapper(*args, **kwargs): From 65b3b488e6a3d3d5d87d56d57e4447f441e4d1ae Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Sun, 22 Nov 2026 00:03:41 +0100 Subject: [PATCH 2/2] fix(redshift): address review feedback on late-binding CTAS fallback - Raise SQLMeshError in VarcharSizeWorkaroundMixin when the temp view's column types are unavailable, instead of creating the table from the LIMIT 0 CTAS with non-deterministic VARCHAR widths (Redshift and MSSQL). - Order BasePostgresEngineAdapter.columns() by attnum so the pg_catalog fallback preserves the view's SELECT order. - Document why Redshift's exists=True path does not forward target types and pin view-derived VARCHAR widths with a test. - Schema-qualify late-binding view sources in the test fixtures. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Vamsi-klu --- sqlmesh/core/engine_adapter/base_postgres.py | 1 + sqlmesh/core/engine_adapter/mixins.py | 36 ++++-- sqlmesh/core/engine_adapter/redshift.py | 3 + .../core/engine_adapter/test_base_postgres.py | 1 + tests/core/engine_adapter/test_mssql.py | 13 +- tests/core/engine_adapter/test_redshift.py | 113 ++++++++++++------ 6 files changed, 111 insertions(+), 56 deletions(-) diff --git a/sqlmesh/core/engine_adapter/base_postgres.py b/sqlmesh/core/engine_adapter/base_postgres.py index e2347b1263..aac1c66711 100644 --- a/sqlmesh/core/engine_adapter/base_postgres.py +++ b/sqlmesh/core/engine_adapter/base_postgres.py @@ -55,6 +55,7 @@ def columns( ) if table.args.get("db"): sql = sql.where(exp.column("nspname").eq(table.args["db"].name)) + sql = sql.order_by("attnum") self.execute(sql) resp = self.cursor.fetchall() diff --git a/sqlmesh/core/engine_adapter/mixins.py b/sqlmesh/core/engine_adapter/mixins.py index 0bf241d4dc..b0118accd3 100644 --- a/sqlmesh/core/engine_adapter/mixins.py +++ b/sqlmesh/core/engine_adapter/mixins.py @@ -282,22 +282,32 @@ def _build_create_table_exp( self.columns(temp_view_name) ) - if columns_to_types_from_view and columns_to_types_all_known( + if not ( columns_to_types_from_view + and columns_to_types_all_known(columns_to_types_from_view) ): - schema = self._build_schema_exp( - exp.to_table(table_name_or_schema), - columns_to_types_from_view, - ) - statement = super()._build_create_table_exp( - schema, - None, - exists=exists, - replace=replace, - target_columns_to_types=columns_to_types_from_view, - table_description=table_description, - **kwargs, + # View metadata is unavailable (e.g. a late-binding view the catalog doesn't expose). + # Never fall back to the LIMIT 0 CTAS since its column types (e.g. VARCHAR widths) + # are non-deterministic and the table could silently truncate data. + raise SQLMeshError( + f"Could not determine column types for table " + f"'{exp.to_table(table_name_or_schema).sql(dialect=self.dialect)}' from " + "its query. Specify the column types explicitly, e.g. by annotating the model." ) + + schema = self._build_schema_exp( + exp.to_table(table_name_or_schema), + columns_to_types_from_view, + ) + statement = super()._build_create_table_exp( + schema, + None, + exists=exists, + replace=replace, + target_columns_to_types=columns_to_types_from_view, + table_description=table_description, + **kwargs, + ) finally: self.drop_view(temp_view_name) diff --git a/sqlmesh/core/engine_adapter/redshift.py b/sqlmesh/core/engine_adapter/redshift.py index 7e14695f0a..482f37c08d 100644 --- a/sqlmesh/core/engine_adapter/redshift.py +++ b/sqlmesh/core/engine_adapter/redshift.py @@ -253,6 +253,9 @@ def _create_table_from_source_queries( ) if self.table_exists(table_name): return + # target_columns_to_types is deliberately not forwarded: inferred types can lack VARCHAR widths, and + # casting to them would truncate data. Keeping the LIMIT 0 CTAS lets VarcharSizeWorkaroundMixin derive + # the exact widths from a temp view instead. super()._create_table_from_source_queries( table_name, source_queries, diff --git a/tests/core/engine_adapter/test_base_postgres.py b/tests/core/engine_adapter/test_base_postgres.py index f286c47c56..8a24cf0d1e 100644 --- a/tests/core/engine_adapter/test_base_postgres.py +++ b/tests/core/engine_adapter/test_base_postgres.py @@ -23,6 +23,7 @@ def test_columns(make_mocked_engine_adapter: t.Callable): 'JOIN "pg_catalog"."pg_class" ON "pg_class"."oid" = "attrelid" ' 'JOIN "pg_catalog"."pg_namespace" ON "pg_namespace"."oid" = "relnamespace" ' """WHERE ("attnum" > 0 AND NOT "attisdropped" AND "relname" = 'table') AND "nspname" = 'db'""" + ' ORDER BY "attnum"' ) assert resp == {"col": exp.DataType.build("INT")} diff --git a/tests/core/engine_adapter/test_mssql.py b/tests/core/engine_adapter/test_mssql.py index 2cd6ad0554..3a6a6a1659 100644 --- a/tests/core/engine_adapter/test_mssql.py +++ b/tests/core/engine_adapter/test_mssql.py @@ -18,6 +18,7 @@ from sqlmesh.core import dialect as d from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType, SourceQuery from sqlmesh.utils.date import to_ds +from sqlmesh.utils.errors import SQLMeshError from tests.core.engine_adapter import to_sql_calls pytestmark = [pytest.mark.engine, pytest.mark.mssql] @@ -1067,14 +1068,14 @@ def test_ctas_varchar_workaround_empty_columns_fallback( ) mocker.patch("sqlmesh.core.engine_adapter.mssql.MSSQLEngineAdapter.columns", return_value={}) - adapter.ctas( - table_name="test_schema.test_table", - query_or_df=parse_one("SELECT a, b FROM tbl WHERE FALSE LIMIT 0"), - exists=False, - ) + with pytest.raises(SQLMeshError, match="Could not determine column types"): + adapter.ctas( + table_name="test_schema.test_table", + query_or_df=parse_one("SELECT a, b FROM tbl WHERE FALSE LIMIT 0"), + exists=False, + ) assert to_sql_calls(adapter) == [ "CREATE VIEW [__temp_ctas_test_random_id] AS SELECT [a], [b] FROM [tbl];", "DROP VIEW IF EXISTS [__temp_ctas_test_random_id];", - "SELECT * INTO [test_schema].[test_table] FROM (SELECT TOP 0 [a] AS [a], [b] AS [b] FROM [tbl] WHERE (1 = 0)) AS temp;", ] diff --git a/tests/core/engine_adapter/test_redshift.py b/tests/core/engine_adapter/test_redshift.py index 1eea84e8a8..37bbe24e3e 100644 --- a/tests/core/engine_adapter/test_redshift.py +++ b/tests/core/engine_adapter/test_redshift.py @@ -990,21 +990,63 @@ def test_ctas_varchar_workaround_empty_or_unknown_columns_fallback( return_value=mocked_columns, ) + with pytest.raises(SQLMeshError, match="Could not determine column types"): + adapter.ctas( + table_name="sqlmesh_xxx.my_example", + query_or_df=parse_one( + "SELECT id, updated_at, trashed_at, CAST(NULL AS TIMESTAMP) AS valid_from, CAST(NULL AS TIMESTAMP) AS valid_to FROM sqlmesh_xxx.my_table WHERE FALSE LIMIT 0" + ), + exists=False, + ) + + # The table must never be created from the LIMIT 0 CTAS (VARCHAR widths are non-deterministic) + # and the temp view must still be cleaned up. + assert to_sql_calls(adapter) == [ + 'CREATE VIEW "__temp_ctas_test_random_id" AS SELECT "id", "updated_at", "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "sqlmesh_xxx"."my_table" WITH NO SCHEMA BINDING', + 'DROP VIEW IF EXISTS "__temp_ctas_test_random_id" CASCADE', + ] + for sql in to_sql_calls(adapter): + parse_one(sql, read="redshift") + + +def test_ctas_exists_true_derives_varchar_widths_from_view_not_target_types( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, +) -> None: + adapter = make_mocked_engine_adapter(RedshiftEngineAdapter) + mocker.patch( + "sqlmesh.core.engine_adapter.base.random_id", + return_value="test_random_id", + ) + mocker.patch( + "sqlmesh.core.engine_adapter.redshift.RedshiftEngineAdapter.table_exists", + return_value=False, + ) + columns_mock = mocker.patch( + "sqlmesh.core.engine_adapter.redshift.RedshiftEngineAdapter.columns", + return_value={ + "id": exp.DataType.build("INT"), + "name": exp.DataType.build("VARCHAR(2000)", dialect="redshift"), + }, + ) + + # Inferred target types can be a plain VARCHAR with no length. They must not be forwarded (default + # exists=True path), otherwise a CAST to VARCHAR would truncate; widths come from the temp view. adapter.ctas( table_name="sqlmesh_xxx.my_example", - query_or_df=parse_one( - "SELECT id, updated_at, trashed_at, CAST(NULL AS TIMESTAMP) AS valid_from, CAST(NULL AS TIMESTAMP) AS valid_to FROM my_table WHERE FALSE LIMIT 0" - ), - exists=False, + query_or_df=parse_one("SELECT id, name FROM sqlmesh_xxx.my_table WHERE FALSE LIMIT 0"), + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "name": exp.DataType.build("VARCHAR"), + }, ) sql_calls = to_sql_calls(adapter) - assert 'CREATE TABLE "sqlmesh_xxx"."my_example"' not in sql_calls - assert all("UNKNOWN" not in sql for sql in sql_calls) + columns_mock.assert_called_once() assert sql_calls == [ - 'CREATE VIEW "__temp_ctas_test_random_id" AS SELECT "id", "updated_at", "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" WITH NO SCHEMA BINDING', + 'CREATE VIEW "__temp_ctas_test_random_id" AS SELECT "id", "name" FROM "sqlmesh_xxx"."my_table" WITH NO SCHEMA BINDING', 'DROP VIEW IF EXISTS "__temp_ctas_test_random_id" CASCADE', - 'CREATE TABLE "sqlmesh_xxx"."my_example" AS SELECT "id", "updated_at", "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" WHERE FALSE LIMIT 0', + 'CREATE TABLE "sqlmesh_xxx"."my_example" ("id" INTEGER, "name" VARCHAR(2000))', ] for sql in sql_calls: parse_one(sql, read="redshift") @@ -1068,6 +1110,7 @@ def test_columns_late_binding_view_and_pg_catalog_fallback( sql_calls_pg = to_sql_calls(adapter) assert len(sql_calls_pg) == 3 assert '"pg_catalog"."pg_attribute"' in sql_calls_pg[2] + assert 'ORDER BY "attnum"' in sql_calls_pg[2] assert resp_pg == { "id": exp.DataType.build("INT", dialect="redshift"), "ts": exp.DataType.build("TIMESTAMP", dialect="redshift"), @@ -1135,7 +1178,7 @@ def test_scd_type_2_by_time_unannotated_lifecycle_issue_5787( id, updated_at, trashed_at - FROM my_table + FROM sqlmesh_xxx.my_table """ ), dialect="redshift", @@ -1181,22 +1224,21 @@ def test_scd_type_2_by_time_unannotated_lifecycle_issue_5787( for sql in create_calls_lbv: parse_one(sql, read="redshift") - # Step 1b: strategy.create when columns("__temp_ctas_*") returns {} completely (all catalog queries empty) + # Step 1b: columns("__temp_ctas_*") returns {} (all catalog queries empty) -> fail loudly instead of + # creating the table from the LIMIT 0 CTAS, which can truncate VARCHARs or lose the schema adapter.cursor.execute.reset_mock() adapter.cursor.fetchall.side_effect = [[], [], []] - strategy.create( - table_name=target_table, - model=model, - is_table_deployable=True, - render_kwargs={}, - skip_grants=True, - ) + with pytest.raises(SQLMeshError, match="Could not determine column types"): + strategy.create( + table_name=target_table, + model=model, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) create_calls_fallback = to_sql_calls(adapter) - assert truncated_ddl not in create_calls_fallback - assert ( - 'CREATE TABLE "sqlmesh__sqlmesh_xxx"."sqlmesh_xxx__my_example__1234567890" AS SELECT "id" AS "id", "updated_at" AS "updated_at", "trashed_at" AS "trashed_at", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "my_table" AS "my_table" WHERE FALSE LIMIT 0' - in create_calls_fallback - ) + assert not any(sql.startswith("CREATE TABLE") for sql in create_calls_fallback) + assert create_calls_fallback[-1] == 'DROP VIEW IF EXISTS "__temp_ctas_test_random_id" CASCADE' for sql in create_calls_fallback: parse_one(sql, read="redshift") @@ -1484,7 +1526,7 @@ def test_scd_type_2_redshift_edge_cases( execution_time_as_valid_from true ) ); - SELECT id, val FROM src + SELECT id, val FROM s.src """ ), dialect="redshift", @@ -1494,19 +1536,16 @@ def test_scd_type_2_redshift_edge_cases( mocker.patch.object(adapter, "table_exists", return_value=False) mocker.patch.object(adapter, "columns", return_value={}) - strategy.create( - table_name="s.t", - model=model_star, - is_table_deployable=True, - render_kwargs={}, - skip_grants=True, - ) - create_calls = to_sql_calls(adapter) - assert 'CREATE TABLE "s"."t"' not in create_calls - assert ( - 'CREATE TABLE "s"."t" AS SELECT "id" AS "id", "val" AS "val", CAST(NULL AS TIMESTAMP) AS "valid_from", CAST(NULL AS TIMESTAMP) AS "valid_to" FROM "src" AS "src" WHERE FALSE LIMIT 0' - in create_calls - ) + with pytest.raises(SQLMeshError, match="Could not determine column types"): + strategy.create( + table_name="s.t", + model=model_star, + is_table_deployable=True, + render_kwargs={}, + skip_grants=True, + ) + assert not any(sql.startswith("CREATE TABLE") for sql in to_sql_calls(adapter)) + adapter.cursor.execute.reset_mock() mocker.patch.object( adapter, @@ -1548,7 +1587,7 @@ def test_scd_type_2_redshift_edge_cases( assert "1970-01-01" not in sub_insert else: assert "CAST('1970-01-01 00:00:00' AS TIMESTAMP)" in sub_insert - sql_calls = create_calls + sql_calls = sub_calls for sql in sql_calls: parse_one(sql, read="redshift")