From 0f97996d8c825ab189b15c992bded5d037cbb8ec Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:02:35 -0800 Subject: [PATCH 01/54] Feat: Add Apache Doris engine adapter with OLAP and Lakehouse multi-catalog support Closes #5875 Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/install-prerequisites.sh | 2 + .github/scripts/wait-for-db.sh | 40 + .github/workflows/pr.yaml | 1 + .readthedocs.yaml | 2 +- Makefile | 3 + docs/concepts/models/model_kinds.md | 5 + docs/concepts/models/overview.md | 1 + docs/guides/configuration.md | 2 + docs/guides/connections.md | 1 + docs/installation.md | 1 + docs/integrations/engines/doris.md | 912 +++ docs/integrations/overview.md | 1 + docs/quickstart/cli.md | 2 + docs/reference/configuration.md | 1 + mkdocs.yml | 1 + pyproject.toml | 2 + sqlmesh/cli/project_init.py | 7 + sqlmesh/core/config/__init__.py | 1 + sqlmesh/core/config/connection.py | 204 +- sqlmesh/core/dialect.py | 490 +- sqlmesh/core/engine_adapter/__init__.py | 2 + sqlmesh/core/engine_adapter/base.py | 62 +- sqlmesh/core/engine_adapter/doris.py | 6999 +++++++++++++++++ sqlmesh/core/model/definition.py | 36 +- .../engine_adapter/integration/__init__.py | 24 +- .../engine_adapter/integration/config.yaml | 10 + .../integration/docker/compose.doris.yaml | 60 + .../integration/test_integration.py | 34 +- .../integration/test_integration_doris.py | 946 +++ tests/core/engine_adapter/test_base.py | 38 +- tests/core/engine_adapter/test_doris.py | 5773 ++++++++++++++ tests/core/engine_adapter/test_starrocks.py | 92 +- tests/core/test_connection_config.py | 172 + 33 files changed, 15896 insertions(+), 31 deletions(-) create mode 100644 docs/integrations/engines/doris.md create mode 100644 sqlmesh/core/engine_adapter/doris.py create mode 100644 tests/core/engine_adapter/integration/docker/compose.doris.yaml create mode 100644 tests/core/engine_adapter/integration/test_integration_doris.py create mode 100644 tests/core/engine_adapter/test_doris.py diff --git a/.github/scripts/install-prerequisites.sh b/.github/scripts/install-prerequisites.sh index 6997633a31..09998836d0 100755 --- a/.github/scripts/install-prerequisites.sh +++ b/.github/scripts/install-prerequisites.sh @@ -28,6 +28,8 @@ elif [ "$ENGINE" == "fabric" ]; then rm packages-microsoft-prod.deb ENGINE_DEPENDENCIES="msodbcsql18" +elif [ "$ENGINE" == "doris" ]; then + sudo sysctl -w vm.max_map_count=2000000 fi ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES" diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index 4a076f31f8..489cdd4583 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -104,6 +104,46 @@ db2_ready() { done } +doris_ready() { + probe_port 9030 + + echo "Waiting for Doris FE to accept SQL connections..." + while true; do + if docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then + echo "Doris FE is alive" + break + fi + sleep 2 + done + + docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>/dev/null || true + + echo "Checking for 1 alive Doris backends..." + while true; do + echo "Checking Doris backends..." + ALIVE_BACKENDS=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true) + + # fallback value if failed to get number + if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then + echo "WARN: Unable to parse number of alive backends, got: '$ALIVE_BACKENDS'" + ALIVE_BACKENDS=0 + fi + + echo "Found $ALIVE_BACKENDS alive backends" + + if [ "$ALIVE_BACKENDS" -ge 1 ]; then + echo "Doris has 1 or more alive backends" + break + fi + + echo "Waiting for more backends to become alive..." + sleep 3 + done + + # set default replication num to 1 (there is only one be in the docker compose file) + docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" +} + echo "Waiting for $ENGINE to be ready..." READINESS_FUNC="${ENGINE}_ready" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2c18fe1e04..75a15f2b88 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -269,6 +269,7 @@ jobs: clickhouse, risingwave, starrocks, + doris, ] env: PYTEST_XDIST_AUTO_NUM_WORKERS: 2 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ee4794538f..09262c1577 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,7 @@ build: python: "3.10" jobs: pre_build: - - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]" + - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,doris,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]" - make api-docs mkdocs: diff --git a/Makefile b/Makefile index 0d8571b33f..b1ce245486 100644 --- a/Makefile +++ b/Makefile @@ -226,6 +226,9 @@ starrocks-test: engine-starrocks-up db2-test: engine-db2-up pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml +doris-test: engine-doris-up + pytest -n auto -m "doris" --reruns 3 --junitxml=test-results/junit-doris.xml + ################# # Cloud Engines # ################# diff --git a/docs/concepts/models/model_kinds.md b/docs/concepts/models/model_kinds.md index cde104790a..dc2ab190d2 100644 --- a/docs/concepts/models/model_kinds.md +++ b/docs/concepts/models/model_kinds.md @@ -444,6 +444,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_TIME_RANGE` kind a | Redshift | DELETE by time range, then INSERT | | Postgres | DELETE by time range, then INSERT | | DuckDB | DELETE by time range, then INSERT | +| Doris | DELETE by time range, then INSERT | ## INCREMENTAL_BY_UNIQUE_KEY @@ -669,6 +670,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_UNIQUE_KEY` kind a | Redshift | MERGE ON unique key | | Postgres | MERGE ON unique key | | DuckDB | DELETE ON matched + INSERT new rows | +| Doris | DELETE ON matched + INSERT new rows | ## FULL Models of the `FULL` kind cause the dataset associated with a model to be fully refreshed (rewritten) upon each model evaluation. @@ -761,6 +763,7 @@ Depending on the target engine, models of the `FULL` kind are materialized using | Redshift | DROP TABLE, CREATE TABLE, INSERT | | Postgres | DROP TABLE, CREATE TABLE, INSERT | | DuckDB | CREATE OR REPLACE TABLE | +| Doris | CREATE TABLE ... AS SELECT (new OLAP), DELETE + INSERT (existing OLAP & Iceberg), INSERT OVERWRITE (unpartitioned Hive) | ## VIEW The model kinds described so far cause the output of a model query to be materialized and stored in a physical table. @@ -837,6 +840,7 @@ MODEL ( * BigQuery * Databricks +* Doris * Snowflake During the evaluation of a model of this kind, the view will be replaced or recreated only if the model's query rendered during evaluation does not match the query used during the previous view creation for this model, or if the target view does not exist. Thus, views are recreated only when necessary in order to realize all the benefits provided by materialized views. @@ -1629,6 +1633,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_PARTITION` kind ar | Redshift | DELETE by partitioning key, then INSERT | | Postgres | DELETE by partitioning key, then INSERT | | DuckDB | DELETE by partitioning key, then INSERT | +| Doris | DELETE by partitioning key, then INSERT | ## INCREMENTAL_UNMANAGED diff --git a/docs/concepts/models/overview.md b/docs/concepts/models/overview.md index 1fd16a96fc..ace9900b62 100644 --- a/docs/concepts/models/overview.md +++ b/docs/concepts/models/overview.md @@ -181,6 +181,7 @@ This table lists each engine's support for `TABLE` and `VIEW` object comments: | BigQuery | Y | Y | | ClickHouse | Y | Y | | Databricks | Y | Y | +| Doris | Y | Y | | DuckDB <=0.9 | N | N | | DuckDB >=0.10 | Y | Y | | MySQL | Y | Y | diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 38f6814c8b..a34fd189f1 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -933,6 +933,7 @@ These pages describe the connection configuration options for each execution eng * [BigQuery](../integrations/engines/bigquery.md) * [ClickHouse](../integrations/engines/clickhouse.md) * [Databricks](../integrations/engines/databricks.md) +* [Doris](../integrations/engines/doris.md) * [DuckDB](../integrations/engines/duckdb.md) * [Fabric](../integrations/engines/fabric.md) * [MotherDuck](../integrations/engines/motherduck.md) @@ -975,6 +976,7 @@ Other state engines with fast and reliable database transactions (less tested th Unsupported state engines, even for development: * [ClickHouse](../integrations/engines/clickhouse.md) +* [Doris](../integrations/engines/doris.md) * [Spark](../integrations/engines/spark.md) * [StarRocks](../integrations/engines/starrocks.md) * [Trino](../integrations/engines/trino.md) diff --git a/docs/guides/connections.md b/docs/guides/connections.md index 1345905bf7..bf6d4811d6 100644 --- a/docs/guides/connections.md +++ b/docs/guides/connections.md @@ -85,6 +85,7 @@ default_gateway: local_db * [ClickHouse](../integrations/engines/clickhouse.md) * [Databricks](../integrations/engines/databricks.md) * [Db2](../integrations/engines/db2.md) +* [Doris](../integrations/engines/doris.md) * [DuckDB](../integrations/engines/duckdb.md) * [Fabric](../integrations/engines/fabric.md) * [MotherDuck](../integrations/engines/motherduck.md) diff --git a/docs/installation.md b/docs/installation.md index 6bcac928e5..221a40fcce 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -54,6 +54,7 @@ Other extras are required to use specific SQL engines, like Bigquery or Postgres | Bigquery | `pip install "sqlmesh[bigquery]"` | | ClickHouse | `pip install "sqlmesh[clickhouse]"` | | Databricks | `pip install "sqlmesh[databricks]"` | + | Doris | `pip install "sqlmesh[doris]"` | | Fabric | `pip install "sqlmesh[fabric]"` | | GCP Postgres | `pip install "sqlmesh[gcppostgres]"` | | MS SQL Server | `pip install "sqlmesh[mssql]"` | diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md new file mode 100644 index 0000000000..2532b17bf8 --- /dev/null +++ b/docs/integrations/engines/doris.md @@ -0,0 +1,912 @@ +# Apache Doris + +!!! info + The Doris engine adapter is a community contribution. Due to this, only limited community support is available. + +This page describes SQLMesh support for the [Apache Doris](https://doris.apache.org/) engine (versions 2.1, 3.x, 4.x, and 5.0), including native Doris OLAP tables and Apache Iceberg multi-catalog lakehouse tables. + +!!! note + Doris may not be used for the SQLMesh [state connection](../../reference/configuration.md#connections). Configure a transactional database such as Postgres, MySQL, or DuckDB (for local development) as your `state_connection`. + +## Local/Built-in Scheduler + +**Engine Adapter Type**: `doris` + +### Installation + +```bash +# Local development (with bundled DuckDB state_connection) +pip install "sqlmesh[doris]" + +# Production (with a PostgreSQL state_connection) +pip install "sqlmesh[doris,postgres]" +``` + +### 5-Minute Quickstart + +1. **Initialize a new Doris project** (generates `config.yaml` with `dialect: doris` and a local DuckDB `state_connection`): + ```bash + mkdir doris_demo && cd doris_demo + sqlmesh init doris + ``` +2. **Configure connection credentials**: Edit `config.yaml` and fill in the required `host` (e.g., `127.0.0.1`) and `user` (e.g., `root`) fields under `gateways.doris.connection`. +3. **Single-BE Docker / Local Dev Tip**: Doris defaults to `replication_num = 3`. When running against a single-BE Docker container, set `replication_num = 1` in `model_defaults.physical_properties` inside `config.yaml`: + ```yaml + model_defaults: + dialect: doris + start: 2024-01-01 + physical_properties: + replication_num: 1 + ``` +4. **Plan and apply**: + ```bash + sqlmesh plan + ``` + +### Connection options + +| Option | Description | Type | Required | +| --- | --- | :---: | :---: | +| `type` | Engine type name - must be `doris` | string | Y | +| `host` | The hostname of the Doris FE (Frontend) server | string | Y | +| `user` | The username to use for authentication with Doris FE | string | Y | +| `password` | The password to use for authentication with Doris FE (Default: `None`) | string | N | +| `port` | The query port number of the Doris FE server (Default: `9030`) | int | N | +| `database` | The optional default database (schema) name | string | N | +| `catalog` | The optional default catalog name (e.g., `internal` or an external `iceberg` catalog) | string | N | +| `charset` | The character set used for the connection (Default: `utf8mb4`) | string | N | +| `collation` | The collation used for the connection | string | N | +| `connect_timeout` | Connection timeout in seconds (Default: `10`) | int | N | +| `read_timeout` | Read timeout in seconds | int | N | +| `write_timeout` | Write timeout in seconds | int | N | +| `pre_ping` | Whether to ping the connection before executing a statement and reconnect if lost (Default: `False`) | bool | N | +| `session_variables` | Session variables to set on each new connection via `SET = ` (e.g., `workload_group`, `query_timeout`) | dict | N | +| `ssl` | Explicit SSL configuration (`True`/`False` or dictionary) passed to `PyMySQL` | bool \| dict | N | +| `ssl_disabled` | Whether SSL is disabled | bool | N | +| `ssl_ca` | Path to the file that contains a PEM-formatted CA certificate | string | N | +| `ssl_cert` | Path to the file that contains a PEM-formatted client certificate | string | N | +| `ssl_key` | Path to the file that contains a PEM-formatted private key for the client certificate | string | N | +| `ssl_verify_cert` | Set to `true` to verify the server's certificate against the CA certificate | bool | N | +| `ssl_verify_identity` | Set to `true` to verify the server's identity in addition to the certificate | bool | N | +| `concurrent_tasks` | The maximum number of tasks that can use this connection concurrently (Default: `4`) | int | N | +| `register_comments` | Whether or not to register model comments with the SQL engine (Default: `True`) | bool | N | + +### Connection configuration example + +=== "Local Development (DuckDB State)" + + ```yaml + gateways: + doris: + connection: + type: doris + host: 127.0.0.1 + port: 9030 + user: root + password: "" + database: analytics + state_connection: + type: duckdb + database: state.db + + default_gateway: doris + + model_defaults: + dialect: doris + physical_properties: + replication_num: 1 + ``` + +=== "Production (Postgres / MySQL State)" + + ```yaml + gateways: + doris: + connection: + type: doris + host: fe.doris.cluster + port: 9030 + user: doris_user + password: {{ env_var('DORIS_PASSWORD') | tojson }} + database: analytics + catalog: internal + connect_timeout: 30 + ssl_ca: /etc/ssl/certs/doris-ca.pem + ssl_verify_cert: true + ssl_verify_identity: true + session_variables: + workload_group: etl_group + query_timeout: 3600 + state_connection: + type: postgres + host: state-db.internal + port: 5432 + user: sqlmesh + password: {{ env_var('POSTGRES_PASSWORD') | tojson }} + database: sqlmesh_state + sslmode: verify-full + # Set PGSSLROOTCERT=/etc/ssl/certs/postgres-ca.pem in the environment to verify + # the PostgreSQL server certificate against your PostgreSQL CA bundle (separate from Doris TLS). + + default_gateway: doris + + model_defaults: + dialect: doris + ``` + +--- + +## OLAP Table Models + +Apache Doris supports three native OLAP table data models: **DUPLICATE KEY**, **UNIQUE KEY**, and **AGGREGATE KEY**. + +!!! tip "Automatic Key Column Prefix Reordering" + Doris requires key columns (`UNIQUE KEY`, `DUPLICATE KEY`, or `AGGREGATE KEY`) to appear as the **first columns** in a `CREATE TABLE` definition in the exact order of the key. SQLMesh's `DorisEngineAdapter` automatically reorders your schema definition during `CREATE TABLE` so key columns appear first, regardless of the column order in your `SELECT` query. + +### DUPLICATE KEY model (default) + +By default, OLAP tables use the `DUPLICATE KEY` model. If no key is specified, Doris uses the first column as the duplicate key and hash distribution column. + +```sql +MODEL ( + name analytics.user_events, + kind FULL, + clustered_by (user_id), + physical_properties ( + duplicate_key = ('user_id', 'event_time'), + buckets = 10 + ) +); + +SELECT + user_id::BIGINT AS user_id, + event_time::DATETIME AS event_time, + event_type::VARCHAR(64) AS event_type +FROM raw.events; +``` + +### UNIQUE KEY model (Merge-on-Write & custom sort keys) + +For `INCREMENTAL_BY_UNIQUE_KEY` models, SQLMesh automatically configures the model's `unique_key` as the Doris table's `UNIQUE KEY` and `DISTRIBUTED BY HASH(...)` key—no extra `physical_properties` required: + +```sql +MODEL ( + name analytics.orders_upsert, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key order_id + ), + physical_properties ( + -- Custom sort keys via order_by require Doris 4.1+; omit order_by on Doris < 4.1 + order_by = ('updated_at', 'order_id'), + enable_unique_key_merge_on_write = 'true' + ) +); + +SELECT + order_id::BIGINT AS order_id, + customer_id::BIGINT AS customer_id, + status::VARCHAR(32) AS status, + updated_at::DATETIME AS updated_at +FROM raw.orders +WHERE CAST(updated_at AS DATE) BETWEEN @start_ds AND @end_ds; +``` + +You can also specify `unique_key` explicitly in `physical_properties` for `FULL` or `INCREMENTAL_BY_TIME_RANGE` models, and on **Doris 4.1+** optionally decouple the sort key from the unique key using `order_by` or `cluster_by` (emitting `ORDER BY (col1, col2)` on Doris Merge-on-Write tables; omit `order_by` on Doris `< 4.1`). + +On Doris 4.1+, `INCREMENTAL_BY_UNIQUE_KEY` models also support `when_matched` and `merge_filter` via Doris's native `MERGE INTO` syntax (automatically excluding `unique_key` columns from `WHEN MATCHED THEN UPDATE SET` to satisfy Doris Merge-on-Write value-column constraints): + +!!! note "Requirements for `when_matched` and `merge_filter`" + - Without `when_matched` or `merge_filter`, `INCREMENTAL_BY_UNIQUE_KEY` uses `DELETE` + `INSERT INTO`, which is supported on Doris 2.1+ across all delete-capable table models. + - When `when_matched` or `merge_filter` is specified, `DorisEngineAdapter` uses native `MERGE INTO`, which requires **Apache Doris 4.1+** and an internal OLAP **`UNIQUE KEY`** target table. Attempting to use `when_matched` or `merge_filter` on Doris `< 4.1`, on a table configured with `duplicate_key` or `aggregate_key` in `physical_properties`, or on an external catalog / lakehouse table raises a `SQLMeshError`. + +```sql +MODEL ( + name analytics.orders_upsert, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key order_id, + when_matched WHEN MATCHED AND source.updated_at > target.updated_at THEN UPDATE SET + target.status = source.status, + target.updated_at = source.updated_at + ) +); +``` + +### AGGREGATE KEY model + +For pre-aggregated rollup tables, specify the key columns using `aggregate_key` in `physical_properties`. Note that Doris requires non-key value columns in an `AGGREGATE KEY` table to specify an aggregation function (`SUM`, `MAX`, `MIN`, `REPLACE`, etc.) in the column definition unless all columns are part of `aggregate_key` (or the table is created via a custom DDL / pre-statement): + +```sql +MODEL ( + name analytics.daily_site_keys, + kind FULL, + clustered_by (user_id), + physical_properties ( + aggregate_key = ('user_id', 'event_type'), + buckets = 16 + ) +); + +SELECT + user_id::BIGINT AS user_id, + event_type::VARCHAR(64) AS event_type +FROM analytics.user_events +GROUP BY user_id, event_type; +``` + +--- + +## Distribution & Bucketing + +Doris requires a `DISTRIBUTED BY` clause on every OLAP table. SQLMesh supports three ways to configure distribution: + +1. **SQLMesh `clustered_by` + optional `buckets`** (recommended for cross-engine portability): + ```sql + MODEL ( + name analytics.events, + kind FULL, + clustered_by (user_id), + physical_properties ( + buckets = 16 + ) + ); + ``` +2. **Shorthand string `distributed_by`**: + ```sql + physical_properties ( + distributed_by = 'HASH(user_id) BUCKETS 16' + -- or 'RANDOM BUCKETS AUTO' + ) + ``` +3. **Structured dictionary or column tuple `distributed_by`**: + ```sql + physical_properties ( + distributed_by = ( + kind = 'HASH', + expressions = 'user_id', + buckets = 'AUTO' + ) + -- Also supports `columns = ('user_id', 'event_time')` or `distributed_by = ('user_id', 'event_time')`, + -- or `random = 'true'` for `DISTRIBUTED BY RANDOM` on DUPLICATE KEY tables. + ) + ``` + +If none of `clustered_by` or `distributed_by` is provided: +- When `unique_key` is present, SQLMesh defaults to `DISTRIBUTED BY HASH() BUCKETS `. +- When `buckets` is specified with `duplicate_key` or `aggregate_key`, SQLMesh emits `DISTRIBUTED BY HASH() BUCKETS ` (or `DISTRIBUTED BY RANDOM BUCKETS ` when no key is specified). + +--- + +## Partitioning + +Doris OLAP tables support **Multi-Range Batch Partitioning**, **Explicit Partition Definitions**, **Dynamic Partitioning**, and **Auto Partitioning** (Doris 2.1+). You can specify the partition expression via the top-level `partitioned_by` model property or via `partitioned_by` / `partition_by` inside `physical_properties`. + +!!! note "OLAP Partitioning Requirements" + - On Doris OLAP tables, `PARTITION BY` requires at least one of `partitions`, `auto_partition = 'true'`, or `'dynamic_partition.enable' = 'true'` in `physical_properties`. If none is provided, SQLMesh omits the `PARTITION BY` clause so Doris does not create a zero-partition table that rejects inserts. + - On `UNIQUE KEY` and `AGGREGATE KEY` tables, Doris requires all `partitioned_by` columns to be a subset of the key columns. + +### Multi-range & explicit partitions + +```sql +MODEL ( + name analytics.partitioned_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_date + ), + partitioned_by RANGE(event_date), + physical_properties ( + partitions = 'FROM ("2023-01-01") TO ("2027-01-01") INTERVAL 1 MONTH' + ) +); +``` + +### Auto partitioning (`auto_partition = true`) + +In Doris 2.1+, `AUTO PARTITION` automatically creates partitions on demand as new data arrives (and SQLMesh automatically marks the range partition column `NOT NULL` and wraps CTAS projections in `NON_NULLABLE(...)`): + +```sql +MODEL ( + name analytics.auto_partitioned_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_time + ), + partitioned_by RANGE(DATE_TRUNC(event_time, 'day')), + physical_properties ( + auto_partition = 'true' + ) +); +``` + +### Dynamic partitioning (`dynamic_partition.enable = 'true'`) + +For rolling retention windows managed by Doris FE's dynamic partition scheduler, enable `'dynamic_partition.create_history_partition' = 'true'` so Doris creates the historical partitions on table creation, set `'dynamic_partition.time_zone' = 'UTC'` so Doris evaluates partition boundaries in the same UTC timezone that SQLMesh uses to resolve `start`, and set `'dynamic_partition.start'` with a buffer beyond the model's `start` window (for example, `'-32'` for `start '30 days ago'`) so backfills that cross a midnight boundary only insert into existing partitions: + +```sql +MODEL ( + name analytics.rolling_daily_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_date + ), + start '30 days ago', + partitioned_by RANGE(event_date), + physical_properties ( + 'dynamic_partition.enable' = 'true', + 'dynamic_partition.time_unit' = 'DAY', + 'dynamic_partition.time_zone' = 'UTC', + 'dynamic_partition.start' = '-32', + 'dynamic_partition.end' = '3', + 'dynamic_partition.create_history_partition' = 'true', + 'dynamic_partition.prefix' = 'p', + 'dynamic_partition.buckets' = '16' + ) +); +``` + +--- + +## Apache Iceberg & Multi-Catalog Lakehouse Integration + +Apache Doris 2.1–5.0 acts as a federated lakehouse engine that can query and write (including `CREATE TABLE`) to external **Apache Iceberg** catalogs backed by REST/Polaris, AWS Glue, Hive Metastore, or Hadoop/S3. Native Alibaba Cloud DLF Iceberg catalogs can be queried as upstream sources, but Doris does not support `CREATE TABLE` in native DLF Iceberg catalogs, so SQLMesh cannot create new model tables there. + +### Pattern 1: Reading Iceberg Bronze tables into Doris OLAP Silver/Gold models + +Because `DorisEngineAdapter` supports multi-catalog 3-part naming (`catalog.db.table`), your SQLMesh models can query external Iceberg tables directly while materializing sub-second serving tables in Doris's `internal` OLAP catalog: + +```sql +MODEL ( + name silver.user_sessions, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column session_date + ) +); + +SELECT + user_id, + session_id, + CAST(event_ts AS DATE) AS session_date +FROM iceberg_catalog.bronze.clickstream +WHERE event_ts BETWEEN @start_ts AND @end_ts; +``` + +### Pattern 2: Materializing Apache Iceberg tables via Doris + +SQLMesh can also create, partition, and overwrite Apache Iceberg tables directly through Doris (`ENGINE=iceberg` and `INSERT OVERWRITE TABLE`). + +!!! important "Virtual Environments & Catalog Bootstrap on External Iceberg Catalogs" + Doris only supports `CREATE VIEW` inside the `internal` catalog; external Iceberg catalogs do not support Doris views, and Doris does not support `environment_suffix_target: catalog`. + When materializing models directly inside an external Iceberg catalog: + + - Set `virtual_environment_mode: dev_only` (and keep `environment_suffix_target: schema` or `table`) in `config.yaml` so production tables are materialized directly in the Iceberg catalog without virtual views. + - Configure `environment_catalog_mapping` in `config.yaml` to route non-`prod` virtual environment views to the `internal` catalog (otherwise `DorisEngineAdapter` will raise an error when attempting to create views in an external catalog): + ```yaml + virtual_environment_mode: dev_only + environment_suffix_target: schema + environment_catalog_mapping: + "^(?!prod$).*": internal + before_all: + - > + CREATE CATALOG IF NOT EXISTS iceberg_catalog PROPERTIES ( + 'type' = 'iceberg', + 'iceberg.catalog.type' = 'rest', + 'uri' = 'http://rest-catalog:8181', + 'warehouse' = 's3://lakehouse/warehouse' + ) + ``` + +!!! warning "Non-Atomic Iceberg `FULL` Refreshes & Recovery Procedure" + `DorisEngineAdapter` disables multi-statement transactions (`SUPPORTS_TRANSACTIONS = False`). When refreshing an existing Iceberg table via `DELETE` followed by `INSERT INTO` (for example, during in-place `FULL` refreshes with `virtual_environment_mode: dev_only`), the two steps are **non-atomic**: + + - **Concurrent reader visibility**: Readers querying the Iceberg table between the `DELETE` snapshot(s) and the `INSERT INTO` snapshot may see an empty or partially cleared table. + - **Insert failure risk**: If the `INSERT INTO` statement fails after the `DELETE` statements commit, the production table remains empty until recovered. + - **Verified recovery procedures** (quiesce concurrent writers before starting recovery): + 1. **Replace or roll back to the prior Iceberg snapshot (`FOR VERSION AS OF`)**: Because `DorisEngineAdapter` clears Iceberg tables using two separate `DELETE` statements (`IS NOT NULL` and `IS NULL`), a failure after the first `DELETE` can leave residual rows in the table. Do not blindly `INSERT` the prior snapshot on top of a partially cleared table (which would duplicate surviving rows), and do not select the recovery snapshot by `committed_at` timestamp order alone (since `$snapshots` includes commits from other branches). Instead, quiesce writers, trace the `parent_id` chain from the production `main` branch head in `$refs` back through the failed refresh's `delete` snapshot(s) in `$snapshots` to find the first failed-refresh `delete` snapshot's `parent_id` (``), verify that `` belongs to the production table's lineage, and either roll back the table metadata to `` in your Iceberg catalog or clear any remaining rows before restoring that snapshot: + ```sql + -- 1. Inspect the production 'main' branch head and follow parent_id through the failed refresh's + -- 'delete' snapshot(s) to identify the first failed-refresh delete snapshot's parent_id (): + SELECT name, type, snapshot_id + FROM iceberg_catalog.silver.curated_events$refs + WHERE name = 'main'; + + SELECT snapshot_id, parent_id, committed_at, operation + FROM iceberg_catalog.silver.curated_events$snapshots; + + -- 2. After quiescing writers, clear any surviving rows from a partial delete so the restore replaces rather than appends: + DELETE FROM iceberg_catalog.silver.curated_events WHERE user_id IS NOT NULL; + DELETE FROM iceberg_catalog.silver.curated_events WHERE user_id IS NULL; + + -- 3. Restore the pre-delete state from the prior snapshot using Iceberg time travel: + INSERT INTO iceberg_catalog.silver.curated_events + SELECT * FROM iceberg_catalog.silver.curated_events FOR VERSION AS OF ; + ``` + 2. **Restate the model or retry the failed plan in SQLMesh**: Once writers are quiesced and the cause of the failure is resolved, run `sqlmesh plan --restate-model iceberg_catalog.silver.curated_events` (or retry the failed `sqlmesh plan`) to replace all rows from the upstream source. Do not rely on `sqlmesh run` alone for recovery, because `sqlmesh run` may skip an interval that was already recorded as processed and leave the table empty. + +```sql +-- On initial creation, SQLMesh runs CREATE TABLE ... ENGINE=iceberg followed by INSERT INTO. +-- On subsequent FULL refreshes, SQLMesh clears all existing rows via non-atomic DELETE + INSERT INTO +-- (rather than dynamic-partition INSERT OVERWRITE) so omitted partitions and empty results +-- replace the entire Iceberg v2 table cleanly; see the warning above for reader visibility and recovery. +MODEL ( + name iceberg_catalog.silver.curated_events, + kind FULL, + table_format iceberg, + partitioned_by (day(event_ts), bucket(16, user_id)), + physical_properties ( + 'write-format' = 'parquet', + 'compression-codec' = 'zstd' + ) +); + +SELECT + user_id::BIGINT AS user_id, + event_ts::DATETIME AS event_ts, + payload::STRING AS payload +FROM iceberg_catalog.bronze.raw_events; +``` + +### Pattern 3: Iceberg Time Travel, Branches/Tags (`@branch` / `@tag`), & Compaction + +SQLMesh's Doris dialect natively parses and transpiles Apache Iceberg snapshot time travel, branch/tag reads and branch/tag DDL (requires Doris 3.1+), and table maintenance actions such as `rewrite_data_files` compaction (requires Doris 4.0.2+) and `expire_snapshots` snapshot expiration (requires Doris 4.1.0+) while preserving upstream table lineage: + +- **Time Travel (`FOR TIME AS OF` / `FOR VERSION AS OF`)**: + ```sql + SELECT user_id, event_ts + FROM iceberg_catalog.bronze.clickstream FOR TIME AS OF '2026-01-15 00:00:00'; + + SELECT user_id, event_ts + FROM iceberg_catalog.bronze.clickstream FOR VERSION AS OF 8493021948102; + ``` +- **Branch & Tag Reads (`@branch(...)` / `@tag(...)`)**: + ```sql + SELECT user_id, event_ts + FROM iceberg_catalog.bronze.clickstream@branch(audit_branch) AS c; + + SELECT user_id, event_ts + FROM iceberg_catalog.bronze.clickstream@tag(v1_release) AS c; + ``` +- **Iceberg Compaction (`rewrite_data_files`) & Branch Management via Post-Statements**: + You can run Iceberg file compaction (`rewrite_data_files`, requires Doris 4.0.2+), snapshot expiration (`expire_snapshots`, requires Doris 4.1.0+), or branch/tag management (requires Doris 3.1+) in model post-statements. Because Doris does not support Iceberg orphan-file cleanup (`remove_orphan_files`), run orphan-file cleanup through a supported external Iceberg maintenance tool (such as Apache Spark's `remove_orphan_files` procedure): + ```sql + MODEL ( + name iceberg_catalog.silver.curated_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_ts + ), + table_format iceberg, + partitioned_by (day(event_ts)) + ); + + SELECT + user_id::BIGINT AS user_id, + event_ts::DATETIME AS event_ts, + payload::STRING AS payload + FROM iceberg_catalog.bronze.raw_events + WHERE event_ts BETWEEN @start_ts AND @end_ts; + + @IF( + @runtime_stage = 'evaluating', + ALTER TABLE @this_model EXECUTE rewrite_data_files("target-file-size-bytes" = "134217728") + ); + ``` + +### Pattern 4: Reading & Writing External Apache Hive and Apache Paimon Catalogs + +In addition to Apache Iceberg, `DorisEngineAdapter` supports reading and writing external **Apache Hive** (`'type' = 'hms'`, `table_format hive`) catalogs and reading external **Apache Paimon** (`'type' = 'paimon'`, `table_format paimon`) catalogs: + +- **Apache Hive (`table_format hive`)**: Supported for reading (`SELECT` on Doris 2.1+) and writing (`CREATE TABLE ... ENGINE=hive`, `INSERT INTO`, and `INSERT OVERWRITE TABLE` on Doris 2.1.6+). Emits `ENGINE=hive` and maps `storage_format` to `'file_format'` (`'parquet'`, `'orc'`, or `'text'`). Supports `FULL` (`INSERT OVERWRITE TABLE` on unpartitioned Hive tables; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available because Doris `INSERT OVERWRITE` only replaces partitions present in the query result and Hive catalogs in Doris do not support `DELETE`) and append workloads (`INCREMENTAL_UNMANAGED`), while row-level `DELETE` and `MERGE INTO` are not supported by Hive catalogs in Doris. Just like external Iceberg targets, because Doris only supports `CREATE VIEW` in the `internal` catalog, materializing models inside an external Hive catalog requires configuring `virtual_environment_mode: dev_only`, `environment_suffix_target: schema` (or `table`), and `environment_catalog_mapping: {"^(?!prod$).*": "internal"}` in `config.yaml` (see the virtual environment configuration in [Pattern 2](#pattern-2-materializing-apache-iceberg-tables-via-doris)). +- **Apache Paimon (`table_format paimon`)**: Officially supported in Doris 2.1–4.1.5 for reading upstream Paimon catalog tables (`SELECT`). While `DorisEngineAdapter` also generates `ENGINE=paimon` DDL (`'file.format'` and `'primary-key'`) for builds that include experimental Paimon write support (tracked upstream in `apache/doris#65086`), Doris 2.1–4.1.5 does not ship general-availability Paimon `INSERT` / `DELETE` / `MERGE INTO` support—use **Apache Iceberg v2** (`table_format iceberg` on Doris 4.1.0+) when your lakehouse target requires `DELETE` or `MERGE INTO`. + +```sql +-- Requires virtual_environment_mode: dev_only, environment_suffix_target: schema (or table), +-- and environment_catalog_mapping: {"^(?!prod$).*": "internal"} in config.yaml because Doris +-- only supports CREATE VIEW inside the internal catalog. +MODEL ( + name hive_catalog.silver.orders_hive, + kind FULL, + table_format hive, + storage_format parquet +); + +SELECT + order_id::BIGINT AS order_id, + amount::DECIMAL(18, 2) AS amount, + ds::DATE AS ds +FROM paimon_catalog.raw.orders; +``` + +### Pattern 5: Reading External Lance, Hudi & Delta Lake Tables (Doris 4.1+) + +Apache Doris 4.1.4+ supports reading multimodal and vector datasets stored in **Lance** format (`'type' = 'lance'` catalog or `s3(...)` / `local(...)` table-valued functions with `'format' = 'lance'`), as well as external **Apache Hudi** (via a Hive Metastore catalog with `'type' = 'hms'`) and **Delta Lake** (`'type' = 'deltalake'`) catalogs. + +!!! warning "Do Not Put Credentials in `before_all`" + SQLMesh renders `before_all` statements and persists them in the `_environment_statements` state table. Provision external catalogs that require static `s3.access_key` / `s3.secret_key` credentials directly in Doris out-of-band, or configure `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in the Doris FE and BE process environments (`'s3.credentials_provider_type' = 'ENV'`, which both `S3Properties` and Lance's `StorageOptions::with_env_s3` resolve automatically in Doris 4.1.4+ without storing secrets in `before_all`; note that Doris 4.1.4's `LanceS3StorageProvider` does not forward `s3.role_arn` to Lance). + +```yaml +# config.yaml +before_all: + - > + CREATE CATALOG IF NOT EXISTS lance_catalog PROPERTIES ( + 'type' = 'lance', + 'lance.catalog.type' = 'filesystem', + 'warehouse' = 's3://ai-lakehouse/lance_warehouse', + 's3.endpoint' = 'https://ai-lakehouse.s3.us-east-1.amazonaws.com', + 's3.region' = 'us-east-1', + 's3.credentials_provider_type' = 'ENV' + ) +``` + +```sql +MODEL ( + name analytics.image_embeddings_olap, + kind FULL, + physical_properties ( + storage_format = 'V3' + ) +); + +SELECT + doc_id::BIGINT AS doc_id, + caption::STRING AS caption, + embedding::ARRAY AS embedding, + L2_DISTANCE(embedding, [0.1, 0.2, 0.3, 0.4]) AS ref_dist +FROM lance_catalog.multimodal.clip_embeddings; +``` + +!!! warning "Lance, Hudi, and Delta Lake Tables are Read-Only in Doris" + Doris supports Lance, Hudi, Delta Lake, and LakeSoul catalogs and `format = 'lance'` table-valued functions for reading only (`CREATE TABLE`, `INSERT`, and `CTAS` are not supported by Doris on those formats). If a SQLMesh model sets `table_format` (or `engine`) to `lance`, `hudi`, `deltalake`, or `lakesoul`, `DorisEngineAdapter` raises a descriptive `SQLMeshError` directing you to materialize into the `internal` OLAP catalog or a writable external catalog (`iceberg`, `paimon`, or `hive`). + +--- + +## DDL-Only / Schema Contract Management for Streaming Tables (`doris_ddl_only`) + +In real-time architectures where **Apache Flink (`flink-doris-connector`)**, **Kafka Routine Load**, **Stream Load**, or **Spark** streams data continuously into a Doris OLAP or Iceberg table, you may want SQLMesh to manage **only the table DDL (`CREATE TABLE`) and schema evolution (`ALTER TABLE`)** without executing `INSERT` or `REPLACE` statements during `sqlmesh run`. + +You can achieve this by adding a lightweight `CustomMaterialization` plugin (`materializations/doris_ddl_only.py`) to your SQLMesh project: + +```python +from __future__ import annotations + +import typing as t +from sqlmesh.core.model import Model +from sqlmesh.core.snapshot.evaluator import CustomMaterialization +from sqlmesh.utils.errors import SQLMeshError + +if t.TYPE_CHECKING: + from sqlmesh.core.engine_adapter._typing import QueryOrDF + + +class DorisDDLOnlyMaterialization(CustomMaterialization): + """Manages CREATE TABLE and ALTER TABLE schema migrations without inserting data.""" + + NAME = "doris_ddl_only" + + def create( + self, + table_name: str, + model: Model, + is_table_deployable: bool, + render_kwargs: t.Dict[str, t.Any], + skip_grants: bool = False, + **kwargs: t.Any, + ) -> None: + if not model.annotated: + raise SQLMeshError( + f"Model '{model.name}' uses 'doris_ddl_only' and must explicitly declare `columns (...)` in MODEL (...)." + ) + super().create( + table_name=table_name, + model=model, + is_table_deployable=is_table_deployable, + render_kwargs=render_kwargs, + skip_grants=skip_grants, + **kwargs, + ) + + def insert( + self, + table_name: str, + query_or_df: QueryOrDF, + model: Model, + is_first_insert: bool, + render_kwargs: t.Dict[str, t.Any], + **kwargs: t.Any, + ) -> None: + # No-op: external Flink / Routine Load / Spark pipelines write data to this table. + return + + def append( + self, + table_name: str, + query_or_df: QueryOrDF, + model: Model, + render_kwargs: t.Dict[str, t.Any], + **kwargs: t.Any, + ) -> None: + # No-op: external Flink / Routine Load / Spark pipelines write data to this table. + return +``` + +Then define your streaming target table in `models/` with `virtual_environment_mode: dev_only`: + +```sql +MODEL ( + name realtime.flink_order_events, + kind CUSTOM ( + materialization 'doris_ddl_only', + forward_only true, + disable_restatement true + ), + columns ( + order_id BIGINT, + user_id BIGINT, + amount DECIMAL(18, 2), + updated_at DATETIME + ), + physical_properties ( + unique_key = ('order_id'), + -- Custom sort keys via order_by require Doris 4.1+; omit order_by on Doris < 4.1 + order_by = ('updated_at', 'order_id'), + buckets = 16, + enable_unique_key_merge_on_write = 'true', + light_schema_change = 'true' + ) +); + +SELECT + CAST(NULL AS BIGINT) AS order_id, + CAST(NULL AS BIGINT) AS user_id, + CAST(NULL AS DECIMAL(18, 2)) AS amount, + CAST(NULL AS DATETIME) AS updated_at; +``` + +!!! tip "Zero-Downtime Schema Evolution with Flink / Routine Load" + Setting `forward_only true` and `disable_restatement true` on a `doris_ddl_only` model ensures that schema additions (`ALTER TABLE ... ADD COLUMN`) are applied in-place on the existing table (`light_schema_change = 'true'`) rather than creating a new empty physical table, and prevents accidental restatement wipes. + +--- + +## Compute-Storage Decoupled Mode (Doris 3.0 / 4.x / 5.0) + +For Doris 3.0+ Compute-Storage Decoupled deployments on object storage (S3, GCS, Azure Blob, OSS) or HDFS: + +- Pass `storage_vault_name` in `physical_properties` (or `model_defaults.physical_properties` in `config.yaml`) to bind tables to a specific storage vault. +- Use `session_variables` in your `doris` connection config to bind SQLMesh workloads to a specific `workload_group` or cloud cluster. + +```sql +MODEL ( + name analytics.cloud_events, + kind FULL, + clustered_by (event_id), + physical_properties ( + storage_vault_name = 'prod_s3_vault', + file_cache_ttl_seconds = '86400' + ) +); +``` + +--- + +## Materialized Views & Indexing + +### Asynchronous Materialized Views (MTMV) + +```sql +MODEL ( + name analytics.user_summary_mv, + kind VIEW ( + materialized true + ), + physical_properties ( + build = 'IMMEDIATE', + refresh = 'AUTO', + refresh_trigger = 'ON SCHEDULE EVERY 1 HOUR', + distributed_by = 'HASH(user_id) BUCKETS 10' + ) +); + +SELECT + user_id, + CAST(event_time AS DATE) AS ds, + COUNT(*) AS event_count +FROM analytics.user_events +GROUP BY user_id, CAST(event_time AS DATE); +``` + +!!! note "Automatic Physical Table Resolution & Non-Destructive Evaluation" + - Doris materialized views are only created on initial deployment or when a rebuild is required (`RECREATE_MATERIALIZED_VIEW_ON_EVALUATION = False`), preserving MTMV refresh state across routine `sqlmesh run` invocations. + - When `build = 'DEFERRED'` is used on an asynchronous MTMV that also defines `audits (...)`, SQLMesh automatically submits `REFRESH MATERIALIZED VIEW AUTO` after creation. Because Doris executes MTMV refresh tasks asynchronously without blocking `REFRESH MATERIALIZED VIEW`, audits may run before the background refresh task finishes or fails; use `build = 'IMMEDIATE'` when audits require the initial refresh to complete before auditing. + - When `excluded_trigger_tables` (supports a single string, comma-separated string, or tuple `('db.t1', 'db.t2')` on async MTMVs) references a SQLMesh-managed model, SQLMesh automatically resolves the model name to its underlying physical snapshot table name. + +### Synchronous Materialized Views (Base-Table Rollup Indexes) + +Doris synchronous materialized views are single-table rollup indexes maintained atomically on a single `internal`-catalog OLAP base table. Unlike asynchronous MTMVs, Doris does **not** allow synchronous materialized views to be queried directly by name (`SELECT * FROM `)—instead, the Doris optimizer transparently routes matching queries against the base table to the synchronous rollup index. + +Because SQLMesh `VIEW (materialized true)` models are promoted via a virtual view (`SELECT * FROM `) and queried directly by downstream models, **SQLMesh `VIEW (materialized true)` models must use asynchronous materialized views (MTMVs)**. To attach a synchronous materialized view to a SQLMesh-managed OLAP base table, create it as an index on `@this_model` in a post-statement (while `DorisEngineAdapter.create_view(..., materialized=True, view_properties={"materialized_type": exp.Literal.string("SYNC"), "source_table": ...})` remains available for direct programmatic DDL management): + +```sql +MODEL ( + name analytics.user_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_time + ), + grain (user_id, event_time) +); + +SELECT + user_id::BIGINT AS user_id, + event_type::VARCHAR(64) AS event_type, + event_time::DATETIME AS event_time +FROM raw.user_events +WHERE event_time BETWEEN @start_ds AND @end_ds; + +-- Note: For INCREMENTAL_BY_TIME_RANGE models, every synchronous MV (rollup) on the +-- base table must include the time_column (event_time) in its SELECT and GROUP BY +-- keys so Doris permits DELETE WHERE event_time ... during interval replacement. +-- For rollups that aggregate across all event_time values, use an asynchronous MTMV instead. +@IF( + @runtime_stage = 'creating', + CREATE MATERIALIZED VIEW user_events_sync_mv AS + SELECT + user_id AS mv_user_id, + event_time AS mv_event_time, + COUNT(event_type) AS event_count + FROM @this_model + GROUP BY user_id, event_time +); +``` + +### Inverted & Vector Indexes, Full-Text Search & Compaction (Doris 2.1–5.0) + +You can attach Doris inverted indexes (`USING INVERTED` with tokenizer `PROPERTIES`), bloom filter indexes (`USING NGRAM_BF`), or Doris 4.x+ ANN vector indexes (`USING ANN` with `hnsw` or `ivf` index types; ANN indexes on `UNIQUE KEY` tables require Doris 4.1.4+ with Merge-on-Write enabled, whereas earlier Doris 4.x versions only support ANN indexes on `DUPLICATE KEY` tables) using post-statements. Because Doris requires ANN vector index columns to be `ARRAY NOT NULL` at table creation time (and Doris does not allow `ALTER TABLE ... MODIFY COLUMN` to change an existing nullable column to `NOT NULL`), declare `not_null_columns = ('embedding')` in `physical_properties` so SQLMesh defines the vector column as `ARRAY NOT NULL` during `CREATE TABLE` (or wraps it in `NON_NULLABLE(...)` during `CTAS`). For `INCREMENTAL_*` models, SQLMesh executes `@runtime_stage = 'creating'` post-statements on the newly created empty table before inserting rows during `@runtime_stage = 'evaluating'`, so all inserted rows are indexed automatically on write in both compute-storage integrated and decoupled (`storage_vault_name`) deployments. For `FULL` models—where initial table creation populates rows via `CTAS` before `@runtime_stage = 'creating'` post-statements run—or when adding an index to a table with existing rows: + +- **Compute-storage integrated deployments**: Run `BUILD INDEX ON @this_model` in `@runtime_stage = 'creating'` (after `ADD INDEX`) to backfill the inverted index on rows written prior to `ADD INDEX`. +- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail evaluation); `ADD INDEX` only indexes newly written rows, so existing rows require a table rewrite or compaction to populate the index. + +You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: + +```sql +MODEL ( + name analytics.searchable_logs, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key log_id + ), + physical_properties ( + -- Required for ANN vector indexes on UNIQUE KEY tables (requires Doris 4.1.4+) + enable_unique_key_merge_on_write = 'true', + -- Define embedding as ARRAY NOT NULL at CREATE TABLE time (required by Doris ANN indexes; + -- Doris does not allow ALTER TABLE ... MODIFY COLUMN to change a nullable column to NOT NULL) + not_null_columns = ('embedding'), + -- Optional Doris 4.1+ Segment V3 format for wide tables + storage_format = 'V3', + -- Use 'size_based' on Merge-on-Write UNIQUE KEY tables ('time_series' is supported on DUPLICATE KEY append/log tables) + compaction_policy = 'size_based', + -- Sequence column prevents out-of-order updates with lower sequence values from overwriting newer rows + 'function_column.sequence_col' = 'updated_at' + ) +); + +SELECT + log_id::BIGINT AS log_id, + message::STRING AS message, + embedding::ARRAY AS embedding, + updated_at::DATETIME AS updated_at, + created_at::DATETIME(6) AS created_at +FROM raw.logs; + +-- DorisEngineAdapter automatically polls SHOW ALTER TABLE COLUMN after each ALTER TABLE +-- statement so asynchronous column/index schema-change jobs finish before the next statement runs. +@IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_message (message) USING INVERTED PROPERTIES ("parser" = "english", "support_phrase" = "true") COMMENT 'Full-text inverted index' +); + +@IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_embedding (embedding) USING ANN PROPERTIES ("index_type" = "hnsw", "metric_type" = "l2_distance", "dim" = "3") COMMENT 'ANN vector index' +); + +-- Only for FULL models (where CTAS writes rows before @runtime_stage = 'creating') in +-- compute-storage integrated deployments; do NOT run BUILD INDEX in compute-storage decoupled mode: +-- @IF( +-- @runtime_stage = 'creating', +-- BUILD INDEX idx_message ON @this_model +-- ); +``` + +!!! note "Recovering from Failed Index Post-Statements & Forward-Only Migrations" + - **Recovering if an `@runtime_stage = 'creating'` index statement fails**: Because `SUPPORTS_TRANSACTIONS = False`, if `CREATE TABLE` (and earlier `ALTER TABLE` post-statements such as `ADD INDEX idx_message`) succeed but a later post-statement such as `ADD INDEX idx_embedding` fails, the partially configured physical snapshot table (`sqlmesh__analytics.analytics__searchable_logs__`) remains in Doris. Retrying `sqlmesh plan` while that incomplete physical table exists can skip table creation (leaving `idx_embedding` missing) or fail on already-created indexes. Before retrying `sqlmesh plan`, either: + 1. Drop the incomplete physical snapshot table (`DROP TABLE IF EXISTS sqlmesh__analytics.analytics__searchable_logs__;`) so the retry recreates the table and runs all `@runtime_stage = 'creating'` statements from scratch, or + 2. Inspect existing indexes (`SHOW INDEX FROM sqlmesh__analytics.analytics__searchable_logs__;`) and manually reconcile the missing `idx_embedding` index (`ALTER TABLE sqlmesh__analytics.analytics__searchable_logs__ ADD INDEX idx_embedding ...`) before retrying. + - **Adding indexes during forward-only migrations**: When SQLMesh creates a new physical snapshot table during a standard `sqlmesh plan`, `@runtime_stage = 'creating'` runs directly on the new physical table. However, during a **forward-only migration** (`sqlmesh plan --forward-only` or `forward_only true`) on an already-existing table, SQLMesh executes `@runtime_stage = 'creating'` against a temporary schema-diff table (`__schema_tmp`) and only applies column-level `ALTER TABLE` diffs to the existing production table—so `ADD INDEX` post-statements in `@runtime_stage = 'creating'` are not applied to the existing production table. + + To add an inverted or ANN index to an existing production table during a forward-only migration: + + 1. Resolve the underlying physical snapshot table backing the production view (`SHOW CREATE VIEW analytics.searchable_logs`, e.g., `sqlmesh__analytics.analytics__searchable_logs__`). + 2. Apply `ALTER TABLE` directly on that physical table and wait for `SHOW ALTER TABLE COLUMN` to report `FINISHED`: + ```sql + ALTER TABLE sqlmesh__analytics.analytics__searchable_logs__ + ADD INDEX idx_message (message) USING INVERTED + PROPERTIES ("parser" = "english", "support_phrase" = "true"); + ``` + 3. Backfill the index for existing rows: + - **Compute-storage integrated deployments**: Run `BUILD INDEX idx_message ON sqlmesh__analytics.analytics__searchable_logs__;` and wait for `SHOW BUILD INDEX` to finish. + - **Compute-storage decoupled (`storage_vault_name`) deployments**: Because Doris does not support `BUILD INDEX` in compute-storage decoupled mode, run a standard non-forward-only `sqlmesh plan` (or `sqlmesh plan --restate-model analytics.searchable_logs` after recreating the table) so all rows are written into the newly indexed table. + +Downstream SQLMesh models can query full-text and vector indexes using Doris's native search operators (`MATCH`, `MATCH_ANY`, `MATCH_ALL`, `MATCH_PHRASE`, `MATCH_PHRASE_PREFIX`, `MATCH_REGEXP`, `MATCH_PHRASE_EDGE`, `SEARCH(...)`) and vector distance functions (`L2_DISTANCE`, `COSINE_DISTANCE`, `INNER_PRODUCT`, `L1_DISTANCE`, and their `_APPROXIMATE` counterparts): + +```sql +SELECT + log_id, + L2_DISTANCE_APPROXIMATE(embedding, [0.12, -0.04, 0.88]) AS dist +FROM analytics.searchable_logs +WHERE message MATCH_PHRASE 'connection timeout' + AND SEARCH('message:timeout') +ORDER BY dist ASC +LIMIT 20; +``` + +--- + +## Audits & Data Diff (`sqlmesh audit` & `sqlmesh table_diff`) + +All built-in and custom SQLMesh [audits](../../concepts/audits.md) (`not_null`, `unique_values`, `number_of_rows`, `accepted_values`, `forall`, etc.) and [`sqlmesh table_diff`](../../guides/tablediff.md) work out of the box on Doris: + +```sql +MODEL ( + name analytics.daily_orders, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column order_date, + partition_by_time_column false + ), + audits ( + not_null(columns := (order_id, customer_id)), + unique_values(columns := (order_id)), + number_of_rows(threshold := 1) + ) +); +``` + +- **Constant folding in `LIMIT` clauses**: Doris requires `LIMIT` arguments to be integer literals rather than arithmetic expressions (such as `LIMIT 10 + 1` emitted by `number_of_rows`). `DorisEngineAdapter` automatically folds constant arithmetic in `LIMIT` clauses before execution. +- **Deferred Materialized Views (`build = 'DEFERRED'`)**: When a materialized view is created with `build = 'DEFERRED'` and has `audits (...)` attached, SQLMesh automatically triggers `REFRESH MATERIALIZED VIEW ... AUTO` before running audits so they evaluate against populated data. +- **`sqlmesh table_diff` on floating-point and string columns**: Temporary diff tables created by `sqlmesh table_diff` automatically promote integer indicator columns into the Doris OLAP short-key prefix, so comparing tables whose leading user columns are `DOUBLE`, `FLOAT`, `STRING`, or `JSON` succeeds without manual key configuration. + +--- + +## Supported Model Kinds & Catalog Matrix + +| SQLMesh Model Kind | Doris Internal OLAP Catalog (`internal` on Local Disk or S3/GCS Storage Vault) | External Iceberg Catalog (`table_format iceberg` on S3/GCS/HDFS) | External Hive Catalog (`table_format hive` on S3/GCS/HDFS) | External Read-Only Lakehouse Catalogs (`paimon`, `lance`, `hudi`, `deltalake`, `lakesoul`) | +| --- | :---: | :---: | :---: | :---: | +| Source / Upstream `SELECT` | Supported | Supported (incl. `FOR TIME/VERSION AS OF`, `@branch`, `@tag`) | Supported (Doris 2.1+) | Supported (Read-Only via corresponding External Catalog; Lance also supports `s3()` / `local()` TVFs) | +| `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `DELETE` + `INSERT`) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | +| `INCREMENTAL_BY_TIME_RANGE` | Supported (`DELETE WHERE time_col >= low AND time_col <= high` + `INSERT`) | Supported (`DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No row-level `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA row-level `DELETE`) | +| `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`, `DELETE ... USING` + `INSERT` or native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | +| `INCREMENTAL_BY_PARTITION` | Supported (Column & time-truncation range `DELETE` + `INSERT`) | Supported (Partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | +| `SCD_TYPE_2_BY_TIME` / `SCD_TYPE_2_BY_COLUMN` | Supported | Supported (Requires Doris 4.1.0+ & Iceberg v2 for row-level `DELETE`) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | +| `VIEW` | Supported (`CREATE OR REPLACE VIEW`) | Requires `internal` catalog (`environment_catalog_mapping`) | Requires `internal` catalog (`environment_catalog_mapping`) | Requires `internal` catalog (`environment_catalog_mapping`) | +| `VIEW (materialized true)` | Supported (Asynchronous MTMVs; synchronous MVs are base-table rollup indexes created separately on the base table) | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | +| `SEED` / Python DataFrame Models | Supported (Automatic `VARCHAR(255)` OLAP key coercion) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Paimon: Read-Only in Doris 2.1–4.1.5; Lance/Hudi/Delta/LakeSoul: Write Guarded | +| `CUSTOM ('doris_ddl_only')` | Supported (DDL + `ALTER TABLE` schema evolution only) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Not supported (Read-Only) | + +--- + +## Property Aliases & StarRocks Compatibility + +If you are migrating StarRocks SQLMesh models or existing Doris DDL conventions, `DorisEngineAdapter` supports these property shorthands in `physical_properties`: + +| Property / Shorthand | SQLMesh Doris Behavior | +| --- | --- | +| `duplicate_key = ('c1', 'c2')` | Emits `DUPLICATE KEY (c1, c2)` (and automatically appends active OLAP `partitioned_by` columns if omitted from `duplicate_key`) | +| `unique_key = ('c1')` | Emits `UNIQUE KEY (c1)` (automatic for `INCREMENTAL_BY_UNIQUE_KEY`) | +| `aggregate_key = ('c1', 'c2')` | Emits `AGGREGATE KEY (c1, c2)` | +| `table_type = 'PRIMARY' \| 'UNIQUE' \| 'DUPLICATE' \| 'AGGREGATE'`, `keys = ('c1', 'c2')` | Maps directly to the corresponding Doris key model (`PRIMARY` maps to Doris `UNIQUE KEY`) | +| `clustered_by (c1)` + `physical_properties (buckets = 10)` | Emits `DISTRIBUTED BY HASH(c1) BUCKETS 10` | +| `physical_properties (partition_by = ...)` | Supported alias for top-level `partitioned_by` | +| `physical_properties (not_null_columns = ('c1', 'c2'))` | Defines specified OLAP columns as `NOT NULL` on `CREATE TABLE` (and wraps `CTAS` projections in `NON_NULLABLE(...)`) | + +--- + +## Troubleshooting & Doris Gotchas + +| Symptom / Doris Error | Cause | Resolution | +| --- | --- | --- | +| `errCode = 2, replication num 3 is larger than available backends 1` | Doris defaults to `replication_num = 3`, which fails on single-BE local/CI clusters. | Set `replication_num: 1` under `model_defaults.physical_properties` in `config.yaml` for local dev. | +| `PARTITION BY` omitted on OLAP table (logged as a warning) | Doris OLAP tables with `PARTITION BY (...) ()` and no partition creation strategy have zero partitions and reject all `INSERT`s. | Add `auto_partition = 'true'`, `'dynamic_partition.enable' = 'true'`, or explicit `partitions = '...'` to `physical_properties`, or set `partition_by_time_column false` in `INCREMENTAL_BY_TIME_RANGE` if you do not want OLAP partitioning. | +| `Doris only supports creating views in the 'internal' catalog` | External Iceberg catalogs do not support Doris `VIEW` or `MATERIALIZED VIEW` objects. | Set `virtual_environment_mode: dev_only` and `environment_catalog_mapping: {"^(?!prod$).*": "internal"}` in `config.yaml`. | +| `Key column cannot be of type STRING / DOUBLE / ARRAY` | Doris forbids `STRING`/`TEXT`, floating-point (`FLOAT`/`DOUBLE`), and complex types as OLAP key columns. | SQLMesh automatically reorders valid key columns first and coerces `STRING`/`TEXT` key columns to `VARCHAR(255)`. For explicit keys on `DOUBLE`/`FLOAT`, use `DECIMAL` or an integer/date key column. | diff --git a/docs/integrations/overview.md b/docs/integrations/overview.md index 1c9d56b7e2..6e0398eb5e 100644 --- a/docs/integrations/overview.md +++ b/docs/integrations/overview.md @@ -17,6 +17,7 @@ SQLMesh supports the following execution engines for running SQLMesh projects (e * [ClickHouse](./engines/clickhouse.md) (clickhouse) * [Databricks](./engines/databricks.md) (databricks) * [Db2](./engines/db2.md) (db2) +* [Doris](./engines/doris.md) (doris) * [DuckDB](./engines/duckdb.md) (duckdb) * [Fabric](./engines/fabric.md) (fabric) * [MotherDuck](./engines/motherduck.md) (motherduck) diff --git a/docs/quickstart/cli.md b/docs/quickstart/cli.md index 6aa4fc34ed..4b300c9018 100644 --- a/docs/quickstart/cli.md +++ b/docs/quickstart/cli.md @@ -118,6 +118,8 @@ Choose your SQL engine: [16] RisingWave [17] Fabric [18] StarRocks + [19] Db2 + [20] Apache Doris Enter a number: 1 ``` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index ae036a7774..a31eea0ccf 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -222,6 +222,7 @@ These pages describe the connection configuration options for each execution eng * [BigQuery](../integrations/engines/bigquery.md) * [ClickHouse](../integrations/engines/clickhouse.md) * [Databricks](../integrations/engines/databricks.md) +* [Doris](../integrations/engines/doris.md) * [DuckDB](../integrations/engines/duckdb.md) * [Fabric](../integrations/engines/fabric.md) * [MotherDuck](../integrations/engines/motherduck.md) diff --git a/mkdocs.yml b/mkdocs.yml index 1a1a34e624..d2c417cd3b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -83,6 +83,7 @@ nav: - integrations/engines/clickhouse.md - integrations/engines/databricks.md - integrations/engines/db2.md + - integrations/engines/doris.md - integrations/engines/duckdb.md - integrations/engines/fabric.md - integrations/engines/motherduck.md diff --git a/pyproject.toml b/pyproject.toml index 596c4197cc..10d6781c15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ db2 = [ "ibm_db", "db2-sqlglot-dialect>=1.1.0;python_version>=\"3.10\"" ] +doris = ["pymysql"] duckdb = [] fabric = ["pyodbc>=5.0.0"] fabric-mssql-python = ["mssql-python>=1.1.0;python_version>=\"3.10\""] @@ -290,6 +291,7 @@ markers = [ "trino: test for Trino (all connectors)", "risingwave: test for Risingwave", "starrocks: test for StarRocks", + "doris: test for Doris", # Other "set_default_connection", diff --git a/sqlmesh/cli/project_init.py b/sqlmesh/cli/project_init.py index e3132a6de3..b149b44bb9 100644 --- a/sqlmesh/cli/project_init.py +++ b/sqlmesh/cli/project_init.py @@ -88,6 +88,13 @@ def _gen_config( " # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection\n" f" # {doc_link.format(engine_link=engine_link)}\n{connection_settings}" ) + if engine_type == "doris": + connection_settings += ( + " # Doris cannot be used as a state_connection; use DuckDB for local dev or Postgres/MySQL in production:\n" + " state_connection:\n" + " type: duckdb\n" + " database: state.db\n" + ) default_configs = { ProjectTemplate.DEFAULT: f"""# --- Gateway Connection --- diff --git a/sqlmesh/core/config/__init__.py b/sqlmesh/core/config/__init__.py index 50d2d9a5a2..3113c61435 100644 --- a/sqlmesh/core/config/__init__.py +++ b/sqlmesh/core/config/__init__.py @@ -12,6 +12,7 @@ BigQueryConnectionConfig as BigQueryConnectionConfig, ConnectionConfig as ConnectionConfig, DatabricksConnectionConfig as DatabricksConnectionConfig, + DorisConnectionConfig as DorisConnectionConfig, DuckDBConnectionConfig as DuckDBConnectionConfig, FabricConnectionConfig as FabricConnectionConfig, GCPPostgresConnectionConfig as GCPPostgresConnectionConfig, diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 29cb1ef467..f223016ac0 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -31,7 +31,7 @@ from sqlmesh.core.engine_adapter.shared import CatalogSupport from sqlmesh.utils import debug_mode_enabled, str_to_bool from sqlmesh.utils.aws import validate_s3_uri -from sqlmesh.utils.errors import ConfigError +from sqlmesh.utils.errors import ConfigError, SQLMeshError from sqlmesh.utils.pydantic import ( ValidationInfo, field_validator, @@ -63,6 +63,8 @@ # Db2 rejects table names starting with underscore (_versions, _snapshots, # _environments, _intervals). Use a separate state_connection (e.g., DuckDB). "db2", + # OLAP primary keys and lack of transactional DDL make Doris unsuitable for state sync + "doris", } MOTHERDUCK_TOKEN_REGEX = re.compile(r"(\?|\&)(motherduck_token=)(\S*)") PASSWORD_REGEX = re.compile(r"(password=)(\S+)") @@ -2692,6 +2694,206 @@ def connect_db2(**kwargs: t.Any) -> t.Any: return connect_db2 +class DorisConnectionConfig(ConnectionConfig): + """Configuration for the Apache Doris connection. + + Apache Doris uses the MySQL network protocol and supports both internal OLAP tables + and external Multi-Catalog lakehouse catalogs (e.g., Apache Iceberg). + + Args: + host: The hostname of the Doris FE (Frontend) node. + user: The Doris username. + password: The optional Doris password. + port: The query port number of the Doris FE node. Default is 9030. + database: The optional database name. + catalog: The optional default catalog name (e.g., 'internal' or an external Iceberg catalog). + charset: The optional character set (defaults to 'utf8mb4'). + collation: The optional collation. + connect_timeout: Timeout in seconds for establishing a connection. Default is 10. + read_timeout: Timeout in seconds for reading from the connection. + write_timeout: Timeout in seconds for writing to the connection. + ssl: Optional dictionary of SSL arguments passed to PyMySQL. + ssl_ca: Path to the file that contains a PEM-formatted CA certificate. + ssl_cert: Path to the file that contains a PEM-formatted client certificate. + ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + ssl_verify_cert: Whether to verify the server certificate against the CA certificate. + ssl_verify_identity: Whether to verify the server hostname against the certificate. + ssl_disabled: Whether to disable SSL connection. + session_variables: Optional dictionary of Doris session variables to set on each connection. + concurrent_tasks: The maximum number of tasks that can use this connection concurrently. + register_comments: Whether or not to register model comments with the SQL engine. + pre_ping: Whether or not to pre-ping the connection before starting a new statement. + """ + + host: str + user: str + password: t.Optional[str] = None + port: t.Optional[int] = 9030 + database: t.Optional[str] = None + catalog: t.Optional[str] = None + charset: t.Optional[str] = None + collation: t.Optional[str] = None + connect_timeout: t.Optional[int] = 10 + read_timeout: t.Optional[int] = None + write_timeout: t.Optional[int] = None + ssl: t.Optional[t.Union[bool, t.Dict[str, t.Any]]] = None + ssl_ca: t.Optional[str] = None + ssl_cert: t.Optional[str] = None + ssl_key: t.Optional[str] = None + ssl_verify_cert: t.Optional[bool] = None + ssl_verify_identity: t.Optional[bool] = None + ssl_disabled: t.Optional[bool] = None + session_variables: t.Optional[t.Dict[str, t.Any]] = None + + concurrent_tasks: int = 4 + register_comments: bool = True + pre_ping: bool = False + + type_: t.Literal["doris"] = Field(alias="type", default="doris") + DIALECT: t.ClassVar[t.Literal["doris"]] = "doris" + DISPLAY_NAME: t.ClassVar[t.Literal["Apache Doris"]] = "Apache Doris" + DISPLAY_ORDER: t.ClassVar[t.Literal[20]] = 20 + + _engine_import_validator = _get_engine_import_validator("pymysql", "doris") + + @field_validator("session_variables", mode="after") + @classmethod + def _validate_session_variables( + cls, v: t.Optional[t.Dict[str, t.Any]] + ) -> t.Optional[t.Dict[str, t.Any]]: + if v: + import math + + for k, val in v.items(): + if not isinstance(k, str) or not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*\Z", k): + raise ConfigError(f"Invalid Doris session variable name: '{k}'") + if not isinstance(val, (str, int, float, bool)) or ( + isinstance(val, float) and (math.isnan(val) or math.isinf(val)) + ): + raise ConfigError(f"Invalid Doris session variable value for '{k}': {val!r}") + return v + + @property + def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: + kwargs: t.Dict[str, t.Any] = { + "autocommit": True, + "charset": self.charset or "utf8mb4", + "local_infile": False, + } + if not self.ssl_disabled and self.ssl is not False: + ssl_config: t.Dict[str, t.Any] = ( + dict(self.ssl) if isinstance(self.ssl, dict) else {} # type: ignore[arg-type] + ) + for key in ("ca", "cert", "key"): + val = getattr(self, f"ssl_{key}", None) + if val is not None: + ssl_config[key] = val + if self.ssl_verify_cert is not None: + ssl_config["verify_mode"] = self.ssl_verify_cert + if ( + self.ssl_verify_cert is False + and self.ssl_verify_identity is None + and "check_hostname" not in ssl_config + ): + ssl_config["check_hostname"] = False + if self.ssl_verify_identity is not None: + if self.ssl_verify_identity and not ( + ssl_config.get("ca") or ssl_config.get("capath") + ): + raise ConfigError( + "Doris `ssl_verify_identity` requires `ssl_ca` (or `ssl.ca` / `ssl.capath`). " + "PyMySQL disables hostname verification when no CA is configured." + ) + ssl_config["check_hostname"] = bool(self.ssl_verify_identity) + verify_mode = ssl_config.get("verify_mode") + if verify_mode is False or ( + isinstance(verify_mode, str) + and verify_mode.strip().lower() in ("false", "0", "no", "none") + ): + ssl_config["check_hostname"] = False + if ssl_config: + kwargs["ssl"] = ssl_config + elif self.ssl is True or (isinstance(self.ssl, dict) and len(self.ssl) == 0): + kwargs["ssl"] = {"check_hostname": False} + return kwargs + + @property + def _extra_engine_config(self) -> t.Dict[str, t.Any]: + return { + "database": self.database, + "catalog": self.catalog, + } + + @property + def _connection_kwargs_keys(self) -> t.Set[str]: + is_external_catalog = bool(self.catalog and self.catalog.lower() != "internal") + return { + k + for k in ( + "host", + "user", + "password", + "port", + "database", + "charset", + "collation", + "connect_timeout", + "read_timeout", + "write_timeout", + "ssl_disabled", + ) + if getattr(self, k) is not None and not (k == "database" and is_external_catalog) + } + + @property + def _engine_adapter(self) -> t.Type[EngineAdapter]: + return engine_adapter.DorisEngineAdapter + + @property + def _connection_factory(self) -> t.Callable: + from pymysql import connect + + if "ssl" not in self._static_connection_kwargs: + return connect + + def connect_with_required_tls(**kwargs: t.Any) -> t.Any: + conn = connect(**kwargs) + if not getattr(conn, "_secure", False): + try: + conn.close() + except Exception: + pass + raise SQLMeshError( + "TLS connection to Doris was requested, but the server did not negotiate TLS." + ) + return conn + + return connect_with_required_tls + + @property + def _cursor_init(self) -> t.Optional[t.Callable[[t.Any], None]]: + if not self.catalog and not self.session_variables: + return None + + def init(cursor: t.Any) -> None: + if self.session_variables: + self._validate_session_variables(self.session_variables) + for k, v in self.session_variables.items(): + val_sql = exp.convert(v).sql("doris", comments=False) + cursor.execute(f"SET `{k}` = {val_sql}") + if self.catalog: + catalog_sql = exp.to_identifier(self.catalog, quoted=True).sql("doris") + cursor.execute(f"SWITCH {catalog_sql}") + if self.database: + db_sql = exp.to_identifier(self.database, quoted=True).sql("doris") + cursor.execute(f"USE {db_sql}") + + return init + + def get_catalog(self) -> t.Optional[str]: + return self.catalog + + CONNECTION_CONFIG_TO_TYPE = { # Map all subclasses of ConnectionConfig to the value of their `type_` field. tpe.all_field_infos()["type_"].default: tpe diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index bcfb300ba1..0a673a421d 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -10,12 +10,14 @@ from enum import Enum, auto from functools import lru_cache -from sqlglot import Dialect, Generator, ParseError, Parser, Tokenizer, TokenType, exp -from sqlglot.dialects.dialect import DialectType -from sqlglot.dialects import DuckDB, Snowflake, TSQL +from sqlglot import Dialect, Generator, ParseError, Parser, Tokenizer, TokenType, exp, transforms +from sqlglot.dialects.dialect import DialectType, rename_func, var_map_sql +from sqlglot.dialects import Doris, DuckDB, Snowflake, TSQL import sqlglot.dialects.athena as athena import sqlglot.generators.athena as athena_generators +from sqlglot.parser import build_var_map from sqlglot.parsers.athena import AthenaTrinoParser +from sqlglot.parsers.hive import _build_named_struct from sqlglot.helper import seq_get from sqlglot.optimizer.normalize_identifiers import normalize_identifiers from sqlglot.optimizer.qualify_columns import quote_identifiers @@ -146,6 +148,15 @@ def _parse_statement(self: Parser) -> t.Optional[exp.Expr]: meta.comments = comments return meta + if ( + isinstance(self, Doris.Parser) + and self._curr.text.upper() == "BUILD" + and self._next + and self._next.token_type == TokenType.INDEX + ): + self._advance() + return _doris_parse_build(self) + try: return self.__parse_statement() # type: ignore except ParseError: @@ -484,6 +495,38 @@ def _parse_types( with_collation: bool = False, ) -> t.Optional[exp.Expr]: start = self._curr + udt_name = ( + self._curr.text.upper() + if self._curr and self._curr.token_type == TokenType.USERDEFINED + else None + ) + if udt_name == "AGG_STATE" and not check_func: + self._advance() + if self._curr and self._curr.token_type in (TokenType.LT, TokenType.L_PAREN): + open_tok = self._curr.token_type + close_tok = TokenType.GT if open_tok == TokenType.LT else TokenType.R_PAREN + depth = 1 + self._advance() + while self._curr and depth > 0: + if self._curr.token_type == open_tok: + depth += 1 + elif self._curr.token_type == close_tok: + depth -= 1 + self._advance() + parsed_type: t.Optional[exp.Expr] = self.expression( + exp.DataType( + this=exp.DataType.Type.USERDEFINED, + kind=self._find_sql(start, self._prev), + ) + ) + else: + parsed_type = self.expression( + exp.DataType(this=exp.DataType.Type.USERDEFINED, kind="AGG_STATE") + ) + if schema and parsed_type: + parsed_type.meta["sql"] = self._find_sql(start, self._prev) + return parsed_type + parsed_type = self.__parse_types( # type: ignore check_func=check_func, schema=schema, @@ -491,6 +534,14 @@ def _parse_types( with_collation=with_collation, ) + if ( + udt_name + and isinstance(parsed_type, exp.DataType) + and parsed_type.this == exp.DataType.Type.USERDEFINED + and not parsed_type.args.get("kind") + ): + parsed_type.set("kind", udt_name) + if schema and parsed_type: parsed_type.meta["sql"] = self._find_sql(start, self._prev) @@ -581,6 +632,351 @@ def altercolumn_sql(self: Generator, expression: exp.AlterColumn) -> str: return sql +def columndef_sql(self: Generator, expression: exp.ColumnDef, sep: str = " ") -> str: + return self._columndef_sql( # type: ignore + expression, + sep=( + ": " + if isinstance(expression.parent, exp.DataType) and expression.parent.is_type("struct") + else sep + ), + ) + + +_DORIS_MATCH_OPS = { + "MATCH", + "MATCH_ANY", + "MATCH_ALL", + "MATCH_PHRASE", + "MATCH_PHRASE_PREFIX", + "MATCH_REGEXP", + "MATCH_PHRASE_EDGE", +} + + +def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Optional[exp.Expr]: + this = self.__parse_range(this) # type: ignore + if this is not None: + idx = self._index + negated = self._match(TokenType.NOT) + if self._curr and self._curr.text.upper() in _DORIS_MATCH_OPS: + op_name = self._curr.text.upper() + self._advance() + rhs = self._parse_bitwise() + expr: exp.Expr = self.expression(exp.Anonymous(this=op_name, expressions=[this, rhs])) + if negated: + expr = self.expression(exp.Not(this=expr)) + return expr + if negated: + self._retreat(idx) + return this + + +_doris_parse_range.__name__ = "_parse_range" + + +def _doris_anonymous_sql(self: Generator, expression: exp.Anonymous) -> str: + if expression.name.upper() in _DORIS_MATCH_OPS and len(expression.expressions) == 2: + left = self.sql(expression.expressions[0]) + right = self.sql(expression.expressions[1]) + return f"{left} {expression.name.upper()} {right}" + return self.anonymous_sql(expression) + + +def _doris_parse_table_parts( + self: Parser, + schema: bool = False, + is_db_reference: bool = False, + wildcard: bool = False, + fast: bool = False, +) -> exp.Table: + table = self.__parse_table_parts( # type: ignore + schema=schema, is_db_reference=is_db_reference, wildcard=wildcard, fast=fast + ) + if not isinstance(table, exp.Table): + return table + if isinstance(table.this, exp.Anonymous): + m = re.match(r"^(.+)@(branch|tag)$", table.this.name, flags=re.IGNORECASE) + if m and table.this.expressions: + real_name = m.group(1) + ref_kind = m.group(2).upper() + ref_arg = table.this.expressions[0] + if isinstance(ref_arg, exp.Column) and not ref_arg.table: + ref_arg = exp.var(ref_arg.name) + ver = exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg) + table.set("this", exp.Identifier(this=real_name, quoted=False)) + table.set("version", ver) + table.meta["doris_version"] = ver + return table + + +_doris_parse_table_parts.__name__ = "_parse_table_parts" + + +def _doris_parse_version(self: Parser) -> t.Optional[exp.Version]: + ver = self.__parse_version() # type: ignore + if ver is not None: + return ver + if ( + self._match_pair(TokenType.PARAMETER, TokenType.VAR, advance=False) + and self._next + and self._next.text.upper() in ("BRANCH", "TAG") + ): + saved_idx = self._index + self._advance() + ref_kind = self._curr.text.upper() + self._advance() + if self._match(TokenType.L_PAREN, advance=False): + ref_args = self._parse_wrapped_csv(lambda: self._parse_id_var(any_token=True)) + if ref_args: + ref_arg = ref_args[0] + if isinstance(ref_arg, exp.Identifier) and not ref_arg.quoted: + ref_arg = exp.var(ref_arg.name) + return self.expression(exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg)) + self._retreat(saved_idx) + return None + + +_doris_parse_version.__name__ = "_parse_version" + + +def _doris_parse_table( + self: Parser, + schema: bool = False, + joins: bool = False, + alias_tokens: t.Optional[t.Collection[TokenType]] = None, + parse_bracket: bool = False, + is_db_reference: bool = False, + parse_partition: bool = False, + consume_pipe: bool = False, +) -> t.Optional[exp.Expr]: + table = self.__parse_table( # type: ignore + schema=schema, + joins=joins, + alias_tokens=alias_tokens, + parse_bracket=parse_bracket, + is_db_reference=is_db_reference, + parse_partition=parse_partition, + consume_pipe=consume_pipe, + ) + if ( + isinstance(table, exp.Table) + and not table.args.get("version") + and table.meta.get("doris_version") + ): + table.set("version", table.meta.pop("doris_version")) + return table + + +_doris_parse_table.__name__ = "_parse_table" + + +def _doris_version_sql(self: Generator, expression: exp.Version) -> str: + this_upper = expression.name.upper() + if this_upper in ("BRANCH", "TAG"): + expr_node = expression.expression + expr_sql = ( + expr_node.name + if isinstance(expr_node, (exp.Var, exp.Identifier)) + and not getattr(expr_node, "quoted", False) + else self.sql(expr_node) + ) + return f"@{this_upper.lower()}({expr_sql})" + kind = expression.text("kind") + this_sql = "TIME" if this_upper in ("TIMESTAMP", "TIME") else this_upper + return f"FOR {this_sql} {kind} {self.sql(expression, 'expression')}" + + +def _doris_parse_index_constraint( + self: Parser, kind: t.Optional[str] = None +) -> exp.IndexColumnConstraint: + constraint = self.__parse_index_constraint(kind=kind) # type: ignore + options = list(constraint.args.get("options") or []) + updated = False + while True: + if self._match(TokenType.PROPERTIES) or self._match_text_seq("PROPERTIES"): + props = exp.Properties(expressions=self._parse_wrapped_properties()) + options.append(self.expression(exp.IndexConstraintOption(parser=props))) + updated = True + elif self._match(TokenType.COMMENT): + options.append(self.expression(exp.IndexConstraintOption(comment=self._parse_string()))) + updated = True + elif self._match(TokenType.USING): + options.append( + self.expression( + exp.IndexConstraintOption(using=self._advance_any() and self._prev.text) + ) + ) + updated = True + else: + break + if updated: + constraint.set("options", options) + return constraint + + +_doris_parse_index_constraint.__name__ = "_parse_index_constraint" + + +def _doris_indexconstraintoption_sql(self: Generator, expression: exp.IndexConstraintOption) -> str: + props = expression.args.get("parser") + if isinstance(props, exp.Properties): + return self.properties(props, prefix="PROPERTIES") + return self._indexconstraintoption_sql(expression) # type: ignore + + +_doris_indexconstraintoption_sql.__name__ = "indexconstraintoption_sql" + + +def _doris_parse_index_params(self: Parser) -> exp.IndexParameters: + params = self.__parse_index_params() # type: ignore + while True: + if self._match(TokenType.USING): + params.set("using", self._parse_var(any_token=True)) + elif self._match(TokenType.PROPERTIES) or self._match_text_seq("PROPERTIES"): + params.set("with_storage", self._parse_wrapped_properties()) + elif self._match(TokenType.COMMENT): + params.set( + "where", + self.expression(exp.CommentColumnConstraint(this=self._parse_string())), + ) + else: + break + return params + + +_doris_parse_index_params.__name__ = "_parse_index_params" + + +def _doris_indexparameters_sql(self: Generator, expression: exp.IndexParameters) -> str: + using = self.sql(expression, "using") + using = f" USING {using}" if using else "" + columns = self.expressions(expression, key="columns", flat=True) + columns = f"({columns})" if columns else "" + partition_by = self.expressions(expression, key="partition_by", flat=True) + partition_by = f" PARTITION BY {partition_by}" if partition_by else "" + where_node = expression.args.get("where") + where = ( + f" {self.sql(where_node)}" + if isinstance(where_node, exp.CommentColumnConstraint) + else self.sql(expression, "where") + ) + include = self.expressions(expression, key="include", flat=True) + if include: + include = f" INCLUDE ({include})" + with_storage = self.expressions(expression, key="with_storage", flat=True) + with_storage = f" PROPERTIES ({with_storage})" if with_storage else "" + tablespace = self.sql(expression, "tablespace") + tablespace = f" USING INDEX TABLESPACE {tablespace}" if tablespace else "" + on = self.sql(expression, "on") + on = f" ON {on}" if on else "" + + return f"{columns}{using}{include}{with_storage}{tablespace}{partition_by}{where}{on}" + + +_doris_indexparameters_sql.__name__ = "indexparameters_sql" + + +def _doris_parse_build(self: Parser) -> t.Optional[exp.Expr]: + start = self._prev + if self._match(TokenType.INDEX): + index_name = self._parse_id_var(any_token=False) + if self._match(TokenType.ON): + table = self._parse_table_parts() + partition_kw: t.Optional[str] = None + if self._match_text_seq("PARTITIONS"): + partition_kw = "PARTITIONS" + elif self._match(TokenType.PARTITION) or self._match_text_seq("PARTITION"): + partition_kw = "PARTITION" + partitions = self._parse_wrapped_id_vars() if partition_kw else [] + alter_expr = self.expression( + exp.Alter( + this=table, + kind="BUILD_INDEX", + actions=[index_name, *partitions], + ) + ) + if partition_kw: + alter_expr.meta["partition_keyword"] = partition_kw + return alter_expr + return self._parse_as_command(start) + + +def _doris_parse_alter_raw_action(self: Parser) -> exp.Var: + start = self._prev + depth = 0 + while self._curr and self._curr.token_type != TokenType.SEMICOLON: + if self._curr.token_type == TokenType.L_PAREN: + depth += 1 + elif self._curr.token_type == TokenType.R_PAREN: + if depth == 0: + break + depth -= 1 + self._advance() + return exp.var(self._find_sql(start, self._prev)) + + +def _doris_parse_alter_add(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], None]: + if self._match(TokenType.INDEX): + return _doris_parse_index_constraint(self) + return self._parse_alter_table_add() + + +def _doris_parse_alter_drop(self: Parser) -> t.Optional[exp.Expr]: + if self._match_texts(("BRANCH", "TAG"), advance=False): + return _doris_parse_alter_raw_action(self) + return self._parse_drop() + + +def _doris_alter_sql(self: Generator, expression: exp.Alter) -> str: + actions = expression.args.get("actions") or [] + if expression.text("kind") == "BUILD_INDEX": + idx_sql = self.sql(actions[0]) if actions else "" + tbl_sql = self.sql(expression, "this") + partition_kw = expression.meta.get("partition_keyword") or "PARTITIONS" + parts_sql = ( + f" {partition_kw} ({', '.join(self.sql(p) for p in actions[1:])})" + if len(actions) > 1 + else "" + ) + return f"BUILD INDEX {idx_sql} ON {tbl_sql}{parts_sql}" + if actions and isinstance(actions[0], exp.IndexColumnConstraint): + return f"ALTER TABLE {self.sql(expression, 'this')} ADD {self.expressions(expression, key='actions', flat=True)}" + return self._alter_sql(expression) # type: ignore + + +_doris_alter_sql.__name__ = "alter_sql" + + +def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: + keys = expression.args.get("keys") + values = expression.args.get("values") + if isinstance(keys, exp.Array) and isinstance(values, exp.Array): + return var_map_sql(self, expression) + if keys is not None or values is not None: + return self.func("MAP", keys, values) + return self.func("MAP", *expression.expressions) + + +def _doris_struct_sql(self: Generator, expression: exp.Struct) -> str: + if any(isinstance(e, exp.PropertyEQ) for e in expression.expressions): + args: t.List[str] = [] + for e in expression.expressions: + if isinstance(e, exp.PropertyEQ): + key_sql = ( + self.sql(exp.Literal.string(e.this.name)) + if isinstance(e.this, (exp.Identifier, exp.Var)) + else self.sql(e.this) + ) + args.append(key_sql) + args.append(self.sql(e.expression)) + else: + args.append(self.sql(e)) + return self.func("NAMED_STRUCT", *args) + values = [e.this if isinstance(e, exp.Alias) else e for e in expression.expressions] + return self.func("STRUCT", *values) + + def _parse_if(self: Parser) -> t.Optional[exp.Expr]: # If we fail to parse an IF function with expressions as arguments, we then try # to parse a statement / command to support the macro @IF(condition, statement) @@ -1343,6 +1739,84 @@ def parse_virtual_statement( def extend_sqlglot() -> None: """Extend SQLGlot with SQLMesh's custom macro aware dialect.""" + Doris.SUPPORTS_USER_DEFINED_TYPES = True + if Doris.tokenizer_class.__name__ != "_DorisTokenizer": + + class _DorisTokenizer(Doris.tokenizer_class): # type: ignore + KEYWORDS = { + **Doris.tokenizer_class.KEYWORDS, + "AGG_STATE": TokenType.USERDEFINED, + "BITMAP": TokenType.USERDEFINED, + "DATETIMEV2": TokenType.DATETIME, + "DATEV2": TokenType.DATE, + "DECIMALV2": TokenType.DECIMAL, + "DECIMALV3": TokenType.DECIMAL, + "FOR TIME": TokenType.TIMESTAMP_SNAPSHOT, + "HLL": TokenType.HLLSKETCH, + "IPV4": TokenType.IPV4, + "IPV6": TokenType.IPV6, + "JSONB": TokenType.JSONB, + "LARGEINT": TokenType.INT128, + "QUANTILE_STATE": TokenType.USERDEFINED, + "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, + "VARIANT": TokenType.VARIANT, + } + + Doris.Tokenizer = _DorisTokenizer + Doris.tokenizer_class = _DorisTokenizer + + Doris.Parser.TABLE_ALIAS_TOKENS = Doris.Parser.TABLE_ALIAS_TOKENS - { + TokenType.ANTI, + TokenType.SEMI, + } + Doris.Parser.FUNCTIONS.update( + { + "INNER_PRODUCT": exp.DotProduct.from_arg_list, + "L1_DISTANCE": exp.ManhattanDistance.from_arg_list, + "MAP": build_var_map, + "NAMED_STRUCT": _build_named_struct, + "SEARCH": lambda args: exp.Anonymous(this="SEARCH", expressions=args), + } + ) + Doris.Parser.ALTER_PARSERS = { + **Doris.Parser.ALTER_PARSERS, + "ADD": _doris_parse_alter_add, + "CREATE": _doris_parse_alter_raw_action, + "DROP": _doris_parse_alter_drop, + "EXECUTE": _doris_parse_alter_raw_action, + "REPLACE": _doris_parse_alter_raw_action, + } + Doris.Generator.TYPE_MAPPING.update( + { + exp.DataType.Type.HLLSKETCH: "HLL", + exp.DataType.Type.INT128: "LARGEINT", + exp.DataType.Type.IPV4: "IPV4", + exp.DataType.Type.IPV6: "IPV6", + exp.DataType.Type.JSONB: "JSONB", + exp.DataType.Type.VARIANT: "VARIANT", + } + ) + Doris.Generator.TRANSFORMS.update( + { + exp.Anonymous: _doris_anonymous_sql, + exp.Array: rename_func("ARRAY"), + exp.DotProduct: rename_func("INNER_PRODUCT"), + exp.ManhattanDistance: rename_func("L1_DISTANCE"), + exp.Map: _doris_map_sql, + exp.ParseJSON: lambda self, e: f"CAST({self.sql(e, 'this')} AS JSON)", + exp.Select: transforms.preprocess( + [ + transforms.eliminate_distinct_on, + transforms.eliminate_qualify, + transforms.unnest_generate_date_array_using_recursive_cte, + ] + ), + exp.Struct: _doris_struct_sql, + exp.VarMap: _doris_map_sql, + exp.Version: _doris_version_sql, + } + ) + tokenizers = {Tokenizer} parsers = {Parser} generators = {Generator} @@ -1432,6 +1906,16 @@ def extend_sqlglot() -> None: _override(Snowflake.Parser, _parse_table_parts) _override(TSQL.Parser, _parse_alter_table_alter) _override(TSQL.Generator, altercolumn_sql) + _override(Doris.Parser, _doris_parse_range) + _override(Doris.Parser, _doris_parse_table_parts) + _override(Doris.Parser, _doris_parse_version) + _override(Doris.Parser, _doris_parse_table) + _override(Doris.Parser, _doris_parse_index_constraint) + _override(Doris.Parser, _doris_parse_index_params) + _override(Doris.Generator, columndef_sql) + _override(Doris.Generator, _doris_indexconstraintoption_sql) + _override(Doris.Generator, _doris_indexparameters_sql) + _override(Doris.Generator, _doris_alter_sql) # DuckDB's prefix absolute power operator `@` clashes with the macro syntax DuckDB.Parser.NO_PAREN_FUNCTION_PARSERS.pop("@", None) diff --git a/sqlmesh/core/engine_adapter/__init__.py b/sqlmesh/core/engine_adapter/__init__.py index 658b149753..bcb70e87a1 100644 --- a/sqlmesh/core/engine_adapter/__init__.py +++ b/sqlmesh/core/engine_adapter/__init__.py @@ -11,6 +11,7 @@ from sqlmesh.core.engine_adapter.bigquery import BigQueryEngineAdapter from sqlmesh.core.engine_adapter.clickhouse import ClickhouseEngineAdapter from sqlmesh.core.engine_adapter.databricks import DatabricksEngineAdapter +from sqlmesh.core.engine_adapter.doris import DorisEngineAdapter from sqlmesh.core.engine_adapter.duckdb import DuckDBEngineAdapter from sqlmesh.core.engine_adapter.mssql import MSSQLEngineAdapter from sqlmesh.core.engine_adapter.mysql import MySQLEngineAdapter @@ -50,6 +51,7 @@ "risingwave": RisingwaveEngineAdapter, "fabric": FabricEngineAdapter, "starrocks": StarRocksEngineAdapter, + "doris": DorisEngineAdapter, } # Add DB2 to the registry only when the dialect package is present diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 2dd234f70a..dd94904993 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -525,10 +525,55 @@ def replace_query( if not target_columns_to_types and table_exists: target_columns_to_types = self.columns(target_table) query = source_queries[0].query_factory() - self_referencing = any( - quote_identifiers(table) == quote_identifiers(target_table) - for table in query.find_all(exp.Table) - ) + + quoted_target = quote_identifiers(target_table.copy()) + default_catalog: t.Optional[str] = None + default_catalog_loaded = False + qualified_target: t.Optional[exp.Table] = None + + def _table_parts( + table: exp.Table, + ) -> t.Tuple[t.Optional[exp.Expr], t.Optional[exp.Expr], t.Optional[exp.Expr]]: + return (table.args.get("catalog"), table.args.get("db"), table.this) + + def _qualified(table: exp.Table, catalog: str) -> exp.Table: + table = table.copy() + if not table.catalog and table.db: + table.set("catalog", exp.parse_identifier(catalog)) + return quote_identifiers(table) + + def _is_target_table(node: exp.Expr) -> bool: + nonlocal default_catalog, default_catalog_loaded, qualified_target + if not isinstance(node, exp.Table): + return False + quoted_node = quote_identifiers(node.copy()) + if _table_parts(quoted_node) == _table_parts(quoted_target): + return True + if quoted_node.this != quoted_target.this or quoted_node.args.get( + "db" + ) != quoted_target.args.get("db"): + return False + if not default_catalog_loaded: + default_catalog = self.default_catalog + default_catalog_loaded = True + if default_catalog: + qualified_target = _qualified(target_table, default_catalog) + return ( + default_catalog is not None + and qualified_target is not None + and _table_parts(_qualified(node, default_catalog)) + == _table_parts(qualified_target) + ) + + def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: + if not _is_target_table(node): + return node + replacement = replacement_table.copy() + if isinstance(node, exp.Table) and node.args.get("alias") is not None: + replacement.set("alias", node.args["alias"].copy()) + return replacement + + self_referencing = any(_is_target_table(table) for table in query.find_all(exp.Table)) # If a query references itself then it must have a table created regardless of approach used. if self_referencing: if not target_columns_to_types: @@ -571,12 +616,7 @@ def replace_query( ) as temp_table: for source_query in source_queries: source_query.add_transform( - lambda node: ( # type: ignore - temp_table # type: ignore - if isinstance(node, exp.Table) - and quote_identifiers(node) == quote_identifiers(target_table) - else node - ) + lambda node: _replace_target_table(node, temp_table) # type: ignore ) return self._insert_overwrite_by_condition( target_table, @@ -2723,7 +2763,7 @@ def temp_table( Yields: The table expression """ - name = exp.to_table(name) + name = exp.to_table(name).copy() # ensure that we use default catalog if none is not specified if isinstance(name, exp.Table) and not name.catalog and name.db and self.default_catalog: name.set("catalog", exp.parse_identifier(self.default_catalog)) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py new file mode 100644 index 0000000000..bc4af509ff --- /dev/null +++ b/sqlmesh/core/engine_adapter/doris.py @@ -0,0 +1,6999 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from collections import defaultdict +import datetime +from decimal import Decimal +import hashlib +import itertools +import logging +import re +import threading +import time +import typing as t + +from sqlglot import exp, parse, parse_one +from sqlglot.helper import ensure_list +from sqlglot.optimizer.qualify_columns import quote_identifiers + +from sqlmesh.core.dialect import to_schema +from sqlmesh.core.engine_adapter.base import ( + EngineAdapterWithIndexSupport, + InsertOverwriteStrategy, + MERGE_SOURCE_ALIAS, + MERGE_TARGET_ALIAS, + _decoded_str, +) +from sqlmesh.core.engine_adapter.mixins import ( + GetCurrentCatalogFromFunctionMixin, + PandasNativeFetchDFSupportMixin, + RowDiffMixin, +) +from sqlmesh.core.engine_adapter.shared import ( + CatalogSupport, + CommentCreationTable, + CommentCreationView, + DataObject, + DataObjectType, + SourceQuery, + set_catalog, +) +from sqlmesh.utils import columns_to_types_all_known, get_source_columns_to_types, random_id +from sqlmesh.utils.errors import SQLMeshError + +if t.TYPE_CHECKING: + from sqlmesh.core._typing import SchemaName, TableName + from sqlmesh.core.engine_adapter._typing import DF, QueryOrDF + from sqlmesh.core.model.kind import TimeColumn + from sqlmesh.core.node import IntervalUnit + from sqlmesh.core.schema_diff import TableAlterOperation + from sqlmesh.utils.date import TimeLike + +logger = logging.getLogger(__name__) + + +@set_catalog() +class DorisEngineAdapter( + EngineAdapterWithIndexSupport, + GetCurrentCatalogFromFunctionMixin, + PandasNativeFetchDFSupportMixin, + RowDiffMixin, +): + DIALECT = "doris" + DEFAULT_BATCH_SIZE = 500 + SUPPORTS_TRANSACTIONS = False + COMMENT_CREATION_TABLE = CommentCreationTable.IN_SCHEMA_DEF_NO_CTAS + COMMENT_CREATION_VIEW = CommentCreationView.IN_SCHEMA_DEF_NO_COMMANDS + MAX_TABLE_COMMENT_LENGTH = 2048 + MAX_COLUMN_COMMENT_LENGTH = 255 + SUPPORTS_INDEXES = True + SUPPORTS_TUPLE_IN = False + SUPPORTS_REPLACE_TABLE = False + MAX_IDENTIFIER_LENGTH = 64 + SUPPORTS_MATERIALIZED_VIEWS = True + SUPPORTS_MATERIALIZED_VIEW_SCHEMA = True + RECREATE_MATERIALIZED_VIEW_ON_EVALUATION = False + SUPPORTS_CREATE_DROP_CATALOG = False + SUPPORTS_QUERY_EXECUTION_TRACKING = True + INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.DELETE_INSERT + CURRENT_CATALOG_EXPRESSION = exp.func("current_catalog") + RESOLVE_TABLE_REFS_IN_PHYSICAL_PROPERTIES: t.FrozenSet[str] = frozenset( + {"excluded_trigger_tables", "source_table"} + ) + SCHEMA_DIFFER_KWARGS = { + "parameterized_type_defaults": { + exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(9, 0), (0,)], + exp.DataType.build("CHAR", dialect=DIALECT).this: [(1,), (255,)], + exp.DataType.build("VARCHAR", dialect=DIALECT).this: [(65533,)], + exp.DataType.build("DATETIME", dialect=DIALECT).this: [(0,)], + exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT).this: [(0,)], + exp.DataType.build("INT", dialect=DIALECT).this: [(11,)], + exp.DataType.build("BIGINT", dialect=DIALECT).this: [(20,)], + exp.DataType.build("SMALLINT", dialect=DIALECT).this: [(6,)], + exp.DataType.build("TINYINT", dialect=DIALECT).this: [(4,), (1,)], + }, + "types_with_unlimited_length": { + exp.DataType.build("TEXT", dialect=DIALECT).this: { + exp.DataType.build("VARCHAR", dialect=DIALECT).this, + exp.DataType.build("CHAR", dialect=DIALECT).this, + }, + exp.DataType.build("VARCHAR", dialect=DIALECT).this: { + exp.DataType.build("VARCHAR", dialect=DIALECT).this, + exp.DataType.build("CHAR", dialect=DIALECT).this, + exp.DataType.build("TEXT", dialect=DIALECT).this, + }, + }, + "compatible_types": { + exp.DataType.build("BOOLEAN", dialect=DIALECT): { + exp.DataType.build("TINYINT", dialect=DIALECT), + exp.DataType.build("SMALLINT", dialect=DIALECT), + exp.DataType.build("INT", dialect=DIALECT), + exp.DataType.build("BIGINT", dialect=DIALECT), + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("TINYINT", dialect=DIALECT): { + exp.DataType.build("SMALLINT", dialect=DIALECT), + exp.DataType.build("INT", dialect=DIALECT), + exp.DataType.build("BIGINT", dialect=DIALECT), + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("DOUBLE", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("SMALLINT", dialect=DIALECT): { + exp.DataType.build("INT", dialect=DIALECT), + exp.DataType.build("BIGINT", dialect=DIALECT), + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("DOUBLE", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("INT", dialect=DIALECT): { + exp.DataType.build("BIGINT", dialect=DIALECT), + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("DOUBLE", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("BIGINT", dialect=DIALECT): { + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("INT128", dialect=DIALECT): { + exp.DataType.build("LARGEINT", dialect=DIALECT, udt=True), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("LARGEINT", dialect=DIALECT, udt=True): { + exp.DataType.build("INT128", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("FLOAT", dialect=DIALECT): { + exp.DataType.build("DOUBLE", dialect=DIALECT), + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("DOUBLE", dialect=DIALECT): { + exp.DataType.build("TEXT", dialect=DIALECT), + }, + exp.DataType.build("DATE", dialect=DIALECT): { + exp.DataType.build("DATETIME", dialect=DIALECT), + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), + }, + exp.DataType.build("DATETIME", dialect=DIALECT): { + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), + }, + exp.DataType.build("DATETIME(0)", dialect=DIALECT): { + exp.DataType.build("DATETIME", dialect=DIALECT), + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(1, 7)), + }, + exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT): { + *(exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") for p in range(0, 7)), + }, + exp.DataType.build("TIMESTAMPTZ(0)", dialect=DIALECT): { + exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT), + *(exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") for p in range(1, 7)), + }, + **{ + exp.DataType.build(f"DATETIME({p})", dialect="doris"): { + exp.DataType.build(f"DATETIME({p2})", dialect="doris") for p2 in range(p + 1, 7) + } + for p in range(1, 6) + }, + **{ + exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris"): { + exp.DataType.build(f"TIMESTAMPTZ({p2})", dialect="doris") + for p2 in range(p + 1, 7) + } + for p in range(1, 6) + }, + }, + } + + INVALID_KEY_TYPES = { + "FLOAT", + "DOUBLE", + "STRING", + "TEXT", + "TINYTEXT", + "MEDIUMTEXT", + "LONGTEXT", + "BLOB", + "TINYBLOB", + "MEDIUMBLOB", + "LONGBLOB", + "BINARY", + "VARBINARY", + "JSON", + "JSONB", + "VARIANT", + "OBJECT", + "ARRAY", + "MAP", + "STRUCT", + "BITMAP", + "HLL", + "HLLSKETCH", + "QUANTILE_STATE", + "AGG_STATE", + } + + def _get_thread_state(self) -> threading.local: + state = getattr(self._connection_pool, "_doris_thread_state", None) + if not isinstance(state, threading.local): + state = self._extra_config.get("_doris_thread_state") + if not isinstance(state, threading.local): + state = getattr(self, "_doris_thread_state", None) + if not isinstance(state, threading.local): + state = threading.local() + self._doris_thread_state = state + if isinstance(getattr(self, "_extra_config", None), dict): + self._extra_config["_doris_thread_state"] = state + try: + setattr(self._connection_pool, "_doris_thread_state", state) + except Exception: + pass + return state + + @property + def _active_catalog(self) -> t.Optional[str]: + return getattr(self._get_thread_state(), "active_catalog", None) + + @_active_catalog.setter + def _active_catalog(self, value: t.Optional[str]) -> None: + self._get_thread_state().active_catalog = value + + @property + def _needs_catalog_restore(self) -> bool: + return bool(getattr(self._get_thread_state(), "needs_catalog_restore", False)) + + @_needs_catalog_restore.setter + def _needs_catalog_restore(self, value: bool) -> None: + self._get_thread_state().needs_catalog_restore = bool(value) + + @property + def _last_described_key_cols(self) -> t.Set[str]: + return getattr(self._get_thread_state(), "last_described_key_cols", set()) + + @_last_described_key_cols.setter + def _last_described_key_cols(self, value: t.Set[str]) -> None: + self._get_thread_state().last_described_key_cols = set(value) + + @property + def _in_iceberg_replace_by_key(self) -> bool: + return bool(getattr(self._get_thread_state(), "in_iceberg_replace_by_key", False)) + + @_in_iceberg_replace_by_key.setter + def _in_iceberg_replace_by_key(self, value: bool) -> None: + self._get_thread_state().in_iceberg_replace_by_key = bool(value) + + @staticmethod + def _get_table_properties(kwargs: t.Dict[str, t.Any]) -> t.Optional[t.Dict[str, t.Any]]: + return kwargs.get("table_properties") or kwargs.get("physical_properties") + + @property + def _sync_mv_source_tables(self) -> t.Dict[str, exp.Table]: + registry = getattr(self._connection_pool, "_doris_sync_mv_source_tables", None) + if not isinstance(registry, dict): + registry = self._extra_config.get("_doris_sync_mv_source_tables") + if not isinstance(registry, dict): + registry = getattr(self, "_doris_sync_mv_source_tables", None) + if not isinstance(registry, dict): + registry = {} + self._doris_sync_mv_source_tables = registry + if isinstance(getattr(self, "_extra_config", None), dict): + self._extra_config["_doris_sync_mv_source_tables"] = registry + try: + setattr(self._connection_pool, "_doris_sync_mv_source_tables", registry) + except Exception: + pass + return registry + + @property + def catalog_support(self) -> CatalogSupport: + return CatalogSupport.FULL_SUPPORT + + @property + def default_catalog(self) -> t.Optional[str]: + return self._default_catalog or self._extra_config.get("catalog") + + def get_current_catalog(self) -> t.Optional[str]: + try: + catalog = super().get_current_catalog() + if catalog: + return _decoded_str(catalog) + except Exception: + logger.debug( + "current_catalog() failed on Doris, falling back to active/default catalog" + ) + return self._active_catalog or self.default_catalog or "internal" + + def set_current_catalog(self, catalog: str) -> None: + catalog_sql = self._to_sql(exp.to_identifier(catalog, quoted=True)) + self.execute(exp.Command(this="SWITCH", expression=catalog_sql)) + self._active_catalog = catalog + self._needs_catalog_restore = False + database = self._extra_config.get("database") + default_cat = self.default_catalog or "internal" + if database and catalog.lower() == str(default_cat).lower(): + db_sql = self._to_sql(exp.to_identifier(str(database), quoted=True)) + self.execute(f"USE {db_sql}") + + def _restore_active_catalog_on_reconnect(self, sql: t.Optional[str] = None) -> None: + active_cat = self._active_catalog or self.default_catalog + database = self._extra_config.get("database") + if not active_cat and not database: + self._needs_catalog_restore = False + return + if sql and re.match( + r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*SWITCH\b", + sql, + flags=re.IGNORECASE | re.DOTALL, + ): + self._needs_catalog_restore = False + return + if active_cat: + catalog_sql = self._to_sql(exp.to_identifier(active_cat, quoted=True)) + self.cursor.execute(f"SWITCH {catalog_sql}") + default_cat = self.default_catalog or "internal" + if database and (not active_cat or active_cat.lower() == str(default_cat).lower()): + db_sql = self._to_sql(exp.to_identifier(str(database), quoted=True)) + self.cursor.execute(f"USE {db_sql}") + self._needs_catalog_restore = False + + _READ_ONLY_EXTERNAL_FORMATS: t.ClassVar[t.Dict[str, str]] = { + "lance": "Lance", + "hudi": "Hudi", + "deltalake": "Delta Lake", + "delta": "Delta Lake", + "lakesoul": "LakeSoul", + } + + def _get_read_only_external_format( + self, + table_format: t.Optional[str] = None, + table_properties: t.Optional[t.Dict[str, t.Any]] = None, + ) -> t.Optional[str]: + if table_format: + fmt_lower = str(table_format).strip().lower() + if fmt_lower in self._READ_ONLY_EXTERNAL_FORMATS: + return fmt_lower + if table_properties: + engine_val = self._extract_str_value(table_properties.get("engine")) + if engine_val: + eng_lower = engine_val.strip().lower() + if eng_lower in self._READ_ONLY_EXTERNAL_FORMATS: + return eng_lower + return None + + def _is_lance_table( + self, + table_format: t.Optional[str] = None, + table_properties: t.Optional[t.Dict[str, t.Any]] = None, + ) -> bool: + return ( + self._get_read_only_external_format( + table_format=table_format, table_properties=table_properties + ) + is not None + ) + + def _ensure_not_lance_table( + self, + table_format: t.Optional[str] = None, + table_properties: t.Optional[t.Dict[str, t.Any]] = None, + ) -> None: + ro_fmt = self._get_read_only_external_format( + table_format=table_format, table_properties=table_properties + ) + if ro_fmt is not None: + fmt_label = self._READ_ONLY_EXTERNAL_FORMATS[ro_fmt] + raise SQLMeshError( + f"Doris {fmt_label} tables are read-only; reference {fmt_label} tables as upstream sources or EXTERNAL models " + f"using a Doris {fmt_label} catalog ('type' = '{ro_fmt}') or file table-valued function ('format' = '{ro_fmt}')." + ) + + def _is_iceberg_table( + self, + table_format: t.Optional[str] = None, + catalog_name: t.Optional[str] = None, + table_properties: t.Optional[t.Dict[str, t.Any]] = None, + ) -> bool: + if self._is_lance_table(table_format=table_format, table_properties=table_properties): + return False + if table_format and str(table_format).strip().lower() in ("iceberg", "paimon", "hive"): + return True + if table_properties: + engine_val = self._extract_str_value(table_properties.get("engine")) + if engine_val and engine_val.strip().lower() in ("iceberg", "paimon", "hive"): + return True + effective_catalog = catalog_name or self.default_catalog + if effective_catalog and str(effective_catalog).strip().lower() != "internal": + return True + return False + + @staticmethod + def _extract_str_value(val: t.Any) -> t.Optional[str]: + if val is None: + return None + if isinstance(val, exp.Paren): + return DorisEngineAdapter._extract_str_value(val.this) + if isinstance(val, (exp.Tuple, exp.Array)): + return ",".join( + s for e in val.expressions if (s := DorisEngineAdapter._extract_str_value(e)) + ) + if isinstance(val, exp.Boolean): + return "true" if val.this else "false" + if isinstance(val, bool): + return "true" if val else "false" + if isinstance(val, exp.Literal): + return str(val.this) + if isinstance(val, exp.Identifier): + return str(val.this) + if isinstance(val, (exp.Column, exp.Table)): + return ".".join(part.name for part in val.parts) + if isinstance(val, exp.Var): + return str(val.this) + if isinstance(val, str): + return val + if isinstance(val, exp.Expr): + return val.name or str(val) + return str(val) + + @staticmethod + def _split_cleaned_column_names(raw: str) -> t.List[str]: + s = raw.strip() + while len(s) >= 2 and ( + (s[0] == "(" and s[-1] == ")") + or (s[0] == "'" and s[-1] == "'") + or (s[0] == '"' and s[-1] == '"') + ): + s = s[1:-1].strip() + result: t.List[str] = [] + for part in s.split(","): + cleaned = part.strip().strip("'\"").strip("()").strip().strip("`'\"").strip() + if cleaned: + result.append(cleaned) + return result + + @classmethod + def _extract_column_names(cls, key_expr: t.Any) -> t.List[str]: + if key_expr is None: + return [] + if isinstance(key_expr, (exp.Tuple, exp.Array)): + names: t.List[str] = [] + for e in key_expr.expressions: + names.extend(cls._extract_column_names(e)) + return names + if isinstance(key_expr, exp.Paren): + return cls._extract_column_names(key_expr.this) + if isinstance(key_expr, (list, tuple)): + names = [] + for e in key_expr: + names.extend(cls._extract_column_names(e)) + return names + if isinstance(key_expr, exp.Column): + return cls._split_cleaned_column_names(key_expr.name) + if isinstance(key_expr, (exp.Identifier, exp.Literal, exp.Var)): + return cls._split_cleaned_column_names(str(key_expr.this)) + if isinstance(key_expr, exp.Expr): + return [] + if isinstance(key_expr, str): + return cls._split_cleaned_column_names(key_expr) + return cls._split_cleaned_column_names(str(key_expr)) + + @classmethod + def _is_valid_olap_key_type(cls, dtype: exp.DataType) -> bool: + type_name = ( + dtype.this.value.upper() + if isinstance(dtype.this, exp.DataType.Type) + else str(dtype.this).upper() + ) + if type_name in cls.INVALID_KEY_TYPES: + return False + if dtype.this == exp.DataType.Type.USERDEFINED: + kind = ( + str(dtype.args.get("kind") or dtype.sql(dialect=cls.DIALECT)) + .upper() + .split("(", 1)[0] + .strip() + .strip("`") + ) + if kind in cls.INVALID_KEY_TYPES: + return False + return True + + def _normalize_nested_value(self, expr: exp.Expr) -> exp.Expr: + return exp.cast(expr, exp.DataType.build("VARCHAR", dialect=self.dialect)) + + def normalize_value( + self, + expr: exp.Expr, + type: exp.DataType, + decimal_precision: int = 3, + timestamp_precision: int = 6, + ) -> exp.Expr: + udt_kind = ( + str(type.args.get("kind") or "").upper().split("(", 1)[0].strip().strip("`") + if type.this == exp.DataType.Type.USERDEFINED + else "" + ) + if type.is_type("int128") or udt_kind == "LARGEINT": + return exp.cast( + exp.cast(expr, exp.DataType.build("LARGEINT", udt=True)), + to=exp.DataType.build("VARCHAR"), + ) + if udt_kind == "BITMAP": + return exp.cast( + exp.func("BITMAP_TO_STRING", expr), + to=exp.DataType.build("VARCHAR"), + ) + if type.is_type("hllsketch") or udt_kind in ("HLL", "HLLSKETCH"): + return exp.cast( + exp.func("HLL_TO_BASE64", expr), + to=exp.DataType.build("VARCHAR"), + ) + return super().normalize_value( + expr, + type, + decimal_precision=decimal_precision, + timestamp_precision=timestamp_precision, + ) + + def _fetch_native_df( + self, query: t.Union[exp.Expr, str], quote_identifiers: bool = False + ) -> DF: + if isinstance(query, exp.Expr): + query = self._normalize_schema_tmp_identifiers(query) + elif isinstance(query, str) and ( + "_schema_tmp" in query + or any( + tok in query.upper() + for tok in ("IS TRUE", "IS FALSE", "IS NOT TRUE", "IS NOT FALSE") + ) + ): + try: + parsed_q = parse_one(query, read=self.dialect) + query = self._to_sql(parsed_q, quote=quote_identifiers) + except Exception: + pass + if self._pre_ping: + try: + self.ping() + except Exception: + logger.info("Connection to the database was lost. Reconnecting...") + self._connection_pool.close() + self._needs_catalog_restore = True + try: + self._connection_pool.get_cursor() + if getattr(self, "_needs_catalog_restore", False): + self._restore_active_catalog_on_reconnect() + return super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + except Exception as e: + err_code = e.args[0] if getattr(e, "args", None) else None + err_name = e.__class__.__name__ + is_disconnect = (err_code in (2006, 2013, 2055) and err_name == "OperationalError") or ( + err_code == 0 and err_name == "InterfaceError" + ) + is_timeout = ( + isinstance(getattr(e, "__context__", None), TimeoutError) + or "timed out" in str(e).lower() + ) + if ( + is_disconnect + and not is_timeout + and not self._connection_pool.is_transaction_active + and self._is_safe_to_retry_sql( + self._to_sql(query, quote=quote_identifiers) + if isinstance(query, exp.Expr) + else query + ) + ): + logger.warning( + "Doris connection lost (%s: %s) during fetchdf, reconnecting and retrying once...", + err_name, + err_code, + ) + self._connection_pool.close() + try: + self._connection_pool.get_cursor() + self._restore_active_catalog_on_reconnect() + return super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + except Exception: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + if is_disconnect and not self._connection_pool.is_transaction_active: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + + _SAFE_RETRY_SQL_PREFIX_RE = re.compile( + r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*(SELECT|WITH|SHOW|DESCRIBE|DESC|EXPLAIN|USE|SWITCH)\b", + re.IGNORECASE | re.DOTALL, + ) + + @classmethod + def _is_safe_to_retry_sql(cls, sql: str) -> bool: + """Returns True only for read-only or idempotent catalog/database switch statements that cannot duplicate writes.""" + match = cls._SAFE_RETRY_SQL_PREFIX_RE.match(sql) + if not match: + return False + if match.group(1).upper() == "WITH": + try: + parsed = parse_one(sql, read=cls.DIALECT) + except Exception: + return False + return isinstance(parsed, (exp.Select, exp.SetOperation)) + return True + + def ping(self) -> None: + self._connection_pool.get().ping(reconnect=False) + + @classmethod + def _shorten_schema_tmp_name(cls, name: str) -> str: + max_len = cls.MAX_IDENTIFIER_LENGTH or 64 + suffix = "_schema_tmp" + if len(name) > max_len and name.endswith(suffix): + digest = hashlib.md5(name.encode("utf-8")).hexdigest()[:8] + prefix_len = max_len - len(suffix) - 9 + return f"{name[:prefix_len]}_{digest}{suffix}" + return name + + @classmethod + def _normalize_schema_tmp_identifiers(cls, expression: exp.Expr) -> exp.Expr: + max_len = cls.MAX_IDENTIFIER_LENGTH or 64 + suffix = "_schema_tmp" + if not any( + len(ident.name) > max_len and ident.name.endswith(suffix) + for ident in expression.find_all(exp.Identifier) + ): + return expression + expression = expression.copy() + for ident in expression.find_all(exp.Identifier): + if len(ident.name) > max_len and ident.name.endswith(suffix): + ident.set("this", cls._shorten_schema_tmp_name(ident.name)) + return expression + + def execute( + self, + expressions: t.Union[str, exp.Expr, t.Sequence[exp.Expr]], + ignore_unsupported_errors: bool = False, + quote_identifiers: bool = True, + track_rows_processed: bool = False, + **kwargs: t.Any, + ) -> None: + if isinstance(expressions, str): + super().execute( + expressions, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + return + + processed_expressions: t.List[exp.Expr] = [] + for e in ensure_list(expressions): + if not isinstance(e, exp.Expr): + super().execute( + expressions, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + return + if isinstance(e, exp.Select) and e.args.get("locks"): + e = e.copy() + e.set("locks", None) + logger.warning( + "[Doris] Removed FOR UPDATE from SELECT statement: %s", + e.sql(dialect=self.dialect, identify=quote_identifiers), + ) + e = self._normalize_schema_tmp_identifiers(e) + processed_expressions.append(e) + + if any( + isinstance(e, exp.Alter) and isinstance(e.this, exp.Table) + for e in processed_expressions + ): + for e in processed_expressions: + if isinstance(e, exp.Alter) and isinstance(e.this, exp.Table): + for attempt in range(300): + try: + super().execute( + e, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + break + except Exception as err: + if ( + "state(schema_change) is not normal" in str(err).lower() + and attempt < 299 + ): + self._wait_for_schema_change(e.this, timeout=5.0) + time.sleep(0.1) + continue + raise + self._wait_for_schema_change(e.this) + else: + super().execute( + e, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + return + + super().execute( + processed_expressions, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + + def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any) -> None: + if self._pre_ping: + try: + self.ping() + except Exception: + logger.info("Connection to the database was lost. Reconnecting...") + self._connection_pool.close() + self._needs_catalog_restore = True + try: + if getattr(self, "_needs_catalog_restore", False): + self._restore_active_catalog_on_reconnect(sql) + super()._execute(sql, track_rows_processed=track_rows_processed, **kwargs) + except Exception as e: + err_code = e.args[0] if getattr(e, "args", None) else None + err_name = e.__class__.__name__ + is_disconnect = (err_code in (2006, 2013, 2055) and err_name == "OperationalError") or ( + err_code == 0 and err_name == "InterfaceError" + ) + is_timeout = ( + isinstance(getattr(e, "__context__", None), TimeoutError) + or "timed out" in str(e).lower() + ) + if ( + is_disconnect + and not is_timeout + and not self._connection_pool.is_transaction_active + and self._is_safe_to_retry_sql(sql) + ): + logger.warning( + "Doris connection lost (%s: %s) during read/session statement, reconnecting and retrying once...", + err_name, + err_code, + ) + self._connection_pool.close_cursor() + try: + conn = self._connection_pool.get() + try: + conn.ping(reconnect=True) + except Exception: + self._connection_pool.close() + self._restore_active_catalog_on_reconnect(sql) + super()._execute(sql, track_rows_processed=track_rows_processed, **kwargs) + except Exception: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + else: + if is_disconnect and not self._connection_pool.is_transaction_active: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + + def columns( + self, table_name: TableName, include_pseudo_columns: bool = False + ) -> t.Dict[str, exp.DataType]: + """Fetches column names and types for the target table, normalizing Doris type aliases.""" + cols, key_cols, has_explicit_keys = self._describe_columns_with_keys(table_name) + self._last_described_key_cols: t.Set[str] = key_cols + self._last_described_has_explicit_keys: bool = has_explicit_keys + return cols + + def _describe_columns_with_keys( + self, table_name: TableName + ) -> t.Tuple[t.Dict[str, exp.DataType], t.Set[str], bool]: + self.execute(exp.Describe(this=exp.to_table(table_name), kind="TABLE")) + describe_output = self.cursor.fetchall() + cols: t.Dict[str, exp.DataType] = {} + key_cols: t.Set[str] = set() + has_explicit_keys = False + for row in itertools.takewhile( + lambda r: not str(r[0]).startswith("#"), + describe_output, + ): + column_name = row[0] if len(row) > 0 else None + column_type = row[1] if len(row) > 1 else None + is_key = row[3] if len(row) > 3 else None + col_name_str = _decoded_str(column_name) if column_name else "" + raw_type = _decoded_str(column_type) if column_type else "" + if not col_name_str.strip() or not raw_type.strip(): + continue + if is_key is not None: + has_explicit_keys = True + if _decoded_str(is_key).strip().lower() in ( + "true", + "pri", + "uni", + "dup", + "agg", + "yes", + ): + key_cols.add(col_name_str.lower()) + is_agg_state = raw_type.strip().upper().startswith("AGG_STATE") + normalized_type = re.sub( + r"\bDECIMALV[23]\b(?!\s*`?\s*:)", "DECIMAL", raw_type, flags=re.IGNORECASE + ) + normalized_type = re.sub( + r"\bDATETIMEV2\b(?!\s*`?\s*:)", "DATETIME", normalized_type, flags=re.IGNORECASE + ) + normalized_type = re.sub( + r"\bDATEV2\b(?!\s*`?\s*:)", "DATE", normalized_type, flags=re.IGNORECASE + ) + if not is_agg_state: + normalized_type = re.sub( + r"\bLARGEINT(?:\(\d+\)|\b)(?!\s*`?\s*:)", + "INT128", + normalized_type, + flags=re.IGNORECASE, + ) + else: + normalized_type = re.sub( + r"\s+(?:NOT\s+)?NULL\b", + "", + normalized_type, + flags=re.IGNORECASE, + ) + normalized_type = re.sub( + r"\bTINYINT\(1\)", "BOOLEAN", normalized_type, flags=re.IGNORECASE + ) + normalized_type = re.sub( + r"\b(TINYINT|SMALLINT|INT|BIGINT)\(\d+\)", + r"\1", + normalized_type, + flags=re.IGNORECASE, + ) + dtype = exp.DataType.build(normalized_type, dialect=self.dialect, udt=True) + for col_def in dtype.find_all(exp.ColumnDef): + if isinstance(col_def.this, exp.Identifier) and col_def.this.quoted: + col_def.this.set("quoted", False) + constraints = [ + c + for c in (col_def.args.get("constraints") or []) + if not ( + isinstance(c, exp.NotNullColumnConstraint) + and bool(c.args.get("allow_null")) + ) + and not ( + isinstance(getattr(c, "kind", None), exp.NotNullColumnConstraint) + and bool(c.kind.args.get("allow_null")) + ) + ] + col_def.set("constraints", constraints or None) + cols[col_name_str] = dtype + if cols and not key_cols and not has_explicit_keys: + first_col = next(iter(cols)) + key_cols.add(first_col.lower()) + return cols, key_cols, has_explicit_keys + + def get_alter_operations( + self, + current_table_name: TableName, + target_table_name: TableName, + *, + ignore_destructive: bool = False, + ignore_additive: bool = False, + ) -> t.List[TableAlterOperation]: + from sqlmesh.core.schema_diff import ( + TableAlterAddColumnOperation, + TableAlterChangeColumnTypeOperation, + TableAlterDropColumnOperation, + TableAlterOperation, + ) + + current_table = exp.to_table(current_table_name) + is_iceberg = self._is_iceberg_table(catalog_name=current_table.catalog) + if is_iceberg: + iceberg_ops = super().get_alter_operations( + current_table_name, + target_table_name, + ignore_destructive=ignore_destructive, + ignore_additive=ignore_additive, + ) + return [ + op + for op in iceberg_ops + if not ( + isinstance(op, TableAlterChangeColumnTypeOperation) + and self._to_sql(op.current_type).upper() + == self._to_sql(op.column_type).upper() + ) + ] + + self._last_described_key_cols = set() + self._last_described_has_explicit_keys = False + current_cols = self.columns(current_table_name) + current_key_cols = set(getattr(self, "_last_described_key_cols", set())) + current_has_explicit_keys = bool(getattr(self, "_last_described_has_explicit_keys", False)) + if current_cols and not current_key_cols and not current_has_explicit_keys: + current_key_cols = {next(iter(current_cols)).lower()} + + self._last_described_key_cols = set() + self._last_described_has_explicit_keys = False + target_cols = self.columns(target_table_name) + target_key_cols = set(getattr(self, "_last_described_key_cols", set())) + target_has_explicit_keys = bool(getattr(self, "_last_described_has_explicit_keys", False)) + if target_cols and not target_key_cols and not target_has_explicit_keys: + target_key_cols = {next(iter(target_cols)).lower()} + + current_by_lower = {c.lower(): c_dtype for c, c_dtype in current_cols.items()} + for t_col, t_dtype in list(target_cols.items()): + curr_dtype = current_by_lower.get(t_col.lower()) + if ( + curr_dtype is not None + and curr_dtype != t_dtype + and self._to_sql(curr_dtype).upper() == self._to_sql(t_dtype).upper() + ): + target_cols[t_col] = curr_dtype.copy() + + key_role_changed_cols: t.Set[str] = set() + if current_has_explicit_keys and target_has_explicit_keys: + common_cols = {c.lower() for c in current_cols} & {c.lower() for c in target_cols} + key_role_changed_cols = (current_key_cols ^ target_key_cols) & common_cols + + invalid_target_key_cols = { + c.lower() + for c, dtype in target_cols.items() + if c.lower() in current_key_cols and not self._is_valid_olap_key_type(dtype) + } + operations = t.cast( + t.List[TableAlterOperation], + self.schema_differ.compare_columns( + current_table_name, + current_cols, + target_cols, + ignore_destructive=ignore_destructive, + ignore_additive=ignore_additive, + ), + ) + dropped_key_cols = { + op.column.name.lower() + for op in operations + if isinstance(op, TableAlterDropColumnOperation) + and op.column.name.lower() in current_key_cols + } + protected_key_cols = invalid_target_key_cols | dropped_key_cols | key_role_changed_cols + return [ + op + for op in operations + if not ( + ( + isinstance(op, TableAlterChangeColumnTypeOperation) + and ( + self._to_sql(op.current_type).upper() + == self._to_sql(op.column_type).upper() + or ( + op.column.name.lower() in current_key_cols + and not self._is_valid_olap_key_type(op.column_type) + ) + or op.column.name.lower() in key_role_changed_cols + ) + ) + or ( + isinstance(op, (TableAlterDropColumnOperation, TableAlterAddColumnOperation)) + and op.column.name.lower() in protected_key_cols + ) + ) + ] + + def _wait_for_schema_change(self, table_name: TableName, timeout: float = 30.0) -> None: + """Wait for any active asynchronous OLAP SCHEMA_CHANGE job on ``table_name`` to complete.""" + table = exp.to_table(table_name) + if not table.name or self._is_iceberg_table(catalog_name=table.catalog): + return + escaped_tbl = table.name.replace("'", "''") + from_clause = f" FROM `{table.db.replace('`', '``')}`" if table.db else "" + show_sql = f"SHOW ALTER TABLE COLUMN{from_clause} WHERE TableName = '{escaped_tbl}'" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + self.cursor.execute(show_sql) + rows = self.cursor.fetchall() + except Exception: + return + if not rows or not isinstance(rows, (list, tuple)): + return + if all( + len(row) <= 9 + or str(_decoded_str(row[9])).strip().upper() in ("FINISHED", "CANCELLED") + for row in rows + ): + return + time.sleep(0.05) + + def alter_table( + self, + alter_expressions: t.Union[t.List[exp.Alter], t.List[TableAlterOperation]], + table_format: t.Optional[str] = None, + ) -> None: + """Execute ALTER TABLE statements on Doris, waiting for asynchronous OLAP SCHEMA_CHANGE jobs.""" + from sqlmesh.core.schema_diff import TableAlterOperation + + with self.transaction(): + for alter_expression in [ + x.expression if isinstance(x, TableAlterOperation) else x for x in alter_expressions + ]: + self.execute(alter_expression) + + def create_schema( + self, + schema_name: SchemaName, + ignore_if_exists: bool = True, + warn_on_error: bool = True, + properties: t.Optional[t.List[exp.Expr]] = None, + ) -> None: + """Create a schema (database) in Doris.""" + properties = properties or [] + return super()._create_schema( + schema_name=schema_name, + ignore_if_exists=ignore_if_exists, + warn_on_error=warn_on_error, + properties=properties, + kind="DATABASE", + ) + + def drop_schema( + self, + schema_name: SchemaName, + ignore_if_not_exists: bool = True, + cascade: bool = False, + **drop_args: t.Dict[str, exp.Expr], + ) -> None: + """Drop schema in Doris. Note: Doris doesn't support CASCADE clause.""" + if cascade: + schema_exp = to_schema(schema_name) if schema_name else exp.Table() + effective_catalog = schema_exp.catalog or self.default_catalog + if effective_catalog and str(effective_catalog).strip().lower() != "internal": + try: + for obj in self._get_data_objects(schema_name): + if obj.type == DataObjectType.TABLE: + self.drop_table(obj.to_table(), exists=True) + except Exception: + logger.debug("Failed to enumerate tables before drop_schema on %s", schema_name) + return self._drop_object( + name=schema_name, + exists=ignore_if_not_exists, + kind="DATABASE", + cascade=False, + **drop_args, + ) + + def _get_data_objects( + self, schema_name: SchemaName, object_names: t.Optional[t.Set[str]] = None + ) -> t.List[DataObject]: + """Returns all the data objects that exist in the given schema and optional catalog.""" + schema_exp = to_schema(schema_name) if schema_name else exp.Table() + catalog = schema_exp.catalog or None + query_catalog = catalog or self.default_catalog + + query = ( + exp.select( + exp.column("table_schema").as_("schema_name"), + exp.column("table_name").as_("name"), + exp.case() + .when( + exp.column("table_type").eq("BASE TABLE"), + exp.Literal.string("table"), + ) + .when( + exp.column("table_type").eq("SYSTEM VIEW"), + exp.Literal.string("view"), + ) + .when( + exp.column("table_type").eq("VIEW"), + exp.Literal.string("view"), + ) + .when( + exp.column("table_type").eq("MATERIALIZED VIEW"), + exp.Literal.string("materialized_view"), + ) + .else_(exp.column("table_type")) + .as_("type"), + ) + .from_(exp.table_("tables", db="information_schema", catalog=query_catalog)) + .where( + exp.column("table_schema").eq( + schema_exp.db if schema_exp.db else exp.func("DATABASE") + ) + ) + ) + short_to_orig: t.Dict[str, str] = {} + if object_names: + lowered_names: t.List[str] = [] + for name in object_names: + short_name = self._shorten_schema_tmp_name(name) + if short_name != name: + short_to_orig[short_name.lower()] = name + lowered_names.append(short_name.lower()) + query = query.where(exp.func("LOWER", exp.column("table_name")).isin(*lowered_names)) + + result: t.List[DataObject] = [] + for row_schema, table_name, table_type in self.fetchall(query): + try: + schema_str = _decoded_str(row_schema) if row_schema is not None else schema_exp.db + name_str = _decoded_str(table_name) if table_name is not None else "unknown" + if name_str.lower() in short_to_orig: + name_str = short_to_orig[name_str.lower()] + obj_type = _decoded_str(table_type) if table_type is not None else "table" + + upper_type = obj_type.upper() + if upper_type in ("BASE TABLE", "EXTERNAL TABLE"): + obj_type = "table" + elif upper_type in ("VIEW", "SYSTEM VIEW"): + obj_type = "view" + elif upper_type in ("MATERIALIZED VIEW", "MATERIALIZED_VIEW"): + obj_type = "materialized_view" + + result.append( + DataObject( + catalog=catalog, + schema=schema_str, + name=name_str, + type=DataObjectType.from_str(obj_type), + ) + ) + except (ValueError, AttributeError) as e: + logger.error( + f"Error processing row: {e}, row: {(row_schema, table_name, table_type)}" + ) + continue + + # In Doris 2.1+, asynchronous materialized views (MTMVs) are stored as OLAP base tables + # and report table_type = 'BASE TABLE' in information_schema.tables. Reclassify them via + # the mv_infos() table-valued function so drop_data_object_on_type_mismatch and metadata + # inspection recognize them as MATERIALIZED_VIEW instead of issuing DROP TABLE. + if (not query_catalog or query_catalog.lower() == "internal") and any( + obj.type.is_table for obj in result + ): + try: + resolved_db = ( + schema_exp.db + or next((obj.schema_name for obj in result if obj.schema_name), "") + or self._extra_config.get("database") + ) + db_arg = ( + self._to_sql(exp.Literal.string(str(resolved_db)), quote=False) + if resolved_db + else "DATABASE()" + ) + mv_rows = self.fetchall( + f"SELECT Name FROM mv_infos('database' = {db_arg})", + quote_identifiers=False, + ) + mv_names = { + _decoded_str(row[0]).lower() for row in mv_rows if len(row) == 1 and row[0] + } + if mv_names: + for obj in result: + if obj.name.lower() in mv_names: + obj.type = DataObjectType.MATERIALIZED_VIEW + except Exception: + logger.debug("mv_infos lookup failed on Doris, skipping MTMV reclassification") + + return result + + def create_view( + self, + view_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + replace: bool = True, + materialized: bool = False, + materialized_properties: t.Optional[t.Dict[str, t.Any]] = None, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + view_properties: t.Optional[t.Dict[str, exp.Expr]] = None, + source_columns: t.Optional[t.List[str]] = None, + **create_kwargs: t.Any, + ) -> None: + """ + Doris view creation behavior: + - Regular VIEW: supports atomic `CREATE OR REPLACE VIEW` via `super().create_view`. + - MATERIALIZED VIEW: does NOT support `CREATE OR REPLACE MATERIALIZED VIEW`, so `replace=True` + drops any existing object/materialized view first (after validating the new query and properties). + """ + target_table = exp.to_table(view_name) + effective_catalog = target_table.catalog or self.default_catalog + if effective_catalog and effective_catalog.lower() != "internal": + view_kind = "materialized views" if materialized else "views" + raise SQLMeshError( + f"Doris only supports creating {view_kind} in the 'internal' catalog, got '{effective_catalog}' " + f"for view '{self._to_sql(target_table)}'. Configure `virtual_environment_mode: dev_only` and " + f"`environment_catalog_mapping` in your SQLMesh configuration to route virtual environment views " + f"to the 'internal' catalog." + ) + + if not materialized: + return super().create_view( + view_name, + query_or_df, + target_columns_to_types, + replace=replace, + materialized=False, + materialized_properties=materialized_properties, + table_description=table_description, + column_descriptions=column_descriptions, + view_properties=view_properties, + source_columns=source_columns, + **create_kwargs, + ) + + self._create_materialized_view( + view_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + replace=replace, + materialized_properties=materialized_properties, + table_description=table_description, + column_descriptions=column_descriptions, + view_properties=view_properties, + source_columns=source_columns, + **create_kwargs, + ) + + def _create_materialized_view( + self, + view_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + replace: bool = True, + materialized_properties: t.Optional[t.Dict[str, t.Any]] = None, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + view_properties: t.Optional[t.Dict[str, exp.Expr]] = None, + source_columns: t.Optional[t.List[str]] = None, + **create_kwargs: t.Any, + ) -> None: + import pandas as pd + + query_or_df = self._native_df_to_pandas_df(query_or_df) + + # Inline DataFrames as a SELECT ... FROM VALUES query rather than delegating DataFrame + # handling to _get_source_queries_and_columns_to_types, which would otherwise materialize + # a temporary table that gets dropped as soon as the SourceQuery context manager exits. + if isinstance(query_or_df, pd.DataFrame): + target_columns_to_types, source_columns = self._columns_to_types( + query_or_df, target_columns_to_types, source_columns + ) + if not target_columns_to_types: + raise SQLMeshError("columns_to_types must be provided for dataframes") + source_columns_to_types = get_source_columns_to_types( + target_columns_to_types, source_columns + ) + ordered_df = query_or_df[list(source_columns_to_types)] + values: t.List[t.Tuple[t.Any, ...]] = list( + ordered_df.itertuples(index=False, name=None) + ) + query_or_df = self._values_to_sql( + values, + source_columns_to_types, + batch_start=0, + batch_end=len(values), + ) + + is_sync_mv = bool( + view_properties + and ( + (self._extract_str_value(view_properties.get("materialized_type")) or "").upper() + == "SYNC" + ) + ) + + source_queries, target_columns_to_types = self._get_source_queries_and_columns_to_types( + query_or_df, + None if is_sync_mv else target_columns_to_types, + batch_size=0, + target_table=view_name, + source_columns=source_columns, + ) + if len(source_queries) != 1: + raise SQLMeshError("Only one source query is supported for creating materialized views") + + with source_queries[0] as query: + target_table = exp.to_table(view_name) + + schema_cols_to_types = target_columns_to_types + if ( + not schema_cols_to_types + and column_descriptions + and not is_sync_mv + and isinstance(query, exp.Query) + and query.named_selects + and "*" not in query.named_selects + ): + schema_cols_to_types = { + col: exp.DataType.build("UNKNOWN") for col in query.named_selects + } + + schema: t.Union[exp.Table, exp.Schema] = target_table + if schema_cols_to_types and not is_sync_mv: + schema = self._build_schema_exp( + target_table, + schema_cols_to_types, + column_descriptions, + is_view=True, + ) + + partitioned_by = None + if not is_sync_mv: + if materialized_properties and materialized_properties.get("partitioned_by"): + partitioned_by = materialized_properties.get("partitioned_by") + elif create_kwargs.get("partitioned_by"): + partitioned_by = create_kwargs.get("partitioned_by") + + clustered_by = None + if not is_sync_mv: + if materialized_properties and materialized_properties.get("clustered_by"): + clustered_by = materialized_properties.get("clustered_by") + elif create_kwargs.get("clustered_by"): + clustered_by = create_kwargs.get("clustered_by") + + properties_exp = self._build_table_properties_exp( + catalog_name=target_table.catalog, + table_properties=view_properties, + target_columns_to_types=target_columns_to_types, + table_description=( + table_description + if not is_sync_mv + and self.COMMENT_CREATION_VIEW.supports_schema_def + and self.comments_enabled + else None + ), + partitioned_by=partitioned_by, + clustered_by=clustered_by, + partition_interval_unit=( + materialized_properties.get("partition_interval_unit") + if materialized_properties and not is_sync_mv + else None + ), + table_kind="MATERIALIZED_VIEW", + ) + + if is_sync_mv: + query = query.copy() + for tbl_node in query.find_all(exp.Table): + if tbl_node.catalog and tbl_node.catalog.lower() == "internal": + tbl_node.set("catalog", None) + if isinstance(query, exp.Select): + base_col_names = { + col_node.name.lower() + for col_node in query.find_all(exp.Column) + if col_node.name + } + aliased_selects: t.List[exp.Expr] = [] + for sel_expr in query.expressions: + if isinstance(sel_expr, exp.Column): + aliased_selects.append( + exp.alias_(sel_expr.copy(), f"mv_{sel_expr.name}", quoted=True) + ) + elif isinstance(sel_expr, exp.Alias) and ( + ( + isinstance(sel_expr.this, exp.Column) + and sel_expr.alias == sel_expr.this.name + ) + or ( + sel_expr.alias + and sel_expr.alias.lower() in base_col_names + and not sel_expr.alias.lower().startswith("mv_") + ) + ): + aliased_selects.append( + exp.alias_( + sel_expr.this.copy(), f"mv_{sel_expr.alias}", quoted=True + ) + ) + else: + aliased_selects.append(sel_expr) + query.set("expressions", aliased_selects) + + sync_source_tbl: t.Optional[exp.Table] = None + if is_sync_mv: + if view_properties and view_properties.get("source_table"): + sync_source_tbl = self._to_source_table(view_properties["source_table"]) + else: + cte_names = {cte.alias.lower() for cte in query.find_all(exp.CTE) if cte.alias} + first_tbl = next( + ( + t_node + for t_node in query.find_all(exp.Table) + if t_node.name and t_node.name.lower() not in cte_names + ), + None, + ) + if first_tbl: + sync_source_tbl = self._to_source_table(first_tbl) + + # Doris does not support `CREATE OR REPLACE MATERIALIZED VIEW`, so drop first when `replace=True` + # after query and table property validation has succeeded. + if replace: + existing_obj = self.get_data_object(view_name) + self.drop_data_object_on_type_mismatch( + existing_obj, DataObjectType.MATERIALIZED_VIEW + ) + drop_view_props = dict(view_properties) if view_properties else {} + if ( + is_sync_mv + and not drop_view_props.get("source_table") + and sync_source_tbl is not None + ): + drop_view_props["source_table"] = sync_source_tbl + elif ( + not is_sync_mv + and not drop_view_props.get("materialized_type") + and existing_obj is not None + and existing_obj.type == DataObjectType.MATERIALIZED_VIEW + ): + drop_view_props["materialized_type"] = exp.Literal.string("ASYNC") + self.drop_view( + view_name, + ignore_if_not_exists=True, + materialized=True, + view_properties=drop_view_props or None, + _skip_full_desc_scan=not is_sync_mv, + ) + + # Doris synchronous materialized views are single-table rollup indexes whose DDL syntax + # (`CREATE MATERIALIZED VIEW AS SELECT ... FROM .`) forbids a + # `.` qualifier on `` and requires an active database on the session. + sync_db_switched: t.Optional[str] = None + if is_sync_mv: + sync_db: t.Optional[str] = target_table.db or ( + sync_source_tbl.db if sync_source_tbl else None + ) + if sync_db: + db_sql = self._to_sql(exp.to_identifier(sync_db, quoted=True)) + self.execute(f"USE {db_sql}") + sync_db_switched = sync_db + unqualified_mv = target_table.copy() + unqualified_mv.set("db", None) + unqualified_mv.set("catalog", None) + schema = unqualified_mv + + try: + self.execute( + exp.Create( + this=schema, + kind="VIEW", + replace=False, + exists=( + bool(create_kwargs.get("exists", False)) if not is_sync_mv else False + ), + expression=query, + properties=properties_exp, + ), + quote_identifiers=self.QUOTE_IDENTIFIERS_IN_VIEWS, + ) + finally: + configured_db = self._extra_config.get("database") + if ( + sync_db_switched + and configured_db + and str(configured_db).lower() != sync_db_switched.lower() + ): + restore_db_sql = self._to_sql( + exp.to_identifier(str(configured_db), quoted=True) + ) + self.execute(f"USE {restore_db_sql}") + registry = self._sync_mv_source_tables + if is_sync_mv and sync_source_tbl is not None: + if target_table.db: + registry[f"{target_table.db.lower()}.{target_table.name.lower()}"] = ( + sync_source_tbl.copy() + ) + else: + registry[target_table.name.lower()] = sync_source_tbl.copy() + if sync_source_tbl.db: + registry[f"{sync_source_tbl.db.lower()}.{target_table.name.lower()}"] = ( + sync_source_tbl.copy() + ) + elif not is_sync_mv: + self._pop_sync_mv_source_table(target_table) + raw_build = ( + view_properties.get("build") + if view_properties and view_properties.get("build") is not None + else (view_properties.get("build_mode") if view_properties else None) + ) + build_mode = self._to_upper_str(raw_build) if raw_build is not None else None + if ( + not is_sync_mv + and build_mode == "DEFERRED" + and bool((materialized_properties or {}).get("has_audits")) + ): + self.execute(f"REFRESH MATERIALIZED VIEW {self._to_sql(target_table)} AUTO") + self._clear_data_object_cache(view_name) + + def _pop_sync_mv_source_table(self, target_table: exp.Table) -> t.Optional[exp.Table]: + registry = self._sync_mv_source_tables + name_lower = target_table.name.lower() + if target_table.db: + db_lower = target_table.db.lower() + popped = registry.pop(f"{db_lower}.{name_lower}", None) + unqual = registry.get(name_lower) + if unqual is not None and (not unqual.db or unqual.db.lower() == db_lower): + registry.pop(name_lower, None) + return popped + popped = registry.pop(name_lower, None) + if popped is not None and popped.db: + registry.pop(f"{popped.db.lower()}.{name_lower}", None) + return popped + + def _to_source_table(self, raw_src: t.Any) -> exp.Table: + if isinstance(raw_src, exp.Table): + tbl = raw_src.copy() + elif isinstance(raw_src, exp.Column): + tbl = exp.table_( + raw_src.name, + db=raw_src.table or None, + catalog=raw_src.db or None, + ) + else: + src_str = self._extract_str_value(raw_src) or str(raw_src) + tbl = exp.to_table(src_str, dialect=self.dialect) + if tbl.catalog and tbl.catalog.lower() == "internal": + tbl.set("catalog", None) + return tbl + + def _resolve_sync_mv_source_from_metadata( + self, + target_table: exp.Table, + is_explicit_sync: bool = False, + skip_full_desc_scan: bool = False, + ) -> t.Tuple[t.Optional[exp.Table], bool, bool]: + db_name: t.Optional[str] = target_table.db or ( + str(self._extra_config["database"]) if self._extra_config.get("database") else None + ) + if not db_name: + try: + db_row = self.fetchone("SELECT DATABASE()") + if ( + isinstance(db_row, (list, tuple)) + and db_row + and isinstance(db_row[0], str) + and db_row[0].strip() + ): + db_name = db_row[0].strip() + except Exception: + db_name = None + if not db_name: + return None, bool(is_explicit_sync), False + + db_sql = self._to_sql(exp.to_identifier(db_name, quoted=True)) + mv_name_lower = target_table.name.lower() + alter_query_failed = False + try: + alter_rows = self.fetchall(f"SHOW ALTER TABLE MATERIALIZED VIEW FROM {db_sql}") + except Exception as exc: + logger.debug( + "Failed to query SHOW ALTER TABLE MATERIALIZED VIEW FROM %s: %s", db_name, exc + ) + alter_rows = None + alter_query_failed = True + + checked_base_tables: t.Set[str] = set() + if isinstance(alter_rows, (list, tuple)): + for row in reversed(list(alter_rows)): + if ( + isinstance(row, (list, tuple)) + and len(row) > 5 + and row[5] is not None + and str(row[5]).strip().lower() == mv_name_lower + ): + state = ( + str(row[8]).strip().upper() if len(row) > 8 and row[8] is not None else "" + ) + if state == "CANCELLED": + continue + base_tbl_name = ( + str(row[1]).strip() if len(row) > 1 and row[1] is not None else "" + ) + if not base_tbl_name: + return None, True, False + if state != "FINISHED": + return exp.table_(base_tbl_name, db=db_name), False, False + if base_tbl_name.lower() in checked_base_tables: + continue + checked_base_tables.add(base_tbl_name.lower()) + base_tbl_sql = self._to_sql(exp.table_(base_tbl_name, db=db_name, quoted=True)) + try: + desc_rows = self.fetchall(f"DESC {base_tbl_sql} ALL") + except Exception as exc: + logger.debug("Failed to query DESC %s ALL: %s", base_tbl_sql, exc) + continue + if not isinstance(desc_rows, (list, tuple)): + continue + if any( + isinstance(d_row, (list, tuple)) + and len(d_row) >= 1 + and d_row[0] is not None + and str(d_row[0]).strip().lower() == mv_name_lower + for d_row in desc_rows + ): + return exp.table_(base_tbl_name, db=db_name), False, False + + if skip_full_desc_scan and not is_explicit_sync and not alter_query_failed: + return None, False, False + + try: + table_rows = self.fetchall(f"SHOW FULL TABLES FROM {db_sql}") + except Exception as exc: + logger.debug("Failed to query SHOW FULL TABLES FROM %s: %s", db_name, exc) + return None, bool(is_explicit_sync or alter_query_failed), False + + if not isinstance(table_rows, (list, tuple)): + return None, bool(is_explicit_sync or alter_query_failed), False + + candidate_tables: t.List[str] = [] + for t_row in table_rows: + if isinstance(t_row, (list, tuple)) and len(t_row) >= 1 and t_row[0] is not None: + t_name = str(t_row[0]).strip() + t_type = ( + str(t_row[1]).strip().upper() + if len(t_row) > 1 and t_row[1] is not None + else "BASE TABLE" + ) + if t_name.lower() == mv_name_lower and not is_explicit_sync: + return None, False, False + if ( + t_type in ("BASE TABLE", "TABLE", "") + and t_name.lower() != mv_name_lower + and t_name.lower() not in checked_base_tables + ): + candidate_tables.append(t_name) + + desc_failed = False + for t_name in candidate_tables: + tbl_sql = self._to_sql(exp.table_(t_name, db=db_name, quoted=True)) + try: + desc_rows = self.fetchall(f"DESC {tbl_sql} ALL") + except Exception as exc: + logger.debug("Failed to query DESC %s ALL: %s", tbl_sql, exc) + desc_failed = True + continue + if not isinstance(desc_rows, (list, tuple)): + desc_failed = True + continue + for d_row in desc_rows: + if ( + isinstance(d_row, (list, tuple)) + and len(d_row) >= 1 + and d_row[0] is not None + and str(d_row[0]).strip().lower() == mv_name_lower + ): + return exp.table_(t_name, db=db_name), False, False + + if desc_failed: + return None, True, False + return None, False, True + + def drop_view( + self, + view_name: TableName, + ignore_if_not_exists: bool = True, + materialized: bool = False, + **kwargs: t.Any, + ) -> None: + """Drop view in Doris.""" + target_table = exp.to_table(view_name) + effective_catalog = target_table.catalog or self.default_catalog + if effective_catalog and effective_catalog.lower() != "internal": + return + + view_properties = kwargs.pop("view_properties", None) + skip_full_desc_scan = bool(kwargs.pop("_skip_full_desc_scan", False)) + sync_source_table: t.Any = None + is_explicit_sync = False + is_explicit_async = False + is_unresolved_sync = False + confirmed_nonexistent = False + if materialized and view_properties: + mat_type = self._extract_str_value(view_properties.get("materialized_type")) + if mat_type and mat_type.upper() == "SYNC": + is_explicit_sync = True + sync_source_table = view_properties.get("source_table") + elif mat_type and mat_type.upper() != "SYNC": + is_explicit_async = True + if materialized and sync_source_table is None and not is_explicit_async: + registry = self._sync_mv_source_tables + if target_table.db: + sync_source_table = registry.get( + f"{target_table.db.lower()}.{target_table.name.lower()}" + ) + else: + sync_source_table = registry.get(target_table.name.lower()) + if materialized and sync_source_table is None and not is_explicit_async: + sync_source_table, is_unresolved_sync, confirmed_nonexistent = ( + self._resolve_sync_mv_source_from_metadata( + target_table, + is_explicit_sync=is_explicit_sync, + skip_full_desc_scan=skip_full_desc_scan, + ) + ) + + if sync_source_table is not None: + source_tbl_exp = self._to_source_table(sync_source_table) + # Synchronous materialized views require the target/source table's database + # to be active on the session and an unqualified `` in `DROP MATERIALIZED VIEW ON `. + sync_db = target_table.db or source_tbl_exp.db + if sync_db: + db_sql = self._to_sql(exp.to_identifier(sync_db, quoted=True)) + self.execute(f"USE {db_sql}") + unqualified_mv = target_table.copy() + unqualified_mv.set("db", None) + unqualified_mv.set("catalog", None) + view_table_sql = self._to_sql(unqualified_mv) + source_table_sql = self._to_sql(source_tbl_exp) + exists_clause = "IF EXISTS " if ignore_if_not_exists else "" + try: + self.execute( + f"DROP MATERIALIZED VIEW {exists_clause}{view_table_sql} ON {source_table_sql}" + ) + finally: + configured_db = self._extra_config.get("database") + if sync_db and configured_db and str(configured_db).lower() != sync_db.lower(): + restore_db_sql = self._to_sql( + exp.to_identifier(str(configured_db), quoted=True) + ) + self.execute(f"USE {restore_db_sql}") + self._pop_sync_mv_source_table(target_table) + self._clear_data_object_cache(view_name) + return + if is_explicit_sync or is_unresolved_sync: + raise SQLMeshError( + f"Cannot drop Doris synchronous materialized view '{view_name}' because its source base table could not be resolved." + ) + super().drop_view(view_name, ignore_if_not_exists, materialized, **kwargs) + + def create_index( + self, + table_name: TableName, + index_name: str, + columns: t.Sequence[str | exp.Expr] | t.Tuple[str, ...], + exists: bool = True, + index_type: t.Optional[str] = None, + index_properties: t.Optional[t.Dict[str, t.Any]] = None, + comment: t.Optional[str] = None, + ) -> None: + if len(columns) != 1: + logger.warning( + f"[Doris] Skipping CREATE INDEX {index_name} on {table_name}: " + f"Doris only supports single-column indexes, got columns {columns}." + ) + return + if index_type or index_properties or comment: + if not self.SUPPORTS_INDEXES: + return + using_var = exp.var(index_type.strip().upper()) if index_type else None + with_storage = ( + [ + exp.Property( + this=exp.Literal.string(str(k)), + value=exp.Literal.string(str(self._extract_str_value(v) or v)), + ) + for k, v in index_properties.items() + ] + if index_properties + else None + ) + comment_node = ( + exp.CommentColumnConstraint( + this=exp.Literal.string(self._truncate_column_comment(comment.strip())) + ) + if comment + else None + ) + self.execute( + exp.Create( + this=exp.Index( + this=exp.to_identifier(index_name), + table=exp.to_table(table_name), + params=exp.IndexParameters( + columns=[ + exp.to_column(c.name if isinstance(c, exp.Expr) else str(c)) + for c in columns + ], + using=using_var, + with_storage=with_storage, + where=comment_node, + ), + ), + kind="INDEX", + exists=exists, + ) + ) + self._wait_for_schema_change(table_name) + return + col_names = tuple(c.name if isinstance(c, exp.Expr) else str(c) for c in columns) + super().create_index(table_name, index_name, col_names, exists=exists) + self._wait_for_schema_change(table_name) + + def _create_table_like( + self, + target_table_name: TableName, + source_table_name: TableName, + exists: bool = True, + **kwargs: t.Any, + ) -> None: + target_table = exp.to_table(target_table_name) + source_table = exp.to_table(source_table_name) + if self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=target_table.catalog or source_table.catalog, + table_properties=self._get_table_properties(kwargs), + ): + self.create_table( + target_table_name, + self.columns(source_table_name), + exists=exists, + **kwargs, + ) + return + self.execute( + exp.Create( + this=target_table, + kind="TABLE", + exists=exists, + properties=exp.Properties( + expressions=[ + exp.LikeProperty( + this=source_table, + ), + ], + ), + ) + ) + + def _build_create_comment_table_exp( + self, table: exp.Table, table_comment: str, table_kind: str = "TABLE" + ) -> exp.Comment | str: + table_sql = self._to_sql(table) + comment_sql = self._to_sql( + exp.Literal.string(self._truncate_table_comment(table_comment.strip())) + ) + return f"ALTER TABLE {table_sql} MODIFY COMMENT {comment_sql}" + + def _build_create_comment_column_exp( + self, table: exp.Table, column_name: str, column_comment: str, table_kind: str = "TABLE" + ) -> exp.Comment | str: + table_sql = self._to_sql(table) + col_sql = self._to_sql(exp.to_identifier(column_name, quoted=True)) + comment_sql = self._to_sql( + exp.Literal.string(self._truncate_column_comment(column_comment.strip())) + ) + return f"ALTER TABLE {table_sql} MODIFY COLUMN {col_sql} COMMENT {comment_sql}" + + def _truncate_table( + self, + table_name: TableName, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + ) -> None: + table = t.cast(exp.Table, self._normalize_schema_tmp_identifiers(exp.to_table(table_name))) + state = self._get_thread_state() + caught_err: t.Optional[Exception] = None + if not self._is_iceberg_table(catalog_name=table.catalog) and not getattr( + state, "truncate_is_iceberg_hint", False + ): + try: + super()._truncate_table(table) + return + except Exception as e: + err_msg = str(e).lower() + if "truncate" not in err_msg and "not support" not in err_msg: + raise e + caught_err = e + + # When TRUNCATE TABLE is unsupported (e.g., on Iceberg tables), clear all rows via DELETE. + # Doris's DeleteHandler rejects DELETE without WHERE, constant WHERE TRUE, OR predicates, + # and floating-point/complex column predicates, so we pick a key-compatible column and + # execute two complementary `IS NOT NULL` and `IS NULL` deletes. + cols_to_types = ( + target_columns_to_types + or getattr(state, "truncate_columns_hint", None) + or self.columns(table) + ) + if cols_to_types: + valid_col_name = next( + ( + col + for col, dtype in cols_to_types.items() + if self._is_valid_olap_key_type(dtype) + ), + next(iter(cols_to_types)), + ) + target_col = exp.column(valid_col_name, quoted=True) + self.execute(exp.delete(table, where=target_col.is_(exp.null()).not_())) + self.execute(exp.delete(table, where=target_col.is_(exp.null()))) + return + if caught_err is not None: + raise caught_err + + def insert_append( + self, + table_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + track_rows_processed: bool = True, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + import pandas as pd + + query_or_df = self._native_df_to_pandas_df(query_or_df) + target_table = exp.to_table(table_name) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=target_table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + if isinstance(query_or_df, pd.DataFrame) and ( + not target_columns_to_types or not columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types, source_columns = self._columns_to_types( + query_or_df, target_columns_to_types, source_columns + ) + elif target_columns_to_types is not None and not columns_to_types_all_known( + target_columns_to_types + ): + target_columns_to_types = self.columns(table_name) + + if ( + not is_iceberg + and target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + + super().insert_append( + table_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + track_rows_processed=track_rows_processed, + source_columns=source_columns, + ) + + def _insert_overwrite_by_condition( + self, + table_name: TableName, + source_queries: t.List[SourceQuery], + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + where: t.Optional[exp.Condition] = None, + insert_overwrite_strategy_override: t.Optional[InsertOverwriteStrategy] = None, + **kwargs: t.Any, + ) -> None: + table_props = self._get_table_properties(kwargs) + self._ensure_not_lance_table( + table_format=kwargs.get("table_format"), + table_properties=table_props, + ) + table_format_str = str(kwargs.get("table_format") or "").strip().lower() + engine_str = ( + (self._extract_str_value(table_props.get("engine")) or "").strip().lower() + if table_props + else "" + ) + is_hive = table_format_str == "hive" or engine_str == "hive" + if insert_overwrite_strategy_override is None and where is None and is_hive: + insert_overwrite_strategy_override = InsertOverwriteStrategy.INSERT_OVERWRITE + table = exp.to_table(table_name) + state = self._get_thread_state() + prev_hint = getattr(state, "truncate_columns_hint", None) + prev_iceberg_hint = getattr(state, "truncate_is_iceberg_hint", False) + state.truncate_columns_hint = target_columns_to_types + state.truncate_is_iceberg_hint = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=table.catalog, + table_properties=table_props, + ) + try: + if not source_queries: + self.delete_from(table_name, where=where or exp.true()) + return + return super()._insert_overwrite_by_condition( + table_name, + source_queries, + target_columns_to_types=target_columns_to_types, + where=where, + insert_overwrite_strategy_override=insert_overwrite_strategy_override, + **kwargs, + ) + finally: + state.truncate_columns_hint = prev_hint + state.truncate_is_iceberg_hint = prev_iceberg_hint + + _TIME_UNIT_PRECEDENCE: t.ClassVar[t.Dict[str, int]] = { + "second": 0, + "minute": 1, + "hour": 2, + "day": 3, + "week": 4, + "month": 5, + "quarter": 6, + "year": 7, + } + + @classmethod + def _extract_time_partition_transform( + cls, expr: exp.Expr + ) -> t.Optional[t.Tuple[exp.Column, str]]: + """Extract `(column, unit)` from time-truncation partition expressions (`DATE_TRUNC`, + `TO_DATE`, `DATE`, `YEAR`/`YEARS`, `QUARTER`/`QUARTERS`, `MONTH`/`MONTHS`, `WEEK`/`WEEKS`, + `DAY`/`DAYS`, `HOUR`/`HOURS`, `MINUTE`/`MINUTES`, `SECOND`/`SECONDS`). + """ + curr = expr.unnest() + col_node: t.Optional[exp.Column] = None + raw_unit: t.Optional[str] = None + + if isinstance(curr, (exp.TimestampTrunc, exp.DateTrunc)): + unit_arg = curr.args.get("unit") + if isinstance(curr.this, exp.Column): + col_node = curr.this + raw_unit = cls._extract_str_value(unit_arg) + elif isinstance(unit_arg, exp.Column): + col_node = unit_arg + raw_unit = cls._extract_str_value(curr.this) + elif isinstance(curr, (exp.TsOrDsToDate, exp.Date)): + if isinstance(curr.this, exp.Column): + col_node = curr.this + raw_unit = "day" + elif isinstance( + curr, + ( + exp.Year, + exp.Quarter, + exp.Month, + exp.Week, + exp.WeekOfYear, + exp.Day, + exp.Hour, + exp.Minute, + exp.Second, + ), + ): + inner_arg = curr.this.unnest() if isinstance(curr.this, exp.Expr) else None + if isinstance(inner_arg, (exp.TsOrDsToDate, exp.Date)) and isinstance( + inner_arg.this, exp.Expr + ): + inner_arg = inner_arg.this.unnest() + if isinstance(inner_arg, exp.Column): + col_node = inner_arg + if isinstance(curr, exp.Year): + raw_unit = "year" + elif isinstance(curr, exp.Quarter): + raw_unit = "quarter" + elif isinstance(curr, exp.Month): + raw_unit = "month" + elif isinstance(curr, (exp.Week, exp.WeekOfYear)): + raw_unit = "week" + elif isinstance(curr, exp.Day): + raw_unit = "day" + elif isinstance(curr, exp.Hour): + raw_unit = "hour" + elif isinstance(curr, exp.Minute): + raw_unit = "minute" + else: + raw_unit = "second" + elif isinstance(curr, exp.Extract): + extract_expr = ( + curr.expression.unnest() if isinstance(curr.expression, exp.Expr) else None + ) + if isinstance(extract_expr, exp.Column): + col_node = extract_expr + raw_unit = cls._extract_str_value(curr.this) + elif isinstance(curr, exp.Anonymous): + fn_name = curr.name.upper() + if fn_name == "DATE_TRUNC" and len(curr.expressions) == 2: + arg0, arg1 = curr.expressions + if isinstance(arg0, exp.Column): + col_node = arg0 + raw_unit = cls._extract_str_value(arg1) + elif isinstance(arg1, exp.Column): + col_node = arg1 + raw_unit = cls._extract_str_value(arg0) + elif ( + fn_name in ("TO_DATE", "DATE") + and len(curr.expressions) == 1 + and isinstance(curr.expressions[0], exp.Column) + ): + col_node = curr.expressions[0] + raw_unit = "day" + elif ( + fn_name + in ( + "YEAR", + "YEARS", + "QUARTER", + "QUARTERS", + "MONTH", + "MONTHS", + "WEEK", + "WEEKS", + "WEEKOFYEAR", + "DAY", + "DAYS", + "HOUR", + "HOURS", + "MINUTE", + "MINUTES", + "SECOND", + "SECONDS", + ) + and len(curr.expressions) == 1 + and isinstance(curr.expressions[0], exp.Column) + ): + col_node = curr.expressions[0] + raw_unit = "week" if fn_name == "WEEKOFYEAR" else fn_name.lower() + + if col_node is not None and raw_unit: + unit_clean = str(raw_unit).strip().strip("'\"").lower() + if unit_clean.endswith("s") and unit_clean[:-1] in cls._TIME_UNIT_PRECEDENCE: + unit_clean = unit_clean[:-1] + if unit_clean in cls._TIME_UNIT_PRECEDENCE: + clean_col = exp.column( + col_node.name, quoted=bool(getattr(col_node.this, "quoted", False)) + ) + return clean_col, unit_clean + return None + + @staticmethod + def _is_date_part_extract_fn(expr: exp.Expr) -> bool: + curr = expr.unnest() + if isinstance( + curr, + ( + exp.Year, + exp.Quarter, + exp.Month, + exp.Week, + exp.WeekOfYear, + exp.Day, + exp.Hour, + exp.Minute, + exp.Second, + exp.Extract, + ), + ): + return True + if isinstance(curr, exp.Anonymous) and curr.name.upper() in ( + "YEAR", + "YEARS", + "QUARTER", + "QUARTERS", + "MONTH", + "MONTHS", + "WEEK", + "WEEKS", + "WEEKOFYEAR", + "DAY", + "DAYS", + "HOUR", + "HOURS", + "MINUTE", + "MINUTES", + "SECOND", + "SECONDS", + ): + return True + return False + + def insert_overwrite_by_partition( + self, + table_name: TableName, + query_or_df: QueryOrDF, + partitioned_by: t.List[exp.Expr], + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + if not partitioned_by: + self.replace_query( + table_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + source_columns=source_columns, + **kwargs, + ) + return + parsed_exprs, _ = self._parse_partition_expressions(partitioned_by) + col_order: t.List[str] = [] + col_to_expr: t.Dict[str, exp.Column] = {} + col_to_unit: t.Dict[str, t.Optional[str]] = {} + + for expr in parsed_exprs or partitioned_by: + if isinstance(expr, exp.Column): + col_lower = expr.name.lower() + if col_lower not in col_to_expr: + col_order.append(col_lower) + col_to_expr[col_lower] = exp.column( + expr.name, quoted=bool(getattr(expr.this, "quoted", False)) + ) + col_to_unit[col_lower] = None + elif isinstance(expr, exp.Identifier): + col_lower = expr.name.lower() + if col_lower not in col_to_expr: + col_order.append(col_lower) + col_to_expr[col_lower] = exp.column(expr.name, quoted=bool(expr.quoted)) + col_to_unit[col_lower] = None + elif (transform := self._extract_time_partition_transform(expr)) is not None: + col_expr, unit = transform + col_lower = col_expr.name.lower() + if col_lower not in col_to_expr: + col_order.append(col_lower) + col_to_expr[col_lower] = col_expr + col_to_unit[col_lower] = unit + elif col_to_unit[col_lower] is not None: + prev_unit = col_to_unit[col_lower] + assert prev_unit is not None + if self._TIME_UNIT_PRECEDENCE[unit] < self._TIME_UNIT_PRECEDENCE[prev_unit]: + col_to_unit[col_lower] = unit + else: + raise SQLMeshError( + f"Doris insert_overwrite_by_partition does not support partition expression '{expr.sql(dialect=self.dialect)}'. " + "Only column names and time-truncation transforms (DATE_TRUNC, TO_DATE, DATE, YEAR, MONTH, DAY, HOUR) are supported." + ) + + key_exprs: t.List[exp.Expr] = [] + for col_lower in col_order: + col_expr = col_to_expr[col_lower] + col_unit = col_to_unit[col_lower] + if col_unit is None: + key_exprs.append(col_expr) + else: + key_exprs.append( + exp.Anonymous( + this="DATE_TRUNC", + expressions=[col_expr, exp.Literal.string(col_unit)], + ) + ) + + target_table = exp.to_table(table_name) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=target_table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + if ( + target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + and not is_iceberg + ): + reorder_kwargs = dict(kwargs) + reorder_kwargs["partitioned_by"] = partitioned_by + target_columns_to_types = self._reorder_olap_columns( + target_columns_to_types, reorder_kwargs + ) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + + if self.INSERT_OVERWRITE_STRATEGY.is_insert_overwrite: + super().insert_overwrite_by_partition( + table_name, + query_or_df, + key_exprs or partitioned_by, + target_columns_to_types=target_columns_to_types, + source_columns=source_columns, + ) + else: + self._replace_by_key( + table_name, + query_or_df, + target_columns_to_types, + key_exprs or partitioned_by, + is_unique_key=False, + source_columns=source_columns, + **kwargs, + ) + + def insert_overwrite_by_time_partition( + self, + table_name: TableName, + query_or_df: QueryOrDF, + start: TimeLike, + end: TimeLike, + time_formatter: t.Callable[[TimeLike, t.Optional[t.Dict[str, exp.DataType]]], exp.Expr], + time_column: TimeColumn | exp.Expr | str, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + target_table = exp.to_table(table_name) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=target_table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + if ( + not is_iceberg + and target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + return super().insert_overwrite_by_time_partition( + table_name, + query_or_df, + start=start, + end=end, + time_formatter=time_formatter, + time_column=time_column, + target_columns_to_types=target_columns_to_types, + source_columns=source_columns, + **kwargs, + ) + + def delete_from( + self, table_name: TableName, where: t.Optional[t.Union[str, exp.Expr]] = None + ) -> None: + """Delete from a table in Doris.""" + if not where: + self._truncate_table(table_name) + return + + if isinstance(where, str): + where = parse_one(where, read=self.dialect) + else: + where = where.copy() + + where = self._simplify_boolean_where(where) + if where == exp.true(): + self._truncate_table(table_name) + return + if where == exp.false(): + return + + where = self._unwrap_literal_casts_in_where(where) + where = self._pushdown_not_in_where(where) + where = self._rewrite_between_in_where(where) + where = self._rewrite_direct_time_transforms_in_where(where) + where = self._simplify_boolean_where(where) + if where == exp.true(): + self._truncate_table(table_name) + return + if where == exp.false(): + return + where = self._normalize_boolean_column_conjuncts(where) + + subquery_info = self._extract_subquery_from_where(where) + if subquery_info is not None: + self._execute_delete_with_subquery(table_name, *subquery_info) + return + + tuple_in_info = self._extract_literal_tuple_in_from_where(where) + if tuple_in_info is not None: + self._delete_key_rows(table_name, *tuple_in_info, is_ast_values=True) + return + + or_key_info = self._extract_or_key_rows_from_where(where) + if or_key_info is not None: + self._delete_key_rows(table_name, *or_key_info, is_ast_values=True) + return + + expanded_or_sets = self._expand_or_free_extra_condition_sets([where]) + if len(expanded_or_sets) > 1: + for branch_conjuncts in expanded_or_sets: + if branch_conjuncts: + self.delete_from(table_name, exp.and_(*branch_conjuncts, copy=False)) + return + + target_table = exp.to_table(table_name) + where = where.unnest() + for col in where.find_all(exp.Column): + if not col.find_ancestor(exp.Subquery, exp.Select) and ( + col.table and col.table.lower() == target_table.name.lower() + ): + col.set("table", None) + col.set("db", None) + col.set("catalog", None) + super().delete_from(table_name, where) + + @classmethod + def _simplify_boolean_where(cls, expr: exp.Expr) -> exp.Expr: + if isinstance(expr, (exp.Subquery, exp.Select)): + return expr + for k, v in list(expr.args.items()): + if isinstance(v, exp.Expr): + expr.set(k, cls._simplify_boolean_where(v)) + elif isinstance(v, list): + expr.set( + k, + [ + cls._simplify_boolean_where(item) if isinstance(item, exp.Expr) else item + for item in v + ], + ) + if isinstance(expr, exp.Paren) and isinstance(expr.this, exp.Boolean): + return expr.this + if ( + isinstance(expr, exp.EQ) + and isinstance(expr.this, exp.Literal) + and isinstance(expr.expression, exp.Literal) + and expr.this.is_number == expr.expression.is_number + ): + return exp.true() if expr.this == expr.expression else exp.false() + if ( + isinstance(expr, exp.NEQ) + and isinstance(expr.this, exp.Literal) + and isinstance(expr.expression, exp.Literal) + and expr.this.is_number == expr.expression.is_number + ): + return exp.false() if expr.this == expr.expression else exp.true() + if isinstance(expr, exp.Not) and isinstance(expr.this, exp.Boolean): + return exp.false() if expr.this == exp.true() else exp.true() + if isinstance(expr, exp.And): + if expr.this == exp.false() or expr.expression == exp.false(): + return exp.false() + if expr.this == exp.true(): + return expr.expression + if expr.expression == exp.true(): + return expr.this + if isinstance(expr, exp.Or): + if expr.this == exp.true() or expr.expression == exp.true(): + return exp.true() + if expr.this == exp.false(): + return expr.expression + if expr.expression == exp.false(): + return expr.this + return expr + + @classmethod + def _flatten_and_conjuncts(cls, expr: exp.Expr) -> t.List[exp.Expr]: + unwrapped = expr.unnest() + if isinstance(unwrapped, exp.And): + result: t.List[exp.Expr] = [] + for child in unwrapped.flatten(): + result.extend(cls._flatten_and_conjuncts(child)) + return result + return [unwrapped] + + @classmethod + def _flatten_or_disjuncts(cls, expr: exp.Expr) -> t.List[exp.Expr]: + unwrapped = expr.unnest() + if isinstance(unwrapped, exp.Or): + result: t.List[exp.Expr] = [] + for child in unwrapped.flatten(): + result.extend(cls._flatten_or_disjuncts(child)) + return result + return [unwrapped] + + def _normalize_extra_conditions(self, extra_conditions: t.List[exp.Expr]) -> t.List[exp.Expr]: + normalized: t.List[exp.Expr] = [] + for raw_cond in extra_conditions: + for cond in self._flatten_and_conjuncts(raw_cond): + unwrapped = cond.unnest() + if isinstance(unwrapped, exp.Or): + or_info = self._extract_or_key_rows_from_where(unwrapped) + if ( + or_info is not None + and len(or_info[0]) == 1 + and not or_info[2] + and all(not isinstance(r[0], exp.Null) for r in or_info[1]) + ): + col_expr = exp.column(or_info[0][0].name, quoted=True) + vals = [r[0].copy() for r in or_info[1]] + normalized.append(col_expr.isin(*vals)) + continue + normalized.append(cond) + return normalized + + @classmethod + def _pushdown_not_in_where(cls, expr: exp.Expr) -> exp.Expr: + if isinstance(expr, (exp.Subquery, exp.Select)): + return expr + if isinstance(expr, exp.Not) and isinstance(expr.this, exp.Expr): + inner = expr.this.unnest() + if isinstance(inner, exp.Not) and isinstance(inner.this, exp.Expr): + return cls._pushdown_not_in_where(inner.this.unnest().copy()) + if isinstance(inner, exp.And): + return exp.Paren( + this=exp.or_( + cls._pushdown_not_in_where(exp.not_(inner.this.copy(), copy=False)), + cls._pushdown_not_in_where(exp.not_(inner.expression.copy(), copy=False)), + copy=False, + ) + ) + if isinstance(inner, exp.Or): + return exp.Paren( + this=exp.and_( + cls._pushdown_not_in_where(exp.not_(inner.this.copy(), copy=False)), + cls._pushdown_not_in_where(exp.not_(inner.expression.copy(), copy=False)), + copy=False, + ) + ) + if isinstance(inner, exp.Between): + return exp.Paren( + this=exp.or_( + exp.LT(this=inner.this.copy(), expression=inner.args["low"].copy()), + exp.GT(this=inner.this.copy(), expression=inner.args["high"].copy()), + copy=False, + ) + ) + if isinstance(inner, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)): + neg_map: t.Dict[t.Type[exp.Binary], t.Type[exp.Binary]] = { + exp.EQ: exp.NEQ, + exp.NEQ: exp.EQ, + exp.GT: exp.LTE, + exp.GTE: exp.LT, + exp.LT: exp.GTE, + exp.LTE: exp.GT, + } + neg_cls = neg_map[type(inner)] + return neg_cls(this=inner.this.copy(), expression=inner.expression.copy()) + if isinstance(inner, (exp.In, exp.Is)) and expr.this is not inner: + return exp.Not(this=inner.copy()) + for k, v in list(expr.args.items()): + if isinstance(v, exp.Expr): + expr.set(k, cls._pushdown_not_in_where(v)) + elif isinstance(v, list): + expr.set( + k, + [ + cls._pushdown_not_in_where(item) if isinstance(item, exp.Expr) else item + for item in v + ], + ) + return expr + + def _expand_or_to_conjunct_sets(self, expr: exp.Expr) -> t.List[t.List[exp.Expr]]: + expr = self._pushdown_not_in_where(expr.copy()) + conjuncts = self._flatten_and_conjuncts(expr) + if len(conjuncts) > 1: + sets: t.List[t.List[exp.Expr]] = [[]] + for c in conjuncts: + sub_sets = self._expand_or_to_conjunct_sets(c) + sets = [[*s, *b] for s in sets for b in sub_sets] + return sets + single = conjuncts[0] + disjuncts = self._flatten_or_disjuncts(single) + if len(disjuncts) > 1: + or_sets: t.List[t.List[exp.Expr]] = [] + for d in disjuncts: + or_sets.extend(self._expand_or_to_conjunct_sets(d)) + return or_sets + leaf = disjuncts[0] + norm = self._normalize_single_boolean_conjunct(leaf) + if norm is not None: + if isinstance(norm.unnest(), exp.Or): + return self._expand_or_to_conjunct_sets(norm) + return [[norm]] + return [[leaf.copy()]] + + def _expand_or_free_extra_condition_sets( + self, extra_conditions: t.Sequence[exp.Expr] + ) -> t.List[t.List[exp.Expr]]: + condition_sets: t.List[t.List[exp.Expr]] = [[]] + for cond in extra_conditions: + sub_sets = self._expand_or_to_conjunct_sets(cond) + condition_sets = [[*cset, *branch] for cset in condition_sets for branch in sub_sets] + return condition_sets + + def _compute_time_transform_bounds( + self, + t_expr: exp.Expr, + unit: str, + val_expr: exp.Expr, + ) -> t.Optional[t.Tuple[exp.Literal, exp.Literal, bool]]: + if self._is_date_part_extract_fn(t_expr) and unit != "year": + raise SQLMeshError( + f"Doris delete_from cannot rewrite non-year date-part predicate '{t_expr.sql(dialect=self.dialect)}' " + "into a contiguous range. Use DATE_TRUNC, TO_DATE, DATE, or YEAR instead." + ) + raw_val = self._extract_str_value(val_expr) + if raw_val is None: + return None + raw_str = raw_val.strip() + if self._is_date_part_extract_fn(t_expr) and unit == "year": + try: + yr_int = ( + int(raw_str[:4]) + if len(raw_str) >= 4 and raw_str[:4].isdigit() + else int(raw_str) + ) + except ValueError: + return None + return ( + exp.Literal.string(f"{yr_int:04d}-01-01"), + exp.Literal.string(f"{yr_int + 1:04d}-01-01"), + True, + ) + try: + dt = datetime.datetime.fromisoformat(raw_str) + except ValueError: + return None + has_time = " " in raw_str or "T" in raw_str or unit in ("hour", "minute", "second") + if unit == "year": + low_dt = dt.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) + high_dt = low_dt.replace(year=low_dt.year + 1) + elif unit == "quarter": + q_month = ((dt.month - 1) // 3) * 3 + 1 + low_dt = dt.replace(month=q_month, day=1, hour=0, minute=0, second=0, microsecond=0) + high_dt = low_dt.replace( + year=low_dt.year + (1 if q_month == 10 else 0), + month=(1 if q_month == 10 else q_month + 3), + ) + elif unit == "month": + low_dt = dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + high_dt = low_dt.replace( + year=low_dt.year + (1 if low_dt.month == 12 else 0), + month=(1 if low_dt.month == 12 else low_dt.month + 1), + ) + elif unit == "week": + low_dt = (dt - datetime.timedelta(days=dt.weekday())).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + high_dt = low_dt + datetime.timedelta(days=7) + elif unit == "day": + low_dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + high_dt = low_dt + datetime.timedelta(days=1) + elif unit == "hour": + low_dt = dt.replace(minute=0, second=0, microsecond=0) + high_dt = low_dt + datetime.timedelta(hours=1) + elif unit == "minute": + low_dt = dt.replace(second=0, microsecond=0) + high_dt = low_dt + datetime.timedelta(minutes=1) + elif unit == "second": + low_dt = dt.replace(microsecond=0) + high_dt = low_dt + datetime.timedelta(seconds=1) + else: + return None + fmt = "%Y-%m-%d %H:%M:%S" if has_time else "%Y-%m-%d" + return ( + exp.Literal.string(low_dt.strftime(fmt)), + exp.Literal.string(high_dt.strftime(fmt)), + dt == low_dt, + ) + + def _build_time_transform_range( + self, + t_expr: exp.Expr, + col_expr: exp.Column, + unit: str, + val_expr: exp.Expr, + ) -> t.Optional[exp.Expr]: + bounds = self._compute_time_transform_bounds(t_expr, unit, val_expr) + if bounds is None: + return None + low_lit, high_lit, _ = bounds + return exp.and_( + exp.GTE( + this=col_expr.copy(), + expression=low_lit, + ), + exp.LT( + this=col_expr.copy(), + expression=high_lit, + ), + copy=False, + ) + + def _rewrite_direct_time_transforms_in_where(self, where: exp.Expr) -> exp.Expr: + for node in list( + where.find_all( + exp.EQ, + exp.NEQ, + exp.GTE, + exp.GT, + exp.LTE, + exp.LT, + exp.In, + exp.Is, + ) + ): + if node.find_ancestor(exp.Subquery, exp.Select): + continue + if isinstance(node, exp.Is) and isinstance(node.expression, exp.Null): + is_trans = self._extract_time_partition_transform(node.this) + if is_trans is not None: + node.set("this", is_trans[0].copy()) + continue + parent_not = node.parent if isinstance(node.parent, exp.Not) else None + if node.find_ancestor(exp.Not) and parent_not is None: + continue + target_node: exp.Expr = parent_not if parent_not is not None else node + if isinstance(node, (exp.EQ, exp.NEQ, exp.GTE, exp.GT, exp.LTE, exp.LT)): + left_trans = self._extract_time_partition_transform(node.this) + right_trans = self._extract_time_partition_transform(node.expression) + t_expr: t.Optional[exp.Expr] = None + col_expr: t.Optional[exp.Column] = None + unit: t.Optional[str] = None + val_expr: t.Optional[exp.Expr] = None + transform_on_left = True + if left_trans is not None and self._is_literal_key_value(node.expression): + t_expr = node.this + col_expr, unit = left_trans + val_expr = node.expression + transform_on_left = True + elif right_trans is not None and self._is_literal_key_value(node.this): + t_expr = node.expression + col_expr, unit = right_trans + val_expr = node.this + transform_on_left = False + if ( + t_expr is not None + and col_expr is not None + and unit is not None + and val_expr is not None + ): + bounds = self._compute_time_transform_bounds(t_expr, unit, val_expr) + if bounds is not None: + low_lit, high_lit, is_aligned = bounds + effective_op: t.Type[exp.Binary] = type(node) + if not transform_on_left: + flip_map: t.Dict[t.Type[exp.Binary], t.Type[exp.Binary]] = { + exp.GTE: exp.LTE, + exp.GT: exp.LT, + exp.LTE: exp.GTE, + exp.LT: exp.GT, + } + effective_op = flip_map.get(effective_op, effective_op) + if parent_not is not None: + neg_map: t.Dict[t.Type[exp.Binary], t.Type[exp.Binary]] = { + exp.EQ: exp.NEQ, + exp.NEQ: exp.EQ, + exp.GTE: exp.LT, + exp.GT: exp.LTE, + exp.LTE: exp.GT, + exp.LT: exp.GTE, + } + effective_op = neg_map.get(effective_op, effective_op) + replacement: t.Optional[exp.Expr] = None + if effective_op is exp.EQ: + if is_aligned: + replacement = exp.and_( + exp.GTE(this=col_expr.copy(), expression=low_lit.copy()), + exp.LT(this=col_expr.copy(), expression=high_lit.copy()), + copy=False, + ) + else: + replacement = exp.false() + elif effective_op is exp.NEQ: + if is_aligned: + replacement = exp.or_( + exp.LT(this=col_expr.copy(), expression=low_lit.copy()), + exp.GTE(this=col_expr.copy(), expression=high_lit.copy()), + copy=False, + ) + else: + replacement = exp.not_( + exp.Is(this=col_expr.copy(), expression=exp.Null()) + ) + elif effective_op is exp.GTE: + bound_lit = low_lit if is_aligned else high_lit + replacement = exp.GTE(this=col_expr.copy(), expression=bound_lit.copy()) + elif effective_op is exp.GT: + replacement = exp.GTE(this=col_expr.copy(), expression=high_lit.copy()) + elif effective_op is exp.LTE: + replacement = exp.LT(this=col_expr.copy(), expression=high_lit.copy()) + elif effective_op is exp.LT: + bound_lit = low_lit if is_aligned else high_lit + replacement = exp.LT(this=col_expr.copy(), expression=bound_lit.copy()) + if replacement is not None: + if target_node is where: + where = replacement + else: + if isinstance(replacement, (exp.And, exp.Or)) and not isinstance( + target_node.parent, (exp.Where, exp.Paren) + ): + replacement = exp.Paren(this=replacement) + target_node.replace(replacement) + elif ( + isinstance(node, exp.In) + and self._get_in_subquery(node) is None + and node.expressions + ): + in_trans = self._extract_time_partition_transform(node.this) + if in_trans is not None and all( + self._is_literal_key_value(v) for v in node.expressions + ): + col_expr, unit = in_trans + bounds_list = [ + self._compute_time_transform_bounds(node.this, unit, v) + for v in node.expressions + ] + if bounds_list and all(b is not None for b in bounds_list): + aligned_ranges = [ + exp.and_( + exp.GTE(this=col_expr.copy(), expression=b[0].copy()), + exp.LT(this=col_expr.copy(), expression=b[1].copy()), + copy=False, + ) + for b in bounds_list + if b is not None and b[2] + ] + if parent_not is not None: + if not aligned_ranges: + replacement_in: exp.Expr = exp.not_( + exp.Is(this=col_expr.copy(), expression=exp.Null()) + ) + else: + not_in_parts: t.List[exp.Expr] = [ + exp.Paren( + this=exp.or_( + exp.LT( + this=col_expr.copy(), + expression=r.this.expression.copy(), + ), + exp.GTE( + this=col_expr.copy(), + expression=r.expression.expression.copy(), + ), + copy=False, + ) + ) + for r in aligned_ranges + ] + replacement_in = exp.and_(*not_in_parts, copy=False) + else: + if not aligned_ranges: + replacement_in = exp.false() + elif len(aligned_ranges) == 1: + replacement_in = aligned_ranges[0] + else: + replacement_in = exp.or_( + *[exp.Paren(this=r) for r in aligned_ranges], + copy=False, + ) + if target_node is where: + where = replacement_in + else: + if isinstance(replacement_in, (exp.And, exp.Or)) and not isinstance( + target_node.parent, (exp.Where, exp.Paren) + ): + replacement_in = exp.Paren(this=replacement_in) + target_node.replace(replacement_in) + return where + + @classmethod + def _unwrap_literal_casts_in_where(cls, where: exp.Expr) -> exp.Expr: + """Unwrap top-level `CAST( AS ...)` expressions in `DELETE WHERE` predicates + (e.g. from `to_time_column` in `insert_overwrite_by_time_partition`) because Doris's OLAP + `DeleteHandler` only accepts plain literals on the RHS of comparison and `IN` predicates. + """ + for cast_node in list(where.find_all(exp.Cast, exp.TryCast)): + if cast_node.find_ancestor(exp.Subquery, exp.Select): + continue + inner = cast_node.this + while isinstance(inner, (exp.Cast, exp.TryCast)): + inner = inner.this + if inner is not None and cls._is_literal_key_value(inner): + cast_node.replace(inner.copy()) + return where + + @staticmethod + def _rewrite_between_in_where(where: exp.Expr) -> exp.Expr: + if isinstance(where, exp.Between): + return exp.and_( + exp.GTE(this=where.this.copy(), expression=where.args["low"].copy()), + exp.LTE(this=where.this.copy(), expression=where.args["high"].copy()), + copy=False, + ) + for between in list(where.find_all(exp.Between)): + if between.find_ancestor(exp.Subquery, exp.Select, exp.Not): + continue + replacement: exp.Expr = exp.and_( + exp.GTE(this=between.this.copy(), expression=between.args["low"].copy()), + exp.LTE(this=between.this.copy(), expression=between.args["high"].copy()), + copy=False, + ) + if not isinstance(between.parent, (exp.Where, exp.Paren)): + replacement = exp.Paren(this=replacement) + between.replace(replacement) + return where + + def _extract_subquery_from_where( + self, where: exp.Expr + ) -> t.Optional[t.Tuple[t.List[exp.Expr], exp.Subquery, t.List[exp.Expr]]]: + """Extract positive IN (SELECT ...) subquery and any sibling AND conditions.""" + conjuncts = self._flatten_and_conjuncts(where) + + for cand_idx, cond in enumerate(conjuncts): + unwrapped_cond = cond.unnest() + if not isinstance(unwrapped_cond, exp.In): + continue + subquery = self._get_in_subquery(unwrapped_cond) + if subquery is None: + continue + + lhs = unwrapped_cond.this + select_expr = subquery.this + key_exprs: t.List[exp.Expr] = [] + + lhs_concat = self._unwrap_concat_ws(lhs) + if lhs_concat is not None: + unwrapped_select_expr = self._unwrap_concat_ws_in_query( + select_expr, len(lhs_concat.expressions) + ) + if unwrapped_select_expr is None: + continue + key_exprs = [e.copy() for e in lhs_concat.expressions[1:]] + subquery = exp.Subquery(this=unwrapped_select_expr) + elif isinstance(lhs, exp.Tuple): + key_exprs = [e.copy() for e in lhs.expressions] + else: + key_exprs = [lhs.copy()] + + normalized_keys: t.List[exp.Expr] = [] + valid_keys = True + for k in key_exprs: + if isinstance(k, exp.Column): + normalized_keys.append(k) + elif isinstance(k, exp.Identifier): + normalized_keys.append(exp.Column(this=k.copy())) + elif self._extract_time_partition_transform(k) is not None: + normalized_keys.append(k.copy()) + else: + valid_keys = False + break + + if not valid_keys or not normalized_keys: + continue + + extra_conditions = [c for i, c in enumerate(conjuncts) if i != cand_idx] + return normalized_keys, subquery, self._normalize_extra_conditions(extra_conditions) + + return None + + def _extract_literal_tuple_in_from_where( + self, where: exp.Expr + ) -> t.Optional[t.Tuple[t.List[exp.Expr], t.List[t.Tuple[exp.Expr, ...]], t.List[exp.Expr]]]: + """Extract positive literal tuple IN predicate `(a, b) IN ((1, 2), (3, 4))` and sibling AND conditions.""" + conjuncts = self._flatten_and_conjuncts(where) + + for cand_idx, cond in enumerate(conjuncts): + unwrapped_cond = cond.unnest() + if not ( + isinstance(unwrapped_cond, exp.In) + and self._get_in_subquery(unwrapped_cond) is None + and isinstance(unwrapped_cond.this, exp.Tuple) + and len(unwrapped_cond.this.expressions) >= 2 + and unwrapped_cond.expressions + and all( + isinstance(item, exp.Tuple) + and len(item.expressions) == len(unwrapped_cond.this.expressions) + for item in unwrapped_cond.expressions + ) + ): + continue + + normalized_keys: t.List[exp.Expr] = [] + valid_keys = True + for k in unwrapped_cond.this.expressions: + if isinstance(k, exp.Column): + normalized_keys.append(k.copy()) + elif isinstance(k, exp.Identifier): + normalized_keys.append(exp.Column(this=k.copy())) + else: + valid_keys = False + break + + if not valid_keys or not normalized_keys: + continue + + tuple_rows: t.List[t.Tuple[exp.Expr, ...]] = [ + tuple(e.copy() for e in item.expressions) + for item in unwrapped_cond.expressions + if isinstance(item, exp.Tuple) + and not any(isinstance(e.unnest(), exp.Null) for e in item.expressions) + ] + + extra_conditions = [c for i, c in enumerate(conjuncts) if i != cand_idx] + return normalized_keys, tuple_rows, self._normalize_extra_conditions(extra_conditions) + + return None + + @staticmethod + def _is_literal_key_value(expr: exp.Expr) -> bool: + return isinstance(expr, (exp.Literal, exp.Boolean, exp.Null)) or ( + isinstance(expr, exp.Neg) and isinstance(expr.this, exp.Literal) + ) + + @classmethod + def _normalize_single_boolean_conjunct(cls, cond: exp.Expr) -> t.Optional[exp.Expr]: + inner = cond.unnest() + if isinstance(inner, exp.Column): + return inner.copy().eq(exp.true()) + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Boolean): + return inner.this.copy().eq(inner.expression.copy()) + if isinstance(inner, exp.Not): + neg_inner = inner.this.unnest() + if isinstance(neg_inner, exp.Column): + return neg_inner.copy().eq(exp.false()) + if isinstance(neg_inner, exp.Is) and isinstance(neg_inner.expression, exp.Boolean): + col_expr = neg_inner.this.copy() + opposite_bool = exp.Boolean(this=not bool(neg_inner.expression.this)) + return exp.Paren( + this=exp.or_( + col_expr.eq(opposite_bool), + col_expr.copy().is_(exp.null()), + copy=False, + ) + ) + return None + + @classmethod + def _normalize_boolean_column_conjuncts(cls, where: exp.Expr) -> exp.Expr: + """Rewrite top-level bare boolean column conjuncts (`AND is_dev`, `AND NOT is_dev`) and + `IS [NOT] TRUE/FALSE` predicates to explicit binary comparisons (`is_dev = TRUE`, + `is_dev = FALSE`) required by Doris DeleteHandler. + """ + single = cls._normalize_single_boolean_conjunct(where) + if single is not None: + return single + unwrapped = where.unnest() + if isinstance(unwrapped, exp.And): + conjuncts = cls._flatten_and_conjuncts(unwrapped) + changed = len(conjuncts) != len(list(unwrapped.flatten())) + new_conjuncts: t.List[exp.Expr] = [] + for cond in conjuncts: + normalized = cls._normalize_single_boolean_conjunct(cond) + if normalized is not None: + new_conjuncts.append(normalized) + changed = True + else: + new_conjuncts.append(cond) + if changed: + return exp.and_(*new_conjuncts, copy=False) + return where + + def _extract_or_key_rows_from_where( + self, where: exp.Expr + ) -> t.Optional[t.Tuple[t.List[exp.Expr], t.List[t.Tuple[exp.Expr, ...]], t.List[exp.Expr]]]: + """Extract a top-level OR of equality/IS NULL key predicates (e.g. from `snapshot_id_filter` + when `SUPPORTS_TUPLE_IN = False`, or `delete_expired_environments`) and sibling AND conditions. + """ + conjuncts = self._flatten_and_conjuncts(where) + + for cand_idx, cond in enumerate(conjuncts): + unwrapped_cond = cond.unnest() + if not isinstance(unwrapped_cond, exp.Or): + continue + + disjuncts = self._flatten_or_disjuncts(unwrapped_cond) + if len(disjuncts) < 2: + continue + + canonical_col_order: t.Optional[t.List[str]] = None + canonical_cols: t.List[exp.Expr] = [] + tuple_rows: t.List[t.Tuple[exp.Expr, ...]] = [] + valid_or = True + + for branch in disjuncts: + branch_conjuncts = self._flatten_and_conjuncts(branch) + col_to_val: t.Dict[str, exp.Expr] = {} + col_to_expr: t.Dict[str, exp.Expr] = {} + for item in branch_conjuncts: + normalized_bool = self._normalize_single_boolean_conjunct(item) + item_unwrapped = ( + normalized_bool if normalized_bool is not None else item.unnest() + ) + if isinstance(item_unwrapped, exp.EQ): + left, right = item_unwrapped.this, item_unwrapped.expression + if isinstance( + left, (exp.Column, exp.Identifier) + ) and self._is_literal_key_value(right): + col_expr = ( + left.copy() + if isinstance(left, exp.Column) + else exp.Column(this=left.copy()) + ) + val_expr = right.copy() + elif isinstance( + right, (exp.Column, exp.Identifier) + ) and self._is_literal_key_value(left): + col_expr = ( + right.copy() + if isinstance(right, exp.Column) + else exp.Column(this=right.copy()) + ) + val_expr = left.copy() + else: + valid_or = False + break + elif isinstance(item_unwrapped, exp.Is) and isinstance( + item_unwrapped.expression, exp.Null + ): + left = item_unwrapped.this + if isinstance(left, (exp.Column, exp.Identifier)): + col_expr = ( + left.copy() + if isinstance(left, exp.Column) + else exp.Column(this=left.copy()) + ) + val_expr = exp.null() + else: + valid_or = False + break + else: + valid_or = False + break + + col_key = col_expr.name.lower() + if not col_key or col_key in col_to_val: + valid_or = False + break + col_to_val[col_key] = val_expr + col_to_expr[col_key] = col_expr + + if not valid_or or not col_to_val: + valid_or = False + break + + if canonical_col_order is None: + canonical_col_order = list(col_to_val.keys()) + canonical_cols = [col_to_expr[k] for k in canonical_col_order] + elif set(col_to_val.keys()) != set(canonical_col_order): + valid_or = False + break + + tuple_rows.append(tuple(col_to_val[k] for k in canonical_col_order)) + + if valid_or and canonical_cols and tuple_rows: + if ( + len(canonical_cols) == 1 + and len([r for r in tuple_rows if not isinstance(r[0], exp.Null)]) <= 1 + ): + continue + extra_conditions = [c for i, c in enumerate(conjuncts) if i != cand_idx] + return ( + canonical_cols, + tuple_rows, + self._normalize_extra_conditions(extra_conditions), + ) + + return None + + @staticmethod + def _unwrap_concat_ws(expr: exp.Expr) -> t.Optional[exp.Expr]: + curr = expr.unnest() + if isinstance(curr, exp.Column) and isinstance(curr.this, exp.Expr): + curr = curr.this.unnest() + if isinstance(curr, exp.ConcatWs) or ( + isinstance(curr, exp.Anonymous) and curr.name.upper() == "CONCAT_WS" + ): + return curr + return None + + @classmethod + def _unwrap_concat_ws_in_query( + cls, query_expr: exp.Expr, expected_arity: int + ) -> t.Optional[exp.Expr]: + if isinstance(query_expr, exp.Subquery): + inner = cls._unwrap_concat_ws_in_query(query_expr.this, expected_arity) + if inner is None: + return None + unwrapped_sq = query_expr.copy() + unwrapped_sq.set("this", inner) + return unwrapped_sq + if isinstance(query_expr, exp.SetOperation): + left = cls._unwrap_concat_ws_in_query(query_expr.this, expected_arity) + right = cls._unwrap_concat_ws_in_query(query_expr.expression, expected_arity) + if left is None or right is None: + return None + unwrapped_set = query_expr.copy() + unwrapped_set.set("this", left) + unwrapped_set.set("expression", right) + return unwrapped_set + if isinstance(query_expr, exp.Select) and len(query_expr.expressions) == 1: + first_sel = query_expr.expressions[0] + raw_sel = first_sel.this if isinstance(first_sel, exp.Alias) else first_sel + sel_concat = cls._unwrap_concat_ws(raw_sel) + if sel_concat is None or len(sel_concat.expressions) != expected_arity: + return None + unwrapped_sel = query_expr.copy() + unwrapped_sel.set("expressions", [e.copy() for e in sel_concat.expressions[1:]]) + return unwrapped_sel + return None + + @staticmethod + def _get_in_subquery(in_expr: exp.In) -> t.Optional[exp.Subquery]: + query = in_expr.args.get("query") + if isinstance(query, exp.Subquery): + return query.copy() + if isinstance(query, (exp.Select, exp.SetOperation)): + return exp.Subquery(this=query.copy()) + if len(in_expr.expressions) == 1: + first = in_expr.expressions[0] + if isinstance(first, exp.Subquery): + return first.copy() + if isinstance(first, (exp.Select, exp.SetOperation)): + return exp.Subquery(this=first.copy()) + return None + + def _execute_delete_with_subquery( + self, + table_name: TableName, + key_exprs: t.List[exp.Expr], + subquery: exp.Subquery, + extra_conditions: t.List[exp.Expr], + ) -> None: + """Execute DELETE FROM ... USING (SELECT ...) in Doris, with literal batch fallback for DUPLICATE KEY/Iceberg.""" + if any(self._extract_time_partition_transform(k) is not None for k in key_exprs): + self._delete_by_time_transform_keys(table_name, key_exprs, subquery, extra_conditions) + return + + target_table = exp.to_table(table_name) + if self._is_iceberg_table(catalog_name=target_table.catalog) or getattr( + self, "_in_iceberg_replace_by_key", False + ): + self._delete_by_literal_keys(table_name, key_exprs, subquery, extra_conditions) + return + + join_conditions: t.List[exp.Expr] = [] + orig_subquery = subquery.copy() + subquery = subquery.copy() + subq_branches: t.List[exp.Select] = [] + + def _collect_subq_branches(node: t.Optional[exp.Expr]) -> None: + if isinstance(node, exp.Subquery): + _collect_subq_branches(node.this) + elif isinstance(node, exp.SetOperation): + _collect_subq_branches(node.this) + _collect_subq_branches(node.expression) + elif isinstance(node, exp.Select): + subq_branches.append(node) + + _collect_subq_branches(subquery.this) + if not subq_branches or any( + len(b.expressions) < len(key_exprs) + or any(isinstance(e, exp.Star) for e in b.expressions) + for b in subq_branches + ): + self._delete_by_literal_keys(table_name, key_exprs, orig_subquery, extra_conditions) + return + first_select = subq_branches[0] + used_t2_aliases: t.Set[str] = set() + for idx, col in enumerate(key_exprs): + col_name = col.name + t1_col = exp.column(col_name, table="_t1") + t2_col_name = col_name + if ( + isinstance(first_select, exp.Select) + and idx < len(first_select.expressions) + and not isinstance(first_select.expressions[idx], exp.Star) + ): + sel_item = first_select.expressions[idx] + candidate_name = ( + sel_item.alias_or_name + if isinstance(sel_item, (exp.Column, exp.Alias, exp.Identifier)) + else "" + ) + if candidate_name and candidate_name.lower() not in used_t2_aliases: + t2_col_name = candidate_name + else: + alias_name = f"_col_{idx}" + new_exprs = list(first_select.expressions) + inner_expr = sel_item.this if isinstance(sel_item, exp.Alias) else sel_item + new_exprs[idx] = exp.alias_(inner_expr.copy(), alias_name, quoted=True) + first_select.set("expressions", new_exprs) + t2_col_name = alias_name + used_t2_aliases.add(t2_col_name.lower()) + t2_col = exp.column(t2_col_name, table="_t2") + join_conditions.append(t1_col.eq(t2_col)) + + extra_condition_sets = self._expand_or_free_extra_condition_sets(extra_conditions) + target_sql = f"{self._to_sql(target_table)} AS `_t1`" + subquery_with_alias = exp.alias_(subquery.copy(), "_t2", table=True) + subquery_sql = self._to_sql(subquery_with_alias) + + try: + for cond_set in extra_condition_sets: + branch_joins = [jc.copy() for jc in join_conditions] + for extra in cond_set: + qualified_extra = extra.copy() + for col in qualified_extra.find_all(exp.Column): + if not col.find_ancestor(exp.Subquery, exp.Select) and ( + not col.table or col.table.lower() == target_table.name.lower() + ): + col.set("table", exp.to_identifier("_t1")) + col.set("db", None) + col.set("catalog", None) + branch_joins.append(qualified_extra) + where_expr = exp.and_(*branch_joins) + where_sql = self._to_sql(where_expr) + delete_sql = f"DELETE FROM {target_sql} USING {subquery_sql} WHERE {where_sql}" + self.execute(delete_sql) + except Exception as e: + err_msg = str(e).lower() + if any( + marker in err_msg + for marker in ( + "unique key", + "olaptable", + "olap table", + "not support", + "only support", + "unsupported", + ) + ): + logger.info( + "DELETE ... USING not supported on table '%s' (%s); falling back to batched literal DELETE.", + table_name, + e, + ) + self._delete_by_literal_keys(table_name, key_exprs, orig_subquery, extra_conditions) + else: + raise + + @staticmethod + def _sanitize_key_value(val: t.Any) -> t.Any: + import pandas as pd + + if val is None: + return None + if isinstance(val, (bytes, bytearray)): + return _decoded_str(val) + try: + if pd.isna(val): + return None + except (TypeError, ValueError): + pass + if hasattr(val, "item") and not isinstance( + val, + ( + str, + bytes, + bytearray, + datetime.date, + datetime.datetime, + datetime.time, + Decimal, + ), + ): + try: + return val.item() + except Exception: + pass + return val + + @staticmethod + def _to_delete_literal_expr(val: t.Any, *, is_ast_values: bool = False) -> exp.Expr: + if is_ast_values and isinstance(val, exp.Expr): + return val.copy() + if isinstance(val, (datetime.date, datetime.datetime, datetime.time)): + return exp.Literal.string(str(val)) + if isinstance(val, Decimal): + return exp.Literal.number(str(val)) + return exp.convert(val) + + def _delete_by_time_transform_keys( + self, + table_name: TableName, + key_exprs: t.List[exp.Expr], + subquery: t.Union[exp.Subquery, exp.Select, exp.SetOperation], + extra_conditions: t.Optional[t.List[exp.Expr]] = None, + ) -> None: + """Delete partition ranges for time-truncation partition keys (`DATE_TRUNC`, `DAY`, `MONTH`, `YEAR`, `HOUR`, `TO_DATE`) + using Doris-compatible half-open bare-column range predicates (`col >= AND col < `). + """ + for k in key_exprs: + k_transform = self._extract_time_partition_transform(k) + if ( + k_transform is not None + and self._is_date_part_extract_fn(k) + and k_transform[1] != "year" + ): + raise SQLMeshError( + f"Doris delete_from cannot rewrite non-year date-part predicate '{k.sql(dialect=self.dialect)}' " + "into a contiguous range. Use DATE_TRUNC, TO_DATE, DATE, or YEAR instead." + ) + + select_query = ( + subquery.this.copy() if isinstance(subquery, exp.Subquery) else subquery.copy() + ) + first_select = select_query + while isinstance(first_select, (exp.SetOperation, exp.Subquery)): + first_select = first_select.this + if not isinstance(first_select, exp.Select) or any( + isinstance(e, exp.Star) for e in first_select.expressions + ): + raise SQLMeshError( + f"Doris delete_from cannot rewrite time-partition subquery '{select_query.sql(dialect=self.dialect)}': " + f"expected a SELECT query with {len(key_exprs)} explicit projection(s)." + ) + + projections: t.List[exp.Expr] = [] + raw_year_indices: t.Set[int] = set() + if ( + isinstance(select_query, exp.Select) + and len(select_query.expressions) == len(key_exprs) + and not select_query.args.get("group") + and not select_query.args.get("having") + and not select_query.args.get("limit") + and not select_query.args.get("offset") + ): + for idx, k in enumerate(key_exprs): + sel_item = select_query.expressions[idx] + raw_sel = sel_item.this if isinstance(sel_item, exp.Alias) else sel_item + transform = self._extract_time_partition_transform(k) + if transform is not None: + _, unit = transform + is_extract_fn = self._is_date_part_extract_fn(k) + is_year_fn = unit == "year" and is_extract_fn + src_transform = self._extract_time_partition_transform(raw_sel) + if ( + src_transform is not None + and src_transform[1] == unit + and (not self._is_date_part_extract_fn(raw_sel) or unit == "year") + ): + src_arg: exp.Expr = src_transform[0].copy() + elif is_year_fn: + raw_year_indices.add(idx) + projections.append(exp.alias_(raw_sel.copy(), f"_val_{idx}", quoted=True)) + continue + elif src_transform is not None or self._is_date_part_extract_fn(raw_sel): + raise SQLMeshError( + f"Doris delete_from cannot rewrite time-partition predicate '{k.sql(dialect=self.dialect)}' " + f"with mismatched subquery projection '{raw_sel.sql(dialect=self.dialect)}' into a contiguous range." + ) + else: + src_arg = raw_sel.copy() + trunc_expr = exp.Anonymous( + this="DATE_TRUNC", + expressions=[src_arg, exp.Literal.string(unit)], + ) + upper_expr = exp.DateAdd( + this=trunc_expr.copy(), + expression=exp.Literal.number(1), + unit=exp.var(unit.upper()), + ) + projections.append(exp.alias_(trunc_expr, f"_low_{idx}", quoted=True)) + projections.append(exp.alias_(upper_expr, f"_high_{idx}", quoted=True)) + else: + projections.append(exp.alias_(raw_sel.copy(), f"_val_{idx}", quoted=True)) + select_query.set("order", None) + select_query.set("expressions", projections) + fetch_query: exp.Select = select_query.distinct() + elif len(first_select.expressions) == len(key_exprs): + all_branches: t.List[exp.Select] = [] + + def _collect_select_branches(node: exp.Expr) -> None: + if isinstance(node, exp.Subquery): + _collect_select_branches(node.this) + elif isinstance(node, exp.SetOperation): + _collect_select_branches(node.this) + _collect_select_branches(node.expression) + elif isinstance(node, exp.Select): + all_branches.append(node) + + _collect_select_branches(select_query) + if any( + len(branch.expressions) != len(key_exprs) + or any(isinstance(e, exp.Star) for e in branch.expressions) + for branch in all_branches + ): + raise SQLMeshError( + f"Doris delete_from cannot rewrite time-partition subquery '{select_query.sql(dialect=self.dialect)}': " + f"expected {len(key_exprs)} explicit projection(s) in every SELECT branch." + ) + transform_indices = { + idx + for idx, k in enumerate(key_exprs) + if self._extract_time_partition_transform(k) is not None + } + unwrapped_indices: t.Set[int] = set(transform_indices) + + # INTERSECT / EXCEPT compare full projection values across branches; unwrapping + # `DATE_TRUNC(ds, 'day')` into raw `ds` inside branches would change set semantics. + if any( + isinstance(op, (exp.Intersect, exp.Except)) + for op in select_query.find_all(exp.SetOperation) + ): + unwrapped_indices.clear() + + # Pass 1: Validate which transform indices can be unwrapped across ALL branches. + for branch in all_branches: + if ( + len(branch.expressions) != len(key_exprs) + or branch.args.get("limit") + or branch.args.get("offset") + ): + unwrapped_indices.clear() + break + has_group = bool(branch.args.get("group")) + can_unwrap_group = ( + not branch.args.get("having") + and not branch.args.get("limit") + and not any(e.find(exp.AggFunc) for e in branch.expressions) + ) + if has_group and not can_unwrap_group: + unwrapped_indices.clear() + break + for idx in list(unwrapped_indices): + expected_transform = self._extract_time_partition_transform(key_exprs[idx]) + expected_unit = expected_transform[1] if expected_transform else "" + sel_item = branch.expressions[idx] + raw_sel = sel_item.this if isinstance(sel_item, exp.Alias) else sel_item + src_transform = self._extract_time_partition_transform(raw_sel) + if ( + src_transform is None + or src_transform[1] != expected_unit + or (self._is_date_part_extract_fn(raw_sel) and expected_unit != "year") + ): + unwrapped_indices.discard(idx) + + if any(branch.args.get("group") for branch in all_branches) and ( + unwrapped_indices != transform_indices + ): + unwrapped_indices.clear() + + # Pass 2: Mutate branch projections only for indices verified across all branches. + if unwrapped_indices: + for branch in all_branches: + branch_exprs = list(branch.expressions) + for idx in unwrapped_indices: + sel_item = branch_exprs[idx] + raw_sel = sel_item.this if isinstance(sel_item, exp.Alias) else sel_item + src_transform = self._extract_time_partition_transform(raw_sel) + assert src_transform is not None + branch_exprs[idx] = src_transform[0].copy() + if branch.args.get("group"): + branch.set("group", None) + branch.set("expressions", branch_exprs) + + for idx, k in enumerate(key_exprs): + transform = self._extract_time_partition_transform(k) + if transform is not None and idx not in unwrapped_indices: + _, unit = transform + is_year_fn = unit == "year" and self._is_date_part_extract_fn(k) + if not is_year_fn: + for branch in all_branches: + if idx < len(branch.expressions): + b_sel = branch.expressions[idx] + b_raw = b_sel.this if isinstance(b_sel, exp.Alias) else b_sel + b_trans = self._extract_time_partition_transform(b_raw) + if self._is_date_part_extract_fn(b_raw) or ( + b_trans is not None and b_trans[1] != unit + ): + raise SQLMeshError( + f"Doris delete_from cannot rewrite time-partition predicate '{k.sql(dialect=self.dialect)}' " + f"with mismatched subquery projection '{b_raw.sql(dialect=self.dialect)}' into a contiguous range." + ) + + new_first_exprs = list(first_select.expressions) + for idx, sel_item in enumerate(new_first_exprs): + inner_expr = sel_item.this if isinstance(sel_item, exp.Alias) else sel_item + new_first_exprs[idx] = exp.alias_(inner_expr.copy(), f"_col_{idx}", quoted=True) + first_select.set("expressions", new_first_exprs) + + if not select_query.args.get("limit") and not select_query.args.get("offset"): + select_query.set("order", None) + for branch in all_branches: + if not branch.args.get("limit") and not branch.args.get("offset"): + branch.set("order", None) + + for idx, k in enumerate(key_exprs): + sq_col = exp.column(f"_col_{idx}", quoted=True) + transform = self._extract_time_partition_transform(k) + if transform is not None: + _, unit = transform + is_extract_fn = self._is_date_part_extract_fn(k) + is_year_fn = unit == "year" and is_extract_fn + if idx not in unwrapped_indices and is_year_fn: + raw_year_indices.add(idx) + projections.append(exp.alias_(sq_col, f"_val_{idx}", quoted=True)) + continue + trunc_expr = exp.Anonymous( + this="DATE_TRUNC", + expressions=[sq_col, exp.Literal.string(unit)], + ) + upper_expr = exp.DateAdd( + this=trunc_expr.copy(), + expression=exp.Literal.number(1), + unit=exp.var(unit.upper()), + ) + projections.append(exp.alias_(trunc_expr, f"_low_{idx}", quoted=True)) + projections.append(exp.alias_(upper_expr, f"_high_{idx}", quoted=True)) + else: + projections.append(exp.alias_(sq_col, f"_val_{idx}", quoted=True)) + fetch_query = ( + exp.select(*projections) + .from_(exp.alias_(exp.Subquery(this=select_query), "_sq", table=True)) + .distinct() + ) + else: + raise SQLMeshError( + f"Doris delete_from cannot rewrite time-partition subquery '{select_query.sql(dialect=self.dialect)}': " + f"expected {len(key_exprs)} projection(s) matching key expressions, got {len(first_select.expressions)}." + ) + + rows = self.fetchall(fetch_query, quote_identifiers=True) + if not rows or not isinstance(rows, (list, tuple)): + return + + target_table = exp.to_table(table_name) + cleaned_extra_conditions: t.List[exp.Expr] = [] + for extra in extra_conditions or []: + cleaned_extra = extra.copy() + for col in cleaned_extra.find_all(exp.Column): + if not col.find_ancestor(exp.Subquery, exp.Select) and ( + col.table and col.table.lower() == target_table.name.lower() + ): + col.set("table", None) + col.set("db", None) + col.set("catalog", None) + cleaned_extra_conditions.append(cleaned_extra) + extra_condition_sets = self._expand_or_free_extra_condition_sets(cleaned_extra_conditions) + executed_delete_sqls: t.Set[str] = set() + + for row in rows: + predicates: t.List[exp.Expr] = [] + pos = 0 + skip_row = False + for idx, k in enumerate(key_exprs): + transform = self._extract_time_partition_transform(k) + if transform is not None: + col_expr, _ = transform + target_col = exp.column(col_expr.name, quoted=True) + if idx in raw_year_indices: + yr_val = self._sanitize_key_value(row[pos]) + pos += 1 + if yr_val is None: + skip_row = True + break + if isinstance(yr_val, (datetime.date, datetime.datetime)): + yr_int = yr_val.year + elif isinstance(yr_val, str): + yr_str = yr_val.strip() + yr_int = ( + int(yr_str[:4]) + if len(yr_str) >= 4 and yr_str[:4].isdigit() + else int(yr_str) + ) + else: + yr_int = int(yr_val) + low_val: t.Any = f"{yr_int:04d}-01-01" + high_val: t.Any = f"{yr_int + 1:04d}-01-01" + predicates.append( + exp.GTE( + this=target_col.copy(), + expression=self._to_delete_literal_expr(low_val), + ) + ) + predicates.append( + exp.LT( + this=target_col.copy(), + expression=self._to_delete_literal_expr(high_val), + ) + ) + continue + low_val = self._sanitize_key_value(row[pos]) + high_val = self._sanitize_key_value(row[pos + 1]) + pos += 2 + if low_val is None or high_val is None: + skip_row = True + break + predicates.append( + exp.GTE( + this=target_col.copy(), + expression=self._to_delete_literal_expr(low_val), + ) + ) + predicates.append( + exp.LT( + this=target_col.copy(), + expression=self._to_delete_literal_expr(high_val), + ) + ) + else: + target_col = exp.column(k.name, quoted=True) + val = self._sanitize_key_value(row[pos]) + pos += 1 + if val is None: + skip_row = True + break + predicates.append(target_col.eq(self._to_delete_literal_expr(val))) + if skip_row or not predicates: + continue + for extra_set in extra_condition_sets: + delete_expr = exp.delete(table_name, where=exp.and_(*predicates, *extra_set)) + delete_sql = self._to_sql(delete_expr) + if delete_sql not in executed_delete_sqls: + executed_delete_sqls.add(delete_sql) + self.execute(delete_expr) + + def _delete_by_literal_keys( + self, + table_name: TableName, + key_exprs: t.List[exp.Expr], + subquery: t.Union[exp.Subquery, exp.Select], + extra_conditions: t.Optional[t.List[exp.Expr]] = None, + ) -> None: + """Fallback for DELETE on DUPLICATE KEY / AGGREGATE KEY / Iceberg tables that prohibits OR predicates.""" + select_query = ( + subquery.this.copy() if isinstance(subquery, exp.Subquery) else subquery.copy() + ) + if isinstance(select_query, exp.Select): + if select_query.args.get("limit") or select_query.args.get("offset"): + if not select_query.args.get("order"): + select_query = select_query.distinct() + else: + select_query.set("order", None) + select_query = select_query.distinct() + rows = self.fetchall(select_query, quote_identifiers=True) + if not rows: + return + num_keys = len(key_exprs) + sanitized_rows = [ + sanitized_row + for row in rows + if len(row) >= num_keys + for sanitized_row in [tuple(self._sanitize_key_value(v) for v in row[:num_keys])] + if all(v is not None for v in sanitized_row) + ] + if not sanitized_rows: + return + self._delete_key_rows( + table_name, + key_exprs, + sanitized_rows, + extra_conditions=extra_conditions, + is_ast_values=False, + ) + + def _delete_key_rows( + self, + table_name: TableName, + key_exprs: t.List[exp.Expr], + rows: t.Sequence[t.Tuple[t.Any, ...]], + extra_conditions: t.Optional[t.List[exp.Expr]] = None, + *, + is_ast_values: bool = False, + ) -> None: + if not rows: + return + + target_table = exp.to_table(table_name) + cleaned_extra_conditions: t.List[exp.Expr] = [] + for extra in extra_conditions or []: + cleaned_extra = extra.copy() + for col in cleaned_extra.find_all(exp.Column): + if not col.find_ancestor(exp.Subquery, exp.Select) and ( + col.table and col.table.lower() == target_table.name.lower() + ): + col.set("table", None) + col.set("db", None) + col.set("catalog", None) + cleaned_extra_conditions.append(cleaned_extra) + extra_condition_sets = self._expand_or_free_extra_condition_sets(cleaned_extra_conditions) + cols = [exp.column(k.name, quoted=True) for k in key_exprs] + batch_size = self.DEFAULT_BATCH_SIZE + + def _is_null_val(val: t.Any) -> bool: + if val is None: + return True + if is_ast_values and isinstance(val, exp.Null): + return True + return False + + def _to_expr(val: t.Any) -> exp.Expr: + return self._to_delete_literal_expr(val, is_ast_values=is_ast_values) + + def _dedup_exprs(exprs: t.List[exp.Expr]) -> t.List[exp.Expr]: + seen: t.Set[str] = set() + unique_exprs: t.List[exp.Expr] = [] + for e in exprs: + key_sql = e.sql(dialect=self.dialect) + if key_sql not in seen: + seen.add(key_sql) + unique_exprs.append(e) + return unique_exprs + + if len(cols) == 1: + col = cols[0] + non_null_vals = _dedup_exprs([_to_expr(r[0]) for r in rows if not _is_null_val(r[0])]) + has_null = any(_is_null_val(r[0]) for r in rows) + + for i in range(0, len(non_null_vals), batch_size): + batch = non_null_vals[i : i + batch_size] + for extra_set in extra_condition_sets: + predicates: t.List[exp.Expr] = [col.isin(*batch), *extra_set] + self.execute(exp.delete(table_name, where=exp.and_(*predicates))) + + if has_null: + for extra_set in extra_condition_sets: + predicates = [col.is_(exp.null()), *extra_set] + self.execute(exp.delete(table_name, where=exp.and_(*predicates))) + return + + # Composite key (N >= 2): dynamically select the column index to batch via IN (...) + # that minimizes the number of distinct prefix groups, breaking ties in favor of the last column. + batch_idx = min( + range(len(cols)), + key=lambda idx: (len({row[:idx] + row[idx + 1 :] for row in rows}), -idx), + ) + grouped: t.Dict[t.Tuple[t.Any, ...], t.List[t.Any]] = defaultdict(list) + for row in rows: + prefix_key = row[:batch_idx] + row[batch_idx + 1 :] + grouped[prefix_key].append(row[batch_idx]) + + batch_col = cols[batch_idx] + prefix_cols = cols[:batch_idx] + cols[batch_idx + 1 :] + + for prefix_vals, batch_vals in grouped.items(): + prefix_preds: t.List[exp.Expr] = [ + p_col.eq(_to_expr(p_val)) if not _is_null_val(p_val) else p_col.is_(exp.null()) + for p_col, p_val in zip(prefix_cols, prefix_vals) + ] + non_null_batch = _dedup_exprs([_to_expr(v) for v in batch_vals if not _is_null_val(v)]) + has_null_batch = any(_is_null_val(v) for v in batch_vals) + + for i in range(0, len(non_null_batch), batch_size): + batch = non_null_batch[i : i + batch_size] + for extra_set in extra_condition_sets: + preds = [*prefix_preds, batch_col.isin(*batch), *extra_set] + self.execute(exp.delete(table_name, where=exp.and_(*preds))) + + if has_null_batch: + for extra_set in extra_condition_sets: + preds = [*prefix_preds, batch_col.is_(exp.null()), *extra_set] + self.execute(exp.delete(table_name, where=exp.and_(*preds))) + + @classmethod + def _resolve_table_type_and_keys( + cls, table_props: t.Dict[str, t.Any], pop: bool = False + ) -> t.Tuple[t.Any, t.Any, t.Any]: + get_fn = table_props.pop if pop else table_props.get + unique_key = get_fn("unique_key", None) + duplicate_key = get_fn("duplicate_key", None) + aggregate_key = get_fn("aggregate_key", None) + raw_table_type = get_fn("table_type", None) + raw_keys = get_fn("keys", None) + + if raw_table_type is not None and raw_keys is None: + raise SQLMeshError( + "Doris property 'table_type' requires 'keys' to be specified in physical_properties." + ) + if raw_keys is not None: + tt_str = ( + (cls._extract_str_value(raw_table_type) or "DUPLICATE") + .strip() + .strip("'\"`") + .upper() + ) + if tt_str in ( + "PRIMARY", + "PRIMARY_KEY", + "PRIMARY KEY", + "UNIQUE", + "UNIQUE_KEY", + "UNIQUE KEY", + ): + if unique_key is not None: + raise SQLMeshError( + "Cannot specify both 'unique_key' and 'table_type'/'keys' in physical_properties." + ) + unique_key = raw_keys + elif tt_str in ("DUPLICATE", "DUPLICATE_KEY", "DUPLICATE KEY"): + if duplicate_key is not None: + raise SQLMeshError( + "Cannot specify both 'duplicate_key' and 'table_type'/'keys' in physical_properties." + ) + duplicate_key = raw_keys + elif tt_str in ("AGGREGATE", "AGGREGATE_KEY", "AGGREGATE KEY"): + if aggregate_key is not None: + raise SQLMeshError( + "Cannot specify both 'aggregate_key' and 'table_type'/'keys' in physical_properties." + ) + aggregate_key = raw_keys + else: + raise SQLMeshError( + f"Invalid Doris table_type '{tt_str}'. Expected PRIMARY, UNIQUE, DUPLICATE, or AGGREGATE." + ) + return unique_key, duplicate_key, aggregate_key + + def adjust_physical_properties_for_incremental( + self, + physical_properties: t.Dict[str, t.Any], + *, + requires_delete_capable_table: bool, + unique_key: t.Optional[t.List[exp.Expr]], + model_name: str, + ) -> t.Dict[str, t.Any]: + """Promote ``unique_key`` from ``INCREMENTAL_BY_UNIQUE_KEY`` models into ``physical_properties`` + when no explicit OLAP key model (``unique_key``, ``duplicate_key``, ``aggregate_key``) is set. + """ + unique_key_prop, duplicate_key_prop, aggregate_key_prop = self._resolve_table_type_and_keys( + physical_properties + ) + if ( + unique_key + and not unique_key_prop + and not duplicate_key_prop + and not aggregate_key_prop + and all(isinstance(k, (exp.Column, exp.Identifier)) for k in unique_key) + ): + cleaned_keys = [exp.column(k.name) for k in unique_key] + physical_properties = dict(physical_properties) + physical_properties["unique_key"] = ( + cleaned_keys[0] if len(cleaned_keys) == 1 else exp.Tuple(expressions=cleaned_keys) + ) + return physical_properties + + def _create_table_from_columns( + self, + table_name: TableName, + target_columns_to_types: t.Dict[str, exp.DataType], + primary_key: t.Optional[t.Tuple[str, ...]] = None, + exists: bool = True, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + **kwargs: t.Any, + ) -> None: + table_properties = dict(self._get_table_properties(kwargs) or {}) + self._ensure_not_lance_table( + table_format=kwargs.get("table_format"), + table_properties=table_properties, + ) + unique_key_prop, duplicate_key_prop, aggregate_key_prop = self._resolve_table_type_and_keys( + table_properties + ) + + if ( + primary_key + and not unique_key_prop + and not duplicate_key_prop + and not aggregate_key_prop + ): + table_properties["unique_key"] = exp.Tuple( + expressions=[exp.to_column(col) for col in primary_key] + ) + + kwargs["table_properties"] = table_properties + + super()._create_table_from_columns( + table_name=table_name, + target_columns_to_types=target_columns_to_types, + primary_key=None, + exists=exists, + table_description=table_description, + column_descriptions=column_descriptions, + **kwargs, + ) + + def replace_query( + self, + table_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + source_columns: t.Optional[t.List[str]] = None, + supports_replace_table_override: t.Optional[bool] = None, + **kwargs: t.Any, + ) -> None: + target_table = exp.to_table(table_name) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=target_table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + if ( + not is_iceberg + and target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + thread_state = self._get_thread_state() + prev_replace_query_target = getattr(thread_state, "replace_query_target", None) + thread_state.replace_query_target = target_table + try: + super().replace_query( + table_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + table_description=table_description, + column_descriptions=column_descriptions, + source_columns=source_columns, + supports_replace_table_override=supports_replace_table_override, + **kwargs, + ) + finally: + thread_state.replace_query_target = prev_replace_query_target + + def _get_doris_version(self) -> t.Optional[t.Tuple[int, int]]: + cached = getattr(self, "_doris_version", None) + if cached is not None: + return cached + row: t.Any = None + try: + row = self.fetchone("SELECT @@version_comment, VERSION()") + except Exception: + try: + row = self.fetchone("SELECT VERSION()") + except Exception: + return None + if not isinstance(row, (list, tuple)): + return None + texts = [_decoded_str(v) for v in row if v is not None] + for text in texts: + m = re.search( + r"doris[-_\s]*(?:version\s*)?(?:doris[-_\s]*)?(\d+)\.(\d+)", + text, + re.IGNORECASE, + ) + if m: + self._doris_version: t.Optional[t.Tuple[int, int]] = ( + int(m.group(1)), + int(m.group(2)), + ) + return self._doris_version + for text in texts: + if text.strip() == "5.7.99" and len(texts) > 1: + continue + m = re.search(r"(\d+)\.(\d+)", text) + if m: + ver = (int(m.group(1)), int(m.group(2))) + if ver != (5, 7): + self._doris_version = ver + return self._doris_version + return None + + def _ensure_native_merge_supported( + self, + target_table: TableName, + resolved_uk: t.Optional[exp.Expr], + duplicate_key_prop: t.Optional[exp.Expr], + aggregate_key_prop: t.Optional[exp.Expr], + is_iceberg: bool, + table_format: t.Optional[str] = None, + ) -> None: + table = exp.to_table(target_table) + effective_catalog = table.catalog or self.default_catalog + fmt_lower = str(table_format).strip().lower() if table_format else "" + if ( + is_iceberg + or (effective_catalog and effective_catalog.lower() != "internal") + or (fmt_lower and fmt_lower not in ("olap", "doris")) + ): + raise SQLMeshError( + "Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + "is only supported on internal OLAP UNIQUE KEY tables." + ) + if duplicate_key_prop is not None or aggregate_key_prop is not None: + key_model = "DUPLICATE KEY" if duplicate_key_prop is not None else "AGGREGATE KEY" + raise SQLMeshError( + f"Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + f"requires a UNIQUE KEY target table, but '{target_table}' is configured as {key_model}." + ) + version = self._get_doris_version() + if version is None: + raise SQLMeshError( + "Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + "requires Apache Doris 4.1+, but the server version could not be determined." + ) + if version < (4, 1): + raise SQLMeshError( + f"Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + f"requires Apache Doris 4.1+ (detected Doris {version[0]}.{version[1]})." + ) + if resolved_uk is None: + try: + ddl_row = self.fetchone(f"SHOW CREATE TABLE {self._to_sql(table)}") + except Exception as exc: + raise SQLMeshError( + f"Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + f"requires verifying that '{target_table}' is a UNIQUE KEY table, but SHOW CREATE TABLE failed: {exc}" + ) from exc + create_ddl = ( + _decoded_str(ddl_row[1]) + if isinstance(ddl_row, (list, tuple)) + and len(ddl_row) >= 2 + and ddl_row[1] is not None + else "" + ) + if not re.search(r"\bUNIQUE\s+KEY\s*\(", create_ddl, re.IGNORECASE): + raise SQLMeshError( + f"Doris native MERGE INTO (required when 'merge_filter' or 'when_matched' is specified) " + f"requires a UNIQUE KEY target table, but '{target_table}' is not a UNIQUE KEY table." + ) + + def merge( + self, + target_table: TableName, + source_table: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]], + unique_key: t.Sequence[exp.Expr], + when_matched: t.Optional[exp.Whens] = None, + merge_filter: t.Optional[exp.Expr] = None, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + if when_matched is None and merge_filter is None: + self._replace_by_key( + target_table, + source_table, + target_columns_to_types, + unique_key, + is_unique_key=True, + source_columns=source_columns, + **kwargs, + ) + return + + self._last_described_key_cols = set() + table = exp.to_table(target_table) + table_props = self._get_table_properties(kwargs) + resolved_uk, duplicate_key_prop, aggregate_key_prop = ( + self._resolve_table_type_and_keys(table_props) if table_props else (None, None, None) + ) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=table.catalog, + table_properties=table_props, + ) + self._ensure_native_merge_supported( + target_table, + resolved_uk=resolved_uk, + duplicate_key_prop=duplicate_key_prop, + aggregate_key_prop=aggregate_key_prop, + is_iceberg=is_iceberg, + table_format=kwargs.get("table_format"), + ) + if not target_columns_to_types or not columns_to_types_all_known(target_columns_to_types): + target_columns_to_types = self.columns(target_table) + else: + if not is_iceberg: + reorder_kwargs = dict(kwargs) + if ( + not duplicate_key_prop + and not aggregate_key_prop + and all(isinstance(k, (exp.Column, exp.Identifier)) for k in unique_key) + ): + reorder_kwargs["unique_key"] = unique_key + target_columns_to_types = self._reorder_olap_columns( + target_columns_to_types, + reorder_kwargs, + ) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + + normalized_unique_key: t.List[exp.Expr] = [ + exp.column(k) if isinstance(k, exp.Identifier) else k for k in unique_key + ] + key_col_names_lower = { + c.name.lower() for k in normalized_unique_key for c in k.find_all(exp.Column) if c.name + } + key_col_names_lower.update(k.lower() for k in self._last_described_key_cols) + if resolved_uk: + key_col_names_lower.update(k.lower() for k in self._extract_column_names(resolved_uk)) + + if not when_matched: + non_key_cols = [ + col for col in target_columns_to_types if col.lower() not in key_col_names_lower + ] + update_cols = non_key_cols or list(target_columns_to_types) + when_matched = exp.Whens( + expressions=[ + exp.When( + matched=True, + source=False, + then=exp.Update( + expressions=[ + exp.column(col, MERGE_TARGET_ALIAS).eq( + exp.column(col, MERGE_SOURCE_ALIAS) + ) + for col in update_cols + ], + ), + ) + ] + ) + else: + when_matched = when_matched.copy() + for when_expr in when_matched.find_all(exp.When): + then_expr = when_expr.args.get("then") + if isinstance(then_expr, exp.Update): + filtered_eqs = [ + eq_expr + for eq_expr in then_expr.expressions + if not ( + isinstance(eq_expr, exp.EQ) + and isinstance(eq_expr.this, exp.Column) + and eq_expr.this.name.lower() in key_col_names_lower + ) + ] + if filtered_eqs: + then_expr.set("expressions", filtered_eqs) + elif then_expr.expressions: + non_key_cols = [ + col + for col in target_columns_to_types + if col.lower() not in key_col_names_lower + ] + if non_key_cols: + fallback_col = non_key_cols[0] + then_expr.set( + "expressions", + [ + exp.column(fallback_col, MERGE_TARGET_ALIAS).eq( + exp.column(fallback_col, MERGE_TARGET_ALIAS) + ) + ], + ) + + super().merge( + target_table, + source_table, + target_columns_to_types, + normalized_unique_key, + when_matched=when_matched, + merge_filter=merge_filter, + source_columns=source_columns, + **kwargs, + ) + + def _replace_by_key( + self, + target_table: TableName, + source_table: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]], + key: t.Sequence[exp.Expr], + is_unique_key: bool, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + if not target_columns_to_types or not columns_to_types_all_known(target_columns_to_types): + target_columns_to_types = self.columns(target_table) + + table = exp.to_table(target_table) + table_props = self._get_table_properties(kwargs) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=table.catalog, + table_properties=table_props, + ) + normalized_key: t.List[exp.Expr] = [ + exp.column(k.name, quoted=bool(k.quoted)) if isinstance(k, exp.Identifier) else k + for k in key + ] + key_cols: t.List[exp.Column] = [] + if is_unique_key and all(isinstance(k, exp.Column) for k in normalized_key): + key_cols = [exp.column(k.name) for k in normalized_key] + + has_sequence_col = bool( + not is_iceberg + and is_unique_key + and table_props + and ( + table_props.get("function_column.sequence_col") is not None + or table_props.get("function_column.sequence_type") is not None + ) + ) + + temp_table_props: t.Optional[t.Dict[str, t.Any]] = None + if not is_iceberg: + col_types_by_lower = { + c.lower(): dtype for c, dtype in (target_columns_to_types or {}).items() + } + norm_key_names_lower = { + k.name.lower() for k in normalized_key if isinstance(k, exp.Column) + } + has_invalid_or_coercible_key_col = any( + k_low in col_types_by_lower + and ( + not self._is_valid_olap_key_type(col_types_by_lower[k_low]) + or self._is_coercible_string_key_type(col_types_by_lower[k_low]) + ) + for k_low in norm_key_names_lower + ) + has_other_valid_olap_key_col = any( + c_low not in norm_key_names_lower + and self._is_valid_olap_key_type(dtype) + and not self._is_coercible_string_key_type(dtype) + for c_low, dtype in col_types_by_lower.items() + ) + avoid_key_coercion = has_invalid_or_coercible_key_col and has_other_valid_olap_key_col + if key_cols and not avoid_key_coercion: + temp_table_props = { + "unique_key": ( + key_cols[0] if len(key_cols) == 1 else exp.Tuple(expressions=key_cols) + ) + } + if table_props: + for seq_k in ("function_column.sequence_col", "function_column.sequence_type"): + if table_props.get(seq_k) is not None: + temp_table_props[seq_k] = table_props[seq_k] + if target_columns_to_types and columns_to_types_all_known(target_columns_to_types): + if avoid_key_coercion: + reorder_kwargs: t.Dict[str, t.Any] = {} + elif temp_table_props: + reorder_kwargs = {"table_properties": temp_table_props} + elif normalized_key: + reorder_kwargs = {"partitioned_by": list(normalized_key)} + else: + reorder_kwargs = {} + target_columns_to_types = self._reorder_olap_columns( + target_columns_to_types, reorder_kwargs + ) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + + temp_table = self._get_temp_table(target_table) + key_exp = ( + exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", *normalized_key, dialect=self.dialect) + if len(normalized_key) > 1 + else normalized_key[0] + ) + column_names = list(target_columns_to_types or []) + + with self.transaction(): + self.ctas( + temp_table, + source_table, + target_columns_to_types=target_columns_to_types, + exists=False, + source_columns=source_columns, + table_properties=temp_table_props, + table_format=kwargs.get("table_format"), + ) + + try: + delete_query = exp.select(key_exp).from_(temp_table) + insert_query = self._select_columns(target_columns_to_types).from_(temp_table) + if not is_unique_key: + delete_query = delete_query.distinct() + else: + insert_query = insert_query.distinct(*normalized_key) + + insert_statement = exp.insert( + insert_query, + target_table, + columns=column_names, + ) + delete_filter = key_exp.isin(query=delete_query) + + if not self.INSERT_OVERWRITE_STRATEGY.is_replace_where: + if not has_sequence_col: + prev_iceberg_ctx = getattr(self, "_in_iceberg_replace_by_key", False) + self._in_iceberg_replace_by_key = is_iceberg + try: + self.delete_from(target_table, delete_filter) + finally: + self._in_iceberg_replace_by_key = prev_iceberg_ctx + else: + insert_statement.set("where", delete_filter) + insert_statement.set("this", exp.to_table(target_table)) + + self.execute(insert_statement, track_rows_processed=True) + finally: + self.drop_table(temp_table) + + def _scd_type_2( + self, + target_table: TableName, + source_table: QueryOrDF, + unique_key: t.Sequence[exp.Expr], + valid_from_col: exp.Column, + valid_to_col: exp.Column, + execution_time: t.Union[TimeLike, exp.Column], + invalidate_hard_deletes: bool = True, + updated_at_col: t.Optional[exp.Column] = None, + check_columns: t.Optional[t.Union[exp.Star, t.Sequence[exp.Expr]]] = None, + updated_at_as_valid_from: bool = False, + execution_time_as_valid_from: bool = False, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + truncate: bool = False, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + table = exp.to_table(target_table) + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + if ( + not target_columns_to_types + or valid_from_col.name not in target_columns_to_types + or valid_to_col.name not in target_columns_to_types + or not columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types = self.columns(target_table) + if ( + not is_iceberg + and target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + ): + target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) + if source_columns: + source_col_set = set(source_columns) + source_columns = [c for c in target_columns_to_types if c in source_col_set] + [ + c for c in source_columns if c not in target_columns_to_types + ] + super()._scd_type_2( + target_table=target_table, + source_table=source_table, + unique_key=unique_key, + valid_from_col=valid_from_col, + valid_to_col=valid_to_col, + execution_time=execution_time, + invalidate_hard_deletes=invalidate_hard_deletes, + updated_at_col=updated_at_col, + check_columns=check_columns, + updated_at_as_valid_from=updated_at_as_valid_from, + execution_time_as_valid_from=execution_time_as_valid_from, + target_columns_to_types=target_columns_to_types, + table_description=table_description, + column_descriptions=column_descriptions, + truncate=truncate, + source_columns=source_columns, + **kwargs, + ) + + def _create_table_from_source_queries( + self, + table_name: TableName, + source_queries: t.List[SourceQuery], + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + exists: bool = True, + replace: bool = False, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + table_kind: t.Optional[str] = None, + track_rows_processed: bool = True, + **kwargs: t.Any, + ) -> None: + self._ensure_not_lance_table( + table_format=kwargs.get("table_format"), + table_properties=self._get_table_properties(kwargs), + ) + table = exp.to_table(table_name) + effective_catalog = table.catalog or self.default_catalog + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=table.catalog, + table_properties=self._get_table_properties(kwargs), + ) + is_internal_iceberg = is_iceberg and ( + not effective_catalog or effective_catalog.lower() == "internal" + ) + + target_columns_to_types_known = bool( + target_columns_to_types and columns_to_types_all_known(target_columns_to_types) + ) + if not is_iceberg and target_columns_to_types and target_columns_to_types_known: + target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) + first_query = source_queries[0].query_factory() if source_queries else None + + def _matches_target_table(t_node: exp.Table) -> bool: + q_node = quote_identifiers(t_node.copy()) + q_target = quote_identifiers(table.copy()) + if (q_node.args.get("catalog"), q_node.args.get("db"), q_node.this) == ( + q_target.args.get("catalog"), + q_target.args.get("db"), + q_target.this, + ): + return True + default_cat = self.default_catalog or "internal" + norm_node = t_node.copy() + if not norm_node.catalog and norm_node.db: + norm_node.set("catalog", exp.parse_identifier(default_cat)) + norm_target = table.copy() + if not norm_target.catalog and norm_target.db: + norm_target.set("catalog", exp.parse_identifier(default_cat)) + q_norm_node = quote_identifiers(norm_node) + q_norm_target = quote_identifiers(norm_target) + return ( + q_norm_node.args.get("catalog"), + q_norm_node.args.get("db"), + q_norm_node.this, + ) == ( + q_norm_target.args.get("catalog"), + q_norm_target.args.get("db"), + q_norm_target.this, + ) + + self_referencing = bool( + first_query + and any(_matches_target_table(t_node) for t_node in first_query.find_all(exp.Table)) + ) + + # 1. Doris's CTAS grammar forbids `ENGINE=iceberg`, so internal/2-part Iceberg tables with known + # column types must be created via schema DDL (`_create_table_from_columns`) + `INSERT INTO`. + # 2. When `replace_query` is called with a self-referencing query on a non-existent table, + # `EngineAdapter.replace_query` creates the empty table via `_create_table_from_columns` + # first and then calls `_create_table_from_source_queries(..., replace=False)`. Executing + # `CREATE TABLE IF NOT EXISTS ... AS SELECT` would be a no-op, so we insert the source queries. + if is_internal_iceberg and not target_columns_to_types_known: + raise SQLMeshError( + f"Doris does not support `CREATE TABLE ... AS SELECT` with `ENGINE=iceberg` on table '{table_name}' " + "without known column types. Either qualify the table with an external Iceberg catalog " + "(`..
`) or specify explicit column types." + ) + + replace_query_target = getattr(self._get_thread_state(), "replace_query_target", None) + from_replace_query = bool( + isinstance(replace_query_target, exp.Table) + and _matches_target_table(replace_query_target) + ) + precreated_by_replace_query = bool(self_referencing and from_replace_query and not replace) + + if (is_internal_iceberg or precreated_by_replace_query) and target_columns_to_types_known: + assert target_columns_to_types is not None + target_already_existed = bool( + is_internal_iceberg + and exists + and not replace + and not from_replace_query + and self.get_data_object(table) is not None + ) + if not precreated_by_replace_query: + if replace: + self.drop_table(table_name, exists=True) + self._create_table_from_columns( + table_name, + target_columns_to_types, + exists=exists, + table_description=table_description, + column_descriptions=column_descriptions, + **kwargs, + ) + if target_already_existed: + return + for source_query in source_queries: + with source_query as query: + query = self._order_projections_and_filter( + query, target_columns_to_types, coerce_types=True + ) + self._insert_append_query( + table_name, + query, + target_columns_to_types, + track_rows_processed=track_rows_processed, + ) + return + + super()._create_table_from_source_queries( + table_name, + source_queries, + target_columns_to_types=target_columns_to_types, + exists=exists, + replace=replace, + table_description=table_description, + column_descriptions=column_descriptions, + table_kind=table_kind, + track_rows_processed=track_rows_processed, + **kwargs, + ) + + @staticmethod + def _remap_positional_group_and_order( + node: exp.Expr, old_to_new_one_based: t.Dict[int, int] + ) -> None: + if all(k == v for k, v in old_to_new_one_based.items()): + return + + def _remap_lit(item: exp.Expr) -> None: + inner = item.this if isinstance(item, exp.Ordered) else item + if isinstance(inner, exp.Literal) and inner.is_int: + old_pos = int(inner.this) + if old_pos in old_to_new_one_based: + inner.set("this", str(old_to_new_one_based[old_pos])) + elif isinstance(inner, (exp.Tuple, exp.Paren)): + for sub in inner.expressions or ([inner.this] if inner.this else []): + _remap_lit(sub) + + group = node.args.get("group") + if isinstance(group, exp.Group): + for item in group.expressions: + _remap_lit(item) + for prop_key in ("rollup", "cube", "grouping_sets"): + prop_val = group.args.get(prop_key) + if isinstance(prop_val, list): + for g_item in prop_val: + if isinstance(g_item, exp.Expr): + _remap_lit(g_item) + elif isinstance(prop_val, exp.Expr): + for g_item in prop_val.expressions: + _remap_lit(g_item) + order = node.args.get("order") + if isinstance(order, exp.Order): + for item in order.expressions: + _remap_lit(item) + + def _build_create_table_exp( + self, + table_name_or_schema: t.Union[exp.Schema, TableName], + expression: t.Optional[exp.Expr], + exists: bool = True, + replace: bool = False, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + table_description: t.Optional[str] = None, + table_kind: t.Optional[str] = None, + **kwargs: t.Any, + ) -> exp.Create: + if kwargs.get("table_properties") is None and kwargs.get("physical_properties") is not None: + kwargs["table_properties"] = kwargs["physical_properties"] + else: + kwargs.setdefault("table_properties", None) + self._ensure_not_lance_table( + table_format=kwargs.get("table_format"), + table_properties=kwargs.get("table_properties"), + ) + kwargs["is_schema_ddl"] = isinstance(table_name_or_schema, exp.Schema) + + catalog_name = None + if isinstance(table_name_or_schema, exp.Schema): + if isinstance(table_name_or_schema.this, exp.Table): + catalog_name = table_name_or_schema.this.catalog + else: + catalog_name = exp.to_table(table_name_or_schema).catalog + + is_iceberg = self._is_iceberg_table( + table_format=kwargs.get("table_format"), + catalog_name=catalog_name, + table_properties=self._get_table_properties(kwargs), + ) + + if ( + not is_iceberg + and target_columns_to_types + and columns_to_types_all_known(target_columns_to_types) + ): + original_cols = dict(target_columns_to_types) + reordered_cols = self._reorder_olap_columns(target_columns_to_types, kwargs) + order_changed = list(reordered_cols) != list(original_cols) + types_changed = any(reordered_cols[c] != original_cols[c] for c in reordered_cols) + query_order_differs = bool( + isinstance(expression, exp.Select) + and len(expression.expressions) == len(reordered_cols) + and [p.alias_or_name.lower() for p in expression.expressions if p.alias_or_name] + != [c.lower() for c in reordered_cols] + ) + if order_changed or types_changed or query_order_differs: + target_columns_to_types = reordered_cols + if isinstance(table_name_or_schema, exp.Schema): + col_def_map = { + col_def.name.lower(): col_def + for col_def in table_name_or_schema.expressions + if isinstance(col_def, exp.ColumnDef) + } + other_exprs = [ + e + for e in table_name_or_schema.expressions + if not isinstance(e, exp.ColumnDef) + ] + if len(col_def_map) == len(target_columns_to_types) and all( + c.lower() in col_def_map for c in target_columns_to_types + ): + new_col_defs: t.List[exp.Expr] = [] + for c, new_dtype in target_columns_to_types.items(): + col_def = col_def_map[c.lower()] + if col_def.args.get("kind") != new_dtype: + col_def = col_def.copy() + col_def.set("kind", new_dtype) + new_col_defs.append(col_def) + table_name_or_schema = table_name_or_schema.copy() + table_name_or_schema.set( + "expressions", + new_col_defs + other_exprs, + ) + if ( + (order_changed or types_changed or query_order_differs) + and expression is not None + and isinstance(expression, exp.Query) + ): + reordered_in_place = False + if isinstance(expression, exp.Select): + proj_by_name = { + proj.alias_or_name.lower(): proj + for proj in expression.expressions + if proj.alias_or_name + } + if len(proj_by_name) == len(expression.expressions) == len( + target_columns_to_types + ) and all(c.lower() in proj_by_name for c in target_columns_to_types): + old_pos_by_lower = { + proj.alias_or_name.lower(): idx + 1 + for idx, proj in enumerate(expression.expressions) + if proj.alias_or_name + } + expression = expression.copy() + updated_projs: t.List[exp.Expr] = [] + old_to_new_one_based: t.Dict[int, int] = {} + for new_idx, (c, new_dtype) in enumerate( + target_columns_to_types.items(), start=1 + ): + old_to_new_one_based[old_pos_by_lower[c.lower()]] = new_idx + proj = proj_by_name[c.lower()].copy() + if types_changed and original_cols[c] != new_dtype: + if isinstance(proj, exp.Alias) and isinstance( + proj.this, exp.Cast + ): + proj.this.set("to", new_dtype.copy()) + elif isinstance(proj, exp.Alias): + proj.set( + "this", + exp.cast(proj.this.copy(), to=new_dtype.copy()), + ) + elif isinstance(proj, exp.Column): + proj = exp.alias_( + exp.cast(proj.copy(), to=new_dtype.copy()), + proj.name, + quoted=bool(getattr(proj.this, "quoted", True)), + ) + updated_projs.append(proj) + expression.set("expressions", updated_projs) + self._remap_positional_group_and_order(expression, old_to_new_one_based) + reordered_in_place = True + if not reordered_in_place: + expression = self._order_projections_and_filter( + expression, target_columns_to_types, coerce_types=types_changed + ) + elif not is_iceberg and isinstance(expression, (exp.Select, exp.SetOperation)): + known_col_types = ( + { + c.lower(): dtype.copy() + for c, dtype in target_columns_to_types.items() + if dtype.this != exp.DataType.Type.UNKNOWN + } + if target_columns_to_types + else None + ) + expression = self._reorder_unannotated_ctas_query( + expression, kwargs, cte_types=known_col_types + ) + if target_columns_to_types: + first_select: t.Optional[exp.Select] = None + curr_node: t.Optional[exp.Expr] = expression + while isinstance(curr_node, (exp.SetOperation, exp.Subquery)): + curr_node = curr_node.this + if isinstance(curr_node, exp.Select): + first_select = curr_node + if first_select is not None: + col_by_lower = {c.lower(): c for c in target_columns_to_types} + reordered_names = [ + col_by_lower[p.alias_or_name.lower()] + for p in first_select.expressions + if p.alias_or_name and p.alias_or_name.lower() in col_by_lower + ] + if len(reordered_names) == len(target_columns_to_types): + target_columns_to_types = { + c: target_columns_to_types[c] for c in reordered_names + } + + # Doris AUTO PARTITION BY RANGE requires the partition column(s) to be defined as NOT NULL, + # and users may also specify physical_properties (not_null_columns = (...)) for columns + # that must be created as NOT NULL (e.g. ARRAY NOT NULL for ANN vector indexes, since + # Doris does not allow ALTER TABLE ... MODIFY COLUMN to change NULL to NOT NULL). + if not is_iceberg: + table_props = self._get_table_properties(kwargs) or {} + raw_not_null_cols = kwargs.get("not_null_columns") or table_props.get( + "not_null_columns" + ) + explicit_not_null_cols = { + c.lower() for c in self._extract_column_names(raw_not_null_cols) + } + if explicit_not_null_cols: + known_cols_lower: t.Set[str] = set() + if target_columns_to_types: + known_cols_lower = {c.lower() for c in target_columns_to_types} + elif isinstance(table_name_or_schema, exp.Schema): + known_cols_lower = { + expr.name.lower() + for expr in table_name_or_schema.expressions + if isinstance(expr, exp.ColumnDef) + } + elif isinstance(expression, exp.Select) and not any( + isinstance(proj, exp.Star) or isinstance(proj.this, exp.Star) + for proj in expression.expressions + ): + known_cols_lower = { + proj.alias_or_name.lower() + for proj in expression.expressions + if proj.alias_or_name + } + if known_cols_lower: + missing_not_null = sorted(explicit_not_null_cols - known_cols_lower) + if missing_not_null: + raise SQLMeshError( + f"not_null_columns {missing_not_null} not found in table columns {sorted(known_cols_lower)}." + ) + + not_null_cols: t.Set[str] = set(explicit_not_null_cols) + auto_partition = table_props.get("auto_partition") + is_auto = bool( + auto_partition is not None + and str(self._extract_str_value(auto_partition) or "").strip().lower() + in ("true", "1") + ) + raw_part_by = ( + kwargs.get("partitioned_by") + or table_props.get("partitioned_by") + or table_props.get("partition_by") + ) + if is_auto and raw_part_by: + part_list = raw_part_by if isinstance(raw_part_by, list) else [raw_part_by] + parsed_part_exprs, part_kind = self._parse_partition_expressions(part_list) + raw_partitions_val = table_props.get("partitions") + inferred_list_part = ( + "VALUES IN" + in str( + self._extract_str_value(raw_partitions_val) or raw_partitions_val or "" + ).upper() + ) + if (part_kind or ("LIST" if inferred_list_part else "RANGE")).upper() != "LIST": + auto_range_cols = { + col.name.lower() + for expr in parsed_part_exprs + for col in expr.find_all(exp.Column) + } + resolved_uk, _, resolved_ak = self._resolve_table_type_and_keys(table_props) + strict_key = resolved_uk or kwargs.get("unique_key") or resolved_ak + strict_key_set = {k.lower() for k in self._extract_column_names(strict_key)} + if strict_key_set and not auto_range_cols.issubset(strict_key_set): + auto_range_cols = set() + not_null_cols |= auto_range_cols + + if not_null_cols and isinstance(table_name_or_schema, exp.Schema): + table_name_or_schema = table_name_or_schema.copy() + updated_exprs: t.List[exp.Expr] = [] + for expr in table_name_or_schema.expressions: + if isinstance(expr, exp.ColumnDef) and expr.name.lower() in not_null_cols: + existing_constraints = list(expr.args.get("constraints") or []) + has_not_null = any( + isinstance(c, exp.NotNullColumnConstraint) + or isinstance(getattr(c, "kind", None), exp.NotNullColumnConstraint) + for c in existing_constraints + ) + if not has_not_null: + expr = expr.copy() + expr.set( + "constraints", + [exp.ColumnConstraint(kind=exp.NotNullColumnConstraint())] + + existing_constraints, + ) + updated_exprs.append(expr) + table_name_or_schema.set("expressions", updated_exprs) + elif not_null_cols and isinstance(expression, (exp.Select, exp.SetOperation)): + expression = expression.copy() + auto_select_branches: t.List[exp.Select] = [] + + def _collect_auto_branches(node: exp.Expr) -> None: + if isinstance(node, exp.Subquery): + _collect_auto_branches(node.this) + elif isinstance(node, exp.SetOperation): + _collect_auto_branches(node.this) + _collect_auto_branches(node.expression) + elif isinstance(node, exp.Select): + auto_select_branches.append(node) + + _collect_auto_branches(expression) + if auto_select_branches: + target_indices = { + idx + for idx, proj in enumerate(auto_select_branches[0].expressions) + if proj.alias_or_name.lower() in not_null_cols + } + for branch_idx, branch in enumerate(auto_select_branches): + new_select_exprs: t.List[exp.Expr] = [] + for idx, proj in enumerate(branch.expressions): + if proj.alias_or_name.lower() in not_null_cols or idx in target_indices: + if isinstance(proj, exp.Alias): + inner = proj.this + if not ( + isinstance(inner, exp.Anonymous) + and inner.name.upper() == "NON_NULLABLE" + ): + proj = proj.copy() + proj.set( + "this", + exp.Anonymous( + this="NON_NULLABLE", + expressions=[inner.copy()], + ), + ) + elif isinstance(proj, exp.Column): + wrapped = exp.Anonymous( + this="NON_NULLABLE", expressions=[proj.copy()] + ) + proj = ( + exp.alias_( + wrapped, + proj.name, + quoted=bool(getattr(proj.this, "quoted", True)), + ) + if branch_idx == 0 + else wrapped + ) + elif not ( + isinstance(proj, exp.Anonymous) + and proj.name.upper() == "NON_NULLABLE" + ): + proj = exp.Anonymous( + this="NON_NULLABLE", expressions=[proj.copy()] + ) + new_select_exprs.append(proj) + branch.set("expressions", new_select_exprs) + + return super()._build_create_table_exp( + table_name_or_schema, + expression=expression, + exists=exists, + replace=replace, + target_columns_to_types=target_columns_to_types, + table_description=table_description, + table_kind=table_kind, + **kwargs, + ) + + @staticmethod + def _is_integer_indicator_projection(proj: exp.Expr) -> bool: + inner = proj.this if isinstance(proj, exp.Alias) else proj + if isinstance(inner, exp.Literal) and inner.is_int: + return True + if isinstance(inner, exp.If): + t_val, f_val = inner.args.get("true"), inner.args.get("false") + return bool( + isinstance(t_val, exp.Literal) + and t_val.is_int + and isinstance(f_val, exp.Literal) + and f_val.is_int + ) + if isinstance(inner, exp.Case): + ifs = inner.args.get("ifs") or [] + default = inner.args.get("default") + return bool( + ifs + and isinstance(default, exp.Literal) + and default.is_int + and all( + isinstance(i.args.get("true"), exp.Literal) and i.args["true"].is_int + for i in ifs + ) + ) + return False + + def _infer_projection_type( + self, proj: exp.Expr, cte_types: t.Optional[t.Dict[str, exp.DataType]] = None + ) -> exp.DataType: + inner = proj.this if isinstance(proj, exp.Alias) else proj + inner = inner.unnest() + while isinstance(inner, exp.Window) and isinstance(inner.this, exp.Expr): + inner = inner.this.unnest() + if isinstance(inner, exp.Column) and cte_types and inner.name.lower() in cte_types: + return cte_types[inner.name.lower()].copy() + if isinstance(inner, (exp.Cast, exp.TryCast)) and isinstance(inner.to, exp.DataType): + return inner.to.copy() + if self._is_integer_indicator_projection(proj): + return exp.DataType.build("INT", dialect=self.dialect) + if isinstance(inner, exp.Literal): + if inner.is_int: + return exp.DataType.build("INT", dialect=self.dialect) + if inner.is_number: + return exp.DataType.build("DOUBLE", dialect=self.dialect) + if inner.is_string: + return exp.DataType.build("VARCHAR(255)", dialect=self.dialect) + if isinstance(inner, exp.Boolean): + return exp.DataType.build("BOOLEAN", dialect=self.dialect) + if isinstance(inner, exp.Div): + return exp.DataType.build("DOUBLE", dialect=self.dialect) + if isinstance(inner, (exp.Add, exp.Sub, exp.Mul)): + left_t = self._infer_projection_type(inner.this, cte_types) + right_t = self._infer_projection_type(inner.expression, cte_types) + if any(t_cand.is_type("float", "double") for t_cand in (left_t, right_t)): + return exp.DataType.build("DOUBLE", dialect=self.dialect) + if isinstance(inner, exp.If): + branch_types = [ + self._infer_projection_type(b_expr, cte_types) + for b_expr in (inner.args.get("true"), inner.args.get("false")) + if isinstance(b_expr, exp.Expr) + ] + known_b = [bt for bt in branch_types if bt.this != exp.DataType.Type.UNKNOWN] + if known_b: + non_key_b = next( + (bt for bt in known_b if not self._is_valid_olap_key_type(bt)), None + ) + return non_key_b or known_b[0] + if isinstance(inner, exp.Case): + case_exprs: t.List[exp.Expr] = [ + i.args["true"] + for i in (inner.args.get("ifs") or []) + if isinstance(i.args.get("true"), exp.Expr) + ] + if isinstance(inner.args.get("default"), exp.Expr): + case_exprs.append(inner.args["default"]) + known_c = [ + ct + for ce in case_exprs + if (ct := self._infer_projection_type(ce, cte_types)).this + != exp.DataType.Type.UNKNOWN + ] + if known_c: + non_key_c = next( + (ct for ct in known_c if not self._is_valid_olap_key_type(ct)), None + ) + return non_key_c or known_c[0] + if isinstance(inner, exp.Coalesce): + coal_exprs = [inner.this] + list(inner.expressions or []) + known_coal = [ + ct + for ce in coal_exprs + if isinstance(ce, exp.Expr) + and (ct := self._infer_projection_type(ce, cte_types)).this + != exp.DataType.Type.UNKNOWN + ] + if known_coal: + non_key_coal = next( + (ct for ct in known_coal if not self._is_valid_olap_key_type(ct)), None + ) + return non_key_coal or known_coal[0] + if isinstance( + inner, + ( + exp.Avg, + exp.Stddev, + exp.StddevPop, + exp.StddevSamp, + exp.Variance, + exp.VariancePop, + exp.EuclideanDistance, + exp.CosineDistance, + exp.ManhattanDistance, + exp.DotProduct, + ), + ): + return exp.DataType.build("DOUBLE", dialect=self.dialect) + if isinstance(inner, (exp.Array, exp.ArrayAgg)): + return exp.DataType.build("ARRAY", dialect=self.dialect) + if isinstance(inner, (exp.Map, exp.VarMap)): + return exp.DataType.build("MAP", dialect=self.dialect) + if isinstance(inner, exp.Struct): + return exp.DataType.build("STRUCT", dialect=self.dialect, udt=True) + if isinstance(inner, (exp.JSONObject, exp.JSONArray, exp.ParseJSON, exp.JSONExtract)): + return exp.DataType.build("JSON", dialect=self.dialect) + if isinstance(inner, exp.Bracket) and isinstance(inner.this, exp.Expr): + base_t = self._infer_projection_type(inner.this, cte_types) + base_type_name = ( + base_t.this.value.upper() + if isinstance(base_t.this, exp.DataType.Type) + else str(base_t.this).upper() + ) + if base_type_name in ("VARIANT", "JSON", "JSONB"): + return base_t + if isinstance(inner, exp.GroupConcat): + return exp.DataType.build("TEXT", dialect=self.dialect) + if isinstance(inner, (exp.Count, exp.RowNumber, exp.Rank, exp.DenseRank, exp.Ntile)): + return exp.DataType.build("BIGINT", dialect=self.dialect) + if isinstance( + inner, + ( + exp.Year, + exp.Month, + exp.Day, + exp.Hour, + exp.Minute, + exp.Second, + exp.DayOfWeek, + exp.DayOfYear, + exp.WeekOfYear, + ), + ): + return exp.DataType.build("INT", dialect=self.dialect) + if isinstance(inner, (exp.CurrentDate, exp.Date, exp.TsOrDsToDate)): + return exp.DataType.build("DATE", dialect=self.dialect) + if isinstance(inner, (exp.CurrentTimestamp, exp.DateTrunc, exp.TimestampTrunc)): + return exp.DataType.build("DATETIME", dialect=self.dialect) + if isinstance(inner, exp.Anonymous): + fn_upper = inner.name.upper() + if fn_upper in ( + "AVG", + "STDDEV", + "STDDEV_POP", + "STDDEV_SAMP", + "VARIANCE", + "VAR_POP", + "VAR_SAMP", + "L2_DISTANCE", + "L2_DISTANCE_APPROXIMATE", + "COSINE_DISTANCE", + "COSINE_DISTANCE_APPROXIMATE", + "INNER_PRODUCT", + "INNER_PRODUCT_APPROXIMATE", + "L1_DISTANCE", + ): + return exp.DataType.build("DOUBLE", dialect=self.dialect) + if fn_upper in ("ARRAY", "ARRAY_AGG", "COLLECT_LIST", "COLLECT_SET"): + return exp.DataType.build("ARRAY", dialect=self.dialect) + if fn_upper in ("MAP", "MAP_AGG", "VARIANT_TYPE", "JSON_OBJECT_FLATTEN"): + return exp.DataType.build("MAP", dialect=self.dialect) + if fn_upper in ("STRUCT", "NAMED_STRUCT"): + return exp.DataType.build("STRUCT", dialect=self.dialect, udt=True) + if fn_upper in ( + "JSON_OBJECT", + "JSON_ARRAY", + "TO_JSON", + "JSON_PARSE", + "JSONB_PARSE", + "JSON_EXTRACT", + ): + return exp.DataType.build("JSON", dialect=self.dialect) + if fn_upper in ( + "BITMAP_UNION", + "BITMAP_EMPTY", + "TO_BITMAP", + "BITMAP_HASH", + "BITMAP_HASH64", + "BITMAP_FROM_STRING", + ): + return exp.DataType.build("BITMAP", dialect=self.dialect, udt=True) + if fn_upper in ("HLL_UNION", "HLL_EMPTY", "HLL_HASH", "HLL_FROM_BASE64"): + return exp.DataType.build("HLL", dialect=self.dialect, udt=True) + if fn_upper in ("TO_QUANTILE_STATE", "QUANTILE_UNION"): + return exp.DataType.build("QUANTILE_STATE", dialect=self.dialect, udt=True) + if fn_upper == "GROUP_CONCAT": + return exp.DataType.build("TEXT", dialect=self.dialect) + if fn_upper in ("COUNT", "ROW_NUMBER", "RANK", "DENSE_RANK", "NTILE"): + return exp.DataType.build("BIGINT", dialect=self.dialect) + if fn_upper == "DATE_TRUNC": + return exp.DataType.build("DATETIME", dialect=self.dialect) + if fn_upper == "TO_DATE": + return exp.DataType.build("DATE", dialect=self.dialect) + if fn_upper == "IS_VALID_UTF8": + return exp.DataType.build("BOOLEAN", dialect=self.dialect) + return exp.DataType.build("UNKNOWN") + + @staticmethod + def _projection_key_name(proj: exp.Expr, idx: int) -> str: + if isinstance(proj, exp.Star): + return "" + if proj.alias_or_name: + return proj.alias_or_name + if ( + isinstance(proj, (exp.Cast, exp.TryCast)) + and isinstance(proj.this, exp.Column) + and proj.this.name + ): + return proj.this.name + return f"__unaliased_col_{idx}" + + def _reorder_unannotated_ctas_query( + self, + expression: exp.Expr, + kwargs: t.Dict[str, t.Any], + cte_types: t.Optional[t.Dict[str, exp.DataType]] = None, + ) -> exp.Expr: + cte_col_types: t.Dict[str, exp.DataType] = dict(cte_types or {}) + with_node = expression.args.get("with_") + if isinstance(with_node, exp.With): + for cte_item in with_node.expressions: + if isinstance(cte_item, exp.CTE) and isinstance(cte_item.this, exp.Select): + for cte_proj in cte_item.this.expressions: + if cte_proj.alias_or_name and not isinstance(cte_proj, exp.Star): + inferred_cte_t = self._infer_projection_type(cte_proj, cte_col_types) + if inferred_cte_t.this != exp.DataType.Type.UNKNOWN: + cte_col_types[cte_proj.alias_or_name.lower()] = inferred_cte_t + if isinstance(expression, exp.SetOperation): + expression = expression.copy() + branches: t.List[exp.Select] = [] + + def _collect_branches(node: exp.Expr) -> None: + if isinstance(node, exp.Subquery): + _collect_branches(node.this) + elif isinstance(node, exp.SetOperation): + _collect_branches(node.this) + _collect_branches(node.expression) + elif isinstance(node, exp.Select): + branches.append(node) + + _collect_branches(expression) + if not branches: + return expression + first_exprs = branches[0].expressions + num_cols = len(first_exprs) + orig_names = [self._projection_key_name(p, i) for i, p in enumerate(first_exprs)] + if ( + num_cols > 0 + and all( + len(b.expressions) == num_cols + and not any(isinstance(p, exp.Star) for p in b.expressions) + for b in branches + ) + and all(orig_names) + and len({n.lower() for n in orig_names}) == num_cols + ): + merged_cols_to_types: t.Dict[str, exp.DataType] = {} + for col_idx, col_name in enumerate(orig_names): + branch_cands = [ + cand + for b in branches + if ( + cand := self._infer_projection_type( + b.expressions[col_idx], cte_col_types + ) + ).this + != exp.DataType.Type.UNKNOWN + ] + non_coercible_invalid = next( + ( + c + for c in branch_cands + if not self._is_valid_olap_key_type(c) + and not self._is_coercible_string_key_type(c) + ), + None, + ) + coercible_str = next( + (c for c in branch_cands if self._is_coercible_string_key_type(c)), + None, + ) + merged_cols_to_types[col_name] = ( + non_coercible_invalid + or coercible_str + or (branch_cands[0] if branch_cands else exp.DataType.build("UNKNOWN")) + ) + reordered_select_cols = self._reorder_olap_columns(merged_cols_to_types, kwargs) + perm = [orig_names.index(c) for c in reordered_select_cols] + coercions = { + orig_names.index(c): reordered_select_cols[c] + for c in reordered_select_cols + if reordered_select_cols[c] != merged_cols_to_types[c] + } + if perm != list(range(num_cols)) or coercions: + old_to_new_one_based = { + orig_idx + 1: new_idx + 1 for new_idx, orig_idx in enumerate(perm) + } + for b_idx, branch in enumerate(branches): + old_projs = list(branch.expressions) + updated_branch_projs: t.List[exp.Expr] = [] + for orig_idx in perm: + proj = old_projs[orig_idx].copy() + new_dtype = coercions.get(orig_idx) + if new_dtype is not None: + col_name = orig_names[orig_idx] + is_synthetic = col_name.startswith("__unaliased_col_") + if isinstance(proj, exp.Alias) and isinstance( + proj.this, (exp.Cast, exp.TryCast) + ): + proj.this.set("to", new_dtype.copy()) + elif isinstance(proj, (exp.Cast, exp.TryCast)): + cast_expr = exp.cast(proj.this.copy(), to=new_dtype.copy()) + proj = ( + exp.alias_(cast_expr, col_name, quoted=True) + if b_idx == 0 and not is_synthetic + else cast_expr + ) + elif isinstance(proj, exp.Alias): + proj.set( + "this", exp.cast(proj.this.copy(), to=new_dtype.copy()) + ) + elif isinstance(proj, exp.Column): + cast_expr = exp.cast(proj.copy(), to=new_dtype.copy()) + proj = ( + exp.alias_( + cast_expr, + proj.name, + quoted=bool(getattr(proj.this, "quoted", True)), + ) + if b_idx == 0 + else cast_expr + ) + else: + cast_expr = exp.cast(proj.copy(), to=new_dtype.copy()) + proj = ( + exp.alias_(cast_expr, col_name, quoted=True) + if b_idx == 0 and not is_synthetic + else cast_expr + ) + updated_branch_projs.append(proj) + branch.set("expressions", updated_branch_projs) + self._remap_positional_group_and_order(branch, old_to_new_one_based) + self._remap_positional_group_and_order(expression, old_to_new_one_based) + return expression + expression.set( + "this", + self._reorder_unannotated_ctas_query(expression.this, kwargs, cte_col_types), + ) + expression.set( + "expression", + self._reorder_unannotated_ctas_query(expression.expression, kwargs, cte_col_types), + ) + return expression + if isinstance(expression, exp.Select): + keyed_projs = [ + (self._projection_key_name(proj, idx), proj) + for idx, proj in enumerate(expression.expressions) + if not isinstance(proj, exp.Star) + ] + proj_by_name = {k.lower(): proj for k, proj in keyed_projs if k} + if proj_by_name and len(proj_by_name) == len(expression.expressions): + select_cols_to_types = { + k: self._infer_projection_type(proj, cte_col_types) for k, proj in keyed_projs + } + reordered_select_cols = self._reorder_olap_columns(select_cols_to_types, kwargs) + order_changed = list(reordered_select_cols) != list(select_cols_to_types) + types_changed = any( + reordered_select_cols[c] != select_cols_to_types[c] + for c in reordered_select_cols + ) + if order_changed or types_changed: + old_pos_by_lower = { + k.lower(): idx + 1 for idx, (k, _) in enumerate(keyed_projs) + } + expression = expression.copy() + updated_projs: t.List[exp.Expr] = [] + old_to_new_one_based = {} + for new_idx, (c, new_dtype) in enumerate( + reordered_select_cols.items(), start=1 + ): + old_to_new_one_based[old_pos_by_lower[c.lower()]] = new_idx + proj = proj_by_name[c.lower()].copy() + if types_changed and select_cols_to_types[c] != new_dtype: + is_synthetic = c.startswith("__unaliased_col_") + if isinstance(proj, exp.Alias) and isinstance( + proj.this, (exp.Cast, exp.TryCast) + ): + proj.this.set("to", new_dtype.copy()) + elif isinstance(proj, (exp.Cast, exp.TryCast)): + cast_expr = exp.cast(proj.this.copy(), to=new_dtype.copy()) + proj = ( + cast_expr + if is_synthetic + else exp.alias_(cast_expr, c, quoted=True) + ) + elif isinstance(proj, exp.Alias): + proj.set("this", exp.cast(proj.this.copy(), to=new_dtype.copy())) + elif isinstance(proj, exp.Column): + proj = exp.alias_( + exp.cast(proj.copy(), to=new_dtype.copy()), + proj.name, + quoted=bool(getattr(proj.this, "quoted", True)), + ) + else: + cast_expr = exp.cast(proj.copy(), to=new_dtype.copy()) + proj = ( + cast_expr + if is_synthetic + else exp.alias_(cast_expr, c, quoted=True) + ) + updated_projs.append(proj) + expression.set("expressions", updated_projs) + self._remap_positional_group_and_order(expression, old_to_new_one_based) + elif any(isinstance(proj, exp.Star) for proj in expression.expressions): + table_props = self._get_table_properties(kwargs) or {} + resolved_uk, resolved_dk, resolved_ak = self._resolve_table_type_and_keys( + table_props + ) + has_explicit_key = bool( + resolved_uk or kwargs.get("unique_key") or resolved_dk or resolved_ak + ) + with_clause = expression.args.get("with_") + from_clause = expression.args.get("from_") + if ( + isinstance(with_clause, exp.With) + and isinstance(from_clause, exp.From) + and isinstance(from_clause.this, exp.Table) + ): + target_cte_name = from_clause.this.name.lower() + for cte_idx, cte in enumerate(with_clause.expressions): + if ( + isinstance(cte, exp.CTE) + and cte.alias.lower() == target_cte_name + and isinstance(cte.this, exp.Select) + and len(cte.this.expressions) >= 2 + ): + indicator_projs = [ + p + for p in cte.this.expressions + if self._is_integer_indicator_projection(p) + ] + other_projs = [ + p + for p in cte.this.expressions + if not self._is_integer_indicator_projection(p) + ] + if ( + not has_explicit_key + and indicator_projs + and list(cte.this.expressions[: len(indicator_projs)]) + != indicator_projs + ): + expression = expression.copy() + copied_cte = expression.args["with_"].expressions[cte_idx] + copied_select = copied_cte.this + copied_select.set("expressions", indicator_projs + other_projs) + elif ( + len(expression.expressions) == 1 + and isinstance(expression.expressions[0], exp.Star) + and not expression.args.get("joins") + ): + reordered_cte_sel = self._reorder_unannotated_ctas_query( + cte.this, kwargs, cte_col_types + ) + if reordered_cte_sel != cte.this: + expression = expression.copy() + copied_cte = expression.args["with_"].expressions[cte_idx] + copied_cte.set("this", reordered_cte_sel) + break + return expression + + @classmethod + def _is_coercible_string_key_type(cls, dtype: exp.DataType) -> bool: + type_name = ( + dtype.this.value.upper() + if isinstance(dtype.this, exp.DataType.Type) + else str(dtype.this).upper() + ) + return type_name in ("TEXT", "STRING", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT") + + def _reorder_olap_columns( + self, + target_columns_to_types: t.Dict[str, exp.DataType], + kwargs: t.Optional[t.Dict[str, t.Any]] = None, + ) -> t.Dict[str, exp.DataType]: + kwargs = kwargs or {} + table_props = self._get_table_properties(kwargs) or {} + if table_props.get("primary_key") is not None or kwargs.get("primary_key") is not None: + raise SQLMeshError( + "Doris does not support 'primary_key' in physical_properties; use 'unique_key', 'duplicate_key', or 'aggregate_key' instead." + ) + if table_props.get("partition") is not None: + raise SQLMeshError( + "Invalid property 'partition' in physical_properties; did you mean 'partition_by', 'partitioned_by', or 'partitions'?" + ) + if ( + sum( + x is not None + for x in ( + kwargs.get("partitioned_by") or None, + table_props.get("partitioned_by"), + table_props.get("partition_by"), + ) + ) + > 1 + ): + raise SQLMeshError( + "Multiple partition definitions found: specify only one of model 'partitioned_by' or physical_properties 'partitioned_by' / 'partition_by'." + ) + resolved_uk, duplicate_key, aggregate_key = self._resolve_table_type_and_keys(table_props) + unique_key = resolved_uk or kwargs.get("unique_key") + + specified_key_types = [ + name + for name, val in ( + ("unique_key", unique_key), + ("duplicate_key", duplicate_key), + ("aggregate_key", aggregate_key), + ) + if self._extract_column_names(val) + ] + if len(specified_key_types) > 1: + raise SQLMeshError( + f"Doris tables support at most one key model (unique_key, duplicate_key, or aggregate_key), " + f"got multiple: {', '.join(specified_key_types)}." + ) + + explicit_key = unique_key or duplicate_key or aggregate_key + order_by = table_props.get("order_by") or table_props.get("cluster_by") + order_by_cols = self._extract_column_names(order_by) + if order_by_cols and not unique_key: + raise SQLMeshError( + "Doris table property 'order_by' is only supported on UNIQUE KEY tables. " + "For DUPLICATE KEY or AGGREGATE KEY tables, sort order is determined by the key columns." + ) + mow_raw = self._extract_str_value(table_props.get("enable_unique_key_merge_on_write")) + if order_by_cols and mow_raw is not None and mow_raw.strip().lower() in ("false", "0"): + raise SQLMeshError( + "Doris table property 'order_by' on UNIQUE KEY tables requires Merge-on-Write " + "('enable_unique_key_merge_on_write' = 'true')." + ) + seq_col_raw = self._extract_str_value(table_props.get("function_column.sequence_col")) + seq_type_raw = self._extract_str_value(table_props.get("function_column.sequence_type")) + if (seq_col_raw or seq_type_raw) and not unique_key: + raise SQLMeshError( + "Doris 'function_column.sequence_col' / 'function_column.sequence_type' is only supported on UNIQUE KEY tables." + ) + + col_names = list(target_columns_to_types) + if not col_names: + return target_columns_to_types + + if explicit_key: + col_by_lower = {c.lower(): c for c in col_names} + raw_key_cols = self._extract_column_names(explicit_key) + missing_cols = [k for k in raw_key_cols if k.lower() not in col_by_lower] + if missing_cols: + raise SQLMeshError( + f"Doris key columns {missing_cols} not found in table schema columns {col_names}." + ) + if seq_col_raw and seq_col_raw.strip("`").lower() not in col_by_lower: + raise SQLMeshError( + f"Doris function_column.sequence_col '{seq_col_raw}' not found in table schema columns {col_names}." + ) + key_cols = [col_by_lower[k.lower()] for k in raw_key_cols if k.lower() in col_by_lower] + if duplicate_key and not unique_key and not aggregate_key: + raw_part_by_dup = ( + kwargs.get("partitioned_by") + or table_props.get("partitioned_by") + or table_props.get("partition_by") + ) + has_active_olap_part = bool( + raw_part_by_dup + and ( + table_props.get("partitions") + or kwargs.get("partitions") + or str(self._extract_str_value(table_props.get("auto_partition")) or "") + .strip() + .lower() + in ("true", "1") + or str( + self._extract_str_value(table_props.get("dynamic_partition.enable")) + or "" + ) + .strip() + .lower() + in ("true", "1") + ) + ) + if has_active_olap_part: + part_list_dup = ( + raw_part_by_dup if isinstance(raw_part_by_dup, list) else [raw_part_by_dup] + ) + parsed_part_dup, _ = self._parse_partition_expressions(part_list_dup) + key_cols_lower = {k.lower() for k in key_cols} + for expr in parsed_part_dup: + for col_node in expr.find_all(exp.Column): + canonical = col_by_lower.get(col_node.name.lower()) + if canonical and canonical.lower() not in key_cols_lower: + key_cols.append(canonical) + key_cols_lower.add(canonical.lower()) + updated_types = dict(target_columns_to_types) + for k in key_cols: + k_dtype = updated_types[k] + if not self._is_valid_olap_key_type(k_dtype): + if self._is_coercible_string_key_type(k_dtype): + updated_types[k] = exp.DataType.build("VARCHAR(255)", dialect=self.dialect) + else: + raise SQLMeshError( + f"Doris key column '{k}' has type '{k_dtype.sql(dialect=self.dialect)}' which is not supported as an OLAP key column. " + "Use an integer, DATE/DATETIME, DECIMAL, CHAR, or VARCHAR(n) column instead." + ) + if order_by_cols: + missing_order_cols = [k for k in order_by_cols if k.lower() not in col_by_lower] + if missing_order_cols: + raise SQLMeshError( + f"Doris order_by columns {missing_order_cols} not found in table schema columns {col_names}." + ) + for ob_raw in order_by_cols: + ob_col = col_by_lower[ob_raw.lower()] + ob_dtype = updated_types[ob_col] + if not self._is_valid_olap_key_type(ob_dtype): + if self._is_coercible_string_key_type(ob_dtype): + updated_types[ob_col] = exp.DataType.build( + "VARCHAR(255)", dialect=self.dialect + ) + else: + raise SQLMeshError( + f"Doris order_by column '{ob_col}' has type '{ob_dtype.sql(dialect=self.dialect)}' which is not supported as an OLAP sort key column. " + "Use an integer, DATE/DATETIME, DECIMAL, CHAR, or VARCHAR(n) column instead." + ) + key_cols_set = set(key_cols) + new_order = ( + key_cols + [c for c in col_names if c not in key_cols_set] + if key_cols and col_names[: len(key_cols)] != key_cols + else col_names + ) + return {c: updated_types[c] for c in new_order} + + updated_types = dict(target_columns_to_types) + raw_part_by = ( + kwargs.get("partitioned_by") + or table_props.get("partitioned_by") + or table_props.get("partition_by") + ) + part_col_names: t.List[str] = [] + if raw_part_by: + part_list = raw_part_by if isinstance(raw_part_by, list) else [raw_part_by] + parsed_part_exprs, _ = self._parse_partition_expressions(part_list) + col_by_lower = {c.lower(): c for c in col_names} + for expr in parsed_part_exprs: + for col in expr.find_all(exp.Column): + canonical = col_by_lower.get(col.name.lower()) + if canonical and canonical not in part_col_names: + part_col_names.append(canonical) + if not self._is_valid_olap_key_type( + updated_types[canonical] + ) and self._is_coercible_string_key_type(updated_types[canonical]): + updated_types[canonical] = exp.DataType.build( + "VARCHAR(255)", dialect=self.dialect + ) + + first_dtype = updated_types[col_names[0]] + if not self._is_valid_olap_key_type(first_dtype): + known_valid_col = next( + ( + c + for c in col_names + if self._is_valid_olap_key_type(updated_types[c]) + and not updated_types[c].is_type("unknown", "null") + ), + None, + ) + coercible_col = next( + (c for c in col_names if self._is_coercible_string_key_type(updated_types[c])), + None, + ) + unknown_valid_col = next( + (c for c in col_names if self._is_valid_olap_key_type(updated_types[c])), + None, + ) + if known_valid_col: + col_names = [known_valid_col] + [c for c in col_names if c != known_valid_col] + elif coercible_col: + updated_types[coercible_col] = exp.DataType.build( + "VARCHAR(255)", dialect=self.dialect + ) + if col_names[0] != coercible_col: + col_names = [coercible_col] + [c for c in col_names if c != coercible_col] + elif unknown_valid_col: + col_names = [unknown_valid_col] + [c for c in col_names if c != unknown_valid_col] + + # Ensure OLAP partition/time columns are included in Doris's implicit DUPLICATE KEY prefix: + # Doris stops implicit key inference at the first non-keyable column, immediately after the + # first VARCHAR column, or after 3 columns (`shortkey_max_column_count`) when no physical + # OLAP PARTITION BY clause is emitted. + if part_col_names: + part_cols = [ + c for c in part_col_names if self._is_valid_olap_key_type(updated_types[c]) + ] + part_cols_set = set(part_cols) + first_non_key_idx = next( + ( + i + for i, c in enumerate(col_names) + if not self._is_valid_olap_key_type(updated_types[c]) + ), + None, + ) + first_varchar_idx = next( + ( + i + for i, c in enumerate(col_names) + if c not in part_cols_set and updated_types[c].is_type("varchar") + ), + None, + ) + auto_str = self._extract_str_value(table_props.get("auto_partition")) + dyn_str = self._extract_str_value(table_props.get("dynamic_partition.enable")) + has_physical_olap_partition = bool( + table_props.get("partitions") + or (auto_str and str(auto_str).strip().lower() in ("true", "1")) + or (dyn_str and str(dyn_str).strip().lower() in ("true", "1")) + ) + shortkey_limit_idx = ( + max(1, 3 - len(part_cols)) if not has_physical_olap_partition else None + ) + cutoff_candidates = [ + idx + for idx in (first_non_key_idx, first_varchar_idx, shortkey_limit_idx) + if idx is not None + ] + cutoff_idx = min(cutoff_candidates) if cutoff_candidates else None + if ( + part_cols + and cutoff_idx is not None + and any(col_names.index(p) >= cutoff_idx for p in part_cols) + ): + moved_parts = [p for p in part_cols if col_names.index(p) >= cutoff_idx] + moved_set = set(moved_parts) + before_cutoff = [c for c in col_names[:cutoff_idx] if c not in moved_set] + after_cutoff = [c for c in col_names[cutoff_idx:] if c not in moved_set] + col_names = before_cutoff + moved_parts + after_cutoff + + if col_names != list(target_columns_to_types) or updated_types != target_columns_to_types: + return {c: updated_types[c] for c in col_names} + + return target_columns_to_types + + def _parse_partition_expressions( + self, partitioned_by: t.List[exp.Expr] + ) -> t.Tuple[t.List[exp.Expr], t.Optional[str]]: + """Parse partition expressions and extract partition kind and normalized expressions.""" + parsed_partitioned_by: t.List[exp.Expr] = [] + partition_kind: t.Optional[str] = None + + flattened_partitioned_by: t.List[exp.Expr] = [] + for item in partitioned_by: + if isinstance(item, (exp.Tuple, exp.Array)): + flattened_partitioned_by.extend(item.expressions) + else: + flattened_partitioned_by.append(item) + + for expr in flattened_partitioned_by: + if isinstance(expr, exp.List) or ( + isinstance(expr, exp.Anonymous) + and expr.this + and str(expr.this).upper() in ("RANGE", "LIST") + ): + partition_kind = "LIST" if isinstance(expr, exp.List) else str(expr.this).upper() + for arg in expr.expressions: + if isinstance(arg, (exp.Column, exp.Func, exp.Anonymous)): + parsed_partitioned_by.append(arg) + else: + parsed_partitioned_by.append(exp.to_column(str(arg))) + continue + + if isinstance(expr, exp.Literal) and getattr(expr, "is_string", False): + text = str(expr.this) + match = re.match(r"^\s*(RANGE|LIST)\s*\((.+)\)\s*$", text, flags=re.IGNORECASE) + if match: + partition_kind = match.group(1).upper() + inner = match.group(2).strip() + if ";" in inner: + raise SQLMeshError(f"Invalid partition expression: {text}") + try: + parsed_stmts = parse(f"SELECT {inner}", read=self.dialect) + if ( + len(parsed_stmts) == 1 + and isinstance(parsed_stmts[0], exp.Select) + and set(parsed_stmts[0].args.keys()) <= {"expressions"} + ): + for sel_expr in parsed_stmts[0].expressions: + parsed_partitioned_by.append(sel_expr) + continue + except Exception: + pass + inner_cols = [c.strip().strip("`") for c in inner.split(",") if c.strip()] + for col in inner_cols: + parsed_partitioned_by.append(exp.to_column(col)) + continue + + if ";" in text or "--" in text or "/*" in text: + raise SQLMeshError(f"Invalid partition expression: {text}") + try: + parsed_stmts = parse(f"SELECT {text}", read=self.dialect) + if ( + len(parsed_stmts) == 1 + and isinstance(parsed_stmts[0], exp.Select) + and set(parsed_stmts[0].args.keys()) <= {"expressions"} + ): + for sel_expr in parsed_stmts[0].expressions: + parsed_partitioned_by.append(sel_expr) + continue + except Exception: + pass + parsed_partitioned_by.append(exp.to_column(text.strip().strip("`"))) + continue + + parsed_partitioned_by.append(expr) + + return parsed_partitioned_by, partition_kind + + @staticmethod + def _to_raw_partition_sql(expr: t.Union[exp.Literal, exp.Var, str, t.Any]) -> exp.Var: + if isinstance(expr, exp.Literal): + raw_text = str(expr.this) + elif isinstance(expr, exp.Var): + raw_text = str(expr.this) + elif isinstance(expr, str): + raw_text = expr + else: + return expr + if ";" in raw_text or "--" in raw_text or "/*" in raw_text: + raise SQLMeshError(f"Invalid partition definition expression: {raw_text}") + return exp.Var(this=raw_text, quoted=False) + + def _build_partitioned_by_exp( + self, + partitioned_by: t.List[exp.Expr], + *, + partition_interval_unit: t.Optional[IntervalUnit] = None, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + catalog_name: t.Optional[str] = None, + **kwargs: t.Any, + ) -> t.Optional[ + t.Union[ + exp.PartitionedByProperty, + exp.PartitionByRangeProperty, + exp.PartitionByListProperty, + exp.Property, + ] + ]: + """Build Doris partitioning expression for OLAP (manual, dynamic, AUTO PARTITION) and Iceberg tables.""" + is_iceberg = bool(kwargs.get("is_iceberg")) + partitions = kwargs.get("partitions") + auto_partition = kwargs.get("auto_partition") + dynamic_partition_enabled = kwargs.get("dynamic_partition_enabled") + + partitioned_by, partition_kind = self._parse_partition_expressions(partitioned_by) + if target_columns_to_types: + col_by_lower = {c.lower(): c for c in target_columns_to_types} + normalized_exprs: t.List[exp.Expr] = [] + for expr in partitioned_by: + expr_copy = expr.copy() + for col_node in expr_copy.find_all(exp.Column): + canonical = col_by_lower.get(col_node.name.lower()) + if canonical and canonical != col_node.name: + col_node.set("this", exp.to_identifier(canonical)) + normalized_exprs.append(expr_copy) + partitioned_by = normalized_exprs + + if is_iceberg: + + def _normalize_iceberg_part_node(node: exp.Expr) -> exp.Expr: + if isinstance(node, exp.PartitionedByBucket): + return exp.Anonymous( + this="BUCKET", + expressions=[node.expression.copy(), node.this.copy()], + ) + if isinstance(node, exp.PartitionByTruncate): + return exp.Anonymous( + this="TRUNCATE", + expressions=[node.expression.copy(), node.this.copy()], + ) + if isinstance(node, exp.Trunc): + dec = node.args.get("decimals") + if isinstance(node.this, exp.Column) and isinstance(dec, exp.Literal): + return exp.Anonymous( + this="TRUNCATE", + expressions=[dec.copy(), node.this.copy()], + ) + if isinstance(node.this, exp.Literal) and isinstance(dec, exp.Column): + return exp.Anonymous( + this="TRUNCATE", + expressions=[node.this.copy(), dec.copy()], + ) + if ( + isinstance(node, exp.Anonymous) + and node.name.upper() in ("BUCKET", "TRUNCATE") + and len(node.expressions) == 2 + and isinstance(node.expressions[0], exp.Column) + and isinstance(node.expressions[1], exp.Literal) + ): + return exp.Anonymous( + this=node.name.upper(), + expressions=[node.expressions[1].copy(), node.expressions[0].copy()], + ) + return node + + iceberg_part_exprs = [ + expr.copy().transform(_normalize_iceberg_part_node) for expr in partitioned_by + ] + return exp.PartitionByListProperty( + partition_expressions=iceberg_part_exprs, + create_expressions=[], + ) + + is_auto = False + if auto_partition is not None: + auto_str = self._extract_str_value(auto_partition) + is_auto = str(auto_str).strip().lower() in ("true", "1") + + # On Doris OLAP tables, if no explicit partitions, auto_partition, or dynamic_partition.enable + # are configured, omit PARTITION BY so Doris does not create a 0-partition table that rejects inserts. + if not partitions and not is_auto and not dynamic_partition_enabled: + logger.warning( + "[Doris] Omitting PARTITION BY on OLAP table because none of 'auto_partition = true', " + "'dynamic_partition.enable = true', or explicit 'partitions' were configured in physical_properties. " + "Add one of those properties to physical_properties, or set 'partition_by_time_column false' in your " + "INCREMENTAL_BY_TIME_RANGE model kind to silence this warning." + ) + return None + + create_expressions: t.Optional[t.List[exp.Expr]] = None + if partitions: + if isinstance(partitions, exp.Tuple): + create_expressions = [self._to_raw_partition_sql(e) for e in partitions.expressions] + else: + create_expressions = [self._to_raw_partition_sql(partitions)] + elif is_auto or dynamic_partition_enabled: + create_expressions = [] + + inferred_list = False + if partition_kind is None and create_expressions: + try: + texts = [getattr(e, "this", "").upper() for e in create_expressions] + inferred_list = any("VALUES IN" in text for text in texts) + except Exception: + inferred_list = False + if partition_kind: + is_list = str(partition_kind).upper() == "LIST" + else: + is_list = inferred_list + + partition_prop: t.Union[exp.PartitionByListProperty, exp.PartitionByRangeProperty] + if is_list: + partition_prop = exp.PartitionByListProperty( + partition_expressions=partitioned_by, + create_expressions=create_expressions, + ) + else: + partition_prop = exp.PartitionByRangeProperty( + partition_expressions=partitioned_by, + create_expressions=create_expressions, + ) + + if is_auto: + return exp.SetConfigProperty(this=exp.var(f"AUTO {self._to_sql(partition_prop)}")) + + return partition_prop + + def _build_table_properties_exp( + self, + catalog_name: t.Optional[str] = None, + table_format: t.Optional[str] = None, + storage_format: t.Optional[str] = None, + partitioned_by: t.Optional[t.List[exp.Expr]] = None, + partition_interval_unit: t.Optional[IntervalUnit] = None, + clustered_by: t.Optional[t.List[exp.Expr]] = None, + table_properties: t.Optional[t.Dict[str, t.Any]] = None, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + table_description: t.Optional[str] = None, + table_kind: t.Optional[str] = None, + **kwargs: t.Any, + ) -> t.Optional[exp.Properties]: + """Creates a SQLGlot table properties expression for Doris DDL.""" + properties: t.List[exp.Expr] = [] + is_materialized_view = table_kind == "MATERIALIZED_VIEW" + is_schema_ddl = bool(kwargs.get("is_schema_ddl")) + + if is_materialized_view: + properties.append(exp.MaterializedProperty()) + + table_properties_copy = dict(table_properties) if table_properties else {} + is_iceberg = not is_materialized_view and self._is_iceberg_table( + table_format=table_format, + catalog_name=catalog_name, + table_properties=table_properties_copy, + ) + + # Pop Iceberg/Paimon/Hive engine property if present so it is not duplicated in PROPERTIES (...) + explicit_engine = self._extract_str_value(table_properties_copy.pop("engine", None)) or ( + str(table_format).strip().lower() if table_format else None + ) + engine_lower = explicit_engine.strip().lower() if explicit_engine else None + if is_iceberg and is_schema_ddl: + engine_name = engine_lower if engine_lower in ("hive", "paimon") else "iceberg" + properties.append(exp.EngineProperty(this=exp.var(engine_name))) + + if storage_format and not is_materialized_view: + sf_str = str(storage_format).strip().strip("'").strip('"') + if sf_str: + if is_iceberg: + if engine_lower == "paimon": + table_properties_copy.setdefault( + "file.format", exp.Literal.string(sf_str.lower()) + ) + elif engine_lower == "hive": + table_properties_copy.setdefault( + "file_format", exp.Literal.string(sf_str.lower()) + ) + elif ( + "write-format" not in table_properties_copy + and "write.format.default" not in table_properties_copy + ): + table_properties_copy["write-format"] = exp.Literal.string(sf_str.lower()) + elif "storage_format" not in table_properties_copy: + table_properties_copy["storage_format"] = exp.Literal.string(sf_str.upper()) + + if is_materialized_view: + build_val = table_properties_copy.pop("build", None) + build_mode_val = table_properties_copy.pop("build_mode", None) + if build_val is None and build_mode_val is not None: + build_val = build_mode_val + if build_val is not None: + build_upper = self._to_upper_str(build_val) + if build_upper not in ("IMMEDIATE", "DEFERRED"): + raise SQLMeshError( + f"Invalid materialized view build mode: {build_upper}. Expected IMMEDIATE or DEFERRED." + ) + properties.append(exp.BuildProperty(this=exp.Var(this=build_upper))) + + refresh_val = table_properties_copy.pop("refresh", None) + for refresh_alias in ("refresh_mode", "refresh_method"): + alias_val = table_properties_copy.pop(refresh_alias, None) + if refresh_val is None and alias_val is not None: + refresh_val = alias_val + refresh_trigger_val = table_properties_copy.pop("refresh_trigger", None) + if refresh_val is not None or refresh_trigger_val is not None: + method_var: exp.Var = exp.Var(this="AUTO") + kind_str: t.Optional[str] = None + every_lit: t.Optional[exp.Expr] = None + unit_var: t.Optional[exp.Var] = None + starts_lit: t.Optional[exp.Literal] = None + + if isinstance(refresh_val, exp.Paren) and isinstance(refresh_val.this, exp.EQ): + refresh_val = exp.Tuple(expressions=[refresh_val.this]) + elif isinstance(refresh_val, exp.EQ): + refresh_val = exp.Tuple(expressions=[refresh_val]) + + if isinstance(refresh_val, exp.Tuple): + method_var, kind_str, every_lit, unit_var, starts_lit = ( + self._parse_refresh_tuple(refresh_val) + ) + if refresh_trigger_val is not None: + trigger_text = self._extract_str_value(refresh_trigger_val) or "" + k, e, u, s = self._parse_trigger_string(trigger_text) + kind_str, every_lit, unit_var, starts_lit = k, e, u, s + else: + raw_refresh = ( + self._extract_str_value(refresh_val) if refresh_val is not None else "AUTO" + ) + raw_refresh_str = (raw_refresh or "AUTO").strip() + if " " in raw_refresh_str: + parts = raw_refresh_str.split(None, 1) + method_upper = parts[0].upper() + if method_upper in ("COMPLETE", "AUTO", "FAST", "FORCE", "NEVER"): + method_var = exp.Var(this=method_upper) + k, e, u, s = self._parse_trigger_string(parts[1]) + kind_str, every_lit, unit_var, starts_lit = k, e, u, s + else: + k, e, u, s = self._parse_trigger_string(raw_refresh_str) + kind_str, every_lit, unit_var, starts_lit = k, e, u, s + else: + method_upper = raw_refresh_str.upper() + if method_upper in ("MANUAL", "COMMIT"): + kind_str = method_upper + elif method_upper in ("COMPLETE", "AUTO", "FAST", "FORCE", "NEVER"): + method_var = exp.Var(this=method_upper) + else: + raise SQLMeshError( + f"Invalid materialized view refresh method: {method_upper}. Expected one of: COMPLETE, AUTO, FAST, FORCE, NEVER, MANUAL, COMMIT." + ) + + if refresh_trigger_val is not None: + trigger_text = self._extract_str_value(refresh_trigger_val) or "" + k, e, u, s = self._parse_trigger_string(trigger_text) + kind_str, every_lit, unit_var, starts_lit = k, e, u, s + + properties.append( + exp.RefreshTriggerProperty( + method=method_var, + kind=kind_str, + every=every_lit, + unit=unit_var, + starts=starts_lit, + ) + ) + + # Pop key models and order_by from table_properties_copy + if ( + table_properties_copy.pop("primary_key", None) is not None + or kwargs.get("primary_key") is not None + ): + raise SQLMeshError( + "Doris does not support 'primary_key' in physical_properties; use 'unique_key', 'duplicate_key', or 'aggregate_key' instead." + ) + if table_properties_copy.get("partition") is not None: + raise SQLMeshError( + "Invalid property 'partition' in physical_properties; did you mean 'partition_by', 'partitioned_by', or 'partitions'?" + ) + resolved_uk, duplicate_key, aggregate_key = self._resolve_table_type_and_keys( + table_properties_copy, pop=True + ) + unique_key = resolved_uk or kwargs.get("unique_key") + order_by = table_properties_copy.pop("order_by", None) or table_properties_copy.pop( + "cluster_by", None + ) + table_properties_copy.pop("not_null_columns", None) + + unique_key_cols = self._extract_column_names(unique_key) + duplicate_key_cols = self._extract_column_names(duplicate_key) + aggregate_key_cols = self._extract_column_names(aggregate_key) + order_by_cols = self._extract_column_names(order_by) + col_by_lower = ( + {c.lower(): c for c in target_columns_to_types} if target_columns_to_types else {} + ) + + # Pop partitioning-related properties before building key properties so explicit duplicate_key + # can automatically include active OLAP partition columns (required by Doris FE validation). + partitions = table_properties_copy.pop("partitions", None) + auto_partition = table_properties_copy.pop("auto_partition", None) + raw_part_by = table_properties_copy.pop("partitioned_by", None) + raw_part_by_alias = table_properties_copy.pop("partition_by", None) + if ( + sum( + x is not None + for x in ( + partitioned_by or None, + raw_part_by, + raw_part_by_alias, + ) + ) + > 1 + ): + raise SQLMeshError( + "Multiple partition definitions found: specify only one of model 'partitioned_by' or physical_properties 'partitioned_by' / 'partition_by'." + ) + if not partitioned_by: + chosen_part_by = raw_part_by if raw_part_by is not None else raw_part_by_alias + if chosen_part_by is not None: + partitioned_by = ( + chosen_part_by if isinstance(chosen_part_by, list) else [chosen_part_by] + ) + + if ( + is_iceberg + and engine_lower == "paimon" + and unique_key_cols + and "primary-key" not in table_properties_copy + ): + paimon_pk_cols = [col_by_lower.get(k.lower(), k) for k in unique_key_cols] + if partitioned_by: + parsed_paimon_part, _ = self._parse_partition_expressions(partitioned_by) + pk_lower_set = {k.lower() for k in paimon_pk_cols} + for expr in parsed_paimon_part: + for col_node in expr.find_all(exp.Column): + part_col_name = col_by_lower.get(col_node.name.lower(), col_node.name) + if part_col_name.lower() not in pk_lower_set: + paimon_pk_cols.append(part_col_name) + pk_lower_set.add(part_col_name.lower()) + table_properties_copy["primary-key"] = exp.Literal.string(",".join(paimon_pk_cols)) + + if not is_iceberg: + specified_key_types = [ + name + for name, cols in ( + ("unique_key", unique_key_cols), + ("duplicate_key", duplicate_key_cols), + ("aggregate_key", aggregate_key_cols), + ) + if cols + ] + if len(specified_key_types) > 1: + raise SQLMeshError( + f"Doris tables support at most one key model (unique_key, duplicate_key, or aggregate_key), " + f"got multiple: {', '.join(specified_key_types)}." + ) + active_key_cols = unique_key_cols or duplicate_key_cols or aggregate_key_cols + if active_key_cols and col_by_lower and not is_materialized_view: + missing_cols = [k for k in active_key_cols if k.lower() not in col_by_lower] + if missing_cols: + raise SQLMeshError( + f"Doris key columns {missing_cols} not found in table schema columns {list(target_columns_to_types or ())}." + ) + if unique_key_cols: + unique_key_cols = [col_by_lower[k.lower()] for k in unique_key_cols] + elif duplicate_key_cols: + duplicate_key_cols = [col_by_lower[k.lower()] for k in duplicate_key_cols] + elif aggregate_key_cols: + aggregate_key_cols = [col_by_lower[k.lower()] for k in aggregate_key_cols] + + if ( + duplicate_key_cols + and partitioned_by + and not is_materialized_view + and ( + partitions + or str(self._extract_str_value(auto_partition) or "").strip().lower() + in ("true", "1") + or str( + self._extract_str_value( + table_properties_copy.get("dynamic_partition.enable") + ) + or "" + ) + .strip() + .lower() + in ("true", "1") + ) + ): + parsed_dup_part_exprs, _ = self._parse_partition_expressions(partitioned_by) + dup_lower_set = {k.lower() for k in duplicate_key_cols} + for expr in parsed_dup_part_exprs: + for col_node in expr.find_all(exp.Column): + part_col_name = col_by_lower.get(col_node.name.lower(), col_node.name) + if part_col_name.lower() not in dup_lower_set: + duplicate_key_cols.append(part_col_name) + dup_lower_set.add(part_col_name.lower()) + + if unique_key_cols: + properties.append( + exp.UniqueKeyProperty(expressions=[exp.to_column(k) for k in unique_key_cols]) + ) + elif duplicate_key_cols: + properties.append( + exp.DuplicateKeyProperty( + expressions=[exp.to_column(k) for k in duplicate_key_cols] + ) + ) + elif aggregate_key_cols: + agg_cols_sql = ", ".join(self._to_sql(exp.to_column(k)) for k in aggregate_key_cols) + properties.append( + exp.SetConfigProperty(this=exp.var(f"AGGREGATE KEY ({agg_cols_sql})")) + ) + + if order_by_cols and not is_materialized_view: + if not unique_key_cols: + raise SQLMeshError( + "Doris table property 'order_by' is only supported on UNIQUE KEY tables. " + "For DUPLICATE KEY or AGGREGATE KEY tables, sort order is determined by the key columns." + ) + mow_val = self._extract_str_value( + table_properties_copy.get("enable_unique_key_merge_on_write") + ) + if mow_val is not None and mow_val.strip().lower() in ("false", "0"): + raise SQLMeshError( + "Doris table property 'order_by' on UNIQUE KEY tables requires Merge-on-Write " + "('enable_unique_key_merge_on_write' = 'true')." + ) + if col_by_lower: + missing_order_cols = [k for k in order_by_cols if k.lower() not in col_by_lower] + if missing_order_cols: + raise SQLMeshError( + f"Doris order_by columns {missing_order_cols} not found in table schema columns {list(target_columns_to_types or ())}." + ) + order_by_cols = [col_by_lower[k.lower()] for k in order_by_cols] + order_cols_sql = ", ".join(self._to_sql(exp.to_column(k)) for k in order_by_cols) + properties.append( + exp.SetConfigProperty(this=exp.var(f"ORDER BY ({order_cols_sql})")) + ) + + if not is_materialized_view: + seq_col_val = self._extract_str_value( + table_properties_copy.get("function_column.sequence_col") + ) + seq_type_val = self._extract_str_value( + table_properties_copy.get("function_column.sequence_type") + ) + if (seq_col_val or seq_type_val) and not unique_key_cols: + raise SQLMeshError( + "Doris 'function_column.sequence_col' / 'function_column.sequence_type' is only supported on UNIQUE KEY tables." + ) + if seq_col_val and col_by_lower: + seq_col_clean = seq_col_val.strip("`") + if seq_col_clean.lower() not in col_by_lower: + raise SQLMeshError( + f"Doris function_column.sequence_col '{seq_col_clean}' not found in table schema columns {list(target_columns_to_types or ())}." + ) + table_properties_copy["function_column.sequence_col"] = exp.Literal.string( + col_by_lower[seq_col_clean.lower()] + ) + compaction_policy_val = self._extract_str_value( + table_properties_copy.get("compaction_policy") + ) + if compaction_policy_val is not None: + cp_lower = compaction_policy_val.strip().lower() + if cp_lower not in ("size_based", "time_series"): + raise SQLMeshError( + f"Invalid Doris compaction_policy '{compaction_policy_val}'. Expected 'size_based' or 'time_series'." + ) + table_properties_copy["compaction_policy"] = exp.Literal.string(cp_lower) + + if table_description: + properties.append( + exp.SchemaCommentProperty( + this=exp.Literal.string(self._truncate_table_comment(table_description)) + ) + ) + + add_partition = True + if partitioned_by: + parsed_part_exprs, _ = self._parse_partition_expressions(partitioned_by) + strict_part_key_cols = unique_key_cols or aggregate_key_cols + if strict_part_key_cols and not is_materialized_view and not is_iceberg: + key_cols_set = {k.lower() for k in strict_part_key_cols} + partition_cols = set() + for expr in parsed_part_exprs: + for col_node in expr.find_all(exp.Column): + partition_cols.add(col_node.name.lower()) + not_in_key = partition_cols - key_cols_set + if not_in_key: + key_label = "UNIQUE KEY" if unique_key_cols else "AGGREGATE KEY" + logger.warning( + f"[Doris] {key_label} model: partitioned_by columns {not_in_key} not in key_cols {key_cols_set}, skip PARTITION BY." + ) + add_partition = False + + if add_partition: + if is_materialized_view: + if parsed_part_exprs: + properties.append( + exp.PartitionedByProperty( + this=exp.Schema(expressions=parsed_part_exprs), + ) + ) + else: + dyn_enable = self._extract_str_value( + table_properties_copy.get("dynamic_partition.enable") + ) + dynamic_partition_enabled = bool( + dyn_enable and dyn_enable.strip().lower() in ("true", "1") + ) + partition_expr = self._build_partitioned_by_exp( + partitioned_by, + partition_interval_unit=partition_interval_unit, + target_columns_to_types=target_columns_to_types, + catalog_name=catalog_name, + partitions=partitions, + auto_partition=auto_partition, + dynamic_partition_enabled=dynamic_partition_enabled, + is_iceberg=is_iceberg, + ) + if partition_expr: + if col_by_lower: + part_col_nodes = [ + col_node + for expr in parsed_part_exprs + for col_node in expr.find_all(exp.Column) + ] + missing_part_cols = [ + c.name for c in part_col_nodes if c.name.lower() not in col_by_lower + ] + if missing_part_cols: + raise SQLMeshError( + f"Doris partitioned_by columns {missing_part_cols} not found in table schema columns {list(target_columns_to_types or ())}." + ) + properties.append(partition_expr) + + # Pop distribution / bucketing properties so they never leak into PROPERTIES (...) + distributed_by = table_properties_copy.pop("distributed_by", None) + top_level_buckets = table_properties_copy.pop("buckets", None) + prop_clustered_by = table_properties_copy.pop("clustered_by", None) + effective_clustered_by = clustered_by or ( + [exp.to_column(c) for c in self._extract_column_names(prop_clustered_by)] + if prop_clustered_by + else None + ) + + if not is_iceberg: + strict_key_cols = ( + (unique_key_cols or aggregate_key_cols) if not is_materialized_view else [] + ) + dist_prop = self._build_distributed_by_exp( + distributed_by=distributed_by, + clustered_by=effective_clustered_by, + unique_key_cols=strict_key_cols, + fallback_key_cols=duplicate_key_cols if not is_materialized_view else [], + top_level_buckets=top_level_buckets, + ) + if dist_prop is not None: + if not is_materialized_view: + dist_kind = str(dist_prop.args.get("kind") or "").upper() + if strict_key_cols and dist_kind == "RANDOM": + key_kind_label = "UNIQUE KEY" if unique_key_cols else "AGGREGATE KEY" + raise SQLMeshError( + f"Doris {key_kind_label} tables do not support DISTRIBUTED BY RANDOM. " + "Use DISTRIBUTED BY HASH (...) instead." + ) + if col_by_lower and dist_prop.expressions: + dist_col_names = [c.name for c in dist_prop.expressions] + missing_dist_cols = [ + c for c in dist_col_names if c.lower() not in col_by_lower + ] + if missing_dist_cols: + raise SQLMeshError( + f"Doris distributed_by columns {missing_dist_cols} not found in table schema columns {list(target_columns_to_types or ())}." + ) + dist_prop.set( + "expressions", + [exp.to_column(col_by_lower[c.lower()]) for c in dist_col_names], + ) + if strict_key_cols and dist_prop.expressions: + strict_key_set = {k.lower() for k in strict_key_cols} + non_key_dist_cols = [ + c.name + for c in dist_prop.expressions + if c.name.lower() not in strict_key_set + ] + if non_key_dist_cols: + key_kind_label = "UNIQUE KEY" if unique_key_cols else "AGGREGATE KEY" + raise SQLMeshError( + f"Doris {key_kind_label} tables require distributed_by columns to be key columns, " + f"got {non_key_dist_cols} which are not in {strict_key_cols}." + ) + properties.append(dist_prop) + + # Default replication_num = 1 for single-BE dev/CI when not explicitly configured on OLAP tables + # Note: Only emit explicit table_properties_copy entries so unit test expectations remain clean. + if table_properties_copy: + properties.extend(self._properties_to_expressions(table_properties_copy)) + + if properties: + return exp.Properties(expressions=properties) + return None + + def _to_upper_str(self, val: t.Any) -> str: + extracted = self._extract_str_value(val) + if extracted is not None: + return extracted.strip().strip("`").upper() + return str(val).strip().strip("`").upper() + + def _parse_refresh_tuple( + self, + tpl: exp.Tuple, + ) -> t.Tuple[ + exp.Var, + t.Optional[str], + t.Optional[exp.Expr], + t.Optional[exp.Var], + t.Optional[exp.Literal], + ]: + method_var: exp.Var = exp.Var(this="AUTO") + kind_str: t.Optional[str] = None + every_lit: t.Optional[exp.Expr] = None + unit_var: t.Optional[exp.Var] = None + starts_lit: t.Optional[exp.Literal] = None + for e in tpl.expressions: + if isinstance(e, exp.EQ): + key = ( + (self._extract_str_value(e.this) or "").strip('"').strip("'").strip("`").lower() + ) + val = e.expression + if key == "method": + method_upper = self._to_upper_str(val) + if method_upper not in ("COMPLETE", "AUTO", "FAST", "FORCE", "NEVER"): + raise SQLMeshError( + f"Invalid materialized view refresh method: {method_upper}. Expected one of: COMPLETE, AUTO, FAST, FORCE, NEVER." + ) + method_var = exp.Var(this=method_upper) + elif key == "kind": + kind_upper = self._to_upper_str(val) + if kind_upper not in ("MANUAL", "COMMIT", "SCHEDULE"): + raise SQLMeshError( + f"Invalid materialized view refresh kind: {kind_upper}. Expected one of: MANUAL, COMMIT, SCHEDULE." + ) + kind_str = kind_upper + elif key == "every": + if isinstance(val, exp.Literal): + val_str = str(val.this).strip() + if not val_str.isdigit(): + raise SQLMeshError( + f"Invalid materialized view refresh every interval: {val_str}. Expected a positive integer." + ) + every_lit = exp.Literal.number(int(val_str)) + elif isinstance(val, (int, float)): + every_lit = exp.Literal.number(int(val)) + elif key == "unit": + unit_upper = self._to_upper_str(val) + if not re.match(r"^[A-Z]+$", unit_upper): + raise SQLMeshError(f"Invalid materialized view refresh unit: {unit_upper}") + unit_var = exp.Var(this=unit_upper) + elif key == "starts": + starts_str = self._extract_str_value(val) + starts_lit = exp.Literal.string( + starts_str if starts_str is not None else str(val) + ) + return method_var, kind_str, every_lit, unit_var, starts_lit + + @staticmethod + def _parse_trigger_string( + trigger_text: str, + ) -> t.Tuple[ + t.Optional[str], + t.Optional[exp.Expr], + t.Optional[exp.Var], + t.Optional[exp.Literal], + ]: + text = trigger_text.strip() + if text.upper().startswith("ON "): + text = text[3:].strip() + kind = None + every_lit: t.Optional[exp.Expr] = None + unit_var = None + starts_lit = None + + m_kind = re.match(r"^(MANUAL|COMMIT|SCHEDULE)\b", text, flags=re.IGNORECASE) + if not m_kind: + raise SQLMeshError( + f"Invalid materialized view refresh trigger: {trigger_text}. Expected MANUAL, COMMIT, or SCHEDULE." + ) + kind = m_kind.group(1).upper() + m_every = re.search(r"\bEVERY\s+(\d+)\s+([A-Za-z]+)", text, flags=re.IGNORECASE) + if m_every: + every_lit = exp.Literal.number(int(m_every.group(1))) + unit_var = exp.Var(this=m_every.group(2).upper()) + m_starts = re.search(r"\bSTARTS\s+'([^']+)'", text, flags=re.IGNORECASE) + if m_starts: + starts_lit = exp.Literal.string(m_starts.group(1)) + return kind, every_lit, unit_var, starts_lit + + def _resolve_buckets(self, raw_buckets: t.Any) -> t.Optional[exp.Expr]: + if raw_buckets is None: + return None + if isinstance(raw_buckets, exp.Literal): + val_str = str(raw_buckets.this).strip() + if val_str.upper() == "AUTO": + return exp.Var(this="AUTO") + if val_str.isdigit(): + return exp.Literal.number(int(val_str)) + if isinstance(raw_buckets, int): + return exp.Literal.number(raw_buckets) + val_str = str(self._extract_str_value(raw_buckets) or "").strip() + if val_str.upper() == "AUTO": + return exp.Var(this="AUTO") + if val_str.isdigit(): + return exp.Literal.number(int(val_str)) + raise SQLMeshError( + f"Invalid buckets value for Doris DISTRIBUTED BY: {raw_buckets}. Expected a positive integer or 'AUTO'." + ) + + @classmethod + def _eval_int_limit_expr(cls, node: t.Optional[exp.Expr]) -> t.Optional[int]: + if node is None: + return None + curr = node.unnest() + if isinstance(curr, exp.Literal) and not curr.is_string: + try: + return int(str(curr.this).strip()) + except ValueError: + return None + if isinstance(curr, (exp.Add, exp.Sub, exp.Mul)): + left = cls._eval_int_limit_expr(curr.this) + right = cls._eval_int_limit_expr(curr.expression) + if left is not None and right is not None: + if isinstance(curr, exp.Add): + return left + right + if isinstance(curr, exp.Sub): + return max(0, left - right) + return left * right + return None + + def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) -> str: + expression = self._normalize_schema_tmp_identifiers(expression) + if expression.find(exp.Lock): + expression = expression.copy() + for select_node in expression.find_all(exp.Select): + if select_node.args.get("locks"): + select_node.set("locks", None) + logger.debug("[Doris] Removed unsupported FOR UPDATE lock from SELECT statement") + if any( + not isinstance(lim.expression, exp.Literal) for lim in expression.find_all(exp.Limit) + ): + expression = expression.copy() + for lim in expression.find_all(exp.Limit): + if not isinstance(lim.expression, exp.Literal): + folded = self._eval_int_limit_expr(lim.expression) + if folded is not None: + lim.set("expression", exp.Literal.number(folded)) + if any( + isinstance(is_node.expression, exp.Boolean) for is_node in expression.find_all(exp.Is) + ): + expression = expression.copy() + for is_node in list(expression.find_all(exp.Is)): + if isinstance(is_node.expression, exp.Boolean): + is_in_delete_where = isinstance( + expression, exp.Delete + ) and not is_node.find_ancestor(exp.Select, exp.Subquery) + if is_in_delete_where: + eq_lhs: exp.Expr = is_node.this.copy() + else: + eq_lhs = exp.Anonymous( + this="COALESCE", + expressions=[ + is_node.this.copy(), + exp.Boolean(this=not bool(is_node.expression.this)), + ], + ) + eq_expr: exp.Expr = exp.EQ(this=eq_lhs, expression=is_node.expression.copy()) + if isinstance(is_node.parent, exp.Not): + eq_expr = exp.Paren(this=eq_expr) + if is_node is expression: + expression = eq_expr + else: + is_node.replace(eq_expr) + if expression.find(exp.AtTimeZone): + expression = expression.copy() + for at_tz in reversed(list(expression.find_all(exp.AtTimeZone))): + zone_expr = at_tz.args.get("zone") + if isinstance(zone_expr, exp.Literal) and zone_expr.is_string: + zone_val = zone_expr.this.strip() + to_tz: exp.Expr = ( + exp.Literal.string("+00:00") + if zone_val.upper() == "UTC" + else exp.Literal.string(zone_val) + ) + elif zone_expr is not None: + to_tz = zone_expr.copy() + else: + to_tz = exp.Literal.string("+00:00") + convert_tz = exp.Anonymous( + this="CONVERT_TZ", + expressions=[ + at_tz.this.copy(), + exp.SessionParameter(this=exp.var("time_zone")), + to_tz, + ], + ) + if at_tz is expression: + expression = convert_tz + else: + at_tz.replace(convert_tz) + if expression.find(exp.TimeStrToTime): + expression = expression.copy() + for ts_node in reversed(list(expression.find_all(exp.TimeStrToTime))): + if ( + ts_node.args.get("zone") + and isinstance(ts_node.this, exp.Literal) + and ts_node.this.is_string + and "." in ts_node.this.this + ): + cast_node = exp.cast( + ts_node.this.copy(), + exp.DataType.build( + "TIMESTAMPTZ", + expressions=[exp.DataTypeParam(this=exp.Literal.number(6))], + ), + ) + if ts_node is expression: + expression = cast_node + else: + ts_node.replace(cast_node) + if expression.find(exp.DataType): + expression = expression.copy() + for dtype in reversed(list(expression.find_all(exp.DataType))): + if dtype.is_type("int128"): + replacement = exp.DataType.build("LARGEINT", udt=True) + if dtype is expression: + expression = replacement + else: + dtype.replace(replacement) + elif ( + dtype.is_type("timestamptz") + and not dtype.expressions + and isinstance(dtype.parent, (exp.Cast, exp.TryCast)) + and isinstance(dtype.parent.this, exp.Literal) + and dtype.parent.this.is_string + and "." in dtype.parent.this.this + ): + dtype.set("expressions", [exp.DataTypeParam(this=exp.Literal.number(6))]) + elif dtype.is_type("struct"): + new_field_sqls: t.List[str] = [] + for field in dtype.expressions: + if isinstance(field, exp.ColumnDef) and field.args.get("kind"): + field_id = field.this.copy() + if quote and isinstance(field_id, exp.Identifier): + field_id.set("quoted", True) + field_name_sql = super()._to_sql( + field_id, quote=False, pretty=False, **kwargs + ) + field_kind_sql = super()._to_sql( + field.args["kind"], quote=quote, pretty=False, **kwargs + ) + new_field_sqls.append(f"{field_name_sql}: {field_kind_sql}") + else: + new_field_sqls.append( + super()._to_sql(field, quote=quote, pretty=False, **kwargs) + ) + struct_replacement = exp.DataType( + this=exp.DataType.Type.USERDEFINED, + kind=f"STRUCT<{', '.join(new_field_sqls)}>", + ) + if dtype is expression: + expression = struct_replacement + else: + dtype.replace(struct_replacement) + return super()._to_sql(expression, quote=quote, **kwargs) + + def _build_distributed_by_exp( + self, + distributed_by: t.Any = None, + clustered_by: t.Optional[t.List[exp.Expr]] = None, + unique_key_cols: t.Optional[t.List[str]] = None, + fallback_key_cols: t.Optional[t.List[str]] = None, + top_level_buckets: t.Any = None, + ) -> t.Optional[exp.DistributedByProperty]: + default_buckets_expr = self._resolve_buckets(top_level_buckets) + + if distributed_by is not None: + distributed_info: t.Dict[str, t.Any] = {} + + # Case 1: Function expression such as HASH(a, b) or RANDOM() + if isinstance(distributed_by, (exp.Anonymous, exp.Func)) and not isinstance( + distributed_by, (exp.Tuple, exp.Array, exp.Paren) + ): + kind_upper = ( + "RANDOM" + if isinstance(distributed_by, exp.Rand) + else distributed_by.name.upper() + ) + if kind_upper not in ("HASH", "RANDOM"): + raise SQLMeshError( + f"Invalid distributed_by kind: {kind_upper}. Expected HASH or RANDOM." + ) + cols = [ + exp.to_column(c) for c in self._extract_column_names(distributed_by.expressions) + ] + if kind_upper == "RANDOM" and cols: + raise SQLMeshError( + "Doris DISTRIBUTED BY RANDOM does not accept column expressions." + ) + if kind_upper == "HASH" and not cols: + raise SQLMeshError( + "Doris DISTRIBUTED BY HASH requires at least one column expression." + ) + return exp.DistributedByProperty( + kind=exp.Var(this=kind_upper), + expressions=cols if cols else None, + buckets=default_buckets_expr or (exp.Literal.number(10) if cols else None), + order=None, + ) + + # Case 2: String shorthand or bare column/identifier such as "HASH(id) BUCKETS 10", "RANDOM BUCKETS AUTO", or id + if isinstance(distributed_by, (str, exp.Column, exp.Identifier, exp.Var)) or ( + isinstance(distributed_by, exp.Literal) + and getattr(distributed_by, "is_string", False) + ): + text = ( + distributed_by.strip() + if isinstance(distributed_by, str) + else ( + str(distributed_by.this).strip() + if isinstance(distributed_by, exp.Literal) + else distributed_by.name.strip() + ) + ) + m = re.match( + r"^(HASH|RANDOM)(?:\s*\(([^)]*)\)|\b)\s*(?:BUCKETS\s+(\S+))?$", + text, + flags=re.IGNORECASE, + ) + if m: + kind_upper = m.group(1).upper() + cols_str = m.group(2) + b_str = m.group(3) + parsed_cols = ( + [exp.to_column(c) for c in self._extract_column_names(cols_str)] + if cols_str + else None + ) + if kind_upper == "RANDOM" and parsed_cols: + raise SQLMeshError( + "Doris DISTRIBUTED BY RANDOM does not accept column expressions." + ) + if kind_upper == "HASH" and not parsed_cols: + raise SQLMeshError( + "Doris DISTRIBUTED BY HASH requires at least one column expression." + ) + b_expr = self._resolve_buckets(b_str) if b_str else default_buckets_expr + return exp.DistributedByProperty( + kind=exp.Var(this=kind_upper), + expressions=parsed_cols, + buckets=b_expr, + order=None, + ) + cols = [exp.to_column(c) for c in self._extract_column_names(text)] + if cols: + return exp.DistributedByProperty( + kind=exp.Var(this="HASH"), + expressions=cols, + buckets=default_buckets_expr or exp.Literal.number(10), + order=None, + ) + + # Case 3: Tuple or Array (either key=value pairs or list of column names from dbt-doris) + if isinstance(distributed_by, (exp.Tuple, exp.Array)): + has_eq = any(isinstance(e, exp.EQ) for e in distributed_by.expressions) + if has_eq: + for expr in distributed_by.expressions: + if isinstance(expr, exp.EQ): + key = ( + (self._extract_str_value(expr.this) or "") + .strip('"') + .strip("'") + .strip("`") + .lower() + ) + if isinstance(expr.expression, exp.Literal): + distributed_info[key] = expr.expression.this + elif isinstance(expr.expression, (exp.Array, exp.Tuple)): + distributed_info[key] = self._extract_column_names(expr.expression) + else: + distributed_info[key] = self._extract_str_value(expr.expression) + else: + cols = [exp.to_column(c) for c in self._extract_column_names(distributed_by)] + if cols: + return exp.DistributedByProperty( + kind=exp.Var(this="HASH"), + expressions=cols, + buckets=default_buckets_expr or exp.Literal.number(10), + order=None, + ) + elif isinstance(distributed_by, exp.EQ) or ( + isinstance(distributed_by, exp.Paren) and isinstance(distributed_by.this, exp.EQ) + ): + expr = ( + distributed_by.this if isinstance(distributed_by, exp.Paren) else distributed_by + ) + key = ( + (self._extract_str_value(expr.this) or "") + .strip('"') + .strip("'") + .strip("`") + .lower() + ) + if isinstance(expr.expression, exp.Literal): + distributed_info[key] = expr.expression.this + else: + distributed_info[key] = self._extract_str_value(expr.expression) + elif isinstance(distributed_by, dict): + distributed_info = distributed_by + + if distributed_info: + raw_kind = distributed_info.get("kind") + raw_random = distributed_info.get("random") + expressions = ( + distributed_info.get("expressions") + if distributed_info.get("expressions") is not None + else distributed_info.get("columns") + ) + buckets = distributed_info.get("buckets", top_level_buckets) + + if raw_random is not None: + rand_str = ( + str(self._extract_str_value(raw_random) or raw_random).strip().lower() + ) + if rand_str in ("true", "1"): + if raw_kind is not None and str(raw_kind).strip().upper() != "RANDOM": + raise SQLMeshError( + f"Conflicting distributed_by configuration: kind='{raw_kind}' with random=true." + ) + raw_kind = "RANDOM" + if raw_kind is None and expressions is not None: + raw_kind = "HASH" + + if raw_kind: + kind_upper = str(raw_kind).strip().upper() + if kind_upper not in ("HASH", "RANDOM"): + raise SQLMeshError( + f"Invalid distributed_by kind: {kind_upper}. Expected HASH or RANDOM." + ) + buckets_expr = self._resolve_buckets(buckets) + + expressions_list = None + if expressions: + col_names = self._extract_column_names(expressions) + expressions_list = [exp.to_column(c) for c in col_names] + + if kind_upper == "RANDOM" and expressions_list: + raise SQLMeshError( + "Doris DISTRIBUTED BY RANDOM does not accept column expressions." + ) + if kind_upper == "HASH" and not expressions_list: + raise SQLMeshError( + "Doris DISTRIBUTED BY HASH requires at least one column expression." + ) + + return exp.DistributedByProperty( + kind=exp.Var(this=kind_upper), + expressions=expressions_list, + buckets=buckets_expr, + order=None, + ) + + # Fallback 1: Top-level clustered_by (or physical_properties clustered_by) + if clustered_by: + col_names = [c for expr in clustered_by for c in self._extract_column_names(expr)] + if unique_key_cols: + strict_key_set = {k.lower() for k in unique_key_cols} + not_in_key = [c for c in col_names if c.lower() not in strict_key_set] + if not_in_key: + logger.warning( + f"[Doris] clustered_by columns {not_in_key} not in key columns {unique_key_cols}, falling back to key column distribution." + ) + col_names = [] + cols = [exp.to_column(c) for c in col_names] + if cols: + return exp.DistributedByProperty( + kind=exp.Var(this="HASH"), + expressions=cols, + buckets=default_buckets_expr or exp.Literal.number(10), + order=None, + ) + + # Fallback 2: Default DISTRIBUTED BY HASH () BUCKETS + if unique_key_cols: + first_col = unique_key_cols[0] + logger.info( + f"[Doris] Adding default distributed_by using unique_key column: {first_col}" + ) + return exp.DistributedByProperty( + expressions=[exp.to_column(first_col)], + kind="HASH", + buckets=default_buckets_expr or exp.Literal.number(10), + ) + + # Fallback 3: Standalone buckets with duplicate_key/aggregate_key or RANDOM + if default_buckets_expr is not None: + if fallback_key_cols: + return exp.DistributedByProperty( + kind=exp.Var(this="HASH"), + expressions=[exp.to_column(fallback_key_cols[0])], + buckets=default_buckets_expr, + order=None, + ) + return exp.DistributedByProperty( + kind=exp.Var(this="RANDOM"), + expressions=None, + buckets=default_buckets_expr, + order=None, + ) + + return None + + def _get_temp_table( + self, table: TableName, table_only: bool = False, quoted: bool = True + ) -> exp.Table: + """Returns the temp table expression, truncated to fit Doris's 64-char identifier limit.""" + table_copy = t.cast(exp.Table, exp.to_table(table).copy()) + table_copy.set( + "this", + exp.to_identifier( + f"__temp_{table_copy.name[:48]}_{random_id(short=True)}", quoted=quoted + ), + ) + + if table_only: + table_copy.set("db", None) + table_copy.set("catalog", None) + + return table_copy + + def _properties_to_expressions(self, properties: t.Dict[str, t.Any]) -> t.List[exp.Expr]: + """Convert a dictionary of properties to a list of Doris PROPERTIES ('k'='v') expressions.""" + expressions: t.List[exp.Expr] = [] + skip_keys = { + "build", + "refresh", + "refresh_trigger", + "distributed_by", + "clustered_by", + "buckets", + "partition_by", + "partitioned_by", + "partitions", + "auto_partition", + "unique_key", + "duplicate_key", + "aggregate_key", + "table_type", + "keys", + "order_by", + "cluster_by", + "not_null_columns", + "engine", + "properties", + "materialized_type", + "source_table", + } + for key, value in properties.items(): + if key in skip_keys or value is None: + continue + raw_str = self._extract_str_value(value) + val_str = raw_str if raw_str is not None else str(value) + if key == "excluded_trigger_tables" and val_str: + val_str = ",".join( + re.sub(r"(?i)^`?internal`?\.", "", part.strip()) + for part in val_str.split(",") + if part.strip() + ) + str_literal = exp.Literal.string(val_str) + expressions.append(exp.Property(this=exp.Literal.string(str(key)), value=str_literal)) + return expressions diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index e8e122dece..6eedc65fd1 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -737,7 +737,10 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An ) for key in keys_to_resolve: rendered[key] = _resolve_model_refs_to_physical_tables( - rendered[key], table_mapping, self.dialect + rendered[key], + table_mapping, + self.dialect, + default_catalog=self.default_catalog, ) return rendered @@ -2830,23 +2833,44 @@ def _split_sql_model_statements( def _resolve_model_refs_to_physical_tables( - value: exp.Expr, table_mapping: t.Dict[str, str], dialect: DialectType + value: exp.Expr, + table_mapping: t.Dict[str, str], + dialect: DialectType, + default_catalog: t.Optional[str] = None, ) -> exp.Literal: """Resolve managed-model references in a property value to their physical table names. - The value is a single table reference or a comma-separated list of them. Each reference that - matches a managed model (via ``table_mapping``) is swapped for its physical ``db.table`` name; - anything else (e.g. a raw source) is kept as written. Returns a single string literal so the - property renders just like a hand-written value. + The value is a single table reference, a comma-separated string of references, or a tuple/array + of references. Each reference that matches a managed model (via ``table_mapping``) is swapped + for its physical ``db.table`` name; anything else (e.g. a raw source) is kept as written. + Returns a single string literal so the property renders just like a hand-written value. """ + if isinstance(value, exp.Paren): + value = value.this + if isinstance(value, exp.Literal) and value.is_string: refs = value.this.split(",") + elif isinstance(value, (exp.Tuple, exp.Array)): + refs = [ + part + for elem in value.expressions + for part in ( + elem.this.split(",") + if isinstance(elem, exp.Literal) and elem.is_string + else [elem.sql(dialect=dialect)] + ) + ] else: refs = [value.sql(dialect=dialect)] def resolve(ref: str) -> str: table = exp.to_table(ref.strip(), dialect=dialect) physical = table_mapping.get(exp.table_name(table, identify=True)) + if not physical and default_catalog and not table.catalog: + normalized_name = d.normalize_model_name( + table, default_catalog=default_catalog, dialect=dialect + ) + physical = table_mapping.get(normalized_name) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). return exp.table_name( exp.to_table(physical, dialect=dialect) if physical else table, identify=False diff --git a/tests/core/engine_adapter/integration/__init__.py b/tests/core/engine_adapter/integration/__init__.py index fe1951add6..3aa2639a92 100644 --- a/tests/core/engine_adapter/integration/__init__.py +++ b/tests/core/engine_adapter/integration/__init__.py @@ -80,6 +80,7 @@ def pytest_marks(self) -> t.List[MarkDecorator]: IntegrationTestEngine("clickhouse", catalog_types=["standalone", "cluster"]), IntegrationTestEngine("risingwave"), IntegrationTestEngine("starrocks"), + IntegrationTestEngine("doris"), # Cloud engines that need paid accounts / special credentials IntegrationTestEngine("clickhouse_cloud", cloud=True), IntegrationTestEngine("redshift", cloud=True), @@ -276,6 +277,7 @@ def timestamp_columns(self) -> t.List[str]: if v.sql().lower().startswith("timestamp") or (v.sql().lower() == "datetime" and self.dialect == "bigquery") or (v.sql().lower() == "datetime" and self.dialect == "starrocks") + or (v.sql().lower() == "datetime" and self.dialect == "doris") ] @property @@ -462,7 +464,7 @@ def get_table_comment( AND pgc.relkind = '{"v" if table_kind == "VIEW" else "r"}' ; """ - elif self.dialect in ["mysql", "snowflake", "starrocks"]: + elif self.dialect in ["mysql", "snowflake", "starrocks", "doris"]: # Snowflake treats all identifiers as uppercase unless they are lowercase and quoted. # They are lowercase and quoted in sushi but not in the inline tests. if self.dialect == "snowflake" and snowflake_capitalize_ids: @@ -473,6 +475,7 @@ def get_table_comment( "mysql": "table_comment", "snowflake": "comment", "starrocks": "table_comment", + "doris": "table_comment", } query = f""" @@ -594,7 +597,7 @@ def get_column_comments( AND pgc.relkind = '{"v" if table_kind == "VIEW" else "r"}' ; """ - elif self.dialect in ["mysql", "snowflake", "trino", "starrocks"]: + elif self.dialect in ["mysql", "snowflake", "trino", "starrocks", "doris"]: # Snowflake treats all identifiers as uppercase unless they are lowercase and quoted. # They are lowercase and quoted in sushi but not in the inline tests. if self.dialect == "snowflake" and snowflake_capitalize_ids: @@ -606,6 +609,7 @@ def get_column_comments( "snowflake": "comment", "trino": "comment", "starrocks": "column_comment", + "doris": "column_comment", } query = f""" @@ -754,6 +758,17 @@ def create_catalog(self, catalog_name: str): self.engine_adapter.execute(f'CREATE DATABASE IF NOT EXISTS "{catalog_name}"') except Exception: pass + elif self.dialect == "doris": + self.engine_adapter.execute( + f""" + CREATE CATALOG IF NOT EXISTS `{catalog_name}` PROPERTIES ( + 'type' = 'iceberg', + 'iceberg.catalog.type' = 'hadoop', + 'warehouse' = 'file:///tmp/doris_iceberg_warehouse/{catalog_name}' + ) + """, + quote_identifiers=False, + ) self._catalogs.append(catalog_name) @@ -765,6 +780,11 @@ def drop_catalog(self, catalog_name: str): elif self.dialect == "fabric": # Use the engine adapter's built-in catalog dropping functionality self.engine_adapter.drop_catalog(catalog_name) + elif self.dialect == "doris": + self.engine_adapter.set_current_catalog("internal") + self.engine_adapter.execute( + f"DROP CATALOG IF EXISTS `{catalog_name}`", quote_identifiers=False + ) else: self.engine_adapter.execute(f'DROP DATABASE IF EXISTS "{catalog_name}"') diff --git a/tests/core/engine_adapter/integration/config.yaml b/tests/core/engine_adapter/integration/config.yaml index 24ba2b2a6d..931f5e0343 100644 --- a/tests/core/engine_adapter/integration/config.yaml +++ b/tests/core/engine_adapter/integration/config.yaml @@ -128,6 +128,16 @@ gateways: check_import: false state_connection: type: duckdb + inttest_doris: + connection: + type: doris + host: {{ env_var('DOCKER_HOSTNAME', 'localhost') }} + port: 9030 + user: root + password: "" + check_import: false + state_connection: + type: duckdb # Cloud databases diff --git a/tests/core/engine_adapter/integration/docker/compose.doris.yaml b/tests/core/engine_adapter/integration/docker/compose.doris.yaml new file mode 100644 index 0000000000..4077193995 --- /dev/null +++ b/tests/core/engine_adapter/integration/docker/compose.doris.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +services: + doris-fe: + image: apache/doris:fe-4.1.5 + container_name: doris-fe + hostname: doris-fe + entrypoint: + - /bin/bash + - -c + command: + - | + sed -i 's/-Xmx8192m -Xms8192m/-Xmx2048m -Xms512m/g' /opt/apache-doris/fe/conf/fe.conf + echo "priority_networks = 172.30.80.0/24" >> /opt/apache-doris/fe/conf/fe.conf + echo "force_olap_table_replication_num = 1" >> /opt/apache-doris/fe/conf/fe.conf + exec bash /opt/apache-doris/fe/bin/start_fe.sh --console + ports: + - "8030:8030" + - "9030:9030" + volumes: + - iceberg-warehouse:/tmp/doris_iceberg_warehouse + networks: + doris_net: + ipv4_address: 172.30.80.2 + + doris-be: + image: apache/doris:be-4.1.5 + container_name: doris-be + hostname: doris-be + depends_on: + - doris-fe + entrypoint: + - /bin/bash + - -c + command: + - | + export SKIP_CHECK_ULIMIT=true + sed -i 's/-Xmx2048m/-Xmx1024m/g' /opt/apache-doris/be/conf/be.conf + echo "priority_networks = 172.30.80.0/24" >> /opt/apache-doris/be/conf/be.conf + echo "mem_limit = 80%" >> /opt/apache-doris/be/conf/be.conf + echo "disable_memory_gc = false" >> /opt/apache-doris/be/conf/be.conf + echo "storage_flood_stage_usage_percent = 99" >> /opt/apache-doris/be/conf/be.conf + echo "storage_flood_stage_left_capacity_bytes = 104857600" >> /opt/apache-doris/be/conf/be.conf + exec bash /opt/apache-doris/be/bin/start_be.sh --console + ports: + - "8040:8040" + - "9050:9050" + volumes: + - iceberg-warehouse:/tmp/doris_iceberg_warehouse + networks: + doris_net: + ipv4_address: 172.30.80.3 + +volumes: + iceberg-warehouse: + +networks: + doris_net: + ipam: + config: + - subnet: 172.30.80.0/24 diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index 6a2905d64d..6098824096 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -158,6 +158,10 @@ def create_objects_and_validate(schema_name: str): pytest.skip( f"Engine adapter {ctx.engine_adapter.dialect} doesn't support catalog operations" ) + if ctx.dialect == "doris": + pytest.skip( + "Doris external catalogs require connector properties and do not support views; covered in test_integration_doris.py" + ) if ctx.dialect == "spark": pytest.skip( "Currently local spark is configured to have iceberg be the testing catalog and drop cascade doesn't work on iceberg. Skipping until we have time to fix." @@ -439,6 +443,12 @@ def test_materialized_view(ctx_query_and_df: TestContext): "which this generic test does not provide; StarRocks MVs are covered by " "test_integration_starrocks.py" ) + if ctx.engine_adapter.dialect == "doris": + pytest.skip( + "Doris materialized views require engine-specific properties (SYNC source_table or MTMV DISTRIBUTED BY), " + "which this generic test does not provide; Doris MVs are covered by " + "test_integration_doris.py" + ) input_data = pd.DataFrame( [ {"id": 1, "ds": "2022-01-01"}, @@ -2684,6 +2694,7 @@ def test_dialects(ctx: TestContext): "spark": pd.Timestamp("2020-01-01 00:00:00"), "databricks": pd.Timestamp("2020-01-01 00:00:00"), "starrocks": pd.Timestamp("2020-01-01 00:00:00"), + "doris": pd.Timestamp("2020-01-01 00:00:00"), }, ), ( @@ -3247,7 +3258,7 @@ def test_value_normalization( if ctx.dialect == "trino" and ctx.engine_adapter.current_catalog_type == "hive": pytest.skip("Trino on Hive doesn't support TIMESTAMP WITH TIME ZONE fields") if ctx.dialect == "fabric": - pytest.skip("Fabric doesn't support TIMESTAMP WITH TIME ZONE fields") + pytest.skip(f"{ctx.dialect} doesn't support TIMESTAMP WITH TIME ZONE fields") if not isinstance(ctx.engine_adapter, RowDiffMixin): pytest.skip( @@ -3258,8 +3269,8 @@ def test_value_normalization( # resolve dialect-specific types if column_type in (DATA_TYPE.DATETIME, DATA_TYPE.TIMESTAMP, DATA_TYPE.TIMESTAMPTZ): - if ctx.dialect in ("mysql", "trino"): - # MySQL needs DATETIME(6) instead of DATETIME or subseconds will be truncated. + if ctx.dialect in ("mysql", "trino", "doris"): + # MySQL and Doris need DATETIME(6) instead of DATETIME or subseconds will be truncated. # It also needs TIMESTAMP(6) as the column type for CREATE TABLE or the truncation will occur full_column_type = exp.DataType.build( column_type, @@ -3283,7 +3294,12 @@ def test_value_normalization( exp.to_table(ctx.table("test_value_normalization")), dialect=ctx.dialect ) columns_to_types_normalized = { - normalize_identifiers(k, dialect=ctx.dialect).sql(dialect=ctx.dialect): v + ( + # "value" is in DorisGenerator.RESERVED_KEYWORDS, so .sql(dialect="doris") returns "`value`" + normalize_identifiers(k, dialect=ctx.dialect).name + if ctx.dialect == "doris" + else normalize_identifiers(k, dialect=ctx.dialect).sql(dialect=ctx.dialect) + ): v for k, v in columns_to_types.items() } @@ -3941,6 +3957,11 @@ def test_materialized_view_evaluation(ctx: TestContext): "which this generic test does not provide; StarRocks MVs are covered by " "test_integration_starrocks.py" ) + elif dialect == "doris": + pytest.skip( + "Doris materialized views require DISTRIBUTED BY or physical_properties configuration; " + "Doris MVs are covered by test_integration_doris.py" + ) model_name = ctx.table("test_tbl") mview_name = ctx.table("test_mview") @@ -4008,6 +4029,11 @@ def test_unicode_characters(ctx: TestContext, tmp_path: Path): # I also think Spark may not support unicode in general but that would need to be verified. if not ctx.engine_adapter.QUOTE_IDENTIFIERS_IN_VIEWS: pytest.skip("Skipping as these engines have issues with unicode characters in model names") + # Doris static fe.conf setting `enable_unicode_name_support` defaults to false (and is not runtime-mutable) + if ctx.dialect == "doris": + pytest.skip( + "Skipping as Doris default setting has issues with unicode characters in model names" + ) model_name = "客户数据" table = ctx.table(model_name).sql(dialect=ctx.dialect) diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py new file mode 100644 index 0000000000..8a4e682a03 --- /dev/null +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -0,0 +1,946 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import typing as t + +import pytest +from pytest import FixtureRequest +from sqlglot import exp, parse_one + +from sqlmesh.core.engine_adapter import DorisEngineAdapter +from tests.core.engine_adapter.integration import ( + ENGINES_BY_NAME, + IntegrationTestEngine, + TestContext, + generate_pytest_params, +) + + +@pytest.fixture(params=list(generate_pytest_params(ENGINES_BY_NAME["doris"]))) +def ctx( + request: FixtureRequest, + create_test_context: t.Callable[[IntegrationTestEngine, str, str], t.Iterable[TestContext]], +) -> t.Iterable[TestContext]: + yield from create_test_context(*request.param) + + +@pytest.fixture +def engine_adapter(ctx: TestContext) -> DorisEngineAdapter: + assert isinstance(ctx.engine_adapter, DorisEngineAdapter) + return ctx.engine_adapter + + +def test_engine_adapter(ctx: TestContext): + assert isinstance(ctx.engine_adapter, DorisEngineAdapter) + assert ctx.engine_adapter.fetchone("select 1") == (1,) + + +def test_doris_iceberg_local_catalog(ctx: TestContext): + adapter = ctx.engine_adapter + assert isinstance(adapter, DorisEngineAdapter) + + catalog_name = f"local_iceberg_{ctx.test_id}" + schema_name = f"iceberg_db_{ctx.test_id}" + table_fqn = f"{catalog_name}.{schema_name}.events" + + try: + adapter.execute( + f""" + CREATE CATALOG IF NOT EXISTS `{catalog_name}` PROPERTIES ( + 'type' = 'iceberg', + 'iceberg.catalog.type' = 'hadoop', + 'warehouse' = 'file:///tmp/doris_iceberg_warehouse/{catalog_name}' + ) + """, + quote_identifiers=False, + ) + adapter.create_schema(f"{catalog_name}.{schema_name}") + adapter.create_table( + table_fqn, + { + "id": exp.DataType.build("int"), + "ds": exp.DataType.build("date"), + }, + partitioned_by=[exp.to_column("ds")], + ) + + cols = adapter.columns(table_fqn) + assert "id" in cols + assert "ds" in cols + + data_objects = adapter.get_data_objects(f"{catalog_name}.{schema_name}") + assert any(obj.name == "events" for obj in data_objects) + + schema_target_fqn = f"{catalog_name}.{schema_name}.events_target" + adapter.create_table( + schema_target_fqn, + { + "id": exp.DataType.build("int"), + "ds": exp.DataType.build("date"), + "region": exp.DataType.build("varchar(32)"), + }, + partitioned_by=[exp.to_column("ds")], + ) + alter_ops = adapter.get_alter_operations(table_fqn, schema_target_fqn) + assert len(alter_ops) == 1 + adapter.alter_table(alter_ops) + + updated_cols = adapter.columns(table_fqn) + assert "region" in updated_cols + + # Test Iceberg table maintenance actions & branch DDL parsed via Doris dialect + create_branch_expr = parse_one( + f"ALTER TABLE {table_fqn} CREATE BRANCH IF NOT EXISTS audit_branch", + read="doris", + ) + adapter.execute(create_branch_expr, quote_identifiers=False) + + branch_read_expr = parse_one( + f"SELECT id FROM {table_fqn}@branch(audit_branch)", + read="doris", + ) + assert list(adapter.fetchall(branch_read_expr, quote_identifiers=False)) == [] + + compact_expr = parse_one( + f"ALTER TABLE {table_fqn} EXECUTE rewrite_data_files()", + read="doris", + ) + adapter.execute(compact_expr, quote_identifiers=False) + + # Also verify writable Apache Paimon local catalog DDL & schema evolution on Doris 4.1+ + paimon_cat = f"local_paimon_{ctx.test_id}" + paimon_db = f"paimon_db_{ctx.test_id}" + paimon_tbl = f"{paimon_cat}.{paimon_db}.orders" + try: + adapter.execute( + f""" + CREATE CATALOG IF NOT EXISTS `{paimon_cat}` PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 'file:///tmp/doris_paimon_warehouse/{paimon_cat}' + ) + """, + quote_identifiers=False, + ) + adapter.create_schema(f"{paimon_cat}.{paimon_db}") + adapter.create_table( + paimon_tbl, + { + "id": exp.DataType.build("int"), + "ds": exp.DataType.build("date"), + }, + table_format="paimon", + storage_format="parquet", + partitioned_by=[exp.to_column("ds")], + unique_key=[exp.to_column("id")], + ) + paimon_cols = adapter.columns(paimon_tbl) + assert "id" in paimon_cols + assert "ds" in paimon_cols + finally: + try: + adapter.drop_schema( + f"{paimon_cat}.{paimon_db}", ignore_if_not_exists=True, cascade=True + ) + except Exception: + pass + try: + adapter.execute(f"DROP CATALOG IF EXISTS `{paimon_cat}`", quote_identifiers=False) + except Exception: + pass + finally: + try: + adapter.drop_schema( + f"{catalog_name}.{schema_name}", ignore_if_not_exists=True, cascade=True + ) + except Exception: + pass + try: + adapter.execute(f"DROP CATALOG IF EXISTS `{catalog_name}`", quote_identifiers=False) + except Exception: + pass + adapter.set_current_catalog("internal") + + +def test_materialized_view_sync(ctx: TestContext): + import pandas as pd # noqa: TID253 + from tests.core.engine_adapter.integration import wait_until + + input_data = pd.DataFrame( + [ + {"id": 1, "ds": "2022-01-01"}, + {"id": 2, "ds": "2022-01-02"}, + {"id": 3, "ds": "2022-01-03"}, + ] + ) + source_table = ctx.table("source_table") + ctx.engine_adapter.ctas(source_table, ctx.input_data(input_data), ctx.columns_to_types) + view = exp.to_table("test_sync_mv") + first_column = next(iter(ctx.columns_to_types)) + view_query = exp.select(first_column).from_(source_table) + ctx.engine_adapter.create_view( + view, + view_query, + materialized=True, + view_properties={ + "materialized_type": exp.Literal.string("SYNC"), + "source_table": source_table, + }, + ) + + def drop_view_success() -> bool: + try: + ctx.engine_adapter.drop_view( + view, + ignore_if_not_exists=False, + materialized=True, + view_properties={ + "materialized_type": exp.Literal.string("SYNC"), + "source_table": source_table, + }, + ) + return True + except Exception: + return False + + wait_until(drop_view_success, attempts=30, wait=1) + + +def test_materialized_view_async_mtmv(ctx: TestContext): + import pandas as pd # noqa: TID253 + from tests.core.engine_adapter.integration import wait_until + + input_data = pd.DataFrame( + [ + {"id": 1, "ds": "2022-01-01"}, + {"id": 2, "ds": "2022-01-02"}, + {"id": 3, "ds": "2022-01-03"}, + ] + ) + source_table = ctx.table("mtmv_source") + ctx.engine_adapter.ctas(source_table, ctx.input_data(input_data), ctx.columns_to_types) + view = ctx.table("test_async_mtmv") + view_query = exp.select(*ctx.columns_to_types).from_(source_table) + ctx.engine_adapter.create_view( + view, + view_query, + materialized=True, + view_properties={ + "build": exp.Literal.string("IMMEDIATE"), + "refresh": exp.Literal.string("COMPLETE"), + "refresh_trigger": exp.Literal.string("ON MANUAL"), + "distributed_by": exp.Literal.string("RANDOM BUCKETS AUTO"), + }, + ) + wait_until(lambda: len(ctx.get_current_data(view)) == len(input_data), attempts=30, wait=1) + ctx.compare_with_current(view, input_data) + results = ctx.get_metadata_results() + assert exp.to_table(view).name in results.materialized_views + assert exp.to_table(view).name not in results.tables + ctx.engine_adapter.drop_view(view, materialized=True) + assert len(ctx.get_metadata_results().materialized_views) == 0 + + +def test_merge_unique_key(ctx: TestContext): + import pandas as pd # noqa: TID253 + + table = ctx.table("test_merge_uk") + ctx.engine_adapter.create_table( + table, + ctx.columns_to_types, + table_properties={"unique_key": exp.Literal.string("id")}, + ) + input_data = pd.DataFrame( + [ + {"id": 1, "ds": "2022-01-01"}, + {"id": 2, "ds": "2022-01-02"}, + {"id": 3, "ds": "2022-01-03"}, + ] + ) + ctx.engine_adapter.merge( + table, + ctx.input_data(input_data), + target_columns_to_types=None, + unique_key=[exp.to_identifier("id")], + ) + merge_data = pd.DataFrame( + [ + {"id": 2, "ds": "2022-01-10"}, + {"id": 4, "ds": "2022-01-04"}, + ] + ) + ctx.engine_adapter.merge( + table, + ctx.input_data(merge_data), + target_columns_to_types=None, + unique_key=[exp.to_identifier("id")], + ) + ctx.compare_with_current( + table, + pd.DataFrame( + [ + {"id": 1, "ds": "2022-01-01"}, + {"id": 2, "ds": "2022-01-10"}, + {"id": 3, "ds": "2022-01-03"}, + {"id": 4, "ds": "2022-01-04"}, + ] + ), + ) + + +def test_auto_partition_and_insert_overwrite_by_partition(ctx: TestContext): + import pandas as pd # noqa: TID253 + from sqlglot import parse_one + + table = ctx.table("test_auto_partition") + columns_to_types = { + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + } + partitioned_by = [parse_one("RANGE(DATE_TRUNC(ds, 'day'))", read="doris")] + ctx.engine_adapter.create_table( + table, + columns_to_types, + partitioned_by=partitioned_by, + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + initial_data = pd.DataFrame( + [ + {"payload": "a", "id": 1, "ds": "2022-01-01"}, + {"payload": "b", "id": 2, "ds": "2022-01-02"}, + ] + ) + ctx.engine_adapter.insert_append( + table, + ctx.input_data(initial_data, columns_to_types), + target_columns_to_types=columns_to_types, + ) + overwrite_data = pd.DataFrame( + [ + {"payload": "b_updated", "id": 20, "ds": "2022-01-02"}, + {"payload": "c", "id": 3, "ds": "2022-01-03"}, + ] + ) + ctx.engine_adapter.insert_overwrite_by_partition( + table, + ctx.input_data(overwrite_data, columns_to_types), + partitioned_by=partitioned_by, + target_columns_to_types=columns_to_types, + ) + rows = ctx.engine_adapter.fetchall( + exp.select("id", exp.cast(exp.column("ds"), "VARCHAR").as_("ds"), "payload") + .from_(table) + .order_by("id") + ) + assert list(rows) == [ + (1, "2022-01-01", "a"), + (3, "2022-01-03", "c"), + (20, "2022-01-02", "b_updated"), + ] + + # Verify DATETIME partition overwrite deletes the entire day range even when timestamps differ + ts_table = ctx.table("test_auto_partition_ts") + ts_cols = { + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ts": exp.DataType.build("DATETIME", dialect="doris"), + } + ts_partitioned_by = [parse_one("RANGE(DATE_TRUNC(ts, 'day'))", read="doris")] + ctx.engine_adapter.create_table( + ts_table, + ts_cols, + partitioned_by=ts_partitioned_by, + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + ts_initial = pd.DataFrame( + [ + {"payload": "d1", "id": 1, "ts": "2022-01-01 09:00:00"}, + {"payload": "d2_morning", "id": 2, "ts": "2022-01-02 08:00:00"}, + {"payload": "d2_evening", "id": 3, "ts": "2022-01-02 18:00:00"}, + ] + ) + ctx.engine_adapter.insert_append( + ts_table, + ctx.input_data(ts_initial, ts_cols), + target_columns_to_types=ts_cols, + ) + ts_overwrite = pd.DataFrame( + [ + {"payload": "d2_replaced", "id": 20, "ts": "2022-01-02 14:30:00"}, + ] + ) + ctx.engine_adapter.insert_overwrite_by_partition( + ts_table, + ctx.input_data(ts_overwrite, ts_cols), + partitioned_by=ts_partitioned_by, + target_columns_to_types=ts_cols, + ) + ts_rows = ctx.engine_adapter.fetchall( + exp.select("id", "payload").from_(ts_table).order_by("id") + ) + assert list(ts_rows) == [ + (1, "d1"), + (20, "d2_replaced"), + ] + + +def test_unique_key_order_by_and_key_reordering(ctx: TestContext): + import pandas as pd # noqa: TID253 + + table = ctx.table("test_uk_order_by") + columns_to_types = { + "payload": exp.DataType.build("VARCHAR(64)", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + "seq": exp.DataType.build("INT", dialect="doris"), + } + ctx.engine_adapter.create_table( + table, + columns_to_types, + unique_key=[exp.to_column("id"), exp.to_column("ds")], + table_properties={ + "order_by": exp.Tuple(expressions=[exp.to_column("seq"), exp.to_column("id")]), + }, + ) + df = pd.DataFrame( + [ + {"payload": "first", "id": 1, "ds": "2022-01-01", "seq": 10}, + {"payload": "second", "id": 2, "ds": "2022-01-02", "seq": 5}, + ] + ) + ctx.engine_adapter.insert_append( + table, + ctx.input_data(df, columns_to_types), + target_columns_to_types=columns_to_types, + ) + rows = ctx.engine_adapter.fetchall( + exp.select("id", exp.cast(exp.column("ds"), "VARCHAR").as_("ds"), "seq", "payload") + .from_(table) + .order_by("id") + ) + assert list(rows) == [ + (1, "2022-01-01", 10, "first"), + (2, "2022-01-02", 5, "second"), + ] + + +def test_duplicate_key_delete_with_subquery_and_between(ctx: TestContext): + import pandas as pd # noqa: TID253 + + target_table = ctx.table("test_dup_delete_target") + src_table = ctx.table("test_dup_delete_src") + ctx.engine_adapter.create_table( + target_table, + ctx.columns_to_types, + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.to_column("id"), exp.to_column("ds")]), + }, + ) + input_data = pd.DataFrame( + [ + {"id": 1, "ds": "2022-01-01"}, + {"id": 2, "ds": "2022-01-02"}, + {"id": 3, "ds": "2022-01-03"}, + {"id": 4, "ds": "2022-01-04"}, + ] + ) + ctx.engine_adapter.insert_append( + target_table, + ctx.input_data(input_data), + target_columns_to_types=ctx.columns_to_types, + ) + # 1. BETWEEN rewrite in DELETE WHERE + ctx.engine_adapter.delete_from(target_table, "ds BETWEEN '2022-01-01' AND '2022-01-01'") + + # 2. Subquery DELETE on DUPLICATE KEY table (triggers literal batch fallback since USING requires UNIQUE KEY) + src_data = pd.DataFrame([{"id": 2, "ds": "2022-01-02"}]) + ctx.engine_adapter.ctas(src_table, ctx.input_data(src_data), ctx.columns_to_types) + key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", exp.column("id"), exp.column("ds")) + ctx.engine_adapter.delete_from( + target_table, + where=key_exp.isin(query=exp.select(key_exp).from_(src_table)), + ) + ctx.compare_with_current( + target_table, + pd.DataFrame( + [ + {"id": 3, "ds": "2022-01-03"}, + {"id": 4, "ds": "2022-01-04"}, + ] + ), + ) + + +def test_insert_overwrite_by_time_partition_and_scd_type_2(ctx: TestContext): + from datetime import datetime + import pandas as pd # noqa: TID253 + from sqlmesh.utils.date import to_time_column + + # 1. insert_overwrite_by_time_partition with CAST(... AS DATE) bounds and partition column after VARCHAR + time_table = ctx.table("test_time_range_overwrite") + time_cols = { + "code": exp.DataType.build("VARCHAR(64)", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + "amount": exp.DataType.build("DOUBLE", dialect="doris"), + } + ctx.engine_adapter.create_table( + time_table, + time_cols, + partitioned_by=[exp.to_column("ds")], + ) + initial_df = pd.DataFrame( + [ + {"code": "a", "ds": "2022-01-01", "amount": 10.0}, + {"code": "b", "ds": "2022-01-02", "amount": 20.0}, + {"code": "c", "ds": "2022-01-03", "amount": 30.0}, + ] + ) + ctx.engine_adapter.insert_append( + time_table, + ctx.input_data(initial_df, time_cols), + target_columns_to_types=time_cols, + ) + replacement_df = pd.DataFrame( + [ + {"code": "b_new", "ds": "2022-01-02", "amount": 25.0}, + ] + ) + ctx.engine_adapter.insert_overwrite_by_time_partition( + time_table, + ctx.input_data(replacement_df, time_cols), + start="2022-01-02", + end="2022-01-02", + time_column="ds", + time_formatter=lambda v, _: to_time_column( + v, exp.DataType.build("DATE", dialect="doris"), "doris" + ), + target_columns_to_types=time_cols, + ) + rows = ctx.engine_adapter.fetchall( + exp.select("code", exp.cast(exp.column("ds"), "VARCHAR").as_("ds"), "amount") + .from_(time_table) + .order_by("ds") + ) + assert list(rows) == [ + ("a", "2022-01-01", 10.0), + ("b_new", "2022-01-02", 25.0), + ("c", "2022-01-03", 30.0), + ] + + # 2. scd_type_2_by_time across initial load and incremental update + scd_table = ctx.table("test_scd_type_2") + scd_src_table = ctx.table("test_scd_type_2_src") + scd_src_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(64)", dialect="doris"), + "updated_at": exp.DataType.build("DATETIME", dialect="doris"), + } + scd_target_cols = { + **scd_src_cols, + "valid_from": exp.DataType.build("DATETIME", dialect="doris"), + "valid_to": exp.DataType.build("DATETIME", dialect="doris"), + } + ctx.engine_adapter.create_table(scd_table, scd_target_cols) + batch1_df = pd.DataFrame( + [ + {"id": 1, "name": "alice_v1", "updated_at": "2022-01-01 00:00:00"}, + {"id": 2, "name": "bob_v1", "updated_at": "2022-01-01 00:00:00"}, + ] + ) + ctx.engine_adapter.ctas(scd_src_table, ctx.input_data(batch1_df, scd_src_cols), scd_src_cols) + ctx.engine_adapter.scd_type_2_by_time( + target_table=scd_table, + source_table=exp.select("id", "name", "updated_at").from_(scd_src_table), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + updated_at_col=exp.column("updated_at", quoted=True), + target_columns_to_types=scd_target_cols, + execution_time=datetime(2022, 1, 2, 0, 0, 0), + truncate=True, + ) + batch2_df = pd.DataFrame( + [ + {"id": 1, "name": "alice_v2", "updated_at": "2022-01-05 00:00:00"}, + {"id": 2, "name": "bob_v1", "updated_at": "2022-01-01 00:00:00"}, + ] + ) + ctx.engine_adapter.replace_query( + scd_src_table, + ctx.input_data(batch2_df, scd_src_cols), + target_columns_to_types=scd_src_cols, + ) + ctx.engine_adapter.scd_type_2_by_time( + target_table=scd_table, + source_table=exp.select("id", "name", "updated_at").from_(scd_src_table), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + updated_at_col=exp.column("updated_at", quoted=True), + target_columns_to_types=scd_target_cols, + execution_time=datetime(2022, 1, 6, 0, 0, 0), + truncate=False, + ) + scd_rows = ctx.engine_adapter.fetchall( + exp.select("id", "name", exp.column("valid_to").is_(exp.null()).as_("is_current")) + .from_(scd_table) + .order_by("id", "valid_from") + ) + assert list(scd_rows) == [ + (1, "alice_v1", 0), + (1, "alice_v2", 1), + (2, "bob_v1", 1), + ] + + +def test_doris_complex_types_and_alter_table_schema_evolution(ctx: TestContext): + from sqlglot import parse_one + + # 1. Create table with LARGEINT, ARRAY, MAP, STRUCT, and BITMAP, insert rows, and run row-diff normalization + rich_table = ctx.table("test_doris_complex_types") + rich_cols = { + "id": exp.DataType.build("INT128", dialect="doris"), + "tags": exp.DataType.build("ARRAY", dialect="doris"), + "attrs": exp.DataType.build("MAP", dialect="doris"), + "profile": exp.DataType.build( + "STRUCT", dialect="doris", udt=True + ), + "bm": exp.DataType.build("BITMAP", dialect="doris", udt=True), + } + ctx.engine_adapter.create_table( + rich_table, + rich_cols, + table_properties={"duplicate_key": exp.Tuple(expressions=[exp.to_column("id")])}, + ) + ctx.engine_adapter.execute( + f"INSERT INTO {ctx.engine_adapter._to_sql(rich_table)} (`id`, `tags`, `attrs`, `profile`, `bm`) " + f"VALUES (100, ARRAY('x', 'y'), MAP('k1', 10, 'k2', 20), STRUCT(7, 'neo'), TO_BITMAP(42))" + ) + described_cols = ctx.engine_adapter.columns(rich_table) + assert described_cols["id"] == exp.DataType.build("INT128") + assert ctx.engine_adapter.get_alter_operations(rich_table, rich_table) == [] + concat_expr = t.cast(DorisEngineAdapter, ctx.engine_adapter).concat_columns(described_cols) + diff_rows = ctx.engine_adapter.fetchall(exp.select(concat_expr).from_(rich_table)) + assert len(diff_rows) == 1 + assert "100" in str(diff_rows[0][0]) + assert "42" in str(diff_rows[0][0]) + + # 2. Live schema evolution (ADD COLUMN + multiple MODIFY COLUMN widenings + suppressed key-column VARCHAR->STRING no-op) + cur_tbl = ctx.table("test_alter_cur") + tgt_tbl = ctx.table("test_alter_tgt") + ctx.engine_adapter.create_table( + cur_tbl, + { + "key_code": exp.DataType.build("VARCHAR(32)", dialect="doris"), + "notes": exp.DataType.build("VARCHAR(32)", dialect="doris"), + "cnt": exp.DataType.build("INT", dialect="doris"), + "dt": exp.DataType.build("DATETIME(0)", dialect="doris"), + "tz": exp.DataType.build("TIMESTAMPTZ(0)", dialect="doris"), + }, + table_properties={"duplicate_key": exp.Tuple(expressions=[exp.to_column("key_code")])}, + ) + ctx.engine_adapter.create_table( + tgt_tbl, + { + "key_code": exp.DataType.build("VARCHAR(32)", dialect="doris"), + "notes": exp.DataType.build("TEXT", dialect="doris"), + "cnt": exp.DataType.build("BIGINT", dialect="doris"), + "dt": exp.DataType.build("DATETIME(6)", dialect="doris"), + "tz": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + "extra_cnt": exp.DataType.build("BIGINT", dialect="doris"), + }, + table_properties={"duplicate_key": exp.Tuple(expressions=[exp.to_column("key_code")])}, + ) + alter_ops = ctx.engine_adapter.get_alter_operations(cur_tbl, tgt_tbl) + ctx.engine_adapter.alter_table(alter_ops) + evolved_cols = ctx.engine_adapter.columns(cur_tbl) + assert "extra_cnt" in evolved_cols + assert evolved_cols["key_code"] == exp.DataType.build("VARCHAR(32)", dialect="doris") + assert evolved_cols["notes"] == exp.DataType.build("TEXT", dialect="doris") + assert evolved_cols["cnt"] == exp.DataType.build("BIGINT", dialect="doris") + assert evolved_cols["dt"] == exp.DataType.build("DATETIME(6)", dialect="doris") + assert evolved_cols["tz"] == exp.DataType.build("DATETIME(6)", dialect="doris") + + # 3. Direct time-transform inequality delete on OLAP table + ev_tbl = ctx.table("test_time_ineq_del") + ev_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + } + ctx.engine_adapter.create_table(ev_tbl, ev_cols) + ctx.engine_adapter.execute( + f"INSERT INTO {ctx.engine_adapter._to_sql(ev_tbl)} (`id`, `ds`) VALUES " + f"(1, '2024-01-14'), (2, '2024-01-15'), (3, '2024-01-16')" + ) + ctx.engine_adapter.delete_from( + ev_tbl, + parse_one( + "DATE_TRUNC(ds, 'day') >= '2024-01-15' AND DATE_TRUNC(ds, 'day') <= '2024-01-15'", + read="doris", + ), + ) + remaining_ids = [ + r[0] for r in ctx.engine_adapter.fetchall(exp.select("id").from_(ev_tbl).order_by("id")) + ] + assert remaining_ids == [1, 3] + + # 4. De Morgan `NOT (id = 3)` and aligned `DATE_TRUNC(ds, 'week')` in `delete_from`, plus `insert_append` OLAP reordering + ctx.engine_adapter.delete_from( + ev_tbl, + parse_one("NOT (id = 3) AND DATE_TRUNC(ds, 'week') = '2024-01-08'", read="doris"), + ) + remaining_after_week = [ + r[0] for r in ctx.engine_adapter.fetchall(exp.select("id").from_(ev_tbl).order_by("id")) + ] + assert remaining_after_week == [3] + + app_tbl = ctx.table("test_insert_append_reorder") + app_cols = { + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + } + ctx.engine_adapter.create_table(app_tbl, app_cols) + ctx.engine_adapter.insert_append( + app_tbl, + parse_one("SELECT 9.5 AS score, 7 AS id", read="doris"), + target_columns_to_types=app_cols, + ) + app_rows = ctx.engine_adapter.fetchall(exp.select("id", "score").from_(app_tbl)) + assert list(app_rows) == [(7, 9.5)] + + +def test_doris_4_1_native_merge_segment_v3_and_ann_index(ctx: TestContext): + # 1. Native MERGE INTO with merge_filter and when_matched on a Merge-on-Write UNIQUE KEY table + merge_tbl = ctx.table("test_doris_41_merge") + merge_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(64)", dialect="doris"), + "val": exp.DataType.build("INT", dialect="doris"), + } + ctx.engine_adapter.create_table( + merge_tbl, + merge_cols, + unique_key=[exp.to_column("id")], + ) + ctx.engine_adapter.execute( + f"INSERT INTO {ctx.engine_adapter._to_sql(merge_tbl)} (`id`, `name`, `val`) VALUES (1, 'orig_a', 10), (2, 'orig_b', 20)" + ) + src_query = parse_one( + "SELECT 1 AS id, 'new_a' AS name, 15 AS val UNION ALL " + "SELECT 2 AS id, 'new_b' AS name, 5 AS val UNION ALL " + "SELECT 3 AS id, 'new_c' AS name, 30 AS val", + read="doris", + ) + ctx.engine_adapter.merge( + target_table=merge_tbl, + source_table=src_query, + target_columns_to_types=merge_cols, + unique_key=[exp.to_column("id")], + merge_filter=parse_one("__MERGE_SOURCE__.id > 0", read="doris"), + when_matched=parse_one( + "WHEN MATCHED AND __MERGE_SOURCE__.val > __MERGE_TARGET__.val THEN UPDATE SET __MERGE_TARGET__.val = __MERGE_SOURCE__.val", + into=exp.Whens, + ), + ) + merged_rows = ctx.engine_adapter.fetchall( + exp.select("id", "name", "val").from_(merge_tbl).order_by("id") + ) + # Row 1 updated val (15 > 10) while keeping name='orig_a'; Row 2 unchanged (5 < 20); Row 3 inserted + assert list(merged_rows) == [ + (1, "orig_a", 15), + (2, "orig_b", 20), + (3, "new_c", 30), + ] + + # 2. Doris 4.1 Segment V3 storage_format = 'V3' via physical_properties + v3_tbl = ctx.table("test_doris_41_segment_v3") + v3_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "payload": exp.DataType.build("VARCHAR(128)", dialect="doris"), + } + ctx.engine_adapter.create_table( + v3_tbl, + v3_cols, + table_properties={"storage_format": exp.Literal.string("V3")}, + ) + v3_row = ctx.engine_adapter.fetchone(f"SHOW CREATE TABLE {ctx.engine_adapter._to_sql(v3_tbl)}") + assert v3_row is not None + create_v3_ddl = v3_row[1] + assert '"storage_format" = "V3"' in create_v3_ddl + + # 3. Doris 4.1 USING ANN IVF vector index creation + ann_tbl = ctx.table("test_doris_41_ann") + ann_tbl_sql = ctx.engine_adapter._to_sql(ann_tbl) + ctx.engine_adapter.execute( + f""" + CREATE TABLE {ann_tbl_sql} ( + `id` INT NOT NULL, + `vec` ARRAY NOT NULL, + INDEX idx_vec (`vec`) USING ANN PROPERTIES ( + 'index_type' = 'ivf', + 'metric_type' = 'l2_distance', + 'dim' = '4', + 'nlist' = '2' + ) + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ('replication_num' = '1') + """ + ) + ann_row = ctx.engine_adapter.fetchone(f"SHOW CREATE TABLE {ann_tbl_sql}") + assert ann_row is not None + create_ann_ddl = ann_row[1] + assert "USING ANN" in create_ann_ddl + assert '"index_type" = "ivf"' in create_ann_ddl + + # 4. Vector distance functions (L2_DISTANCE, COSINE_DISTANCE, INNER_PRODUCT, L1_DISTANCE) + ctx.engine_adapter.execute( + f"INSERT INTO {ann_tbl_sql} (`id`, `vec`) VALUES (1, [1.0, 0.0, 0.0, 0.0]), (2, [0.0, 1.0, 0.0, 0.0])" + ) + vec_query = parse_one( + f"SELECT id, L2_DISTANCE(vec, [1.0, 0.0, 0.0, 0.0]) AS l2, " + f"COSINE_DISTANCE(vec, [1.0, 0.0, 0.0, 0.0]) AS cos_d, " + f"INNER_PRODUCT(vec, [1.0, 0.0, 0.0, 0.0]) AS ip, " + f"L1_DISTANCE(vec, [1.0, 0.0, 0.0, 0.0]) AS l1 " + f"FROM {ann_tbl_sql} ORDER BY l2 ASC LIMIT 1", + read="doris", + ) + vec_row = ctx.engine_adapter.fetchone(vec_query) + assert vec_row is not None + assert vec_row[0] == 1 + assert float(vec_row[1]) == 0.0 + assert float(vec_row[3]) == 1.0 + + # 5. Inverted index creation via create_index, BUILD INDEX, MATCH_ANY, and SEARCH() + doris_adapter = t.cast(DorisEngineAdapter, ctx.engine_adapter) + fts_tbl = ctx.table("test_doris_41_fts") + fts_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "msg": exp.DataType.build("STRING", dialect="doris"), + } + doris_adapter.create_table( + fts_tbl, + fts_cols, + table_properties={"compaction_policy": exp.Literal.string("time_series")}, + ) + doris_adapter.create_index( + fts_tbl, + "idx_msg", + [exp.to_column("msg")], + index_type="INVERTED", + index_properties={"parser": "english"}, + comment="fts index", + ) + doris_adapter.execute( + f"INSERT INTO {doris_adapter._to_sql(fts_tbl)} (`id`, `msg`) VALUES " + f"(1, 'apache doris vector search'), (2, 'iceberg table compaction')" + ) + fts_query = parse_one( + f"SELECT id FROM {doris_adapter._to_sql(fts_tbl)} WHERE msg MATCH_ANY 'doris vector' ORDER BY id", + read="doris", + ) + assert [r[0] for r in doris_adapter.fetchall(fts_query)] == [1] + + # 6. UNIQUE KEY table with function_column.sequence_col and _replace_by_key sequence protection + seq_tbl = ctx.table("test_doris_41_seq") + seq_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "val": exp.DataType.build("VARCHAR(32)", dialect="doris"), + "ver": exp.DataType.build("BIGINT", dialect="doris"), + } + doris_adapter.create_table( + seq_tbl, + seq_cols, + unique_key=[exp.to_column("id")], + table_properties={"function_column.sequence_col": exp.Literal.string("ver")}, + ) + doris_adapter._replace_by_key( + seq_tbl, + parse_one("SELECT 1 AS id, 'v10' AS val, 10 AS ver", read="doris"), + seq_cols, + [exp.to_column("id")], + is_unique_key=True, + table_properties={"function_column.sequence_col": exp.Literal.string("ver")}, + ) + # Stale update with lower sequence number (ver = 5 < 10) must not overwrite 'v10' + doris_adapter._replace_by_key( + seq_tbl, + parse_one("SELECT 1 AS id, 'v5_stale' AS val, 5 AS ver", read="doris"), + seq_cols, + [exp.to_column("id")], + is_unique_key=True, + table_properties={"function_column.sequence_col": exp.Literal.string("ver")}, + ) + seq_rows = doris_adapter.fetchall(exp.select("id", "val", "ver").from_(seq_tbl)) + assert list(seq_rows) == [(1, "v10", 10)] + + # 7. Self-referencing replace_query with aliased, catalog-qualified `internal.. AS t` + self_ref_tbl = ctx.table("test_doris_self_ref") + self_ref_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "val": exp.DataType.build("INT", dialect="doris"), + } + doris_adapter.create_table(self_ref_tbl, self_ref_cols) + doris_adapter.execute( + f"INSERT INTO {doris_adapter._to_sql(self_ref_tbl)} (`id`, `val`) VALUES (1, 10), (2, 20)" + ) + prev_default_cat = doris_adapter._default_catalog + try: + doris_adapter._default_catalog = "internal" + doris_adapter.replace_query( + self_ref_tbl, + parse_one( + f"SELECT t.id, t.val + 5 AS val FROM `internal`.`{self_ref_tbl.db}`.`{self_ref_tbl.name}` AS t", + read="doris", + ), + target_columns_to_types=self_ref_cols, + ) + finally: + doris_adapter._default_catalog = prev_default_cat + self_ref_rows = doris_adapter.fetchall( + exp.select("id", "val").from_(self_ref_tbl).order_by("id") + ) + assert list(self_ref_rows) == [(1, 15), (2, 25)] + + # 8. NGRAM_BF index creation + BUILD INDEX on INVERTED index + TIMESTAMPTZ cross-timezone normalize_value equivalence + doris_adapter.create_index( + fts_tbl, + "idx_msg_ngram", + [exp.to_column("msg")], + index_type="NGRAM_BF", + index_properties={"gram_size": "3", "bf_size": "256"}, + ) + from tests.core.engine_adapter.integration import wait_until + + build_idx_expr = parse_one( + f"BUILD INDEX idx_msg ON {doris_adapter._to_sql(fts_tbl)}", + read="doris", + ) + + def _build_index() -> bool: + try: + doris_adapter.execute(build_idx_expr) + return True + except Exception as e: + if "is not NORMAL" in str(e): + return False + raise + + wait_until(_build_index, attempts=30, wait=1) + norm_tz = doris_adapter.normalize_value( + exp.to_column("ts"), + exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + timestamp_precision=6, + ) + tz_check_rows = doris_adapter.fetchall( + exp.select(norm_tz.as_("utc_str")).from_( + parse_one( + "(SELECT CAST('2024-06-01 12:00:00.123456+08:00' AS TIMESTAMPTZ(6)) AS ts " + "UNION ALL " + "SELECT CAST('2024-06-01 04:00:00.123456+00:00' AS TIMESTAMPTZ(6)) AS ts) t", + read="doris", + ) + ) + ) + assert len(tz_check_rows) == 2 + assert tz_check_rows[0][0] == tz_check_rows[1][0] == "2024-06-01 04:00:00.123456" diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index 1971ba3bbc..d7bd74aa45 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -13,7 +13,7 @@ from sqlmesh.core import dialect as d from sqlmesh.core.dialect import normalize_model_name from sqlmesh.core.engine_adapter import EngineAdapter, EngineAdapterWithIndexSupport -from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObject +from sqlmesh.core.engine_adapter.shared import CatalogSupport, InsertOverwriteStrategy, DataObject from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation, NestedSupport from sqlmesh.utils import columns_to_types_to_struct from sqlmesh.utils.date import to_ds @@ -3289,6 +3289,42 @@ def test_replace_query_self_referencing_not_exists_known( ] +def test_replace_query_self_referencing_two_part_and_aliased( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + mocker.patch.object( + EngineAdapter, + "catalog_support", + new_callable=mocker.PropertyMock, + return_value=CatalogSupport.FULL_SUPPORT, + ) + adapter = make_mocked_engine_adapter(EngineAdapter, default_catalog="catalog") + adapter.SUPPORTS_REPLACE_TABLE = False + adapter.INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.DELETE_INSERT + + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject(catalog="catalog", schema="db", name="test", type="table"), + ) + mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="abcdefgh") + + adapter.replace_query( + "catalog.db.test", + parse_one("SELECT t.a FROM db.test AS t"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "catalog"."db"."test" ("a" INT)', + 'CREATE SCHEMA IF NOT EXISTS "catalog"."db"', + 'CREATE TABLE IF NOT EXISTS "catalog"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "catalog"."db"."test") AS "_subquery"', + 'DELETE FROM "catalog"."db"."test" WHERE TRUE', + 'INSERT INTO "catalog"."db"."test" ("a") SELECT "t"."a" FROM "catalog"."db"."__temp_test_abcdefgh" AS "t"', + 'DROP TABLE IF EXISTS "catalog"."db"."__temp_test_abcdefgh"', + ] + + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py new file mode 100644 index 0000000000..11fba0677d --- /dev/null +++ b/tests/core/engine_adapter/test_doris.py @@ -0,0 +1,5773 @@ +# SPDX-License-Identifier: Apache-2.0 +import typing as t + +import pandas as pd # noqa: TID253 +import pytest +from pytest_mock.plugin import MockerFixture +from sqlglot import expressions as exp +from sqlglot import parse_one + +from sqlmesh.core.engine_adapter.doris import DorisEngineAdapter +from sqlmesh.core.engine_adapter.shared import CatalogSupport, DataObject, DataObjectType +from sqlmesh.utils.errors import SQLMeshError +from tests.core.engine_adapter import to_sql_calls + +pytestmark = [pytest.mark.doris, pytest.mark.engine] + + +@pytest.fixture +def make_mocked_engine_adapter( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + mocker: MockerFixture, +) -> t.Callable[..., DorisEngineAdapter]: + def _make_function( + klass: t.Type[DorisEngineAdapter] = DorisEngineAdapter, + *args: t.Any, + **kwargs: t.Any, + ) -> DorisEngineAdapter: + adapter = make_mocked_engine_adapter(klass, *args, **kwargs) + if isinstance(adapter, DorisEngineAdapter): + setattr(adapter, "_wait_for_schema_change", mocker.MagicMock()) + orig_resolve = adapter._resolve_sync_mv_source_from_metadata + bound_default_fetchall = DorisEngineAdapter.fetchall.__get__( + adapter, DorisEngineAdapter + ) + + def _resolve_if_configured( + target_table: exp.Table, + is_explicit_sync: bool = False, + skip_full_desc_scan: bool = False, + ) -> t.Tuple[t.Optional[exp.Table], bool, bool]: + fetchall_fn = getattr(adapter, "fetchall", None) + cursor_fetchall = getattr(adapter.cursor, "fetchall", None) + fetchall_patched = fetchall_fn != bound_default_fetchall + cursor_configured = getattr(cursor_fetchall, "side_effect", None) is not None or ( + isinstance(getattr(cursor_fetchall, "return_value", None), (list, tuple)) + ) + if not fetchall_patched and not cursor_configured: + return None, bool(is_explicit_sync), False + return orig_resolve( + target_table, + is_explicit_sync=is_explicit_sync, + skip_full_desc_scan=skip_full_desc_scan, + ) + + setattr(adapter, "_resolve_sync_mv_source_from_metadata", _resolve_if_configured) + return adapter + + return _make_function + + +def test_create_view(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_view("test_view", parse_one("SELECT a FROM tbl")) + adapter.create_view("test_view", parse_one("SELECT a FROM tbl"), replace=False) + # check view properties + adapter.create_view( + "test_view", + parse_one("SELECT a FROM tbl"), + replace=False, + view_properties={"a": exp.convert(1)}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE OR REPLACE VIEW `test_view` AS SELECT `a` FROM `tbl`", + "CREATE VIEW `test_view` AS SELECT `a` FROM `tbl`", + "CREATE VIEW `test_view` AS SELECT `a` FROM `tbl`", + ] + + +def test_create_view_with_comment(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_view( + "test_view", + parse_one("SELECT a FROM tbl"), + replace=True, + target_columns_to_types={"a": exp.DataType.build("INT")}, + table_description="test_description", + column_descriptions={"a": "test_column_description"}, + ) + adapter.create_view( + "test_view", + parse_one("SELECT a FROM tbl"), + replace=False, + target_columns_to_types={"a": exp.DataType.build("INT")}, + table_description="test_description", + column_descriptions={"a": "test_column_description"}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE OR REPLACE VIEW `test_view` (`a` COMMENT 'test_column_description') COMMENT 'test_description' AS SELECT `a` FROM `tbl`", + "CREATE VIEW `test_view` (`a` COMMENT 'test_column_description') COMMENT 'test_description' AS SELECT `a` FROM `tbl`", + ] + + +def test_create_materialized_view(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_view( + "test_view", + parse_one("SELECT a FROM tbl"), + materialized=True, + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + adapter.create_view( + "test_view", + parse_one("SELECT a FROM tbl"), + replace=False, + materialized=True, + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + assert to_sql_calls(adapter) == [ + "DROP MATERIALIZED VIEW IF EXISTS `test_view`", + "CREATE MATERIALIZED VIEW `test_view` (`a`) AS SELECT `a` FROM `tbl`", + "CREATE MATERIALIZED VIEW `test_view` (`a`) AS SELECT `a` FROM `tbl`", + ] + + adapter.cursor.reset_mock() + adapter.create_view( + "test_view", + parse_one("SELECT a, b FROM tbl"), + replace=False, + materialized=True, + target_columns_to_types={"a": exp.DataType.build("INT"), "b": exp.DataType.build("INT")}, + column_descriptions={"a": "test_column_description", "b": "test_column_description"}, + ) + adapter.create_view( + "test_view", parse_one("SELECT a, b FROM tbl"), replace=False, materialized=True + ) + + assert to_sql_calls(adapter) == [ + "CREATE MATERIALIZED VIEW `test_view` (`a` COMMENT 'test_column_description', `b` COMMENT 'test_column_description') AS SELECT `a`, `b` FROM `tbl`", + "CREATE MATERIALIZED VIEW `test_view` AS SELECT `a`, `b` FROM `tbl`", + ] + + +def test_create_table(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT")}, + column_descriptions={"a": "test_column_description"}, + table_properties={"unique_key": exp.Column(this=exp.Identifier(this="a", quoted=True))}, + ) + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT")}, + column_descriptions={"a": "test_column_description"}, + table_properties={"duplicate_key": exp.Column(this=exp.Identifier(this="a", quoted=True))}, + ) + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT")}, + table_description="test_description", + column_descriptions={"a": "test_column_description"}, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT COMMENT 'test_column_description') UNIQUE KEY (`a`) DISTRIBUTED BY HASH (`a`) BUCKETS 10", + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT COMMENT 'test_column_description') DUPLICATE KEY (`a`)", + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT COMMENT 'test_column_description') COMMENT 'test_description'", + ] + + adapter.cursor.reset_mock() + + +def test_create_table_like(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.create_table_like("target_table", "source_table") + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `target_table` LIKE `source_table`", + ] + + +def test_create_schema(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_schema("test_schema") + + assert to_sql_calls(adapter) == [ + "CREATE DATABASE IF NOT EXISTS `test_schema`", + ] + + +def test_drop_schema(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.drop_schema("test_schema") + adapter.drop_schema("test_schema", ignore_if_not_exists=False) + + assert to_sql_calls(adapter) == [ + "DROP DATABASE IF EXISTS `test_schema`", + "DROP DATABASE `test_schema`", + ] + + +def test_rename_table(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.rename_table("old_table", "new_table") + adapter.cursor.execute.assert_called_once_with("ALTER TABLE `old_table` RENAME `new_table`") + + +def test_replace_by_key( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + temp_table = parse_one("temp_table") + mocker.patch.object(adapter, "_get_temp_table", return_value=temp_table) + mocker.patch.object(adapter, "ctas") + mocker.patch.object(adapter, "delete_from") + mocker.patch.object(adapter, "drop_table") + + columns_to_types = {"a": exp.DataType.build("INT"), "b": exp.DataType.build("INT")} + source_query = parse_one("SELECT a, b FROM src") + target_table = "target_table" + + adapter._replace_by_key(target_table, source_query, columns_to_types, [exp.column("a")], True) + adapter._replace_by_key(target_table, source_query, columns_to_types, [exp.column("a")], False) + adapter._replace_by_key( + target_table, source_query, columns_to_types, [exp.column("a"), exp.column("b")], True + ) + adapter._replace_by_key( + target_table, source_query, columns_to_types, [exp.column("a"), exp.column("b")], False + ) + + assert to_sql_calls(adapter, identify=True) == [ + "INSERT INTO `target_table` (`a`, `b`) SELECT `a`, `b` FROM (SELECT `a` AS `a`, `b` AS `b`, ROW_NUMBER() OVER (PARTITION BY `a` ORDER BY `a`) AS _row_number FROM `temp_table`) AS _t WHERE _row_number = 1", + "INSERT INTO `target_table` (`a`, `b`) SELECT `a`, `b` FROM `temp_table`", + "INSERT INTO `target_table` (`a`, `b`) SELECT `a`, `b` FROM (SELECT `a` AS `a`, `b` AS `b`, ROW_NUMBER() OVER (PARTITION BY `a`, `b` ORDER BY `a`, `b`) AS _row_number FROM `temp_table`) AS _t WHERE _row_number = 1", + "INSERT INTO `target_table` (`a`, `b`) SELECT `a`, `b` FROM `temp_table`", + ] + + +def test_create_index(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.create_index("test_table", "test_index", ("cola",)) + adapter.cursor.execute.assert_called_once_with( + "CREATE INDEX IF NOT EXISTS `test_index` ON `test_table`(`cola`)" + ) + + +def test_create_table_with_distributed_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + distributed_by = { + "expressions": ["a", "b"], + "kind": "HASH", + "buckets": 8, + } + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT"), "b": exp.DataType.build("INT")}, + table_properties={"distributed_by": distributed_by}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT, `b` INT) DISTRIBUTED BY HASH (`a`, `b`) BUCKETS 8", + ] + + adapter.cursor.execute.reset_mock() + + distributed_by = { + "expressions": None, + "kind": "RANDOM", + "buckets": None, + } + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT"), "b": exp.DataType.build("INT")}, + table_properties={"distributed_by": distributed_by}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT, `b` INT) DISTRIBUTED BY RANDOM", + ] + + adapter.cursor.execute.reset_mock() + + distributed_by = { + "expressions": ["a"], + "kind": "HASH", + "buckets": "AUTO", + } + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT"), "b": exp.DataType.build("INT")}, + table_properties={"distributed_by": distributed_by}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT, `b` INT) DISTRIBUTED BY HASH (`a`) BUCKETS AUTO", + ] + + +def test_create_table_with_properties( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT")}, + table_properties={ + "refresh_interval": "86400", + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT) PROPERTIES ('refresh_interval'='86400')", + ] + + +def test_create_table_with_partitioned_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={"a": exp.DataType.build("INT"), "b": exp.DataType.build("DATE")}, + partitioned_by=[exp.Literal.string("RANGE(b)")], + table_properties={ + "partitions": exp.Literal.string( + "FROM ('2000-11-14') TO ('2021-11-14') INTERVAL 2 YEAR" + ) + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`a` INT, `b` DATE) PARTITION BY RANGE (`b`) (FROM ('2000-11-14') TO ('2021-11-14') INTERVAL 2 YEAR)", + ] + + +def test_create_table_with_range_partitioned_by_with_partitions( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "waiter_id": exp.DataType.build("INT"), + "customer_id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATETIME"), + }, + partitioned_by=[exp.Literal.string("RANGE(ds)")], + table_properties={ + "partitions": exp.Tuple( + expressions=[ + exp.Literal.string('PARTITION `p2023` VALUES [("2023-01-01"), ("2024-01-01"))'), + exp.Literal.string('PARTITION `p2024` VALUES [("2024-01-01"), ("2025-01-01"))'), + exp.Literal.string('PARTITION `p2025` VALUES [("2025-01-01"), ("2026-01-01"))'), + exp.Literal.string("PARTITION `other` VALUES LESS THAN MAXVALUE"), + ] + ), + "distributed_by": exp.Tuple( + expressions=[ + exp.EQ( + this=exp.Column(this=exp.Identifier(this="kind", quoted=True)), + expression=exp.Literal.string("HASH"), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="expressions", quoted=True)), + expression=exp.Column(this=exp.Identifier(this="id", quoted=True)), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="buckets", quoted=True)), + expression=exp.Literal.number(10), + ), + ] + ), + "replication_allocation": exp.Literal.string("tag.location.default: 3"), + "in_memory": exp.Literal.string("false"), + "storage_format": exp.Literal.string("V2"), + "disable_auto_compaction": exp.Literal.string("false"), + }, + ) + + expected_sql = ( + "CREATE TABLE IF NOT EXISTS `test_table` " + "(`id` INT, `waiter_id` INT, `customer_id` INT, `ds` DATETIME) " + "PARTITION BY RANGE (`ds`) " + '(PARTITION `p2023` VALUES [("2023-01-01"), ("2024-01-01")), ' + 'PARTITION `p2024` VALUES [("2024-01-01"), ("2025-01-01")), ' + 'PARTITION `p2025` VALUES [("2025-01-01"), ("2026-01-01")), ' + "PARTITION `other` VALUES LESS THAN MAXVALUE) " + "DISTRIBUTED BY HASH (`id`) BUCKETS 10 " + "PROPERTIES (" + "'replication_allocation'='tag.location.default: 3', " + "'in_memory'='false', " + "'storage_format'='V2', " + "'disable_auto_compaction'='false')" + ) + + assert to_sql_calls(adapter) == [expected_sql] + + +def test_create_table_with_list_partitioned_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "status": exp.DataType.build("VARCHAR(10)"), + }, + partitioned_by=[exp.Literal.string("LIST(status)")], + table_properties={ + "partitions": exp.Tuple( + expressions=[ + exp.Literal.string('PARTITION `active` VALUES IN ("active", "pending")'), + exp.Literal.string('PARTITION `inactive` VALUES IN ("inactive", "disabled")'), + ] + ), + }, + ) + + expected_sql = ( + "CREATE TABLE IF NOT EXISTS `test_table` " + "(`id` INT, `status` VARCHAR(10)) " + "PARTITION BY LIST (`status`) " + '(PARTITION `active` VALUES IN ("active", "pending"), ' + 'PARTITION `inactive` VALUES IN ("inactive", "disabled"))' + ) + + assert to_sql_calls(adapter) == [expected_sql] + + +def test_create_table_with_range_partitioned_by_anonymous_function( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + """Test that RANGE(ds) function call syntax generates correct SQL without duplicate RANGE.""" + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "test_table", + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATETIME"), + }, + # This simulates how partitioned_by RANGE(ds) gets parsed from model definition + partitioned_by=[exp.Anonymous(this="RANGE", expressions=[exp.to_column("ds")])], + table_properties={ + "partitions": exp.Literal.string( + 'FROM ("2000-11-14") TO ("2099-11-14") INTERVAL 1 MONTH' + ) + }, + ) + + expected_sql = ( + "CREATE TABLE IF NOT EXISTS `test_table` " + "(`id` INT, `ds` DATETIME) " + "PARTITION BY RANGE (`ds`) " + '(FROM ("2000-11-14") TO ("2099-11-14") INTERVAL 1 MONTH)' + ) + + assert to_sql_calls(adapter) == [expected_sql] + + +def test_create_materialized_view_with_duplicate_key( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_view( + "test_mv", + parse_one("SELECT id, status, COUNT(*) as cnt FROM orders GROUP BY id, status"), + materialized=True, + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "status": exp.DataType.build("VARCHAR(10)"), + "cnt": exp.DataType.build("BIGINT"), + }, + view_properties={ + "duplicate_key": exp.Tuple( + expressions=[ + exp.to_column("id"), + exp.to_column("status"), + ] + ), + }, + ) + + expected_sqls = [ + "DROP MATERIALIZED VIEW IF EXISTS `test_mv`", + ( + "CREATE MATERIALIZED VIEW `test_mv` " + "(`id`, `status`, `cnt`) " + "DUPLICATE KEY (`id`, `status`) " + "AS SELECT `id`, `status`, COUNT(*) AS `cnt` FROM `orders` GROUP BY `id`, `status`" + ), + ] + + assert to_sql_calls(adapter) == expected_sqls + + +def test_create_full_materialized_view( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + view_properties: t.Dict[str, exp.Expr] = { + "build": exp.Literal.string("IMMEDIATE"), + "refresh": exp.Tuple( + expressions=[ + exp.EQ( + this=exp.Column(this=exp.Identifier(this="method", quoted=True)), + expression=exp.Literal.string("AUTO"), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="kind", quoted=True)), + expression=exp.Literal.string("SCHEDULE"), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="every", quoted=True)), + expression=exp.Literal.number(1), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="unit", quoted=True)), + expression=exp.Literal.string("DAY"), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="starts", quoted=True)), + expression=exp.Literal.string("2024-12-01 20:30:00"), + ), + ], + ), + "distributed_by": exp.Tuple( + expressions=[ + exp.EQ( + this=exp.Column(this=exp.Identifier(this="kind", quoted=True)), + expression=exp.Literal.string("HASH"), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="expressions", quoted=True)), + expression=exp.Column(this=exp.Identifier(this="orderkey", quoted=True)), + ), + exp.EQ( + this=exp.Column(this=exp.Identifier(this="buckets", quoted=True)), + expression=exp.Literal.number(2), + ), + ] + ), + "unique_key": exp.to_column("orderkey"), + "replication_num": exp.Literal.string("1"), + } + materialized_properties = { + "partitioned_by": [ + parse_one("DATE_TRUNC(o_orderdate, 'MONTH')", dialect="doris"), + ], + "clustered_by": [], + "partition_interval_unit": None, + } + columns_to_types = { + "orderdate": exp.DataType.build("DATE"), + "orderkey": exp.DataType.build("INT"), + "partkey": exp.DataType.build("INT"), + } + column_descriptions = { + "orderdate": "order date", + "orderkey": "order key", + "partkey": "part key", + } + query = parse_one( + """ + SELECT + o_orderdate, + l_orderkey, + l_partkey + FROM + orders + LEFT JOIN lineitem ON l_orderkey = o_orderkey + LEFT JOIN partsupp ON ps_partkey = l_partkey + and l_suppkey = ps_suppkey + """ + ) + adapter.create_view( + "complete_mv", + query, + replace=False, + materialized=True, + target_columns_to_types=columns_to_types, + column_descriptions=column_descriptions, + table_description="test_description", + view_properties=view_properties, + materialized_properties=materialized_properties, + ) + expected_sqls = [ + "CREATE MATERIALIZED VIEW `complete_mv` (`orderdate` COMMENT 'order date', `orderkey` COMMENT 'order key', `partkey` COMMENT 'part key') " + "BUILD IMMEDIATE REFRESH AUTO ON SCHEDULE EVERY 1 DAY STARTS '2024-12-01 20:30:00' KEY (`orderkey`) COMMENT 'test_description' " + "PARTITION BY (DATE_TRUNC(`o_orderdate`, 'MONTH')) " + "DISTRIBUTED BY HASH (`orderkey`) BUCKETS 2 PROPERTIES ('replication_num'='1') " + "AS SELECT `o_orderdate`, `l_orderkey`, `l_partkey` FROM `orders` LEFT JOIN `lineitem` ON `l_orderkey` = `o_orderkey` LEFT JOIN `partsupp` ON `ps_partkey` = `l_partkey` AND `l_suppkey` = `ps_suppkey`", + ] + sql_calls = to_sql_calls(adapter) + + # Remove extra spaces for comparison + def norm(s): + return " ".join(s.split()) + + for expected_sql in expected_sqls: + assert any(norm(expected_sql) == norm(sql) for sql in sql_calls), ( + f"Expected SQL not found.\nExpected: {expected_sql}\nGot: {sql_calls}" + ) + + +def test_create_table_with_single_string_distributed_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + """Test creating table with distributed_by where expressions is a single string.""" + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + distributed_by = { + "expressions": "recordid", # Single string instead of array + "kind": "HASH", + "buckets": 10, + } + adapter.create_table( + "test_table", + target_columns_to_types={ + "recordid": exp.DataType.build("INT"), + "name": exp.DataType.build("VARCHAR"), + }, + table_properties={"distributed_by": distributed_by}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_table` (`recordid` INT, `name` VARCHAR) DISTRIBUTED BY HASH (`recordid`) BUCKETS 10", + ] + + +def test_delete_from_with_subquery(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # Test DELETE FROM with IN subquery + adapter.delete_from( + "test_schema_xtll416o.test_table", + "id IN (SELECT id FROM test_schema_xtll416o.temp_test_table_ik29031e)", + ) + + # Test DELETE FROM with NOT IN subquery (must NOT rewrite to USING ... WHERE <>) + adapter.delete_from( + "test_schema_xtll416o.test_table", + "id NOT IN (SELECT id FROM test_schema_xtll416o.temp_test_table_ik29031e)", + ) + + # Test simple DELETE FROM (should use base implementation) + adapter.delete_from("test_schema_xtll416o.test_table", "id = 1") + + assert to_sql_calls(adapter) == [ + "DELETE FROM `test_schema_xtll416o`.`test_table` AS `_t1` USING (SELECT `id` FROM `test_schema_xtll416o`.`temp_test_table_ik29031e`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id`", + "DELETE FROM `test_schema_xtll416o`.`test_table` WHERE NOT `id` IN (SELECT `id` FROM `test_schema_xtll416o`.`temp_test_table_ik29031e`)", + "DELETE FROM `test_schema_xtll416o`.`test_table` WHERE `id` = 1", + ] + + +def test_delete_from_with_complex_subquery( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # Test DELETE FROM with complex subquery (multiple columns) + adapter.delete_from( + "test_schema_xtll416o.test_table", + "(id, name) IN (SELECT id, name FROM test_schema_xtll416o.temp_test_table_ik29031e WHERE active = 1)", + ) + + # Test DELETE FROM with subquery that has WHERE clause + adapter.delete_from( + "test_schema_xtll416o.test_table", + "id IN (SELECT id FROM test_schema_xtll416o.temp_test_table_ik29031e WHERE created_date > '2024-01-01')", + ) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `test_schema_xtll416o`.`test_table` AS `_t1` USING (SELECT `id`, `name` FROM `test_schema_xtll416o`.`temp_test_table_ik29031e` WHERE `active` = 1) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`name` = `_t2`.`name`", + "DELETE FROM `test_schema_xtll416o`.`test_table` AS `_t1` USING (SELECT `id` FROM `test_schema_xtll416o`.`temp_test_table_ik29031e` WHERE `created_date` > '2024-01-01') AS `_t2` WHERE `_t1`.`id` = `_t2`.`id`", + ] + + +def test_delete_from_fallback_to_base( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # Test DELETE FROM with simple conditions (should use base implementation) + adapter.delete_from("test_table", "id = 1") + adapter.delete_from("test_table", "name = 'test' AND status = 'active'") + adapter.delete_from("test_table", "id IN (1, 2, 3)") # Simple IN with values, not subquery + + assert to_sql_calls(adapter) == [ + "DELETE FROM `test_table` WHERE `id` = 1", + "DELETE FROM `test_table` WHERE `name` = 'test' AND `status` = 'active'", + "DELETE FROM `test_table` WHERE `id` IN (1, 2, 3)", + ] + + +def test_delete_from_full_table(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # Test DELETE FROM with WHERE TRUE (should use TRUNCATE) + adapter.delete_from("test_table", exp.true()) + + assert to_sql_calls(adapter) == [ + "TRUNCATE TABLE `test_table`", + ] + + +def test_create_table_olap_key_prefix_reordering_and_order_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "events", + target_columns_to_types={ + "payload": exp.DataType.build("VARCHAR(255)"), + "id": exp.DataType.build("BIGINT"), + "ds": exp.DataType.build("DATE"), + "updated_at": exp.DataType.build("DATETIME"), + }, + unique_key=[exp.to_column("id"), exp.to_column("ds")], + table_properties={ + "order_by": exp.Tuple(expressions=[exp.to_column("updated_at"), exp.to_column("id")]), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `events` (`id` BIGINT, `ds` DATE, `payload` VARCHAR(255), `updated_at` DATETIME) UNIQUE KEY (`id`, `ds`) ORDER BY (`updated_at`, `id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10", + ] + + +def test_create_table_aggregate_key_and_clustered_by_buckets( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "metrics_agg", + target_columns_to_types={ + "total": exp.DataType.build("BIGINT"), + "tenant_id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATE"), + }, + clustered_by=[exp.to_column("tenant_id")], + table_properties={ + "aggregate_key": exp.Tuple( + expressions=[exp.to_column("tenant_id"), exp.to_column("ds")] + ), + "buckets": exp.Literal.number(16), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `metrics_agg` (`tenant_id` INT, `ds` DATE, `total` BIGINT) AGGREGATE KEY (`tenant_id`, `ds`) DISTRIBUTED BY HASH (`tenant_id`) BUCKETS 16", + ] + + +def test_create_table_auto_partition_and_unannotated_partitioned_by_guard( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + # Without partitions or auto_partition, OLAP partitioned_by is safely omitted + adapter.create_table( + "olap_no_partitions", + target_columns_to_types={"id": exp.DataType.build("INT"), "ds": exp.DataType.build("DATE")}, + partitioned_by=[exp.to_column("ds")], + ) + # With auto_partition = true, emits AUTO PARTITION BY RANGE (...) () and marks range partition column NOT NULL + adapter.create_table( + "olap_auto_partition", + target_columns_to_types={ + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + partitioned_by=[parse_one("RANGE(DATE_TRUNC(ds, 'DAY'))", dialect="doris")], + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `olap_no_partitions` (`id` INT, `ds` DATE)", + "CREATE TABLE IF NOT EXISTS `olap_auto_partition` (`id` INT, `ds` DATE NOT NULL, `payload` STRING) AUTO PARTITION BY RANGE (DATE_TRUNC(`ds`, 'DAY')) ()", + ] + + +def test_iceberg_multi_catalog_ddl_and_dml( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + assert adapter.catalog_support == CatalogSupport.FULL_SUPPORT + assert adapter.default_catalog is None + + # 1. Explicit table_format='iceberg' on 2-part table name emits ENGINE=iceberg and PARTITION BY LIST (...) () + adapter.create_table( + "silver.iceberg_events", + target_columns_to_types={ + "id": exp.DataType.build("BIGINT"), + "ds": exp.DataType.build("DATE"), + "region": exp.DataType.build("STRING"), + }, + table_format="iceberg", + partitioned_by=[parse_one("DAY(ds)", dialect="doris"), exp.to_column("region")], + unique_key=[exp.to_column("id")], + table_properties={"write-format": exp.Literal.string("parquet")}, + ) + + # 2. 3-part external Iceberg catalog CTAS omits ENGINE=iceberg (invalid in Doris CTAS grammar) + adapter.ctas( + "iceberg_cat.silver.events_ctas", + parse_one("SELECT id, ds FROM bronze.events"), + target_columns_to_types={ + "id": exp.DataType.build("BIGINT"), + "ds": exp.DataType.build("DATE"), + }, + ) + + # 3. replace_query on existing Iceberg table uses full-table DELETE + INSERT INTO + # (avoiding Doris dynamic partition overwrite retaining stale partitions or no-oping on empty results) + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="iceberg_cat", schema="silver", name="events_ctas", type=DataObjectType.TABLE + ), + ) + adapter.replace_query( + "iceberg_cat.silver.events_ctas", + parse_one("SELECT id, ds FROM bronze.events"), + target_columns_to_types={ + "id": exp.DataType.build("BIGINT"), + "ds": exp.DataType.build("DATE"), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `silver`.`iceberg_events` (`id` BIGINT, `ds` DATE, `region` STRING) ENGINE=iceberg PARTITION BY LIST (DAY(`ds`), `region`) () PROPERTIES ('write-format'='parquet')", + "CREATE TABLE IF NOT EXISTS `iceberg_cat`.`silver`.`events_ctas` AS SELECT CAST(`id` AS BIGINT) AS `id`, CAST(`ds` AS DATE) AS `ds` FROM (SELECT `id`, `ds` FROM `bronze`.`events`) AS `_subquery`", + "DELETE FROM `iceberg_cat`.`silver`.`events_ctas` WHERE NOT `id` IS NULL", + "DELETE FROM `iceberg_cat`.`silver`.`events_ctas` WHERE `id` IS NULL", + "INSERT INTO `iceberg_cat`.`silver`.`events_ctas` (`id`, `ds`) SELECT `id`, `ds` FROM `bronze`.`events`", + ] + + +def test_columns_normalization_largeint_and_struct( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.cursor.fetchall.return_value = [ + ("id", "LARGEINT", "No", "true", None, ""), + ("price", "DECIMALV3(9, 0)", "Yes", "false", None, ""), + ("is_active", "TINYINT(1)", "Yes", "false", None, ""), + ("count_val", "INT(11)", "Yes", "false", None, ""), + ("meta", "STRUCT", "Yes", "false", None, ""), + ] + + cols = adapter.columns("test_db.test_table") + assert cols == { + "id": exp.DataType.build("INT128"), + "price": exp.DataType.build("DECIMAL(9, 0)", dialect="doris"), + "is_active": exp.DataType.build("BOOLEAN", dialect="doris"), + "count_val": exp.DataType.build("INT", dialect="doris"), + "meta": exp.DataType.build("STRUCT", dialect="doris"), + } + + adapter.cursor.reset_mock() + adapter.create_table( + "test_db.struct_tbl", + target_columns_to_types={"id": cols["id"], "meta": cols["meta"]}, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `test_db`.`struct_tbl` (`id` LARGEINT, `meta` STRUCT<`f1`: INT, `f2`: VARCHAR(20)>)", + ] + + +def test_get_data_objects_with_catalog_and_mv( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter, patch_get_data_objects=False) + mocker.patch.object( + adapter, + "fetchall", + return_value=[ + ("silver", "tbl1", "table"), + ("silver", "mv1", "materialized_view"), + ], + ) + + objs = adapter._get_data_objects("iceberg_cat.silver") + assert len(objs) == 2 + assert objs[0].type == DataObjectType.TABLE + assert objs[0].catalog == "iceberg_cat" + assert objs[0].schema_name == "silver" + assert objs[0].name == "tbl1" + assert objs[1].type == DataObjectType.MATERIALIZED_VIEW + assert objs[1].name == "mv1" + + +def test_comments_escaping(make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter]): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + table_comment_sql = adapter._build_create_comment_table_exp( + exp.to_table("db.tbl"), " O'Reilly's \"table\" \\ test " + ) + col_comment_sql = adapter._build_create_comment_column_exp( + exp.to_table("db.tbl"), "col'1", ' Col\'s "comment" ' + ) + assert ( + table_comment_sql + == "ALTER TABLE `db`.`tbl` MODIFY COMMENT 'O''Reilly''s \"table\" \\\\ test'" + ) + assert ( + col_comment_sql + == "ALTER TABLE `db`.`tbl` MODIFY COLUMN `col'1` COMMENT 'Col''s \"comment\"'" + ) + + +def test_delete_from_between_concat_ws_and_literal_fallback( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. BETWEEN rewrite in delete_from + adapter.delete_from("test_table", "ds BETWEEN '2024-01-01' AND '2024-01-31'") + + # 2. _replace_by_key / LogicalMergeMixin CONCAT_WS unwrap into multi-column USING + key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", exp.column("id"), exp.column("ds")) + merge_where = key_exp.isin(query=exp.select(key_exp).from_("src_table")) + adapter.delete_from("target_table", where=merge_where) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `test_table` WHERE `ds` >= '2024-01-01' AND `ds` <= '2024-01-31'", + "DELETE FROM `target_table` AS `_t1` USING (SELECT `id`, `ds` FROM `src_table`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`ds` = `_t2`.`ds`", + ] + + # 3. Fallback to zero-OR batched _delete_by_literal_keys when USING fails on DUPLICATE KEY / Iceberg + # skips NULL subquery keys to preserve SQL IN semantics, while explicit IS NULL in OR predicates is preserved. + adapter.cursor.reset_mock() + mocker.patch.object( + adapter, + "fetchall", + return_value=[("tenant_a", 1), ("tenant_a", 2), ("tenant_a", None)], + ) + adapter._delete_by_literal_keys( + "dup_table", + [exp.column("tenant"), exp.column("id")], + parse_one("SELECT tenant, id FROM src_table", into=exp.Select), + ) + adapter.delete_from( + "dup_table", + where=parse_one( + "(tenant = 'tenant_a' AND id = 1) OR (tenant = 'tenant_a' AND id IS NULL)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `dup_table` WHERE `tenant` = 'tenant_a' AND `id` IN (1, 2)", + "DELETE FROM `dup_table` WHERE `tenant` = 'tenant_a' AND `id` IN (1)", + "DELETE FROM `dup_table` WHERE `tenant` = 'tenant_a' AND `id` IS NULL", + ] + + +def test_execute_operational_error_reconnect( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + class OperationalError(Exception): + pass + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + conn_mock = mocker.MagicMock() + mocker.patch.object(adapter._connection_pool, "get", return_value=conn_mock) + close_spy = mocker.spy(adapter._connection_pool, "close") + + call_count = 0 + + def side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise OperationalError(2013, "Lost connection to MySQL server during query") + + adapter.cursor.execute.side_effect = side_effect + adapter._execute("WITH `cte` AS (SELECT 1) SELECT * FROM `cte`") + + assert call_count == 2 + conn_mock.ping.assert_called_once_with(reconnect=True) + + # Read timeout (2013 with TimeoutError / "timed out") must NOT be retried even for SELECT + conn_mock.reset_mock() + close_spy.reset_mock() + timeout_calls = 0 + + def timeout_side_effect(*args, **kwargs): + nonlocal timeout_calls + timeout_calls += 1 + raise OperationalError( + 2013, "Lost connection to MySQL server during query (timed out)" + ) from TimeoutError("timed out") + + adapter.cursor.execute.side_effect = timeout_side_effect + with pytest.raises(OperationalError): + adapter._execute("SELECT 1") + + assert timeout_calls == 1 + conn_mock.ping.assert_not_called() + close_spy.assert_called_once() + + # Write statements (e.g. INSERT or CTE-prefixed INSERT) must NOT be retried after OperationalError(2013) to avoid duplicating rows + for unsafe_write_sql in ( + "INSERT INTO `tbl` VALUES (1)", + "WITH `cte` AS (SELECT 1) INSERT INTO `tbl` SELECT * FROM `cte`", + ): + conn_mock.reset_mock() + close_spy.reset_mock() + write_calls = 0 + + def write_side_effect(*args, **kwargs): + nonlocal write_calls + write_calls += 1 + raise OperationalError(2013, "Lost connection to MySQL server during query") + + adapter.cursor.execute.side_effect = write_side_effect + with pytest.raises(OperationalError): + adapter._execute(unsafe_write_sql) + + assert write_calls == 1 + conn_mock.ping.assert_not_called() + close_spy.assert_called_once() + + +def test_external_catalog_view_error_and_drop_noop( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.utils.errors import SQLMeshError + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # Regular view creation on an external catalog raises SQLMeshError pointing to virtual_environment_mode and environment_catalog_mapping + with pytest.raises( + SQLMeshError, + match=r"Doris only supports creating views in the 'internal' catalog, got 'iceberg_cat'.*virtual_environment_mode: dev_only.*environment_catalog_mapping", + ): + adapter.create_view("iceberg_cat.silver__dev.events", parse_one("SELECT 1 AS id")) + + # Drop view on external catalog remains a safe no-op during cleanup + adapter.drop_view("iceberg_cat.silver__dev.events") + assert to_sql_calls(adapter) == [] + + # Materialized view on external catalog also raises SQLMeshError + with pytest.raises( + SQLMeshError, + match="Doris only supports creating materialized views in the 'internal' catalog", + ): + adapter.create_view( + "iceberg_cat.silver.events_mv", + parse_one("SELECT 1 AS id"), + materialized=True, + ) + + # Materialized view with invalid property raises SQLMeshError BEFORE dropping existing MV + with pytest.raises(SQLMeshError, match="Invalid materialized view build mode"): + adapter.create_view( + "silver.events_mv", + parse_one("SELECT 1 AS id"), + replace=True, + materialized=True, + view_properties={"build": exp.Literal.string("INVALID_MODE")}, + ) + assert to_sql_calls(adapter) == [] + + # Invalid refresh_trigger also raises SQLMeshError instead of silently falling back to AUTO + with pytest.raises(SQLMeshError, match="Invalid materialized view refresh trigger"): + adapter.create_view( + "silver.events_mv", + parse_one("SELECT 1 AS id"), + replace=True, + materialized=True, + view_properties={"refresh_trigger": exp.Literal.string("ON DEMAND")}, + ) + assert to_sql_calls(adapter) == [] + + +def test_duplicate_key_with_buckets_and_random_buckets_fallback( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "dup_bucketed", + target_columns_to_types={"id": exp.DataType.build("INT"), "val": exp.DataType.build("INT")}, + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.to_column("id")]), + "buckets": exp.Literal.number(16), + }, + ) + adapter.create_table( + "random_bucketed", + target_columns_to_types={"id": exp.DataType.build("INT"), "val": exp.DataType.build("INT")}, + table_properties={ + "buckets": exp.Literal.number(8), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `dup_bucketed` (`id` INT, `val` INT) DUPLICATE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 16", + "CREATE TABLE IF NOT EXISTS `random_bucketed` (`id` INT, `val` INT) DISTRIBUTED BY RANDOM BUCKETS 8", + ] + + +def test_truncate_table_fallback_on_unsupported_truncate( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object( + adapter, + "columns", + return_value={"id": exp.DataType.build("BIGINT"), "ds": exp.DataType.build("DATE")}, + ) + + call_count = 0 + + def side_effect(sql, *args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1 and "TRUNCATE" in str(sql): + raise Exception("errCode = 2, detailMessage = Not support truncate table on iceberg") + + adapter.cursor.execute.side_effect = side_effect + adapter._truncate_table("silver.iceberg_two_part") + + assert to_sql_calls(adapter) == [ + "TRUNCATE TABLE `silver`.`iceberg_two_part`", + "DELETE FROM `silver`.`iceberg_two_part` WHERE NOT `id` IS NULL", + "DELETE FROM `silver`.`iceberg_two_part` WHERE `id` IS NULL", + ] + + +def test_row_diff_mixin_concat_columns( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + concat_expr = adapter.concat_columns( + { + "id": exp.DataType.build("INT"), + "flag": exp.DataType.build("BOOLEAN"), + "ts": exp.DataType.build("DATETIME"), + } + ) + assert ( + concat_expr.sql("doris") + == "CONCAT(COALESCE(CAST(CAST(id AS BIGINT) AS VARCHAR), ''), ',', COALESCE(CAST(CAST(flag AS INT) AS VARCHAR), ''), ',', COALESCE(CAST(DATE_FORMAT(ts, '%Y-%m-%d %T.%f') AS VARCHAR), ''))" + ) + + +def test_temp_table_length_and_catalog_switch_and_wrapped_concat_ws( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. _get_temp_table never exceeds Doris's 64-character identifier limit + long_table = "db." + ("a" * 60) + temp_tbl = adapter._get_temp_table(long_table) + assert len(temp_tbl.name) <= 64 + + # 2. set_current_catalog preserves backtick quoting on hyphenated catalog names + adapter.set_current_catalog("iceberg-cat") + + # 3. delete_from unwraps parenthesized CONCAT_WS(...) + key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", exp.column("id"), exp.column("ds")) + wrapped_where = exp.Paren(this=key_exp).isin(query=exp.select(key_exp).from_("src_table")) + adapter.delete_from("target_table", where=wrapped_where) + + assert to_sql_calls(adapter) == [ + "SWITCH `iceberg-cat`", + "DELETE FROM `target_table` AS `_t1` USING (SELECT `id`, `ds` FROM `src_table`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`ds` = `_t2`.`ds`", + ] + + +def test_insert_overwrite_by_partition_unwraps_wrappers_and_transforms( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object( + adapter, + "_get_temp_table", + return_value=exp.table_("__temp_tbl", db="db"), + ) + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[("2024-01-15 00:00:00", "2024-01-16 00:00:00")], + ) + + adapter.insert_overwrite_by_partition( + "db.events", + parse_one("SELECT id, ds FROM db.src", read="doris"), + partitioned_by=[ + parse_one("RANGE(DATE_TRUNC(ds, 'day'))", read="doris"), + ], + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATETIME"), + }, + ) + + sql_calls = to_sql_calls(adapter) + assert ( + "DELETE FROM `db`.`events` WHERE DATE_TRUNC(`ds`, 'day') IN (SELECT DISTINCT DATE_TRUNC(`ds`, 'day') FROM `db`.`__temp_tbl`)" + not in sql_calls + ) + assert ( + "SELECT DISTINCT DATE_TRUNC(`ds`, 'day') AS `_low_0`, DATE_ADD(DATE_TRUNC(`ds`, 'day'), INTERVAL 1 DAY) AS `_high_0` FROM `db`.`__temp_tbl`" + in sql_calls + ) + assert ( + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-15 00:00:00' AND `ds` < '2024-01-16 00:00:00'" + in sql_calls + ) + + # Unsupported non-range partition transforms (e.g. BUCKET) raise a clear SQLMeshError + with pytest.raises( + SQLMeshError, + match="Doris insert_overwrite_by_partition does not support partition expression", + ): + adapter.insert_overwrite_by_partition( + "db.events", + parse_one("SELECT id, ds FROM db.src", read="doris"), + partitioned_by=[parse_one("BUCKET(16, id)", read="doris")], + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATETIME"), + }, + ) + + +def test_sync_materialized_view_drop_and_replace( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.create_view( + "db.sync_mv", + parse_one( + "SELECT region, SUM(amount) AS total FROM db.orders GROUP BY region", read="doris" + ), + replace=True, + materialized=True, + view_properties={ + "materialized_type": exp.Literal.string("sync"), + "source_table": exp.Literal.string("db.orders"), + }, + ) + + assert to_sql_calls(adapter) == [ + "USE `db`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_mv` ON `db`.`orders`", + "USE `db`", + "CREATE MATERIALIZED VIEW `sync_mv` AS SELECT `region` AS `mv_region`, SUM(`amount`) AS `total` FROM `db`.`orders` GROUP BY `region`", + ] + + adapter.cursor.execute.reset_mock() + + adapter.create_view( + "sync_mv", + parse_one("SELECT id FROM db.orders", read="doris"), + replace=True, + materialized=True, + view_properties={ + "materialized_type": exp.Literal.string("SYNC"), + "source_table": exp.to_table("db.orders"), + }, + ) + + assert to_sql_calls(adapter) == [ + "USE `db`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_mv` ON `db`.`orders`", + "USE `db`", + "CREATE MATERIALIZED VIEW `sync_mv` AS SELECT `id` AS `mv_id` FROM `db`.`orders`", + ] + + adapter.cursor.execute.reset_mock() + + # source_table parsed as exp.Column (from MODEL physical_properties) and 3-part internal.db.orders + adapter.create_view( + "sync_mv", + parse_one("SELECT id FROM orders", read="doris"), + replace=True, + materialized=True, + view_properties={ + "materialized_type": exp.Literal.string("SYNC"), + "source_table": parse_one("internal.db.orders", read="doris"), + }, + ) + + assert to_sql_calls(adapter) == [ + "USE `db`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_mv` ON `db`.`orders`", + "USE `db`", + "CREATE MATERIALIZED VIEW `sync_mv` AS SELECT `id` AS `mv_id` FROM `orders`", + ] + + +def test_delete_with_subquery_fallback_and_cardinality_aware_composite_batching( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # When DELETE ... USING fails on DUPLICATE/AGGREGATE KEY OLAP tables, fallback to literal batching + # and pick the highest-cardinality column (id: 3 values) to batch in IN (...) while grouping by low-cardinality ds (1 value). + mocker.patch.object( + adapter, + "fetchall", + return_value=[(1, "2024-01-01"), (2, "2024-01-01"), (3, "2024-01-01")], + ) + + first_delete = True + + def execute_side_effect(sql, *args, **kwargs): + nonlocal first_delete + if first_delete and "USING" in str(sql): + first_delete = False + raise Exception("errCode = 2, detailMessage = delete using only support unique key") + + adapter.cursor.execute.side_effect = execute_side_effect + + key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", exp.column("id"), exp.column("ds")) + where = exp.Paren(this=key_exp).isin(query=exp.select(key_exp).from_("src_table")) + adapter.delete_from("target_table", where=where) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `target_table` AS `_t1` USING (SELECT `id`, `ds` FROM `src_table`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`ds` = `_t2`.`ds`", + "DELETE FROM `target_table` WHERE `ds` = '2024-01-01' AND `id` IN (1, 2, 3)", + ] + + +def test_rewrite_between_in_where_preserves_not_between_and_wraps_or( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + raw_where = parse_one( + "(ds BETWEEN '2024-01-01' AND '2024-01-07' OR status = 'expired') AND id NOT BETWEEN 10 AND 20", + read="doris", + ) + rewritten_where = adapter._rewrite_between_in_where(raw_where.copy()) + assert ( + adapter._to_sql(rewritten_where) + == "((`ds` >= '2024-01-01' AND `ds` <= '2024-01-07') OR `status` = 'expired') AND NOT `id` BETWEEN 10 AND 20" + ) + + adapter.delete_from( + "target_table", + where=raw_where, + ) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `target_table` WHERE `ds` >= '2024-01-01' AND `ds` <= '2024-01-07' AND `id` < 10", + "DELETE FROM `target_table` WHERE `ds` >= '2024-01-01' AND `ds` <= '2024-01-07' AND `id` > 20", + "DELETE FROM `target_table` WHERE `status` = 'expired' AND `id` < 10", + "DELETE FROM `target_table` WHERE `status` = 'expired' AND `id` > 20", + ] + + +@pytest.mark.parametrize( + "key_cols,rows", + [ + (["id"], [(1,), (None,), (2,), (3,)]), + (["id", "ds"], [(1, "2024-01-01"), (2, "2024-01-01"), (None, "2024-01-01"), (1, None)]), + ( + ["tenant", "region", "id"], + [ + ("t1", "us", 10), + ("t1", "us", 20), + ("t1", "us", None), + ("t1", None, 30), + (None, None, None), + ], + ), + ], +) +def test_delete_by_literal_keys_doris_delete_handler_invariants( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + mocker: MockerFixture, + key_cols: t.List[str], + rows: t.List[t.Tuple[t.Any, ...]], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object(adapter, "fetchall", return_value=rows) + + adapter._delete_by_literal_keys( + exp.to_table("db.target_tbl"), + [exp.column(c) for c in key_cols], + parse_one(f"SELECT {', '.join(key_cols)} FROM db.src_tbl", read="doris", into=exp.Select), + ) + + sql_calls = to_sql_calls(adapter) + assert len(sql_calls) >= 1 + for sql in sql_calls: + parsed = parse_one(sql, read="doris") + assert isinstance(parsed, exp.Delete) + # Doris DeleteHandler forbids OR and BETWEEN in DELETE WHERE predicates + assert parsed.find(exp.Or) is None + assert parsed.find(exp.Between) is None + + +@pytest.mark.parametrize( + "non_key_type", + [ + "FLOAT", + "DOUBLE", + "ARRAY", + "MAP", + "STRUCT", + "JSON", + "VARIANT", + "BITMAP", + "HLL", + "QUANTILE_STATE", + "AGG_STATE(sum(INT))", + ], +) +def test_olap_key_type_and_prefix_invariants( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + non_key_type: str, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.create_table( + "db.complex_tbl", + target_columns_to_types={ + "bad_first_col": exp.DataType.build(non_key_type, dialect="doris", udt=True), + "valid_key": exp.DataType.build("BIGINT", dialect="doris"), + }, + ) + + sql_calls = to_sql_calls(adapter) + assert len(sql_calls) == 1 + # valid_key must be moved to column index 0 ahead of non-keyable bad_first_col + assert sql_calls[0].startswith( + "CREATE TABLE IF NOT EXISTS `db`.`complex_tbl` (`valid_key` BIGINT, `bad_first_col`" + ) + + +def test_adjust_physical_properties_for_incremental( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + assert adapter.RECREATE_MATERIALIZED_VIEW_ON_EVALUATION is False + assert adapter.RESOLVE_TABLE_REFS_IN_PHYSICAL_PROPERTIES == frozenset( + {"excluded_trigger_tables", "source_table"} + ) + + # Single unique_key promoted when no explicit key is present (table qualifiers stripped) + props = adapter.adjust_physical_properties_for_incremental( + {}, + requires_delete_capable_table=True, + unique_key=[exp.to_column("src.id")], + model_name="db.incremental_model", + ) + assert props == {"unique_key": exp.to_column("id")} + + # Composite unique_key promoted as exp.Tuple + props_composite = adapter.adjust_physical_properties_for_incremental( + {}, + requires_delete_capable_table=True, + unique_key=[exp.to_column("id"), exp.to_column("ds")], + model_name="db.incremental_model", + ) + assert props_composite == { + "unique_key": exp.Tuple(expressions=[exp.to_column("id"), exp.to_column("ds")]) + } + + # Explicit duplicate_key in physical_properties is preserved without overwriting + props_explicit = adapter.adjust_physical_properties_for_incremental( + {"duplicate_key": exp.to_column("id")}, + requires_delete_capable_table=True, + unique_key=[exp.to_column("id")], + model_name="db.incremental_model", + ) + assert props_explicit == {"duplicate_key": exp.to_column("id")} + + # Verify end-to-end CREATE TABLE with promoted unique_key reorders key columns first and emits UNIQUE KEY + adapter.create_table( + "db.incremental_model", + target_columns_to_types={ + "name": exp.DataType.build("VARCHAR(50)", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + table_properties=props, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`incremental_model` (`id` INT, `name` VARCHAR(50)) UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10" + ] + + +def test_merge_with_identifier_unique_key( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + mocker: MockerFixture, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + adapter.merge( + target_table="db.target", + source_table=parse_one("SELECT id, ds FROM db.src"), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + unique_key=[exp.to_identifier("id")], + ) + sql_calls = to_sql_calls(adapter) + assert any( + call.startswith( + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id` FROM `db`.`__temp_target_abcd1234`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id`" + ) + for call in sql_calls + ) + + +def test_list_partition_ast_and_random_distributed_by( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.create_table( + "db.list_tbl", + target_columns_to_types={ + "region": exp.DataType.build("VARCHAR(16)", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + partitioned_by=[parse_one("LIST(region)", read="doris")], + table_properties={ + "partitions": exp.Literal.string("PARTITION p_na VALUES IN ('US', 'CA')"), + "distributed_by": parse_one("RANDOM()", read="doris"), + "buckets": exp.Literal.number(8), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`list_tbl` (`region` VARCHAR(16), `id` INT) PARTITION BY LIST (`region`) (PARTITION p_na VALUES IN ('US', 'CA')) DISTRIBUTED BY RANDOM BUCKETS 8" + ] + + +def test_key_model_mutual_exclusion_and_missing_column_validation( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + with pytest.raises(SQLMeshError, match="at most one key model"): + adapter.create_table( + "db.bad_keys", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("id"), + "duplicate_key": exp.to_column("id"), + }, + ) + + with pytest.raises(SQLMeshError, match="not found in table schema columns"): + adapter.create_table( + "db.missing_key", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("nonexistent_col"), + }, + ) + + with pytest.raises(SQLMeshError, match="only supported on UNIQUE KEY tables"): + adapter.create_table( + "db.order_by_without_uk", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "duplicate_key": exp.to_column("id"), + "order_by": exp.to_column("name"), + }, + ) + + # Case-insensitive key column matching resolves to the canonical schema column name + adapter.create_table( + "db.case_insensitive_uk", + target_columns_to_types={ + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("ID"), + "distributed_by": "HASH(ID) BUCKETS 8", + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`case_insensitive_uk` (`id` INT, `name` VARCHAR(32)) UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 8" + ] + + with pytest.raises( + SQLMeshError, match="Doris UNIQUE KEY tables do not support DISTRIBUTED BY RANDOM" + ): + adapter.create_table( + "db.uk_random_dist", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("id"), + "distributed_by": "RANDOM BUCKETS 8", + }, + ) + + with pytest.raises( + SQLMeshError, match="Doris distributed_by columns .* not found in table schema columns" + ): + adapter.create_table( + "db.missing_dist_col", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "distributed_by": "HASH(missing_col) BUCKETS 8", + }, + ) + + # Explicit distributed_by on UNIQUE KEY / AGGREGATE KEY must be a subset of key columns + with pytest.raises( + SQLMeshError, + match="Doris UNIQUE KEY tables require distributed_by columns to be key columns", + ): + adapter.create_table( + "db.uk_non_key_dist", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("id"), + "distributed_by": "HASH(name) BUCKETS 8", + }, + ) + + # Cross-engine clustered_by on non-key column for UNIQUE KEY table falls back to key column + adapter.cursor.execute.reset_mock() + adapter.create_table( + "db.uk_clustered_by_fallback", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + clustered_by=[exp.to_column("name")], + table_properties={ + "unique_key": exp.to_column("id"), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`uk_clustered_by_fallback` (`id` INT, `name` VARCHAR(32)) UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10" + ] + + # Missing partitioned_by column raises SQLMeshError when partitioning is active + with pytest.raises( + SQLMeshError, match="Doris partitioned_by columns .* not found in table schema columns" + ): + adapter.create_table( + "db.missing_part_col", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + partitioned_by=[parse_one("RANGE(missing_ds)", read="doris")], + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + + +def test_ctas_reorders_projections_in_place_without_double_subquery( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + adapter.ctas( + "db.ctas_reordered", + parse_one("SELECT name, id FROM db.src", read="doris"), + target_columns_to_types={ + "name": exp.DataType.build("VARCHAR(50)", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + table_properties={"unique_key": exp.to_column("id")}, + ) + + # Unannotated CTAS (target_columns_to_types=None) also reorders Select projections for key prefix + adapter.ctas( + "db.ctas_unannotated_reordered", + parse_one("SELECT name, id FROM db.src", read="doris"), + table_properties={"unique_key": exp.to_column("id")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`ctas_reordered` UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 AS SELECT CAST(`id` AS INT) AS `id`, CAST(`name` AS VARCHAR(50)) AS `name` FROM (SELECT `name`, `id` FROM `db`.`src`) AS `_subquery`", + "CREATE TABLE IF NOT EXISTS `db`.`ctas_unannotated_reordered` UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 AS SELECT `id`, `name` FROM `db`.`src`", + ] + + +def test_get_data_objects_reclassifies_mtmv_via_mv_infos( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter, patch_get_data_objects=False) + mocker.patch.object( + adapter, + "fetchall", + side_effect=[ + [("silver", "regular_tbl", "BASE TABLE"), ("silver", "async_mv", "BASE TABLE")], + [("async_mv",)], + ], + ) + + objects = adapter._get_data_objects("silver") + assert [(o.name, o.type) for o in objects] == [ + ("regular_tbl", DataObjectType.TABLE), + ("async_mv", DataObjectType.MATERIALIZED_VIEW), + ] + + +def test_delete_from_boolean_literal_simplification( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # FALSE is a no-op + adapter.delete_from("db.tbl", where=exp.false()) + assert to_sql_calls(adapter) == [] + + # Compound AND TRUE is simplified away so Doris DeleteHandler does not reject boolean literal + adapter.delete_from("db.tbl", where="id = 1 AND TRUE") + assert to_sql_calls(adapter) == ["DELETE FROM `db`.`tbl` WHERE `id` = 1"] + + adapter.cursor.execute.reset_mock() + adapter.delete_from("db.tbl", where="TRUE") + assert to_sql_calls(adapter) == ["TRUNCATE TABLE `db`.`tbl`"] + + # Target-qualified extra condition in DELETE ... USING is rewritten to alias _t1 + adapter.cursor.execute.reset_mock() + adapter.delete_from( + "db.target", + where="target.ds = '2024-01-01' AND id IN (SELECT id FROM db.src)", + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id` FROM `db`.`src`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`ds` = '2024-01-01'" + ] + + # Target-qualified condition in simple DELETE FROM is stripped to unqualified column for Doris DeleteHandler + adapter.cursor.execute.reset_mock() + adapter.delete_from( + "db.target", + where="target.ds = '2024-01-01' AND target.id = 5", + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`target` WHERE `ds` = '2024-01-01' AND `id` = 5" + ] + + +def test_resolve_table_refs_in_physical_properties_for_mv_and_sync_mv( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.core.dialect import parse + from sqlmesh.core.model import SqlModel, load_sql_based_model + from sqlmesh.core.snapshot import Snapshot, SnapshotChangeCategory + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + base_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name doris_db.orders, + kind FULL, + dialect doris, + columns (id INT, amount INT) + ); + SELECT 1 AS id, 10 AS amount; + """, + default_dialect="doris", + ) + ), + ) + base_snapshot = Snapshot.from_node(base_model, nodes={}, ttl="in 1 week") + base_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + physical_tbl = exp.to_table(base_snapshot.table_name()) + expected_physical = f"{physical_tbl.db}.{physical_tbl.name}" + snapshots = {base_snapshot.name: base_snapshot} + + # 1. Async MTMV with excluded_trigger_tables resolves model ref to physical snapshot table + async_mv_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name doris_db.orders_mtmv, + kind VIEW (materialized true), + dialect doris, + columns (id INT, amount INT), + physical_properties ( + build = IMMEDIATE, + refresh = AUTO, + excluded_trigger_tables = doris_db.orders + ) + ); + SELECT id, amount FROM doris_db.orders; + """, + default_dialect="doris", + ) + ), + ) + rendered_async_props = async_mv_model.render_physical_properties( + snapshots=snapshots, engine_adapter=adapter + ) + adapter.create_view( + async_mv_model.name, + async_mv_model.render_query() or parse_one("SELECT id, amount FROM doris_db.orders"), + replace=False, + materialized=True, + target_columns_to_types=async_mv_model.columns_to_types, + view_properties=rendered_async_props, + ) + async_ddl = to_sql_calls(adapter)[-1] + assert f"'excluded_trigger_tables'='{expected_physical}'" in async_ddl + + # 2. Sync MV with source_table resolves model ref to physical snapshot table for DROP ... ON + adapter.cursor.execute.reset_mock() + sync_mv_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name doris_db.orders_sync_mv, + kind VIEW (materialized true), + dialect doris, + physical_properties ( + materialized_type = SYNC, + source_table = doris_db.orders + ) + ); + SELECT id, SUM(amount) AS total FROM doris_db.orders GROUP BY id; + """, + default_dialect="doris", + ) + ), + ) + rendered_sync_props = sync_mv_model.render_physical_properties( + snapshots=snapshots, engine_adapter=adapter + ) + adapter.create_view( + sync_mv_model.name, + sync_mv_model.render_query(snapshots=snapshots) + or parse_one("SELECT id, SUM(amount) AS total FROM doris_db.orders GROUP BY id"), + replace=True, + materialized=True, + view_properties=rendered_sync_props, + ) + sync_calls = to_sql_calls(adapter) + assert sync_calls[0] == "USE `doris_db`" + assert sync_calls[1] == ( + f"DROP MATERIALIZED VIEW IF EXISTS `orders_sync_mv` ON `{physical_tbl.db}`.`{physical_tbl.name}`" + ) + assert sync_calls[-1].startswith("CREATE MATERIALIZED VIEW `orders_sync_mv` AS ") + + +def test_dataframe_seed_all_string_coercion_and_batch_size( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + import pandas as pd + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + assert adapter.DEFAULT_BATCH_SIZE == 500 + + # 1. All-TEXT schema (e.g. from columns_to_types_from_df on a string CSV seed) coerces first column to VARCHAR(255) + adapter.create_table( + "db.country_seed", + target_columns_to_types={ + "country_code": exp.DataType.build("text"), + "country_name": exp.DataType.build("text"), + }, + ) + + # 2. Schema with DOUBLE + TEXT moves TEXT column to position 0 and coerces it to VARCHAR(255) + adapter.create_table( + "db.metric_seed", + target_columns_to_types={ + "score": exp.DataType.build("double"), + "metric_name": exp.DataType.build("text"), + }, + ) + + # 3. Explicit unique_key on a TEXT column coerces the key column to VARCHAR(255) + adapter.create_table( + "db.explicit_text_uk", + target_columns_to_types={ + "payload": exp.DataType.build("text"), + "code": exp.DataType.build("text"), + }, + table_properties={"unique_key": exp.to_column("code")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`country_seed` (`country_code` VARCHAR(255), `country_name` STRING)", + "CREATE TABLE IF NOT EXISTS `db`.`metric_seed` (`metric_name` VARCHAR(255), `score` DOUBLE)", + "CREATE TABLE IF NOT EXISTS `db`.`explicit_text_uk` (`code` VARCHAR(255), `payload` STRING) UNIQUE KEY (`code`) DISTRIBUTED BY HASH (`code`) BUCKETS 10", + ] + + # 4. Explicit unique_key on a non-coercible invalid key type (DOUBLE) raises SQLMeshError + with pytest.raises(SQLMeshError, match="not supported as an OLAP key column"): + adapter.create_table( + "db.bad_double_uk", + target_columns_to_types={ + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + table_properties={"unique_key": exp.to_column("score")}, + ) + + # 5. DataFrame CTAS with pretty_sql=True and 500 rows succeeds without RecursionError + pretty_adapter = make_mocked_engine_adapter(DorisEngineAdapter, pretty_sql=True) + df = pd.DataFrame({"code": [f"c_{i}" for i in range(500)], "val": list(range(500))}) + pretty_adapter.ctas("db.df_seed", df) + df_sql_calls = to_sql_calls(pretty_adapter) + assert len(df_sql_calls) == 1 + assert "CREATE TABLE IF NOT EXISTS `db`.`df_seed`" in df_sql_calls[0] + + +def test_iceberg_bucket_and_truncate_partition_transforms( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.core.dialect import parse + from sqlmesh.core.model import SqlModel, load_sql_based_model + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + iceberg_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name iceberg_catalog.silver.curated_events, + kind FULL, + dialect doris, + table_format iceberg, + partitioned_by (day(event_ts), bucket(16, user_id), truncate(10, region)) + ); + SELECT + user_id::BIGINT AS user_id, + event_ts::DATETIME AS event_ts, + region::STRING AS region + FROM iceberg_catalog.bronze.raw_events; + """, + default_dialect="doris", + ) + ), + ) + + adapter.create_table( + iceberg_model.name, + target_columns_to_types=iceberg_model.columns_to_types or {}, + table_format=iceberg_model.table_format, + partitioned_by=iceberg_model.partitioned_by, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `iceberg_catalog`.`silver`.`curated_events` (`user_id` BIGINT, `event_ts` DATETIME, `region` STRING) ENGINE=iceberg PARTITION BY LIST (DAY(`event_ts`), BUCKET(16, `user_id`), TRUNCATE(10, `region`)) ()" + ] + + +def test_ctas_auto_partition_non_nullable_and_union_key_reordering( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. CTAS with AUTO PARTITION BY RANGE wraps partition column projection in NON_NULLABLE(...) + adapter.ctas( + "db.ctas_auto_part", + parse_one("SELECT payload, id, ds FROM db.src", read="doris"), + target_columns_to_types={ + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + partitioned_by=[parse_one("RANGE(DATE_TRUNC(ds, 'DAY'))", read="doris")], + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + + # 2. Unannotated CTAS with UNION ALL reorders projections in all branches for unique_key prefix + adapter.ctas( + "db.ctas_union_reordered", + parse_one( + "SELECT name, id FROM db.src1 UNION ALL SELECT name, id FROM db.src2", read="doris" + ), + table_properties={"unique_key": exp.to_column("id")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`ctas_auto_part` AUTO PARTITION BY RANGE (DATE_TRUNC(`ds`, 'DAY')) () AS SELECT CAST(`id` AS INT) AS `id`, NON_NULLABLE(CAST(`ds` AS DATE)) AS `ds`, CAST(`payload` AS STRING) AS `payload` FROM (SELECT `payload`, `id`, `ds` FROM `db`.`src`) AS `_subquery`", + "CREATE TABLE IF NOT EXISTS `db`.`ctas_union_reordered` UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 AS SELECT `id`, `name` FROM `db`.`src1` UNION ALL SELECT `id`, `name` FROM `db`.`src2`", + ] + + +def test_delete_tautology_and_union_subquery_and_dedup_partition_overwrite( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Literal tautology 1=1 routes to TRUNCATE TABLE; contradiction 1=0 is a no-op + adapter.delete_from("db.tbl", where="1 = 1") + adapter.delete_from("db.tbl", where="1 = 0") + + # 2. IN subquery with UNION resolves the projected column name from the first SELECT branch + adapter.delete_from( + "db.tbl", + where="id IN (SELECT src_id FROM db.a UNION SELECT src_id FROM db.b)", + ) + + # 3. IN subquery with unaliased expression aliases the projection for _t2 join + adapter.delete_from( + "db.tbl", + where="id IN (SELECT other_id + 1 FROM db.a)", + ) + + assert to_sql_calls(adapter) == [ + "TRUNCATE TABLE `db`.`tbl`", + "DELETE FROM `db`.`tbl` AS `_t1` USING (SELECT `src_id` FROM `db`.`a` UNION SELECT `src_id` FROM `db`.`b`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`src_id`", + "DELETE FROM `db`.`tbl` AS `_t1` USING (SELECT `other_id` + 1 AS `_col_0` FROM `db`.`a`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`_col_0`", + ] + + # 4. insert_overwrite_by_partition deduplicates time transforms to finest unit (MONTH) and strips table qualifiers + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "_get_temp_table", + return_value=exp.table_("__temp_tbl", db="db"), + ) + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[("2024-01-01", "2024-02-01")], + ) + adapter.insert_overwrite_by_partition( + "db.events", + parse_one("SELECT id, ds FROM db.src", read="doris"), + partitioned_by=[ + parse_one("YEAR(src.ds)", read="doris"), + parse_one("MONTH(src.ds)", read="doris"), + ], + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATE"), + }, + ) + overwrite_calls = to_sql_calls(adapter) + assert ( + "SELECT DISTINCT DATE_TRUNC(`ds`, 'month') AS `_low_0`, DATE_ADD(DATE_TRUNC(`ds`, 'month'), INTERVAL 1 MONTH) AS `_high_0` FROM `db`.`__temp_tbl`" + in overwrite_calls + ) + assert ( + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2024-02-01'" + in overwrite_calls + ) + + +def test_sync_mv_inferred_source_table_on_replace_and_unqualified_get_data_objects( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter, patch_get_data_objects=False) + + # 1. Sync MV with replace=True and no explicit source_table in view_properties infers base table from query + mocker.patch.object(adapter, "get_data_object", return_value=None) + adapter.create_view( + "sync_rollup_mv", + parse_one("SELECT id, SUM(amount) AS total FROM prod_db.orders GROUP BY id", read="doris"), + replace=True, + materialized=True, + view_properties={"materialized_type": exp.var("SYNC")}, + ) + assert to_sql_calls(adapter) == [ + "USE `prod_db`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_rollup_mv` ON `prod_db`.`orders`", + "USE `prod_db`", + "CREATE MATERIALIZED VIEW `sync_rollup_mv` AS SELECT `id` AS `mv_id`, SUM(`amount`) AS `total` FROM `prod_db`.`orders` GROUP BY `id`", + ] + + # 2. _get_data_objects with empty schema uses DATABASE() instead of empty string literal + adapter.cursor.execute.reset_mock() + fetchall_spy = mocker.patch.object( + adapter, + "fetchall", + side_effect=[ + [("active_db", "tbl1", "BASE TABLE")], + [], + ], + ) + objs = adapter._get_data_objects("") + assert len(objs) == 1 + assert objs[0].schema_name == "active_db" + first_query_sql = fetchall_spy.call_args_list[0].args[0].sql("doris") + assert "table_schema = DATABASE()" in first_query_sql + second_query_sql = fetchall_spy.call_args_list[1].args[0] + assert "mv_infos('database' = 'active_db')" in second_query_sql + + +def test_ctas_string_key_coercion_and_internal_iceberg_ctas( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. CTAS with explicit unique_key on TEXT column coerces CAST(code AS VARCHAR(255)) and reorders key first + adapter.ctas( + "db.seed_uk_ctas", + parse_one("SELECT payload, code FROM db.raw_seed", read="doris"), + target_columns_to_types={ + "payload": exp.DataType.build("text"), + "code": exp.DataType.build("text"), + }, + table_properties={"unique_key": exp.to_column("code")}, + ) + + # 2. CTAS with all-TEXT columns (implicit DUPLICATE KEY on first column) coerces first column CAST to VARCHAR(255) + adapter.ctas( + "db.seed_dup_ctas", + parse_one("SELECT country_code, country_name FROM db.raw_countries", read="doris"), + target_columns_to_types={ + "country_code": exp.DataType.build("text"), + "country_name": exp.DataType.build("text"), + }, + ) + + # 3. 2-part internal Iceberg CTAS (table_format="iceberg") creates table with ENGINE=iceberg and inserts rows + adapter.ctas( + "silver.iceberg_events", + parse_one("SELECT id, ds FROM bronze.events", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("BIGINT"), + "ds": exp.DataType.build("DATE"), + }, + table_format="iceberg", + partitioned_by=[parse_one("DAY(ds)", dialect="doris")], + table_properties={"write-format": exp.Literal.string("parquet")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`seed_uk_ctas` UNIQUE KEY (`code`) DISTRIBUTED BY HASH (`code`) BUCKETS 10 AS SELECT CAST(`code` AS VARCHAR(255)) AS `code`, CAST(`payload` AS STRING) AS `payload` FROM (SELECT `payload`, `code` FROM `db`.`raw_seed`) AS `_subquery`", + "CREATE TABLE IF NOT EXISTS `db`.`seed_dup_ctas` AS SELECT CAST(`country_code` AS VARCHAR(255)) AS `country_code`, CAST(`country_name` AS STRING) AS `country_name` FROM (SELECT `country_code`, `country_name` FROM `db`.`raw_countries`) AS `_subquery`", + "CREATE TABLE IF NOT EXISTS `silver`.`iceberg_events` (`id` BIGINT, `ds` DATE) ENGINE=iceberg PARTITION BY LIST (DAY(`ds`)) () PROPERTIES ('write-format'='parquet')", + "INSERT INTO `silver`.`iceberg_events` (`id`, `ds`) SELECT CAST(`id` AS BIGINT) AS `id`, CAST(`ds` AS DATE) AS `ds` FROM (SELECT `id`, `ds` FROM `bronze`.`events`) AS `_subquery`", + ] + + +def test_self_referencing_replace_query_on_new_table( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object(adapter, "get_data_object", return_value=None) + + adapter.replace_query( + "db.self_ref_tbl", + parse_one("SELECT id, name FROM db.self_ref_tbl", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "name": exp.DataType.build("VARCHAR(50)"), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`self_ref_tbl` (`id` INT, `name` VARCHAR(50))", + "INSERT INTO `db`.`self_ref_tbl` (`id`, `name`) SELECT CAST(`id` AS INT) AS `id`, CAST(`name` AS VARCHAR(50)) AS `name` FROM (SELECT `id`, `name` FROM `db`.`self_ref_tbl`) AS `_subquery`", + ] + + +def test_delete_literal_tuple_in_and_subquery_duplicate_aliases_and_concat_ws_guard( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Literal tuple IN (e.g. from SQLMesh state sync snapshot_id_filter) rewrites to grouped single-column IN + adapter.delete_from( + "sqlmesh._snapshots", + parse_one( + "(name, identifier) IN (('model_a', 'id_1'), ('model_a', 'id_2'), ('model_b', 'id_3'))", + read="doris", + ), + ) + + # 2. Composite subquery selecting duplicate short column names (`u.id, m.id`) aliases the second column (`_col_1`) + adapter.delete_from( + "db.pair_links", + parse_one( + "(a_id, b_id) IN (SELECT u.id, m.id FROM db.users u JOIN db.members m ON u.org_id = m.org_id)", + read="doris", + ), + ) + + # 3. CONCAT_WS on LHS with a single non-CONCAT_WS column in RHS subquery does not unwrap into mismatched arity + adapter.delete_from( + "db.target_tbl", + parse_one( + "CONCAT_WS('__', a, b) IN (SELECT combined_key FROM db.staging_keys)", + read="doris", + ), + ) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `sqlmesh`.`_snapshots` WHERE `name` = 'model_a' AND `identifier` IN ('id_1', 'id_2')", + "DELETE FROM `sqlmesh`.`_snapshots` WHERE `name` = 'model_b' AND `identifier` IN ('id_3')", + "DELETE FROM `db`.`pair_links` AS `_t1` USING (SELECT `u`.`id`, `m`.`id` AS `_col_1` FROM `db`.`users` AS `u` JOIN `db`.`members` AS `m` ON `u`.`org_id` = `m`.`org_id`) AS `_t2` WHERE `_t1`.`a_id` = `_t2`.`id` AND `_t1`.`b_id` = `_t2`.`_col_1`", + "DELETE FROM `db`.`target_tbl` WHERE CONCAT_WS('__', `a`, `b`) IN (SELECT `combined_key` FROM `db`.`staging_keys`)", + ] + + +def test_to_sql_root_int128_and_nested_struct_pretty_sql_and_fetchdf_reconnect( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + import pandas as pd + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Root exp.DataType INT128 replacement + assert adapter._to_sql(exp.DataType.build("INT128")) == "LARGEINT" + + # 2. Nested STRUCT with INT128 under _pretty_sql = True + adapter._pretty_sql = True + nested_struct = exp.DataType.build( + "STRUCT>", + dialect="doris", + ) + assert ( + adapter._to_sql(nested_struct) + == "STRUCT<`outer_id`: LARGEINT, `nested`: STRUCT<`inner_id`: LARGEINT, `label`: VARCHAR(20)>>" + ) + adapter._pretty_sql = False + + # 3. _fetch_native_df retries once on non-timeout OperationalError(2013) disconnect + class OperationalError(Exception): + pass + + close_spy = mocker.spy(adapter._connection_pool, "close") + fetch_calls = 0 + + def fake_read_sql_query(sql, con, *args, **kwargs): + nonlocal fetch_calls + fetch_calls += 1 + if fetch_calls == 1: + raise OperationalError(2013, "Lost connection to MySQL server during query") + return pd.DataFrame({"id": [1]}) + + mocker.patch("pandas.io.sql.read_sql_query", side_effect=fake_read_sql_query) + df = adapter.fetchdf("SELECT 1 AS id") + assert len(df) == 1 + assert fetch_calls == 2 + assert close_spy.call_count == 1 + + # 4. _fetch_native_df does NOT retry non-read-only queries on disconnect + fetch_calls = 0 + close_spy.reset_mock() + with pytest.raises(OperationalError): + adapter.fetchdf("WITH `cte` AS (SELECT 1) INSERT INTO `tbl` SELECT * FROM `cte`") + assert fetch_calls == 1 + assert close_spy.call_count == 1 + + +def test_strip_for_update_and_multi_col_index_and_partition_by_alias_and_deferred_mv_audit_refresh( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Strip FOR UPDATE locks from SELECT statements (e.g. EnvironmentState.finalize) + locked_select = ( + exp.select("plan_id") + .from_("sqlmesh._environments") + .where(exp.column("name").eq("prod")) + .lock() + ) + adapter.execute(locked_select) + + # 2. Multi-column composite index creation is skipped gracefully (Doris only supports single-column indexes) + adapter.create_index("sqlmesh._snapshots", "idx_snapshots", ("name", "identifier")) + + # 3. partition_by alias in table_properties works for OLAP AUTO PARTITION BY RANGE + adapter.create_table( + "db.alias_part_tbl", + target_columns_to_types={ + "id": exp.DataType.build("INT"), + "ds": exp.DataType.build("DATE"), + }, + table_properties={ + "partition_by": parse_one("RANGE(DATE_TRUNC(ds, 'day'))", read="doris"), + "auto_partition": exp.Literal.string("true"), + }, + ) + + # 4. Async materialized view with build='DEFERRED' and has_audits=True triggers REFRESH MATERIALIZED VIEW ... AUTO + adapter.create_view( + "db.deferred_audited_mv", + parse_one("SELECT id, ds FROM db.alias_part_tbl", read="doris"), + replace=False, + materialized=True, + materialized_properties={"has_audits": True}, + view_properties={ + "build": exp.Literal.string("DEFERRED"), + "refresh": exp.Literal.string("COMPLETE ON MANUAL"), + "partition_by": exp.Tuple(expressions=[exp.column("ds")]), + }, + ) + + assert to_sql_calls(adapter) == [ + "SELECT `plan_id` FROM `sqlmesh`.`_environments` WHERE `name` = 'prod'", + "CREATE TABLE IF NOT EXISTS `db`.`alias_part_tbl` (`id` INT, `ds` DATE NOT NULL) AUTO PARTITION BY RANGE (DATE_TRUNC(`ds`, 'DAY')) ()", + "CREATE MATERIALIZED VIEW `db`.`deferred_audited_mv` BUILD DEFERRED REFRESH COMPLETE ON MANUAL PARTITION BY (`ds`) AS SELECT `id`, `ds` FROM `db`.`alias_part_tbl`", + "REFRESH MATERIALIZED VIEW `db`.`deferred_audited_mv` AUTO", + ] + + +def test_state_sync_or_key_delete_and_boolean_column_and_alter_key_coercion_guard( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + from sqlmesh.core.snapshot import SnapshotId, SnapshotNameVersion + from sqlmesh.core.state_sync.db.utils import snapshot_id_filter, snapshot_name_version_filter + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + assert adapter.SUPPORTS_TUPLE_IN is False + + # 1. snapshot_id_filter with SUPPORTS_TUPLE_IN=False produces OR of AND equalities; + # delete_from decomposes it into zero-OR grouped IN deletes. + sids = [ + SnapshotId(name="db.model_a", identifier="id_1"), + SnapshotId(name="db.model_a", identifier="id_2"), + SnapshotId(name="db.model_b", identifier="id_3"), + ] + for where in snapshot_id_filter(adapter, sids, batch_size=100): + adapter.delete_from("sqlmesh._snapshots", where=where) + + # 2. IntervalState._delete_intervals_by_dev_version passes `where.and_(exp.column("is_dev"))` + snvs = [ + SnapshotNameVersion(name="db.model_a", version="v1"), + SnapshotNameVersion(name="db.model_a", version="v2"), + ] + for where in snapshot_name_version_filter( + adapter, snvs, version_column_name="dev_version", alias=None, batch_size=100 + ): + adapter.delete_from("sqlmesh._intervals", where.and_(exp.column("is_dev"))) + + # 3. EnvironmentState.delete_expired_environments passes single-column OR `environment_name = 'dev1' OR environment_name = 'dev2'` + adapter.delete_from( + "sqlmesh._environment_statements", + where=exp.or_( + exp.column("environment_name").eq(exp.Literal.string("dev_1")), + exp.column("environment_name").eq(exp.Literal.string("dev_2")), + ), + ) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `sqlmesh`.`_snapshots` WHERE `name` = 'db.model_a' AND `identifier` IN ('id_1', 'id_2')", + "DELETE FROM `sqlmesh`.`_snapshots` WHERE `name` = 'db.model_b' AND `identifier` IN ('id_3')", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `name` = 'db.model_a' AND `dev_version` IN ('v1', 'v2') AND `is_dev` = TRUE", + "DELETE FROM `sqlmesh`.`_environment_statements` WHERE `environment_name` IN ('dev_1', 'dev_2')", + ] + + # 4. get_alter_operations suppresses invalid VARCHAR(255) -> STRING type change on an OLAP key column + # while preserving ADD COLUMN and non-key column widening. + adapter.cursor.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + # DESCRIBE db.current_tbl: `code` is the OLAP key column coerced to VARCHAR(255), `notes` is a non-key VARCHAR(100) + [ + ("code", "VARCHAR(255)", "Yes", "true", None, ""), + ("notes", "VARCHAR(100)", "Yes", "false", None, ""), + ], + # DESCRIBE db.target_tbl: `id` was added, `code` is STRING, `notes` widened to STRING + [ + ("id", "INT", "Yes", "true", None, ""), + ("code", "TEXT", "Yes", "false", None, ""), + ("notes", "TEXT", "Yes", "false", None, ""), + ], + ], + ) + ops = adapter.get_alter_operations("db.current_tbl", "db.target_tbl") + adapter.cursor.reset_mock() + adapter.alter_table(ops) + assert to_sql_calls(adapter) == [ + "ALTER TABLE `db`.`current_tbl` ADD COLUMN `id` INT", + "ALTER TABLE `db`.`current_tbl` MODIFY COLUMN `notes` STRING", + ] + + +def test_unannotated_view_column_comments_and_starrocks_property_parity( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Unannotated VIEW and MATERIALIZED VIEW with column_descriptions (as called by ViewStrategy.create + # where target_columns_to_types=None) infer projection column names to emit column comments. + adapter.create_view( + "db.unannotated_view", + parse_one("SELECT id, name FROM db.users", read="doris"), + replace=False, + table_description="Users view", + column_descriptions={"id": "Primary user ID"}, + ) + adapter.create_view( + "db.unannotated_mv", + parse_one("SELECT id, name FROM db.users", read="doris"), + replace=False, + materialized=True, + table_description="Users MV", + column_descriptions={"id": "Primary user ID"}, + ) + + # 2. Parenthesized string forms for unique_key and order_by, structured distributed_by with `columns=...`, + # and tuple generic properties (e.g. excluded_trigger_tables = ('db.t1', 'db.t2')). + adapter.create_table( + "db.paren_keys_tbl", + target_columns_to_types={ + "name": exp.DataType.build("VARCHAR(50)"), + "id": exp.DataType.build("INT"), + "dt": exp.DataType.build("DATE"), + }, + table_properties={ + "unique_key": exp.Literal.string("(id, dt)"), + "order_by": exp.Literal.string("'(dt, id)'"), + "distributed_by": exp.Tuple( + expressions=[ + exp.EQ(this=exp.column("columns"), expression=exp.Literal.string("(id, dt)")), + exp.EQ(this=exp.column("buckets"), expression=exp.Literal.number(12)), + ] + ), + "excluded_trigger_tables": exp.Tuple( + expressions=[exp.Literal.string("db.t1"), exp.Literal.string("db.t2")] + ), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE VIEW `db`.`unannotated_view` COMMENT 'Users view' AS SELECT `id`, `name` FROM `db`.`users`", + "CREATE MATERIALIZED VIEW `db`.`unannotated_mv` (`id` COMMENT 'Primary user ID', `name`) COMMENT 'Users MV' AS SELECT `id`, `name` FROM `db`.`users`", + "CREATE TABLE IF NOT EXISTS `db`.`paren_keys_tbl` (`id` INT, `dt` DATE, `name` VARCHAR(50)) UNIQUE KEY (`id`, `dt`) ORDER BY (`dt`, `id`) DISTRIBUTED BY HASH (`id`, `dt`) BUCKETS 12 PROPERTIES ('excluded_trigger_tables'='db.t1,db.t2')", + ] + + # 3. Fail-fast validation for unsupported primary_key, typo `partition`, conflicting partition definitions, + # and invalid RANDOM(col) / column-less HASH distribution. + cols = {"id": exp.DataType.build("INT"), "dt": exp.DataType.build("DATE")} + + with pytest.raises(SQLMeshError, match="does not support 'primary_key'"): + adapter.create_table( + "db.bad_pk", + target_columns_to_types=cols, + table_properties={"primary_key": exp.to_column("id")}, + ) + + with pytest.raises(SQLMeshError, match="Invalid property 'partition'"): + adapter.create_table( + "db.bad_partition_typo", + target_columns_to_types=cols, + table_properties={"partition": exp.to_column("dt")}, + ) + + with pytest.raises(SQLMeshError, match="Multiple partition definitions"): + adapter.create_table( + "db.conflict_part", + target_columns_to_types=cols, + partitioned_by=[exp.to_column("dt")], + table_properties={"partition_by": exp.to_column("dt")}, + ) + + with pytest.raises( + SQLMeshError, match="DISTRIBUTED BY RANDOM does not accept column expressions" + ): + adapter.create_table( + "db.bad_random_cols", + target_columns_to_types=cols, + table_properties={"distributed_by": exp.Literal.string("RANDOM(id)")}, + ) + + with pytest.raises( + SQLMeshError, match="DISTRIBUTED BY HASH requires at least one column expression" + ): + adapter.create_table( + "db.bad_hash_no_cols", + target_columns_to_types=cols, + table_properties={"distributed_by": exp.Literal.string("HASH BUCKETS 10")}, + ) + + +def test_is_bool_rewrites_negative_or_keys_union_delete_and_table_diff_ctas( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `IS TRUE` / `IS FALSE` / `IS NOT TRUE` in UPDATE (e.g. SnapshotState.unpause_snapshots) and DELETE + adapter.update_table( + "sqlmesh._snapshots", + {"unpaused_ts": 1700000000}, + where=exp.and_( + exp.column("forward_only").is_(exp.true()), + exp.column("name").eq(exp.Literal.string("db.m1")), + ), + ) + adapter.delete_from( + "sqlmesh._intervals", + where=exp.and_( + exp.column("is_dev").is_(exp.true()).not_(), + exp.column("name").eq(exp.Literal.string("db.m1")), + ), + ) + + # 2. Top-level OR of key rows with negative number literals and bare boolean / IS TRUE conjuncts + adapter.delete_from( + "sqlmesh._intervals", + where=parse_one( + "(id = -1 AND is_dev) OR (id = -2 AND NOT is_dev)", + read="doris", + ), + ) + + # 3. DELETE ... USING with a UNION ALL subquery containing duplicate/unaliased projection names + union_where = exp.Tuple(expressions=[exp.column("k1"), exp.column("k2")]).isin( + query=parse_one( + "SELECT a.id, b.id FROM db.a AS a JOIN db.b AS b ON a.id = b.id " + "UNION ALL " + "SELECT c.id, d.id FROM db.c AS c JOIN db.d AS d ON c.id = d.id", + read="doris", + ) + ) + adapter.delete_from("db.target_tbl", where=union_where) + + # 4. TableDiff-style unannotated CTAS (`SELECT *, CASE ... AS row_full_match FROM __stats`) + # promotes the integer indicator column (`s_exists`) to index 0 of `__stats` so string/float + # first columns do not fail Doris's implicit DUPLICATE KEY rule. + table_diff_query = parse_one( + "WITH `__source` AS (SELECT `s`.`name` AS `name`, `s`.`id` AS `__sqlmesh_join_key` FROM `db`.`src` AS `s`), " + "`__target` AS (SELECT `t`.`name` AS `name`, `t`.`id` AS `__sqlmesh_join_key` FROM `db`.`tgt` AS `t`), " + "`__stats` AS (" + "SELECT `s`.`name` AS `s__name`, `t`.`name` AS `t__name`, " + "IF(NOT `s`.`__sqlmesh_join_key` IS NULL, 1, 0) AS `s_exists`, " + "IF(NOT `t`.`__sqlmesh_join_key` IS NULL, 1, 0) AS `t_exists` " + "FROM `__source` AS `s` LEFT JOIN `__target` AS `t` ON `s`.`__sqlmesh_join_key` = `t`.`__sqlmesh_join_key`" + ") " + "SELECT *, CASE WHEN `s__name` = `t__name` THEN 1 ELSE 0 END AS `row_full_match` FROM `__stats`", + read="doris", + ) + adapter.ctas("sqlmesh_temp.diff", table_diff_query) + + assert to_sql_calls(adapter) == [ + "UPDATE `sqlmesh`.`_snapshots` SET `unpaused_ts` = 1700000000 WHERE COALESCE(`forward_only`, FALSE) = TRUE AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` = FALSE AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` IS NULL AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `id` = -1 AND `is_dev` IN (TRUE)", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `id` = -2 AND `is_dev` IN (FALSE)", + "DELETE FROM `db`.`target_tbl` AS `_t1` USING (SELECT `a`.`id`, `b`.`id` AS `_col_1` FROM `db`.`a` AS `a` JOIN `db`.`b` AS `b` ON `a`.`id` = `b`.`id` UNION ALL SELECT `c`.`id`, `d`.`id` FROM `db`.`c` AS `c` JOIN `db`.`d` AS `d` ON `c`.`id` = `d`.`id`) AS `_t2` WHERE `_t1`.`k1` = `_t2`.`id` AND `_t1`.`k2` = `_t2`.`_col_1`", + "CREATE TABLE IF NOT EXISTS `sqlmesh_temp`.`diff` AS WITH `__source` AS (SELECT `s`.`name` AS `name`, `s`.`id` AS `__sqlmesh_join_key` FROM `db`.`src` AS `s`), `__target` AS (SELECT `t`.`name` AS `name`, `t`.`id` AS `__sqlmesh_join_key` FROM `db`.`tgt` AS `t`), `__stats` AS (SELECT CASE WHEN NOT `s`.`__sqlmesh_join_key` IS NULL THEN 1 ELSE 0 END AS `s_exists`, CASE WHEN NOT `t`.`__sqlmesh_join_key` IS NULL THEN 1 ELSE 0 END AS `t_exists`, `s`.`name` AS `s__name`, `t`.`name` AS `t__name` FROM `__source` AS `s` LEFT JOIN `__target` AS `t` ON `s`.`__sqlmesh_join_key` = `t`.`__sqlmesh_join_key`) SELECT *, CASE WHEN `s__name` = `t__name` THEN 1 ELSE 0 END AS `row_full_match` FROM `__stats`", + ] + + # 5. Unannotated 2-part/internal Iceberg CTAS raises a clear SQLMeshError instead of silently dropping ENGINE=iceberg + with pytest.raises( + SQLMeshError, + match="Doris does not support `CREATE TABLE ... AS SELECT` with `ENGINE=iceberg`", + ): + adapter.ctas( + "silver.unannotated_iceberg", + parse_one("SELECT id, ds FROM bronze.events", read="doris"), + table_format="iceberg", + ) + + +def test_table_type_keys_random_distribution_mv_exists_and_tuple_ref_resolution( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.core.dialect import parse + from sqlmesh.core.model import SqlModel, load_sql_based_model + from sqlmesh.core.snapshot import Snapshot, SnapshotChangeCategory + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + cols = { + "payload": exp.DataType.build("VARCHAR(50)"), + "id": exp.DataType.build("INT"), + "dt": exp.DataType.build("DATE"), + } + + # 1. StarRocks-style `table_type` ('PRIMARY' | 'UNIQUE' | 'DUPLICATE' | 'AGGREGATE') + `keys` + adapter.create_table( + "db.sr_primary", + target_columns_to_types=cols, + table_properties={ + "table_type": exp.Literal.string("PRIMARY"), + "keys": exp.Tuple(expressions=[exp.to_column("id"), exp.to_column("dt")]), + "buckets": exp.Literal.number(8), + }, + ) + adapter.create_table( + "db.sr_duplicate_random", + target_columns_to_types=cols, + table_properties={ + "table_type": exp.Literal.string("DUPLICATE"), + "keys": exp.Tuple(expressions=[exp.to_column("id")]), + "distributed_by": exp.Tuple( + expressions=[ + exp.EQ(this=exp.column("random"), expression=exp.Literal.string("true")), + exp.EQ(this=exp.column("buckets"), expression=exp.Literal.number(6)), + ] + ), + }, + ) + adapter.create_table( + "db.sr_aggregate", + target_columns_to_types={"id": exp.DataType.build("INT"), "dt": exp.DataType.build("DATE")}, + table_properties={ + "table_type": exp.Literal.string("AGGREGATE"), + "keys": exp.Literal.string("(id, dt)"), + "buckets": exp.Literal.number(4), + }, + ) + + # 2. `create_view(..., materialized=True, replace=False, exists=True)` emits IF NOT EXISTS + adapter.create_view( + "db.mv_if_not_exists", + parse_one("SELECT id FROM db.sr_primary", read="doris"), + replace=False, + materialized=True, + exists=True, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`sr_primary` (`id` INT, `dt` DATE, `payload` VARCHAR(50)) UNIQUE KEY (`id`, `dt`) DISTRIBUTED BY HASH (`id`) BUCKETS 8", + "CREATE TABLE IF NOT EXISTS `db`.`sr_duplicate_random` (`id` INT, `payload` VARCHAR(50), `dt` DATE) DUPLICATE KEY (`id`) DISTRIBUTED BY RANDOM BUCKETS 6", + "CREATE TABLE IF NOT EXISTS `db`.`sr_aggregate` (`id` INT, `dt` DATE) AGGREGATE KEY (`id`, `dt`) DISTRIBUTED BY HASH (`id`) BUCKETS 4", + "CREATE MATERIALIZED VIEW IF NOT EXISTS `db`.`mv_if_not_exists` AS SELECT `id` FROM `db`.`sr_primary`", + ] + + # 3. Validation errors for invalid `table_type` / missing `keys` / conflicting `random=true` + with pytest.raises(SQLMeshError, match="requires 'keys' to be specified"): + adapter.create_table( + "db.bad_tt_no_keys", + target_columns_to_types=cols, + table_properties={"table_type": exp.Literal.string("UNIQUE")}, + ) + + with pytest.raises(SQLMeshError, match="Invalid Doris table_type"): + adapter.create_table( + "db.bad_tt_val", + target_columns_to_types=cols, + table_properties={ + "table_type": exp.Literal.string("UNKNOWN_MODEL"), + "keys": exp.to_column("id"), + }, + ) + + with pytest.raises(SQLMeshError, match="Conflicting distributed_by configuration"): + adapter.create_table( + "db.bad_dist_conflict", + target_columns_to_types=cols, + table_properties={ + "distributed_by": exp.Tuple( + expressions=[ + exp.EQ(this=exp.column("kind"), expression=exp.Literal.string("HASH")), + exp.EQ(this=exp.column("random"), expression=exp.Literal.string("true")), + ] + ) + }, + ) + + # 4. Tuple `excluded_trigger_tables` + `default_catalog="internal"` resolves 2-part model refs + # and strips the leading `internal.` catalog prefix from the physical table name in Doris MTMV DDL. + adapter.cursor.execute.reset_mock() + base_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name doris_db.orders, + kind FULL, + dialect doris, + columns (id INT, amount INT) + ); + SELECT 1 AS id, 10 AS amount; + """, + default_dialect="doris", + ), + default_catalog="internal", + ), + ) + base_snapshot = Snapshot.from_node(base_model, nodes={}, ttl="in 1 week") + base_snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + physical_tbl = exp.to_table(base_snapshot.table_name()) + expected_physical = f"{physical_tbl.db}.{physical_tbl.name}" + snapshots = {base_snapshot.name: base_snapshot} + + tuple_mv_model = t.cast( + SqlModel, + load_sql_based_model( + parse( + """ + MODEL ( + name doris_db.orders_tuple_mtmv, + kind VIEW (materialized true), + dialect doris, + columns (id INT, amount INT), + physical_properties ( + build = IMMEDIATE, + refresh = AUTO, + excluded_trigger_tables = ('doris_db.orders', 'ext_cat.ext_db.raw_events') + ) + ); + SELECT id, amount FROM doris_db.orders; + """, + default_dialect="doris", + ), + default_catalog="internal", + ), + ) + rendered_tuple_props = tuple_mv_model.render_physical_properties( + snapshots=snapshots, engine_adapter=adapter + ) + adapter.create_view( + tuple_mv_model.name, + tuple_mv_model.render_query() or parse_one("SELECT id, amount FROM doris_db.orders"), + replace=False, + materialized=True, + target_columns_to_types=tuple_mv_model.columns_to_types, + view_properties=rendered_tuple_props, + ) + tuple_mv_ddl = to_sql_calls(adapter)[-1] + assert ( + f"'excluded_trigger_tables'='{expected_physical},ext_cat.ext_db.raw_events'" in tuple_mv_ddl + ) + + +def test_time_range_delete_cast_unwrap_partition_key_prefix_scd_type_2_and_reconnect_catalog( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + from datetime import datetime + from sqlmesh.utils.date import to_time_column + + adapter = make_mocked_engine_adapter(DorisEngineAdapter, default_catalog="internal") + + # 1. insert_overwrite_by_time_partition with `to_time_column` (`CAST('...' AS DATE)`) unwraps + # CAST literals in top-level DELETE WHERE while preserving CAST in INSERT SELECT WHERE. + adapter.insert_overwrite_by_time_partition( + "db.events", + parse_one("SELECT id, ds FROM db.src_events", read="doris"), + start="2024-01-01", + end="2024-01-31", + time_column="ds", + time_formatter=lambda v, _: to_time_column( + v, exp.DataType.build("DATE", dialect="doris"), "doris" + ), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + ) + + time_overwrite_calls = to_sql_calls(adapter) + assert time_overwrite_calls == [ + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` <= '2024-01-31'", + "INSERT INTO `db`.`events` (`id`, `ds`) SELECT `id`, `ds` FROM (SELECT `id`, `ds` FROM `db`.`src_events`) AS `_subquery` WHERE `ds` BETWEEN CAST('2024-01-01' AS DATE) AND CAST('2024-01-31' AS DATE)", + ] + + # 2. Implicit DUPLICATE KEY prefix ordering for partitioned_by / time_column: + # (a) TEXT partition column coerced to VARCHAR(255) and moved before DOUBLE; + # (b) DATE partition column moved before VARCHAR column (which cuts off Doris implicit key); + # (c) DATE partition column moved within the first 3 columns (`shortkey_max_column_count = 3`). + adapter.cursor.execute.reset_mock() + adapter.create_table( + "db.part_text_col", + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "amount": exp.DataType.build("DOUBLE", dialect="doris"), + "ds": exp.DataType.build("TEXT", dialect="doris"), + }, + partitioned_by=[exp.to_column("ds")], + ) + adapter.create_table( + "db.part_after_varchar", + target_columns_to_types={ + "code": exp.DataType.build("VARCHAR(100)", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + "amount": exp.DataType.build("DOUBLE", dialect="doris"), + }, + partitioned_by=[exp.to_column("ds")], + ) + adapter.create_table( + "db.part_fourth_col", + target_columns_to_types={ + "c1": exp.DataType.build("INT", dialect="doris"), + "c2": exp.DataType.build("INT", dialect="doris"), + "c3": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + partitioned_by=[exp.to_column("ds")], + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`part_text_col` (`id` INT, `ds` VARCHAR(255), `amount` DOUBLE)", + "CREATE TABLE IF NOT EXISTS `db`.`part_after_varchar` (`ds` DATE, `code` VARCHAR(100), `amount` DOUBLE)", + "CREATE TABLE IF NOT EXISTS `db`.`part_fourth_col` (`c1` INT, `c2` INT, `ds` DATE, `c3` INT)", + ] + + # 3. SCD Type 2 on an existing table when `default_catalog="internal"` does not mutate `target_table` + # in `temp_table` and properly rewrites `db.scd_tbl` references to `__temp_scd_tbl_...`. + adapter.cursor.execute.reset_mock() + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", schema="db", name="scd_tbl", type=DataObjectType.TABLE + ), + ) + adapter.scd_type_2_by_time( + target_table="db.scd_tbl", + source_table=t.cast( + exp.Select, + parse_one("SELECT id, name, updated_at FROM db.src_tbl", read="doris"), + ), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + updated_at_col=exp.column("updated_at", quoted=True), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(50)", dialect="doris"), + "updated_at": exp.DataType.build("DATETIME", dialect="doris"), + "valid_from": exp.DataType.build("DATETIME", dialect="doris"), + "valid_to": exp.DataType.build("DATETIME", dialect="doris"), + }, + execution_time=datetime(2024, 1, 2, 0, 0, 0), + ) + scd_calls = to_sql_calls(adapter) + insert_scd_sql = next(c for c in scd_calls if c.startswith("INSERT INTO `db`.`scd_tbl`")) + assert "FROM `internal`.`db`.`__temp_scd_tbl_abcd1234`" in insert_scd_sql + assert "FROM `db`.`scd_tbl`" not in insert_scd_sql + + # 4. Reconnect in `_execute` restores active catalog set via `set_current_catalog` and falls back + # to `self._connection_pool.close()` when `conn.ping(reconnect=True)` raises on a closed socket. + class InterfaceError(Exception): + pass + + adapter.cursor.execute.reset_mock() + adapter.set_current_catalog("iceberg_cat") + adapter.cursor.execute.reset_mock() + + conn_mock = mocker.MagicMock() + conn_mock.ping.side_effect = InterfaceError(0, "Already closed") + mocker.patch.object(adapter._connection_pool, "get", return_value=conn_mock) + close_spy = mocker.spy(adapter._connection_pool, "close") + + exec_attempt = 0 + + def reconnect_side_effect(sql, *args, **kwargs): + nonlocal exec_attempt + exec_attempt += 1 + if exec_attempt == 1: + raise InterfaceError(0, "Already closed") + + adapter.cursor.execute.side_effect = reconnect_side_effect + adapter._execute("SELECT 1") + + assert close_spy.call_count == 1 + assert exec_attempt == 1 + assert to_sql_calls(adapter) == [ + "SWITCH `iceberg_cat`", + "SELECT 1", + ] + + +def test_thread_local_state_isolation_time_transform_partition_delete_table_diff_and_pm_features( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + from datetime import date, datetime + from decimal import Decimal + import threading + import pandas as pd + from sqlmesh.cli.project_init import InitCliMode, ProjectTemplate, _gen_config + + adapter = make_mocked_engine_adapter( + DorisEngineAdapter, default_catalog="internal", database="analytics" + ) + assert adapter.SUPPORTS_QUERY_EXECUTION_TRACKING is True + + # 1. Thread-local state isolation for `_active_catalog`, `_needs_catalog_restore`, and + # `_last_described_key_cols` across worker threads, `with_settings()`, and `pool.close()`. + adapter._active_catalog = "internal" + adapter._needs_catalog_restore = False + adapter._last_described_key_cols = {"main_key"} + + cloned = t.cast(DorisEngineAdapter, adapter.with_settings(correlation_id="test_corr_id")) + assert cloned._active_catalog == "internal" + assert cloned._last_described_key_cols == {"main_key"} + + worker_results: t.Dict[str, t.Any] = {} + + def worker_fn() -> None: + worker_results["initial_cat"] = adapter._active_catalog + worker_results["initial_keys"] = set(adapter._last_described_key_cols) + adapter._active_catalog = "iceberg_worker" + adapter._needs_catalog_restore = True + adapter._last_described_key_cols = {"worker_key"} + worker_results["updated_cat"] = cloned._active_catalog + worker_results["updated_keys"] = set(cloned._last_described_key_cols) + + thread = threading.Thread(target=worker_fn) + thread.start() + thread.join() + + assert worker_results == { + "initial_cat": None, + "initial_keys": set(), + "updated_cat": "iceberg_worker", + "updated_keys": {"worker_key"}, + } + # Main thread state was not overwritten by worker thread and survives connection pool close + adapter._connection_pool.close() + assert adapter._active_catalog == "internal" + assert adapter._needs_catalog_restore is False + assert adapter._last_described_key_cols == {"main_key"} + + # 2. Reconnect in `_restore_active_catalog_on_reconnect` restores both `SWITCH ` + # and `USE ` when `database` is configured on the default catalog. + adapter.cursor.execute.reset_mock() + adapter._needs_catalog_restore = True + adapter._restore_active_catalog_on_reconnect("SELECT 1") + assert to_sql_calls(adapter) == [ + "SWITCH `internal`", + "USE `analytics`", + ] + + # 3. TableDiff-style query with 4 integer indicators and DOUBLE/STRING user columns promotes all + # indicators to the front of `__stats` (covering all 3 slots of `shortkey_max_column_count = 3`). + adapter.cursor.execute.reset_mock() + four_indicator_diff = parse_one( + "WITH `__stats` AS (" + "SELECT `s`.`score` AS `s__score`, `t`.`score` AS `t__score`, `s`.`bio` AS `s__bio`, " + "IF(NOT `s`.`k` IS NULL, 1, 0) AS `s_exists`, " + "IF(NOT `t`.`k` IS NULL, 1, 0) AS `t_exists`, " + "IF(NOT `s`.`k` IS NULL AND NOT `t`.`k` IS NULL, 1, 0) AS `row_joined`, " + "IF(`s`.`id` IS NULL AND `t`.`id` IS NULL, 1, 0) AS `null_grain` " + "FROM `db`.`s` LEFT JOIN `db`.`t` ON `s`.`k` = `t`.`k`" + ") SELECT *, CASE WHEN `s__score` = `t__score` THEN 1 ELSE 0 END AS `row_full_match` FROM `__stats`", + read="doris", + ) + adapter.ctas("sqlmesh_temp.diff_4", four_indicator_diff) + diff_sql = to_sql_calls(adapter)[0] + assert ( + "`__stats` AS (SELECT CASE WHEN NOT `s`.`k` IS NULL THEN 1 ELSE 0 END AS `s_exists`, " + "CASE WHEN NOT `t`.`k` IS NULL THEN 1 ELSE 0 END AS `t_exists`, " + "CASE WHEN NOT `s`.`k` IS NULL AND NOT `t`.`k` IS NULL THEN 1 ELSE 0 END AS `row_joined`, " + "CASE WHEN `s`.`id` IS NULL AND `t`.`id` IS NULL THEN 1 ELSE 0 END AS `null_grain`, " + "`s`.`score` AS `s__score`" in diff_sql + ) + + # 4. Non-star unannotated CTAS infers explicit CAST types (`DOUBLE`, `STRING`, `INT`) and + # prefers known valid key columns (`id` INT) or coerces `STRING` to `VARCHAR(255)` at column 0. + adapter.cursor.execute.reset_mock() + adapter.ctas( + "db.unannotated_cast_reordered", + parse_one( + "SELECT CAST(score AS DOUBLE) AS score, raw_col, CAST(id AS INT) AS id FROM db.src", + read="doris", + ), + ) + adapter.ctas( + "db.unannotated_cast_string", + parse_one( + "SELECT CAST(score AS DOUBLE) AS score, CAST(label AS STRING) AS label FROM db.src", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`unannotated_cast_reordered` AS SELECT CAST(`id` AS INT) AS `id`, CAST(`score` AS DOUBLE) AS `score`, `raw_col` FROM `db`.`src`", + "CREATE TABLE IF NOT EXISTS `db`.`unannotated_cast_string` AS SELECT CAST(`label` AS VARCHAR(255)) AS `label`, CAST(`score` AS DOUBLE) AS `score` FROM `db`.`src`", + ] + + # 5. `replace_query` on an existing table and `scd_type_2_by_column` when column 0 is `DOUBLE` + # or `TEXT` reorder/coerce `target_columns_to_types` (and `source_columns`). + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", schema="db", name="existing_seed", type=DataObjectType.TABLE + ), + ) + seed_df = pd.DataFrame([{"score": 9.5, "code": "us"}]) + adapter.replace_query( + "db.existing_seed", + seed_df, + target_columns_to_types={ + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "code": exp.DataType.build("TEXT", dialect="doris"), + }, + source_columns=["score", "code"], + ) + replace_calls = to_sql_calls(adapter) + assert replace_calls == [ + "TRUNCATE TABLE `db`.`existing_seed`", + "INSERT INTO `db`.`existing_seed` (`code`, `score`) SELECT CAST(`code` AS VARCHAR(255)) AS `code`, CAST(`score` AS DOUBLE) AS `score` FROM (SELECT 'us' AS `code`, 9.5 AS `score`) AS `t`", + ] + + adapter.cursor.execute.reset_mock() + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="efgh5678") + adapter.scd_type_2_by_column( + target_table="db.scd_col_tbl", + source_table=t.cast( + exp.Select, + parse_one("SELECT score, id FROM db.src_tbl", read="doris"), + ), + unique_key=[exp.column("id")], + valid_from_col=exp.column("valid_from", quoted=True), + valid_to_col=exp.column("valid_to", quoted=True), + check_columns=[exp.column("score")], + target_columns_to_types={ + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "valid_from": exp.DataType.build("DATETIME", dialect="doris"), + "valid_to": exp.DataType.build("DATETIME", dialect="doris"), + }, + execution_time=datetime(2024, 1, 2, 0, 0, 0), + ) + scd_col_calls = to_sql_calls(adapter) + assert scd_col_calls[0].startswith( + "CREATE TABLE IF NOT EXISTS `db`.`scd_col_tbl` (`id` INT, `score` DOUBLE, `valid_from` DATETIME, `valid_to` DATETIME)" + ) + + # 6. `_delete_by_literal_keys` with Python `date`, `datetime`, and `Decimal` values from PyMySQL + # emits plain string/number literals (`'2024-01-01'`, `12.5`) instead of `CAST('...' AS DATE)`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "fetchall", + return_value=[(date(2024, 1, 1), datetime(2024, 1, 1, 12, 30, 0), Decimal("12.50"))], + ) + adapter._delete_by_literal_keys( + "db.typed_keys", + [exp.column("dt"), exp.column("ts"), exp.column("amt")], + parse_one("SELECT dt, ts, amt FROM db.staging", read="doris", into=exp.Select), + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`typed_keys` WHERE `dt` = '2024-01-01' AND `ts` = '2024-01-01 12:30:00' AND `amt` IN (12.50)" + ] + + # 7. `sqlmesh init doris` generates a working `state_connection` block (`type: duckdb`). + doris_init_cfg = _gen_config( + engine_type="doris", + settings=None, + start="2024-01-01", + template=ProjectTemplate.DEFAULT, + cli_mode=InitCliMode.DEFAULT, + ) + assert "state_connection:\n type: duckdb\n database: state.db" in doris_init_cfg + assert "dialect: doris" in doris_init_cfg + + +def test_union_time_transform_delete_agg_ctas_mv_df_order_and_limit_fold( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + import pandas as pd + from sqlmesh.core.config.connection import DorisConnectionConfig + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `_delete_by_time_transform_keys` uses the source subquery's projection column (`src_ds`) + # when it differs from the target column (`target_ds`), and supports `UNION ALL` subqueries. + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [("2024-01-15", "2024-01-16")], + [("2024-01-01", "2025-01-01")], + ], + ) + adapter.delete_from( + "db.events", + where=parse_one( + "DATE_TRUNC(target_ds, 'day') IN (SELECT DATE_TRUNC(src_ds, 'day') FROM db.staging)", + read="doris", + ), + ) + adapter.delete_from( + "db.events", + where=parse_one( + "YEAR(ds) IN (SELECT YEAR(ds) FROM db.s1 UNION ALL SELECT YEAR(ds) FROM db.s2)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT DATE_TRUNC(`src_ds`, 'day') AS `_low_0`, DATE_ADD(DATE_TRUNC(`src_ds`, 'day'), INTERVAL 1 DAY) AS `_high_0` FROM `db`.`staging`", + "DELETE FROM `db`.`events` WHERE `target_ds` >= '2024-01-15' AND `target_ds` < '2024-01-16'", + "SELECT DISTINCT DATE_TRUNC(`_col_0`, 'year') AS `_low_0`, DATE_ADD(DATE_TRUNC(`_col_0`, 'year'), INTERVAL 1 YEAR) AS `_high_0` FROM (SELECT `ds` AS `_col_0` FROM `db`.`s1` UNION ALL SELECT `ds` FROM `db`.`s2`) AS `_sq`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + ] + + # 2. `CONCAT_WS` multi-column unwrap in `delete_from` supports `UNION ALL` subqueries, + # `_execute_delete_with_subquery` aliases unaliased string literals (`'us' AS _col_0`), + # and `_delete_key_rows` deduplicates repeated literal keys. + adapter.cursor.execute.reset_mock() + key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", exp.column("id"), exp.column("ds")) + union_subq = exp.union( + exp.select(key_exp).from_("db.s1"), + exp.select(key_exp).from_("db.s2"), + distinct=False, + ) + adapter.delete_from("db.target", where=key_exp.isin(query=union_subq)) + adapter.delete_from( + "db.target", + where=parse_one("(region, id) IN (SELECT 'us', id FROM db.staging)", read="doris"), + ) + adapter.delete_from( + "db.target", + where=parse_one( + "(region, id) IN (('us', 1), ('us', 1), ('us', 2))", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id`, `ds` FROM `db`.`s1` UNION ALL SELECT `id`, `ds` FROM `db`.`s2`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`ds` = `_t2`.`ds`", + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT 'us' AS `_col_0`, `id` FROM `db`.`staging`) AS `_t2` WHERE `_t1`.`region` = `_t2`.`_col_0` AND `_t1`.`id` = `_t2`.`id`", + "DELETE FROM `db`.`target` WHERE `region` = 'us' AND `id` IN (1, 2)", + ] + + # 3. Unannotated CTAS with aggregate/complex expressions at column 0 (`AVG(...)`, `ARRAY(...)`) + # infers non-key types (`DOUBLE`, `ARRAY`) and promotes valid key columns (`store_id`, `cnt`) to column 0. + adapter.cursor.execute.reset_mock() + adapter.ctas( + "db.agg_ctas", + parse_one( + "SELECT AVG(price) AS avg_price, CAST(store_id AS INT) AS store_id FROM db.sales GROUP BY store_id", + read="doris", + ), + ) + adapter.ctas( + "db.array_ctas", + parse_one("SELECT ARRAY(1, 2) AS arr, COUNT(*) AS cnt FROM db.src", read="doris"), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`agg_ctas` AS SELECT CAST(`store_id` AS INT) AS `store_id`, AVG(`price`) AS `avg_price` FROM `db`.`sales` GROUP BY `store_id`", + "CREATE TABLE IF NOT EXISTS `db`.`array_ctas` AS SELECT COUNT(*) AS `cnt`, ARRAY(1, 2) AS `arr` FROM `db`.`src`", + ] + + # 4. `_create_materialized_view` with a DataFrame whose column order (`b`, `a`) differs from + # `target_columns_to_types` (`a`, `b`) reorders the DataFrame columns before `itertuples()`. + adapter.cursor.execute.reset_mock() + mv_df = pd.DataFrame([{"b": "hello", "a": 42}]) + adapter.create_view( + "db.df_mv", + mv_df, + replace=False, + materialized=True, + target_columns_to_types={ + "a": exp.DataType.build("INT", dialect="doris"), + "b": exp.DataType.build("VARCHAR(20)", dialect="doris"), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE MATERIALIZED VIEW `db`.`df_mv` (`a`, `b`) AS SELECT CAST(`a` AS INT) AS `a`, CAST(`b` AS VARCHAR(20)) AS `b` FROM (SELECT 42 AS `a`, 'hello' AS `b`) AS `t`", + ] + + # 5. `get_alter_operations` suppresses destructive `DROP` + `ADD` pairs on OLAP key columns + # when the target column type (`TEXT`) is an invalid OLAP key type. + adapter.cursor.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [("code", "VARCHAR(255)", "Yes", "true", None, "")], + [("code", "TEXT", "Yes", "false", None, "")], + ], + ) + ops_destructive = adapter.get_alter_operations( + "db.current_tbl", "db.target_tbl", ignore_destructive=False + ) + assert ops_destructive == [] + + # 6. `_to_sql` folds constant integer arithmetic inside `exp.Limit` (`LIMIT 10 + 1` -> `LIMIT 11`) + # so built-in `number_of_rows` audit queries execute cleanly on Doris. + audit_query = parse_one( + "SELECT COUNT(*) FROM (SELECT 1 FROM db.events WHERE TRUE LIMIT 10 + 1) AS _q HAVING COUNT(*) <= 10", + read="doris", + ) + assert ( + adapter._to_sql(audit_query) + == "SELECT COUNT(*) FROM (SELECT 1 FROM `db`.`events` WHERE TRUE LIMIT 11) AS `_q` HAVING COUNT(*) <= 10" + ) + + # 7. Lockstep `UNION ALL` CTAS reordering when only one branch has explicit types (`AVG(score)`, `CAST(id AS INT)`) + # plus `NON_NULLABLE(...)` wrapping across all `UNION ALL` branches when `auto_partition = true`. + adapter.cursor.execute.reset_mock() + adapter.ctas( + "db.union_lockstep_ctas", + parse_one( + "SELECT AVG(score) AS score, CAST(id AS INT) AS id FROM db.s1 GROUP BY id " + "UNION ALL SELECT score, id FROM db.s2", + read="doris", + ), + ) + adapter.ctas( + "db.union_auto_part_ctas", + parse_one( + "SELECT id, ds FROM db.s1 UNION ALL SELECT id, ds FROM db.s2", + read="doris", + ), + partitioned_by=[parse_one("RANGE(ds)", read="doris")], + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`union_lockstep_ctas` AS SELECT CAST(`id` AS INT) AS `id`, AVG(`score`) AS `score` FROM `db`.`s1` GROUP BY `id` UNION ALL SELECT `id`, `score` FROM `db`.`s2`", + "CREATE TABLE IF NOT EXISTS `db`.`union_auto_part_ctas` AUTO PARTITION BY RANGE (`ds`) () AS SELECT `id`, NON_NULLABLE(`ds`) AS `ds` FROM `db`.`s1` UNION ALL SELECT `id`, NON_NULLABLE(`ds`) FROM `db`.`s2`", + ] + + # 8. Explicit `duplicate_key` with active OLAP `partitioned_by` automatically includes the + # partition column in `DUPLICATE KEY` and the leading schema prefix (required by Doris FE). + adapter.cursor.execute.reset_mock() + adapter.create_table( + "db.dup_part_tbl", + { + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + partitioned_by=[parse_one("RANGE(ds)", read="doris")], + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.Literal.string("id")]), + "auto_partition": exp.Literal.string("true"), + "buckets": exp.Literal.number(8), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`dup_part_tbl` (`id` INT, `ds` DATE NOT NULL, `score` DOUBLE) DUPLICATE KEY (`id`, `ds`) AUTO PARTITION BY RANGE (`ds`) () DISTRIBUTED BY HASH (`id`) BUCKETS 8" + ] + + # 9. `_delete_by_time_transform_keys` handles pre-extracted integer `YEAR` columns (`SELECT yr`), + # `GROUP BY` inside `UNION ALL` branches, and skips `NULL` subquery keys while preserving explicit `YEAR(ds) IS NULL`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [(2024,), (None,)], + [("2024-01-01", "2025-01-01")], + ], + ) + adapter.delete_from( + "db.events", + where=parse_one("YEAR(ds) IN (SELECT yr FROM db.staging)", read="doris"), + ) + adapter.delete_from( + "db.events", + where=parse_one("YEAR(ds) IS NULL", read="doris"), + ) + adapter.delete_from( + "db.events", + where=parse_one( + "YEAR(ds) IN (SELECT YEAR(ds) FROM db.s1 GROUP BY YEAR(ds) UNION ALL SELECT YEAR(ds) FROM db.s2)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT `yr` AS `_val_0` FROM `db`.`staging`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + "DELETE FROM `db`.`events` WHERE `ds` IS NULL", + "SELECT DISTINCT DATE_TRUNC(`_col_0`, 'year') AS `_low_0`, DATE_ADD(DATE_TRUNC(`_col_0`, 'year'), INTERVAL 1 YEAR) AS `_high_0` FROM (SELECT `ds` AS `_col_0` FROM `db`.`s1` UNION ALL SELECT `ds` FROM `db`.`s2`) AS `_sq`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + ] + + # 10. `insert_overwrite_by_partition` with empty `partitioned_by=[]` delegates cleanly to `replace_query`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", schema="db", name="unpart_tbl", type=DataObjectType.TABLE + ), + ) + adapter.insert_overwrite_by_partition( + "db.unpart_tbl", + parse_one("SELECT 1 AS id", read="doris"), + partitioned_by=[], + target_columns_to_types={"id": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(adapter) == [ + "TRUNCATE TABLE `db`.`unpart_tbl`", + "INSERT INTO `db`.`unpart_tbl` (`id`) SELECT 1 AS `id`", + ] + + # 11. `DorisConnectionConfig` with `ssl=True` populates a non-empty `ssl` dict so PyMySQL enables TLS. + mocker.patch("sqlmesh.core.config.connection.importlib.import_module") + ssl_conn_cfg = DorisConnectionConfig(host="fe.internal", user="root", ssl=True) + assert ssl_conn_cfg._static_connection_kwargs["ssl"] == {"check_hostname": False} + assert bool(ssl_conn_cfg._static_connection_kwargs["ssl"]) is True + + +def test_schema_differ_text_widening_largeint_and_key_protection( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Widening non-key value columns (`INT -> TEXT`, `DATE -> DATETIME(6)`) emits in-place + # `MODIFY COLUMN` operations instead of destructive `DROP` + `ADD`. + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [ + ("id", "INT", "Yes", "true", None, ""), + ("val", "INT", "Yes", "false", None, ""), + ("dt", "DATE", "Yes", "false", None, ""), + ], + [ + ("id", "INT", "Yes", "true", None, ""), + ("val", "TEXT", "Yes", "false", None, ""), + ("dt", "DATETIME(6)", "Yes", "false", None, ""), + ], + ], + ) + ops_widen = adapter.get_alter_operations( + "db.current_tbl", "db.target_tbl", ignore_destructive=True + ) + assert len(ops_widen) == 2 + adapter.cursor.execute.reset_mock() + adapter.alter_table(ops_widen) + assert to_sql_calls(adapter) == [ + "ALTER TABLE `db`.`current_tbl` MODIFY COLUMN `val` STRING", + "ALTER TABLE `db`.`current_tbl` MODIFY COLUMN `dt` DATETIME(6)", + ] + + # 2. `LARGEINT` vs `INT128` is treated as mutually compatible and produces zero alter operations. + mocker.patch.object( + adapter, + "columns", + side_effect=[ + {"id": exp.DataType.build("LARGEINT", udt=True)}, + {"id": exp.DataType.build("INT128")}, + ], + ) + assert adapter.get_alter_operations("db.current_tbl", "db.target_tbl") == [] + + # 3. Changing an OLAP key column to an incompatible valid key type (`DATETIME -> INT`) + # is blocked from emitting a destructive `DROP COLUMN` + `ADD COLUMN` on a key column, + # while non-key column destructive replacements (`DATETIME -> INT`) are still emitted when `ignore_destructive=False`. + mocker.stopall() + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [ + ("k_ts", "DATETIME", "Yes", "true", None, ""), + ("v_ts", "DATETIME", "Yes", "false", None, ""), + ], + [ + ("k_ts", "INT", "Yes", "true", None, ""), + ("v_ts", "INT", "Yes", "false", None, ""), + ], + ], + ) + ops_key_protected = adapter.get_alter_operations( + "db.current_tbl", "db.target_tbl", ignore_destructive=False + ) + adapter.cursor.execute.reset_mock() + adapter.alter_table(ops_key_protected) + assert to_sql_calls(adapter) == [ + "ALTER TABLE `db`.`current_tbl` DROP COLUMN `v_ts`", + "ALTER TABLE `db`.`current_tbl` ADD COLUMN `v_ts` INT", + ] + + +def test_union_ctas_string_coercion_and_duplicate_key_list_partition_coercion( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Lockstep `UNION ALL` unannotated CTAS where column 0 is non-key (`AVG(score)`) and column 1 + # is `CAST(label AS STRING)`: both branches are reordered in lockstep AND cast `label` to `VARCHAR(255)`. + adapter.ctas( + "db.union_str_ctas", + parse_one( + "SELECT AVG(score) AS score, CAST(label AS STRING) AS label FROM db.s1 GROUP BY label " + "UNION ALL SELECT score, label FROM db.s2", + read="doris", + ), + ) + + # 2. Explicit `duplicate_key = ('id')` combined with `PARTITION BY LIST(region)` where `region` is `TEXT`: + # `region` is automatically appended to `DUPLICATE KEY`, moved into the key prefix, AND coerced to `VARCHAR(255)`. + adapter.create_table( + "db.dup_list_part_tbl", + { + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "region": exp.DataType.build("TEXT", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + partitioned_by=[parse_one("LIST(region)", read="doris")], + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.Literal.string("id")]), + "partitions": exp.Literal.string("PARTITION p_us VALUES IN ('us')"), + "buckets": exp.Literal.number(4), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`union_str_ctas` AS SELECT CAST(`label` AS VARCHAR(255)) AS `label`, AVG(`score`) AS `score` FROM `db`.`s1` GROUP BY `label` UNION ALL SELECT CAST(`label` AS VARCHAR(255)), `score` FROM `db`.`s2`", + "CREATE TABLE IF NOT EXISTS `db`.`dup_list_part_tbl` (`id` INT, `region` VARCHAR(255), `score` DOUBLE) DUPLICATE KEY (`id`, `region`) PARTITION BY LIST (`region`) (PARTITION p_us VALUES IN ('us')) DISTRIBUTED BY HASH (`id`) BUCKETS 4", + ] + + +def test_doris_connection_config_tls_matrix(mocker: MockerFixture): + from sqlmesh.core.config.connection import DorisConnectionConfig + from sqlmesh.utils.errors import ConfigError + + mocker.patch("sqlmesh.core.config.connection.importlib.import_module") + + # 1. `ssl_verify_identity=True` requires `ssl_ca` (or `ssl.ca` / `ssl.capath`) because PyMySQL + # disables hostname verification when no CA is configured; with `ssl_ca`, it populates `check_hostname=True`. + with pytest.raises(ConfigError, match="requires `ssl_ca`"): + _ = DorisConnectionConfig( + host="fe.internal", user="root", ssl_verify_identity=True + )._static_connection_kwargs + + cfg_identity = DorisConnectionConfig( + host="fe.internal", + user="root", + ssl_ca="/path/to/ca.pem", + ssl_verify_identity=True, + ) + assert cfg_identity._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "check_hostname": True, + } + + # 2. `ssl_verify_cert=True` populates `verify_mode=True` in `kwargs["ssl"]`. + cfg_cert = DorisConnectionConfig(host="fe.internal", user="root", ssl_verify_cert=True) + assert cfg_cert._static_connection_kwargs["ssl"] == {"verify_mode": True} + + # 3. `ssl_disabled=True` overrides `ssl=True` and omits the `ssl` dictionary. + cfg_disabled = DorisConnectionConfig( + host="fe.internal", user="root", ssl=True, ssl_disabled=True + ) + assert "ssl" not in cfg_disabled._static_connection_kwargs + + +def test_doris_snapshot_evaluator_end_to_end_model_kinds( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + make_snapshot: t.Callable, + mocker: MockerFixture, +): + from sqlmesh.core.model import load_sql_based_model + from sqlmesh.core.dialect import parse as d_parse + from sqlmesh.core.snapshot import SnapshotChangeCategory, SnapshotEvaluator + + adapter = make_mocked_engine_adapter(DorisEngineAdapter, default_catalog="internal") + adapter.cursor.fetchall.return_value = [] + mocker.patch.object(DorisEngineAdapter, "_get_data_objects", return_value=[]) + mocker.patch.object(DorisEngineAdapter, "table_exists", return_value=True) + mocker.patch.object( + DorisEngineAdapter, + "get_data_object", + side_effect=lambda t: DataObject( + catalog="internal", + schema=exp.to_table(t).db or "db", + name=exp.to_table(t).name, + type=DataObjectType.TABLE, + ), + ) + evaluator = SnapshotEvaluator(adapter) + + # 1. INCREMENTAL_BY_TIME_RANGE model evaluation through SnapshotEvaluator + inc_time_model = load_sql_based_model( + d_parse( + """ + MODEL ( + name db.inc_time, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column ds + ), + dialect doris, + physical_properties ( + duplicate_key = (id), + auto_partition = true, + buckets = 4 + ), + partitioned_by RANGE(ds) + ); + SELECT CAST(id AS INT) AS id, CAST(ds AS DATE) AS ds, CAST(amount AS DOUBLE) AS amount + FROM db.raw_events + WHERE ds BETWEEN @start_ds AND @end_ds + """, + default_dialect="doris", + ), + default_catalog="internal", + ) + inc_time_snap = make_snapshot(inc_time_model) + inc_time_snap.categorize_as(SnapshotChangeCategory.BREAKING) + inc_time_snap.add_interval("2023-12-31", "2023-12-31") + + adapter.cursor.execute.reset_mock() + evaluator.create([inc_time_snap], {}) + evaluator.evaluate( + inc_time_snap, + start="2024-01-01", + end="2024-01-02", + execution_time="2024-01-03", + snapshots={inc_time_snap.name: inc_time_snap}, + ) + time_sql = to_sql_calls(adapter) + assert any( + "DUPLICATE KEY (`id`, `ds`) AUTO PARTITION BY RANGE (`ds`) () DISTRIBUTED BY HASH (`id`) BUCKETS 4" + in s + for s in time_sql + ) + assert any( + "DELETE FROM `internal`.`sqlmesh__db`.`db__inc_time__" in s + and "WHERE `ds` >= '2024-01-01' AND `ds` <= '2024-01-02'" in s + for s in time_sql + ) + assert any("INSERT INTO `internal`.`sqlmesh__db`.`db__inc_time__" in s for s in time_sql) + + # 2. INCREMENTAL_BY_UNIQUE_KEY model evaluation through SnapshotEvaluator + inc_uk_model = load_sql_based_model( + d_parse( + """ + MODEL ( + name db.inc_uk, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key id + ), + dialect doris, + physical_properties ( + buckets = 8 + ) + ); + SELECT CAST(id AS INT) AS id, CAST(status AS VARCHAR(32)) AS status + FROM db.raw_users + """, + default_dialect="doris", + ), + default_catalog="internal", + ) + inc_uk_snap = make_snapshot(inc_uk_model) + inc_uk_snap.categorize_as(SnapshotChangeCategory.BREAKING) + inc_uk_snap.add_interval("2023-12-31", "2023-12-31") + + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.return_value = [ + ("id", "INT", "Yes", "true", None, ""), + ("status", "VARCHAR(32)", "Yes", "false", None, ""), + ] + evaluator.create([inc_uk_snap], {}) + evaluator.evaluate( + inc_uk_snap, + start="2024-01-01", + end="2024-01-02", + execution_time="2024-01-03", + snapshots={inc_uk_snap.name: inc_uk_snap}, + ) + uk_sql = to_sql_calls(adapter) + assert any("UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 8" in s for s in uk_sql) + assert any("INSERT INTO `internal`.`sqlmesh__db`.`db__inc_uk__" in s for s in uk_sql) + assert not any("MERGE INTO" in s for s in uk_sql) + + # 3. SCD_TYPE_2_BY_TIME model evaluation through SnapshotEvaluator + scd_model = load_sql_based_model( + d_parse( + """ + MODEL ( + name db.scd_users, + kind SCD_TYPE_2_BY_TIME ( + unique_key id, + updated_at_name updated_at, + time_data_type DATETIME + ), + columns ( + id INT, + status VARCHAR(32), + updated_at DATETIME, + valid_from DATETIME, + valid_to DATETIME + ), + dialect doris, + physical_properties ( + buckets = 4 + ) + ); + SELECT CAST(id AS INT) AS id, CAST(status AS VARCHAR(32)) AS status, CAST(updated_at AS DATETIME) AS updated_at + FROM db.raw_users + """, + default_dialect="doris", + ), + default_catalog="internal", + ) + scd_snap = make_snapshot(scd_model) + scd_snap.categorize_as(SnapshotChangeCategory.BREAKING) + scd_snap.add_interval("2023-12-31", "2023-12-31") + + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.return_value = [ + ("id", "INT", "Yes", "true", None, ""), + ("status", "VARCHAR(32)", "Yes", "true", None, ""), + ("updated_at", "DATETIME", "Yes", "false", None, ""), + ("valid_from", "DATETIME", "Yes", "false", None, ""), + ("valid_to", "DATETIME", "Yes", "false", None, ""), + ] + evaluator.create([scd_snap], {}) + evaluator.evaluate( + scd_snap, + start="2024-01-01", + end="2024-01-02", + execution_time="2024-01-03", + snapshots={scd_snap.name: scd_snap}, + ) + scd_sql = to_sql_calls(adapter) + assert any( + "CREATE TABLE IF NOT EXISTS `internal`.`sqlmesh__db`.`db__scd_users__" in s + and "DISTRIBUTED BY RANDOM BUCKETS 4" in s + for s in scd_sql + ) + assert any("TRUNCATE TABLE `internal`.`sqlmesh__db`.`db__scd_users__" in s for s in scd_sql) + assert any("INSERT INTO `internal`.`sqlmesh__db`.`db__scd_users__" in s for s in scd_sql) + + # 4. INCREMENTAL_BY_PARTITION model evaluation through SnapshotEvaluator + inc_part_model = load_sql_based_model( + d_parse( + """ + MODEL ( + name db.inc_part, + kind INCREMENTAL_BY_PARTITION, + dialect doris, + partitioned_by LIST(region), + physical_properties ( + duplicate_key = (id), + partitions = "PARTITION p_us VALUES IN ('us')", + buckets = 4 + ) + ); + SELECT CAST(id AS INT) AS id, CAST(region AS VARCHAR(16)) AS region, CAST(amount AS DOUBLE) AS amount + FROM db.raw_regional_sales + """, + default_dialect="doris", + ), + default_catalog="internal", + ) + inc_part_snap = make_snapshot(inc_part_model) + inc_part_snap.categorize_as(SnapshotChangeCategory.BREAKING) + inc_part_snap.add_interval("2023-12-31", "2023-12-31") + + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.return_value = [ + ("id", "INT", "Yes", "true", None, ""), + ("region", "VARCHAR(16)", "Yes", "true", None, ""), + ("amount", "DOUBLE", "Yes", "false", None, ""), + ] + evaluator.create([inc_part_snap], {}) + evaluator.evaluate( + inc_part_snap, + start="2024-01-01", + end="2024-01-02", + execution_time="2024-01-03", + snapshots={inc_part_snap.name: inc_part_snap}, + ) + part_sql = to_sql_calls(adapter) + assert any( + "DUPLICATE KEY (`id`, `region`) PARTITION BY LIST (`region`) (PARTITION p_us VALUES IN ('us')) DISTRIBUTED BY HASH (`id`) BUCKETS 4" + in s + for s in part_sql + ) + assert any( + "DELETE FROM `internal`.`sqlmesh__db`.`db__inc_part__" in s + and "USING (SELECT DISTINCT `region` FROM `internal`.`sqlmesh__db`.`__temp_db__inc_part__" + in s + and "WHERE `_t1`.`region` = `_t2`.`region`" in s + for s in part_sql + ) + assert any("INSERT INTO `internal`.`sqlmesh__db`.`db__inc_part__" in s for s in part_sql) + + +def test_unannotated_unknown_columns_ctas_and_sync_mv_and_schema_tmp_truncation( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Unannotated SqlModel passes `target_columns_to_types` with `UNKNOWN` types to `ctas`: + # must reorder the `SELECT` projections so `id` (the `unique_key`) is column 0. + adapter.ctas( + "db.unannotated_uk_model", + parse_one("SELECT name, id FROM db.src", read="doris"), + target_columns_to_types={ + "name": exp.DataType.build("UNKNOWN"), + "id": exp.DataType.build("UNKNOWN"), + }, + table_properties={"unique_key": exp.to_column("id")}, + ) + + # 2. Unannotated SqlModel with `UNKNOWN` column types and `AVG(score)` at column 0: + # falls through to `_reorder_unannotated_ctas_query` and moves `CAST(id AS INT)` to column 0. + adapter.ctas( + "db.unannotated_agg_model", + parse_one( + "SELECT AVG(score) AS score, CAST(id AS INT) AS id FROM db.src GROUP BY id", + read="doris", + ), + target_columns_to_types={ + "score": exp.DataType.build("UNKNOWN"), + "id": exp.DataType.build("UNKNOWN"), + }, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`unannotated_uk_model` UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 AS SELECT `id`, `name` FROM `db`.`src`", + "CREATE TABLE IF NOT EXISTS `db`.`unannotated_agg_model` AS SELECT CAST(`id` AS INT) AS `id`, AVG(`score`) AS `score` FROM `db`.`src` GROUP BY `id`", + ] + + # 3. Sync MV (`materialized_type='SYNC'`) with `target_columns_to_types` and `table_description`: + # must NOT wrap the query in `SELECT CAST(...) FROM (...) AS _subquery`, must NOT emit `COMMENT`, + # and `drop_view(view_name, materialized=True)` without `view_properties` still emits `ON `. + adapter.cursor.execute.reset_mock() + adapter.create_view( + "db.sync_mv_typed", + parse_one("SELECT id, SUM(amount) AS total FROM db.orders GROUP BY id", read="doris"), + replace=False, + materialized=True, + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "total": exp.DataType.build("BIGINT", dialect="doris"), + }, + table_description="Sync rollup MV", + view_properties={ + "materialized_type": exp.Literal.string("SYNC"), + "source_table": exp.Literal.string("db.orders"), + }, + ) + adapter.drop_view("db.sync_mv_typed", materialized=True) + + assert to_sql_calls(adapter) == [ + "USE `db`", + "CREATE MATERIALIZED VIEW `sync_mv_typed` AS SELECT `id` AS `mv_id`, SUM(`amount`) AS `total` FROM `db`.`orders` GROUP BY `id`", + "USE `db`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_mv_typed` ON `db`.`orders`", + ] + + # 4. Overlong `_schema_tmp` table identifier (> 64 chars) is deterministically shortened to <= 64 chars + # across `CREATE TABLE`, `DESCRIBE`, and `DROP TABLE`. + adapter.cursor.execute.reset_mock() + long_schema_tmp = f"db.{'very_long_model_name_' * 3}_schema_tmp" + assert len(exp.to_table(long_schema_tmp).name) > 64 + adapter.cursor.fetchall.return_value = [("id", "INT", "Yes", "true", None, "")] + adapter.ctas(long_schema_tmp, parse_one("SELECT 1 AS id", read="doris")) + adapter.columns(long_schema_tmp) + adapter.drop_table(long_schema_tmp) + + schema_tmp_calls = to_sql_calls(adapter) + assert len(schema_tmp_calls) == 3 + created_tbl = parse_one(schema_tmp_calls[0], read="doris").this + assert len(created_tbl.name) <= 64 + assert created_tbl.name.endswith("_schema_tmp") + assert f"`{created_tbl.name}`" in schema_tmp_calls[1] + assert f"`{created_tbl.name}`" in schema_tmp_calls[2] + + +def test_nested_delete_and_mixed_union_time_transform_and_merge_text_preservation( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], mocker: MockerFixture +): + from sqlmesh.core.config.connection import DorisConnectionConfig + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Nested parenthesized `AND`s with `IN (SELECT ...)` and extra `OR` condition: + # extracts the subquery and normalizes `(status = 'x' OR status = 'y')` into `status IN ('x', 'y')`. + adapter.delete_from( + "db.target", + where=parse_one( + "(tenant_id = 1 AND id IN (SELECT id FROM db.staging)) AND (status = 'x' OR status = 'y')", + read="doris", + ), + ) + + # 2. Single-column `id IN (SELECT * FROM db.staging_ids)` falls back to literal key batching + # instead of emitting invalid `SELECT * AS _col_0`. + mocker.patch.object(adapter, "fetchall", return_value=[(10,), (20,)]) + adapter.delete_from( + "db.target", + where=parse_one("id IN (SELECT * FROM db.staging_ids)", read="doris"), + ) + + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id` FROM `db`.`staging`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`tenant_id` = 1 AND `_t1`.`status` IN ('x', 'y')", + "DELETE FROM `db`.`target` WHERE `id` IN (10, 20)", + ] + + # 3. `_delete_by_time_transform_keys`: + # (a) Mixed `UNION ALL` branches (`SELECT YEAR(ds) FROM db.s1 UNION ALL SELECT yr FROM db.s2`) + # use `branch_is_raw_col` instead of wrapping integer `yr` in `DATE_TRUNC(yr, 'year')`. + # (b) `GROUP BY` with aggregate `MAX(region)` in `WHERE`/`HAVING` is preserved instead of stripping `GROUP BY`. + adapter.cursor.execute.reset_mock() + mocker.stopall() + mocker.patch.object( + adapter.cursor, + "fetchall", + side_effect=[ + [(2024,)], + [("2024-01-01", "2024-01-02")], + ], + ) + adapter.delete_from( + "db.events", + where=parse_one( + "YEAR(ds) IN (SELECT YEAR(ds) FROM db.s1 UNION ALL SELECT yr FROM db.s2)", + read="doris", + ), + ) + adapter.delete_from( + "db.events", + where=parse_one( + "DATE_TRUNC(ds, 'day') IN (SELECT DATE_TRUNC(ds, 'day') FROM db.s1 GROUP BY DATE_TRUNC(ds, 'day') HAVING COUNT(*) > 1)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT `_col_0` AS `_val_0` FROM (SELECT YEAR(`ds`) AS `_col_0` FROM `db`.`s1` UNION ALL SELECT `yr` FROM `db`.`s2`) AS `_sq`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + "SELECT DISTINCT DATE_TRUNC(`_col_0`, 'day') AS `_low_0`, DATE_ADD(DATE_TRUNC(`_col_0`, 'day'), INTERVAL 1 DAY) AS `_high_0` FROM (SELECT DATE_TRUNC(`ds`, 'DAY') AS `_col_0` FROM `db`.`s1` GROUP BY DATE_TRUNC(`ds`, 'DAY') HAVING COUNT(*) > 1) AS `_sq`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2024-01-02'", + ] + + # 4. `merge` with `{"notes": TEXT, "code": TEXT}` and `unique_key=[code]`: + # `temp_table` uses `UNIQUE KEY (code)` so `code` is coerced to `VARCHAR(255)` while non-key `notes` stays `STRING` (`TEXT`). + adapter.cursor.execute.reset_mock() + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + adapter.merge( + target_table="db.notes_tbl", + source_table=parse_one("SELECT notes, code FROM db.staging_notes", read="doris"), + target_columns_to_types={ + "notes": exp.DataType.build("TEXT", dialect="doris"), + "code": exp.DataType.build("TEXT", dialect="doris"), + }, + unique_key=[exp.to_column("code")], + ) + merge_calls = to_sql_calls(adapter) + assert ( + merge_calls[0] + == "CREATE TABLE `db`.`__temp_notes_tbl_abcd1234` UNIQUE KEY (`code`) DISTRIBUTED BY HASH (`code`) BUCKETS 10 AS SELECT CAST(`code` AS VARCHAR(255)) AS `code`, CAST(`notes` AS STRING) AS `notes` FROM (SELECT `notes`, `code` FROM `db`.`staging_notes`) AS `_subquery`" + ) + + # 5. `DorisConnectionConfig` TLS dictionary with `ssl_verify_cert=False` sets `check_hostname=False` + # (preventing Python 3.13 `SSLContext` `ValueError`) and preserves `ca` when `ssl` is a dict. + mocker.patch("sqlmesh.core.config.connection.importlib.import_module") + cfg_ca_no_verify = DorisConnectionConfig( + host="fe.internal", + user="root", + ssl={"ca": "/path/to/ca.pem"}, + ssl_verify_cert=False, + ) + assert cfg_ca_no_verify._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "verify_mode": False, + "check_hostname": False, + } + + +def test_doris_schema_tmp_shortening_across_fetchall_fetchdf_and_data_objects( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter, patch_get_data_objects=False) + long_tmp_name = "temp_very_long_model_name_exceeding_sixty_four_characters_abcdefgh_schema_tmp" + assert len(long_tmp_name) > 64 + short_tmp_name = adapter._shorten_schema_tmp_name(long_tmp_name) + assert len(short_tmp_name) <= 64 + assert short_tmp_name.endswith("_schema_tmp") + + # 1. `fetchall` / `_to_sql` rewrites Table, TableAlias, and Column.table qualifiers for >64-char `_schema_tmp` + mocker.patch.object(adapter.cursor, "fetchall", return_value=[(1,)]) + query_with_qualifiers = parse_one( + f"SELECT `{long_tmp_name}`.`id` FROM `db`.`{long_tmp_name}` AS `{long_tmp_name}`", + read="doris", + ) + assert adapter.fetchall(query_with_qualifiers) == [(1,)] + assert to_sql_calls(adapter) == [ + f"SELECT `{short_tmp_name}`.`id` FROM `db`.`{short_tmp_name}` AS `{short_tmp_name}`" + ] + + # 2. `fetchdf` (`_fetch_native_df`) normalizes >64-char `_schema_tmp` identifiers before `pd.read_sql_query` + mock_read_sql = mocker.patch( + "pandas.io.sql.read_sql_query", + return_value=pd.DataFrame({"id": [1]}), + ) + df = adapter.fetchdf(query_with_qualifiers) + assert list(df["id"]) == [1] + assert ( + mock_read_sql.call_args[0][0] + == f"SELECT `{short_tmp_name}`.`id` FROM `db`.`{short_tmp_name}` AS `{short_tmp_name}`" + ) + + # 3. `_get_data_objects` queries `information_schema.tables` using the shortened `_schema_tmp` name + # and restores the caller's original unshortened name on the returned `DataObject`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "fetchall", + side_effect=[ + [("db", short_tmp_name, "BASE TABLE")], + [], + ], + ) + objs = adapter._get_data_objects("db", object_names={long_tmp_name}) + assert len(objs) == 1 + assert objs[0].name == long_tmp_name + query_arg = adapter.fetchall.call_args_list[0][0][0] + rendered_info_schema_sql = adapter._to_sql(query_arg) + assert short_tmp_name.lower() in rendered_info_schema_sql.lower() + assert long_tmp_name.lower() not in rendered_info_schema_sql.lower() + + +def test_doris_delete_by_time_transform_validation_and_deduplication( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Non-YEAR date-part extract `MONTH(ds) IN (SELECT MONTH(src_ds) ...)` raises `SQLMeshError` + # instead of silently deleting an entire calendar month. + with pytest.raises( + SQLMeshError, + match="cannot rewrite non-year date-part predicate", + ): + adapter.delete_from( + "db.events", + where=parse_one( + "MONTH(ds) IN (SELECT MONTH(src_ds) FROM db.staging)", + read="doris", + ), + ) + + # 2. Mismatched subquery time-transform unit (`DATE_TRUNC(ds, 'day') IN (SELECT YEAR(src_ds) ...)`) + # raises `SQLMeshError`. + with pytest.raises( + SQLMeshError, + match="with mismatched subquery projection", + ): + adapter.delete_from( + "db.events", + where=parse_one( + "DATE_TRUNC(ds, 'day') IN (SELECT YEAR(src_ds) FROM db.staging)", + read="doris", + ), + ) + + # 3. `YEAR(ds) IN (SELECT ds FROM db.staging)` where `staging.ds` returns ISO date strings + # in the same year parses the year prefix and deduplicates identical `DELETE` statements. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[("2024-05-10",), ("2024-08-20",)], + ) + adapter.delete_from( + "db.events", + where=parse_one( + "YEAR(ds) IN (SELECT ds FROM db.staging)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT `ds` AS `_val_0` FROM `db`.`staging`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + ] + + +def test_doris_replace_by_key_merge_and_partition_overwrite_edge_cases( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + + # 1. `insert_overwrite_by_partition` / `_replace_by_key` with `{"notes": TEXT, "region": VARCHAR(16)}` + # and `partitioned_by=[region]` moves `region` to column 0 in the temp table so `notes` stays `STRING` + # (never truncated to `VARCHAR(255)`). + adapter.insert_overwrite_by_partition( + table_name="db.notes_by_region", + query_or_df=parse_one("SELECT notes, region FROM db.src_notes", read="doris"), + partitioned_by=[exp.to_column("region")], + target_columns_to_types={ + "notes": exp.DataType.build("TEXT", dialect="doris"), + "region": exp.DataType.build("VARCHAR(16)", dialect="doris"), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE `db`.`__temp_notes_by_region_abcd1234` AS SELECT CAST(`region` AS VARCHAR(16)) AS `region`, CAST(`notes` AS STRING) AS `notes` FROM (SELECT `notes`, `region` FROM `db`.`src_notes`) AS `_subquery`", + "DELETE FROM `db`.`notes_by_region` AS `_t1` USING (SELECT DISTINCT `region` FROM `db`.`__temp_notes_by_region_abcd1234`) AS `_t2` WHERE `_t1`.`region` = `_t2`.`region`", + "INSERT INTO `db`.`notes_by_region` (`region`, `notes`) SELECT `region`, `notes` FROM `db`.`__temp_notes_by_region_abcd1234`", + "DROP TABLE IF EXISTS `db`.`__temp_notes_by_region_abcd1234`", + ] + + # 2. `merge` when `target_columns_to_types` contains `UNKNOWN` falls back to `self.columns(target_table)`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "columns", + return_value={ + "id": exp.DataType.build("INT", dialect="doris"), + "val": exp.DataType.build("VARCHAR(64)", dialect="doris"), + }, + ) + adapter.merge( + target_table="db.unknown_cols_target", + source_table=parse_one("SELECT id, val FROM db.staging", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("UNKNOWN"), + "val": exp.DataType.build("UNKNOWN"), + }, + unique_key=[exp.to_column("id")], + ) + assert ( + to_sql_calls(adapter)[0] + == "CREATE TABLE `db`.`__temp_unknown_cols_target_abcd1234` UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 AS SELECT CAST(`id` AS INT) AS `id`, CAST(`val` AS VARCHAR(64)) AS `val` FROM (SELECT `id`, `val` FROM `db`.`staging`) AS `_subquery`" + ) + + # 3. `merge` with 2-part table name and `table_format="iceberg"` routes to Iceberg `_replace_by_key` + # (no OLAP `UNIQUE KEY` or `DISTRIBUTED BY` on the temp table, and literal-key DELETE instead of DELETE ... USING). + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[(10,)], + ) + adapter.merge( + target_table="iceberg_db.events", + source_table=parse_one("SELECT id, payload FROM iceberg_db.staging", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "payload": exp.DataType.build("STRING", dialect="doris"), + }, + unique_key=[exp.to_column("id")], + table_format="iceberg", + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE `iceberg_db`.`__temp_events_abcd1234` (`id` INT, `payload` STRING) ENGINE=iceberg", + "INSERT INTO `iceberg_db`.`__temp_events_abcd1234` (`id`, `payload`) SELECT CAST(`id` AS INT) AS `id`, CAST(`payload` AS STRING) AS `payload` FROM (SELECT `id`, `payload` FROM `iceberg_db`.`staging`) AS `_subquery`", + "SELECT DISTINCT `id` FROM `iceberg_db`.`__temp_events_abcd1234`", + "DELETE FROM `iceberg_db`.`events` WHERE `id` IN (10)", + "INSERT INTO `iceberg_db`.`events` (`id`, `payload`) SELECT `id`, `payload` FROM (SELECT `id` AS `id`, `payload` AS `payload`, ROW_NUMBER() OVER (PARTITION BY `id` ORDER BY `id`) AS _row_number FROM `iceberg_db`.`__temp_events_abcd1234`) AS _t WHERE _row_number = 1", + "DROP TABLE IF EXISTS `iceberg_db`.`__temp_events_abcd1234`", + ] + + # 4. `insert_overwrite_by_time_partition` with DataFrame and `{"val": DOUBLE, "ds": DATE, "id": INT}` + # reorders `target_columns_to_types` and `source_columns` so the temp table created by `_values_to_sql` + # places a valid OLAP key column (`ds` / `id`) before `val DOUBLE`. + adapter.cursor.execute.reset_mock() + df = pd.DataFrame( + { + "val": [3.14], + "ds": ["2024-01-01"], + "id": [42], + } + ) + adapter.insert_overwrite_by_time_partition( + table_name="db.metrics", + query_or_df=df, + start="2024-01-01", + end="2024-01-01", + time_formatter=lambda x, _: exp.Literal.string(str(x)), + time_column="ds", + target_columns_to_types={ + "val": exp.DataType.build("DOUBLE", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + ) + time_part_calls = to_sql_calls(adapter) + assert time_part_calls == [ + "DELETE FROM `db`.`metrics` WHERE `ds` >= '2024-01-01 00:00:00+00:00' AND `ds` <= '2024-01-01 23:59:59.999999+00:00'", + "INSERT INTO `db`.`metrics` (`ds`, `val`, `id`) SELECT `ds`, `val`, `id` FROM (SELECT CAST(`ds` AS DATE) AS `ds`, CAST(`val` AS DOUBLE) AS `val`, CAST(`id` AS INT) AS `id` FROM (SELECT '2024-01-01' AS `ds`, 3.14 AS `val`, 42 AS `id`) AS `t`) AS `_subquery` WHERE `ds` BETWEEN '2024-01-01 00:00:00+00:00' AND '2024-01-01 23:59:59.999999+00:00'", + ] + + +def test_doris_unannotated_ctas_union_all_and_cte_column_reordering( + make_mocked_engine_adapter: t.Callable, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `UNION ALL` where Branch 1 has `1 AS metric` (INT) and Branch 2 has `1.5 AS metric` (DOUBLE): + # Doris widens `metric` to DOUBLE across the UNION, so `id` (INT across all branches) is promoted to col 0. + adapter.ctas( + table_name="db.union_metrics", + query_or_df=parse_one( + "SELECT 1 AS metric, 10 AS id UNION ALL SELECT 1.5 AS metric, 20 AS id", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`union_metrics` AS SELECT 10 AS `id`, 1 AS `metric` UNION ALL SELECT 20 AS `id`, 1.5 AS `metric`" + ] + + # 2. CTE (`WITH agg AS (...) SELECT avg_score, id FROM agg`) propagates CTE column types to the outer SELECT + # and reorders `id` before `avg_score` (`AVG` -> `DOUBLE`). + adapter.cursor.execute.reset_mock() + adapter.ctas( + table_name="db.cte_explicit_cols", + query_or_df=parse_one( + "WITH agg AS (SELECT AVG(score) AS avg_score, CAST(id AS INT) AS id FROM db.src GROUP BY id) " + "SELECT avg_score, id FROM agg", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`cte_explicit_cols` AS WITH `agg` AS (SELECT AVG(`score`) AS `avg_score`, CAST(`id` AS INT) AS `id` FROM `db`.`src` GROUP BY `id`) SELECT `id`, `avg_score` FROM `agg`" + ] + + # 3. CTE with `SELECT * FROM agg` reorders the inner CTE projection when the CTE starts with `AVG(score)`. + adapter.cursor.execute.reset_mock() + adapter.ctas( + table_name="db.cte_star", + query_or_df=parse_one( + "WITH agg AS (SELECT AVG(score) AS avg_score, CAST(id AS INT) AS id FROM db.src GROUP BY id) " + "SELECT * FROM agg", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`cte_star` AS WITH `agg` AS (SELECT CAST(`id` AS INT) AS `id`, AVG(`score`) AS `avg_score` FROM `db`.`src` GROUP BY `id`) SELECT * FROM `agg`" + ] + + # 4. `_reorder_olap_columns` prefers coercible `STRING` over `UNKNOWN` when column 0 is `DOUBLE`. + reordered_cols = adapter._reorder_olap_columns( + { + "metric": exp.DataType.build("DOUBLE", dialect="doris"), + "mystery": exp.DataType.build("UNKNOWN"), + "label": exp.DataType.build("STRING", dialect="doris"), + } + ) + assert list(reordered_cols.keys()) == ["label", "metric", "mystery"] + assert reordered_cols["label"] == exp.DataType.build("VARCHAR(255)", dialect="doris") + + +def test_doris_sync_mv_source_tables_persistence_and_isolation( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, +): + from sqlmesh.core.config.connection import DorisConnectionConfig + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Creating a sync MV in `db1` and `db2` with the same view name does not collide, + # survives `adapter.with_settings()`, and `drop_view` with `materialized_type='SYNC'` + # (without `source_table` in `view_properties`) still finds the recorded base table. + adapter.create_view( + "db1.shared_mv", + parse_one("SELECT id, SUM(val) AS total FROM db1.orders GROUP BY id", read="doris"), + replace=False, + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + adapter.create_view( + "db2.shared_mv", + parse_one("SELECT id, SUM(val) AS total FROM db2.events GROUP BY id", read="doris"), + replace=False, + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + + cloned_adapter = adapter.with_settings() + adapter.cursor.execute.reset_mock() + cloned_adapter.drop_view( + "db1.shared_mv", + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + cloned_adapter.drop_view( + "db2.shared_mv", + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + assert to_sql_calls(adapter) == [ + "USE `db1`", + "DROP MATERIALIZED VIEW IF EXISTS `shared_mv` ON `db1`.`orders`", + "USE `db2`", + "DROP MATERIALIZED VIEW IF EXISTS `shared_mv` ON `db2`.`events`", + ] + + # 2. Recreating a view name as an async MTMV pops any stale sync MV entry so subsequent + # `drop_view(materialized=True)` emits `DROP MATERIALIZED VIEW IF EXISTS db1.shared_mv`. + adapter.create_view( + "db1.shared_mv", + parse_one("SELECT id, SUM(val) AS total FROM db1.orders GROUP BY id", read="doris"), + replace=False, + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + adapter.create_view( + "db1.shared_mv", + parse_one("SELECT id, val FROM db1.orders", read="doris"), + replace=False, + materialized=True, + ) + adapter.cursor.execute.reset_mock() + adapter.drop_view("db1.shared_mv", materialized=True) + assert to_sql_calls(adapter) == [ + "DROP MATERIALIZED VIEW IF EXISTS `db1`.`shared_mv`", + ] + + # 3. After a process restart (`_sync_mv_source_tables` is empty), `drop_view(materialized=True)` + # resolves the sync MV source table from Doris metadata (`SHOW ALTER TABLE MATERIALIZED VIEW` + # verified by `DESC ALL`, or `SHOW FULL TABLES` + `DESC ALL`), skips stale `FINISHED` + # jobs whose rollup was already dropped, and raises `SQLMeshError` when a sync MV source is unresolved. + fresh_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object( + fresh_adapter, + "fetchall", + side_effect=[ + # 3a. Resolved from SHOW ALTER TABLE MATERIALIZED VIEW FROM `db1` + DESC `db1`.`orders_base` ALL + [ + ( + 1001, + "orders_base", + "2026-01-01", + "2026-01-01", + "orders_base", + "restarted_sync_mv", + 2001, + 3001, + "FINISHED", + "", + "100%", + 3600, + ) + ], + [ + ("orders_base", "DUP_KEYS", "id", "INT", "Yes", "true", None, ""), + ("restarted_sync_mv", "AGG_KEYS", "mv_id", "INT", "Yes", "true", None, ""), + ], + # 3b. Alter history empty -> resolved from SHOW FULL TABLES + DESC `db1`.`fact_tbl` ALL + [], + [("fact_tbl", "BASE TABLE")], + [ + ("fact_tbl", "DUP_KEYS", "id", "INT", "Yes", "true", None, ""), + ("desc_sync_mv", "AGG_KEYS", "mv_id", "INT", "Yes", "true", None, ""), + ], + # 3c. Stale FINISHED job whose rollup was already dropped (`DESC` only has base table), + # followed by async MV with the same name -> drops async MV instead of stale base table + [ + ( + 1002, + "orders_base", + "2026-01-01", + "2026-01-01", + "orders_base", + "reused_async_mv", + 2002, + 3002, + "FINISHED", + "", + "100%", + 3600, + ) + ], + [ + ("orders_base", "DUP_KEYS", "id", "INT", "Yes", "true", None, ""), + ], + [("reused_async_mv", "BASE TABLE")], + # 3d. Unresolved sync MV raises SQLMeshError instead of silently skipping + [], + [], + ], + ) + fresh_adapter.drop_view("db1.restarted_sync_mv", materialized=True) + fresh_adapter.drop_view("db1.desc_sync_mv", materialized=True) + fresh_adapter.drop_view("db1.reused_async_mv", materialized=True) + assert to_sql_calls(fresh_adapter) == [ + "USE `db1`", + "DROP MATERIALIZED VIEW IF EXISTS `restarted_sync_mv` ON `db1`.`orders_base`", + "USE `db1`", + "DROP MATERIALIZED VIEW IF EXISTS `desc_sync_mv` ON `db1`.`fact_tbl`", + "DROP MATERIALIZED VIEW IF EXISTS `db1`.`reused_async_mv`", + ] + + with pytest.raises( + SQLMeshError, + match="Cannot drop Doris synchronous materialized view 'db1.unresolved_sync_mv'", + ): + fresh_adapter.drop_view( + "db1.unresolved_sync_mv", + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + + # 4. `DorisConnectionConfig._connection_factory` enforces TLS when SSL options are configured + pymysql = pytest.importorskip("pymysql") + + tls_cfg = DorisConnectionConfig( + host="fe.doris.cluster", + user="doris_user", + ssl_ca="/etc/ssl/certs/doris-ca.pem", + ssl_verify_cert=True, + ssl_verify_identity=True, + ) + insecure_conn = mocker.MagicMock(_secure=False) + secure_conn = mocker.MagicMock(_secure=True) + mocker.patch.object(pymysql, "connect", side_effect=[insecure_conn, secure_conn]) + with pytest.raises(SQLMeshError, match="server did not negotiate TLS"): + tls_cfg._connection_factory(host="fe.doris.cluster", user="doris_user") + insecure_conn.close.assert_called_once() + assert tls_cfg._connection_factory(host="fe.doris.cluster", user="doris_user") is secure_conn + + +def test_doris_reconnect_catalog_db_restore_union_star_delete_and_extra_or_conditions( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + class OperationalError(Exception): + pass + + adapter = make_mocked_engine_adapter( + DorisEngineAdapter, + default_catalog=None, + database="analytics", + ) + + # 1. `set_current_catalog("internal")` restores `USE \`analytics\`` when `database` is configured. + adapter.set_current_catalog("iceberg_cat") + adapter.set_current_catalog("internal") + assert to_sql_calls(adapter) == [ + "SWITCH `iceberg_cat`", + "SWITCH `internal`", + "USE `analytics`", + ] + + # 2. When a query fails with an operational error and the reconnect's `SWITCH` also fails, + # `_needs_catalog_restore` stays `True` until restoration succeeds on a subsequent call. + adapter.set_current_catalog("iceberg_cat") + adapter.cursor.execute.reset_mock() + adapter.cursor.execute.side_effect = [ + OperationalError(2013, "Lost connection to MySQL server during query"), + OperationalError(2006, "MySQL server has gone away"), + None, + None, + ] + with pytest.raises(OperationalError, match="MySQL server has gone away"): + adapter.execute("SELECT 1") + assert adapter._needs_catalog_restore is True + + adapter.execute("SELECT 1") + assert adapter._needs_catalog_restore is False + assert to_sql_calls(adapter) == [ + "SELECT 1", + "SWITCH `iceberg_cat`", + "SWITCH `iceberg_cat`", + "SELECT 1", + ] + + # 3. `delete_from` with `UNION ALL` where the second branch uses `SELECT *` expands/projects key columns + # and slices extra trailing columns in `_delete_by_literal_keys`. + adapter.cursor.execute.reset_mock() + adapter.cursor.execute.side_effect = None + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[(101, "ignored_extra_col"), (102, "ignored_extra_col")], + ) + adapter.delete_from( + "db.target", + where=parse_one( + "id IN (SELECT id FROM db.s1 UNION ALL SELECT * FROM db.s2)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT `id` FROM `db`.`s1` UNION ALL SELECT * FROM `db`.`s2`", + "DELETE FROM `db`.`target` WHERE `id` IN (101, 102)", + ] + + # 4. Extra `OR` condition alongside `IN (SELECT ...)` (`id IN (SELECT id FROM db.s1) AND (status = 'a' OR status IS NULL)`) + # on an Iceberg or literal-key delete path is distributed into separate `OR`-free `DELETE` statements so Doris does not + # reject the `DELETE` with `WhereClause only supports compound predicate, binary predicate, is_null predicate or in predicate`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[(1,), (2,)], + ) + adapter.delete_from( + "iceberg_cat.db.target", + where=parse_one( + "id IN (SELECT id FROM db.s1) AND (status = 'a' OR status IS NULL)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT `id` FROM `db`.`s1`", + "DELETE FROM `iceberg_cat`.`db`.`target` WHERE `id` IN (1, 2) AND `status` = 'a'", + "DELETE FROM `iceberg_cat`.`db`.`target` WHERE `id` IN (1, 2) AND `status` IS NULL", + ] + + +def test_doris_dialect_native_types_and_complex_constructors_and_row_diff( + make_mocked_engine_adapter: t.Callable, +): + from sqlmesh.core.dialect import parse as d_parse + from sqlmesh.core.model import load_sql_based_model + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Doris-native data types parse cleanly in `MODEL (columns (...))` and round-trip in Doris SQL: + # `LARGEINT`, `IPV4`, `IPV6`, `JSONB`, `VARIANT`, `BITMAP`, `HLL`, `QUANTILE_STATE`, `AGG_STATE`, + # `DATEV2`, `DATETIMEV2(3)`, `DECIMALV3(18, 4)`, `ARRAY`, `MAP`, and `STRUCT`. + model = load_sql_based_model( + d_parse( + """ + MODEL ( + name db.doris_rich_types, + kind FULL, + dialect doris, + columns ( + id LARGEINT, + ip4 IPV4, + ip6 IPV6, + jb JSONB, + var_col VARIANT, + bm BITMAP, + hll_col HLL, + qs QUANTILE_STATE, + agg_s AGG_STATE, + dt2 DATEV2, + ts2 DATETIMEV2(3), + dec3 DECIMALV3(18, 4), + tags ARRAY, + attrs MAP, + profile STRUCT + ) + ); + SELECT + CAST(1 AS LARGEINT) AS id, + CAST('127.0.0.1' AS IPV4) AS ip4, + CAST('::1' AS IPV6) AS ip6, + CAST('{}' AS JSONB) AS jb, + CAST('{}' AS VARIANT) AS var_col, + BITMAP_EMPTY() AS bm, + HLL_EMPTY() AS hll_col, + TO_QUANTILE_STATE(1, 2048) AS qs, + max_by_state(1, 2) AS agg_s, + CAST('2024-01-01' AS DATEV2) AS dt2, + CAST('2024-01-01 00:00:00' AS DATETIMEV2(3)) AS ts2, + CAST(12.34 AS DECIMALV3(18, 4)) AS dec3, + ARRAY('a', 'b') AS tags, + MAP('k1', 1, 'k2', 2) AS attrs, + STRUCT(100, 'alice') AS profile + """, + default_dialect="doris", + ) + ) + cols = model.columns_to_types + assert cols is not None + adapter.create_table("db.doris_rich_types", cols) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`doris_rich_types` (" + "`id` LARGEINT, `ip4` IPV4, `ip6` IPV6, `jb` JSONB, `var_col` VARIANT, " + "`bm` BITMAP, `hll_col` HLL, `qs` QUANTILE_STATE, `agg_s` AGG_STATE, " + "`dt2` DATE, `ts2` DATETIME(3), `dec3` DECIMAL(18, 4), `tags` ARRAY, " + "`attrs` MAP, `profile` STRUCT<`uid`: BIGINT, `name`: VARCHAR(64)>)" + ] + + # 2. Complex literal constructors (`ARRAY`, `MAP`, `STRUCT`, `NAMED_STRUCT`), `LEFT SEMI JOIN`, + # `LEFT ANTI JOIN` on unaliased tables, and `DISTINCT ON` transpile cleanly in Doris dialect. + query_sql = parse_one( + "SELECT [1, 2], MAP('a', 1, 'b', 2), STRUCT(1, 'x'), NAMED_STRUCT('id', 1, 'name', 'x') " + "FROM db.t1 LEFT ANTI JOIN db.t2 ON db.t1.id = db.t2.id " + "LEFT SEMI JOIN db.t3 ON db.t1.id = db.t3.id", + read="doris", + ).sql("doris") + assert ( + query_sql + == "SELECT ARRAY(1, 2), MAP('a', 1, 'b', 2), STRUCT(1, 'x'), NAMED_STRUCT('id', 1, 'name', 'x') " + "FROM db.t1 LEFT ANTI JOIN db.t2 ON db.t1.id = db.t2.id " + "LEFT SEMI JOIN db.t3 ON db.t1.id = db.t3.id" + ) + distinct_on_sql = parse_one( + "SELECT DISTINCT ON (id) id, ts FROM db.events ORDER BY id, ts DESC", + read="doris", + ).sql("doris") + assert "ROW_NUMBER() OVER (PARTITION BY id ORDER BY id, ts DESC)" in distinct_on_sql + + # 3. `RowDiffMixin.concat_columns` (`_normalize_value`) casts complex & special Doris types + # to `VARCHAR` instead of `DataType.Type.STRUCT` / `ARRAY` / `USERDEFINED`. + concat_expr = adapter.concat_columns( + { + "id": cols["id"], + "tags": cols["tags"], + "attrs": cols["attrs"], + "profile": cols["profile"], + "bm": cols["bm"], + "var_col": cols["var_col"], + } + ) + rendered_concat = concat_expr.sql("doris") + assert "AS STRUCT" not in rendered_concat + assert "AS ARRAY" not in rendered_concat + assert "AS MAP" not in rendered_concat + assert "AS BITMAP" not in rendered_concat + assert "CAST(tags AS VARCHAR)" in rendered_concat + assert "CAST(`profile` AS VARCHAR)" in rendered_concat + assert ( + adapter._normalize_nested_value(exp.column("tags", "s")).sql("doris") + == "CAST(s.tags AS VARCHAR)" + ) + + +def test_doris_ctas_window_partially_annotated_positional_group_order_and_subquery_union( + make_mocked_engine_adapter: t.Callable, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Unannotated CTAS with `ROW_NUMBER() OVER (...)` at col 1 and `AVG(score)` at col 0 + # recognizes `exp.Window` (`ROW_NUMBER` -> `BIGINT`) and promotes `rn` to col 0. + adapter.ctas( + "db.win_ctas", + parse_one( + "SELECT AVG(score) AS avg_score, ROW_NUMBER() OVER (ORDER BY id) AS rn FROM db.src GROUP BY id", + read="doris", + ), + ) + + # 2. Partially annotated CTAS (`score: DOUBLE`, `id: INT`, `extra: UNKNOWN`) uses known column + # types to promote `id` before `score` (`DOUBLE`) and remaps positional `GROUP BY 2 ORDER BY 2 DESC` -> `1`. + adapter.ctas( + "db.partial_annotated_ctas", + parse_one( + "SELECT AVG(score) AS score, id, extra FROM db.src GROUP BY 2, 3 ORDER BY 2 DESC", + read="doris", + ), + target_columns_to_types={ + "score": exp.DataType.build("DOUBLE", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "extra": exp.DataType.build("UNKNOWN"), + }, + ) + + # 3. Parenthesized `UNION ALL` branches (`(SELECT ...) UNION ALL (SELECT ...)`) with `auto_partition = true` + # unwrap `exp.Subquery` nodes, reorder projections in lockstep, and wrap range partition column in `NON_NULLABLE`. + adapter.ctas( + "db.paren_union_auto_ctas", + parse_one( + "(SELECT AVG(score) AS score, CAST(id AS INT) AS id, CAST(ds AS DATE) AS ds FROM db.s1 GROUP BY 2, 3) " + "UNION ALL " + "(SELECT CAST(score AS DOUBLE) AS score, CAST(id AS INT) AS id, CAST(ds AS DATE) AS ds FROM db.s2)", + read="doris", + ), + partitioned_by=[parse_one("RANGE(ds)", read="doris")], + table_properties={"auto_partition": exp.Literal.string("true")}, + ) + + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`win_ctas` AS SELECT ROW_NUMBER() OVER (ORDER BY `id`) AS `rn`, AVG(`score`) AS `avg_score` FROM `db`.`src` GROUP BY `id`", + "CREATE TABLE IF NOT EXISTS `db`.`partial_annotated_ctas` AS SELECT `id`, AVG(`score`) AS `score`, `extra` FROM `db`.`src` GROUP BY 1, 3 ORDER BY 1 DESC", + "CREATE TABLE IF NOT EXISTS `db`.`paren_union_auto_ctas` AUTO PARTITION BY RANGE (`ds`) () AS (SELECT CAST(`id` AS INT) AS `id`, NON_NULLABLE(CAST(`ds` AS DATE)) AS `ds`, AVG(`score`) AS `score` FROM `db`.`s1` GROUP BY 1, 2) UNION ALL (SELECT CAST(`id` AS INT) AS `id`, NON_NULLABLE(CAST(`ds` AS DATE)) AS `ds`, CAST(`score` AS DOUBLE) AS `score` FROM `db`.`s2`)", + ] + + +def test_doris_replace_by_key_text_preservation_intersect_delete_and_direct_time_delete( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + + # 1. `merge` / `_replace_by_key` when `unique_key` is a `TEXT`/`STRING` column (`doc_id`) and + # another valid OLAP key column (`tenant_id INT`) exists: does NOT coerce `doc_id` to `VARCHAR(255)` + # on the temp table (preventing 255-byte truncation of long keys), while promoting `tenant_id` to col 0. + adapter.merge( + target_table="db.docs_tbl", + source_table=parse_one("SELECT doc_id, tenant_id FROM db.staging_docs", read="doris"), + target_columns_to_types={ + "doc_id": exp.DataType.build("TEXT", dialect="doris"), + "tenant_id": exp.DataType.build("INT", dialect="doris"), + }, + unique_key=[exp.to_column("doc_id")], + ) + merge_sqls = to_sql_calls(adapter) + assert ( + merge_sqls[0] + == "CREATE TABLE `db`.`__temp_docs_tbl_abcd1234` AS SELECT CAST(`tenant_id` AS INT) AS `tenant_id`, CAST(`doc_id` AS STRING) AS `doc_id` FROM (SELECT `doc_id`, `tenant_id` FROM `db`.`staging_docs`) AS `_subquery`" + ) + + # 2. `_delete_by_time_transform_keys` with `INTERSECT` or `EXCEPT` does NOT unwrap `DATE_TRUNC(ds, 'day')` + # into raw `ds` inside the branches (preserving day-level set intersection/difference semantics), + # and unwraps parenthesized `(SELECT ...)` branches in `DELETE ... USING`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter.cursor, + "fetchall", + return_value=[("2024-01-15", "2024-01-16")], + ) + adapter.delete_from( + "db.events", + where=parse_one( + "DATE_TRUNC(ds, 'day') IN ((SELECT DATE_TRUNC(ds, 'day') FROM db.s1) INTERSECT (SELECT DATE_TRUNC(ds, 'day') FROM db.s2))", + read="doris", + ), + ) + adapter.delete_from( + "db.events", + where=parse_one( + "id IN ((SELECT id FROM db.s1) UNION ALL (SELECT id FROM db.s2))", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "SELECT DISTINCT DATE_TRUNC(`_col_0`, 'day') AS `_low_0`, DATE_ADD(DATE_TRUNC(`_col_0`, 'day'), INTERVAL 1 DAY) AS `_high_0` FROM ((SELECT DATE_TRUNC(`ds`, 'DAY') AS `_col_0` FROM `db`.`s1`) INTERSECT (SELECT DATE_TRUNC(`ds`, 'DAY') FROM `db`.`s2`)) AS `_sq`", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-16'", + "DELETE FROM `db`.`events` AS `_t1` USING ((SELECT `id` FROM `db`.`s1`) UNION ALL (SELECT `id` FROM `db`.`s2`)) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id`", + ] + + # 3. Direct time-transform predicates (`DATE_TRUNC(ds, 'day') = '2024-01-15'`, `YEAR(ds) = 2024`, + # `DATE_TRUNC(ds, 'day') IN ('2024-01-15', '2024-01-16')`) and heterogeneous `OR` predicates + # (`a = 1 OR b = 2`) in `delete_from` are rewritten into `OR`-free bare-column predicates. + adapter.cursor.execute.reset_mock() + adapter.delete_from("db.events", "DATE_TRUNC(ds, 'day') = '2024-01-15'") + adapter.delete_from("db.events", "YEAR(ds) = 2024") + adapter.delete_from("db.events", "DATE_TRUNC(ds, 'day') IN ('2024-01-15', '2024-01-16')") + adapter.delete_from("db.events", "a = 1 OR b = 2") + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-16'", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-01' AND `ds` < '2025-01-01'", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-16'", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-16' AND `ds` < '2024-01-17'", + "DELETE FROM `db`.`events` WHERE `a` = 1", + "DELETE FROM `db`.`events` WHERE `b` = 2", + ] + + +def test_doris_physical_properties_kwarg_drop_schema_cascade_and_mv_df_source_columns( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `physical_properties={'engine': exp.Literal.string('iceberg')}` (as passed by SnapshotEvaluator) + # is honored across `replace_query` / `_insert_overwrite_by_condition`. + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", schema="db", name="iceberg_by_prop", type=DataObjectType.TABLE + ), + ) + adapter.replace_query( + "db.iceberg_by_prop", + parse_one("SELECT 1 AS id", read="doris"), + target_columns_to_types={"id": exp.DataType.build("INT", dialect="doris")}, + physical_properties={"engine": exp.Literal.string("iceberg")}, + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`iceberg_by_prop` WHERE NOT `id` IS NULL", + "DELETE FROM `db`.`iceberg_by_prop` WHERE `id` IS NULL", + "INSERT INTO `db`.`iceberg_by_prop` (`id`) SELECT 1 AS `id`", + ] + + # 2. `drop_schema(..., cascade=True)` on an external Iceberg catalog drops member tables first + # before issuing `DROP DATABASE`. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "_get_data_objects", + return_value=[ + DataObject( + catalog="iceberg_cat", schema="silver", name="t1", type=DataObjectType.TABLE + ), + DataObject( + catalog="iceberg_cat", schema="silver", name="t2", type=DataObjectType.TABLE + ), + ], + ) + adapter.drop_schema("iceberg_cat.silver", ignore_if_not_exists=True, cascade=True) + assert to_sql_calls(adapter) == [ + "DROP TABLE IF EXISTS `iceberg_cat`.`silver`.`t1`", + "DROP TABLE IF EXISTS `iceberg_cat`.`silver`.`t2`", + "DROP DATABASE IF EXISTS `iceberg_cat`.`silver`", + ] + + # 3. `_create_materialized_view` with a DataFrame containing extra `source_columns` not in + # `target_columns_to_types` slices only `source_columns_to_types` so `_values_to_sql` never raises `ValueError`. + adapter.cursor.execute.reset_mock() + df_extra = pd.DataFrame([{"id": 1, "name": "alice", "extra_ignored": 999}]) + adapter.create_view( + "db.mv_from_df_subset", + df_extra, + replace=False, + materialized=True, + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(32)", dialect="doris"), + }, + source_columns=["id", "name", "extra_ignored"], + ) + assert to_sql_calls(adapter) == [ + "CREATE MATERIALIZED VIEW `db`.`mv_from_df_subset` (`id`, `name`) AS SELECT CAST(`id` AS INT) AS `id`, CAST(`name` AS VARCHAR(32)) AS `name` FROM (SELECT 1 AS `id`, 'alice' AS `name`) AS `t`" + ] + + # 4. `_truncate_table` normalizes >64-char `_schema_tmp` table names before `TRUNCATE TABLE` and fallback `DELETE`. + adapter.cursor.execute.reset_mock() + long_tmp = "db." + ("very_long_table_name_" * 3) + "_schema_tmp" + short_tmp = adapter._shorten_schema_tmp_name(exp.to_table(long_tmp).name) + adapter._truncate_table(long_tmp) + assert to_sql_calls(adapter) == [f"TRUNCATE TABLE `db`.`{short_tmp}`"] + + # 5. `_fetch_native_df` with a raw string query containing `IS TRUE` or >64-char `_schema_tmp` + # normalizes the SQL string before `pd.read_sql_query`. + mock_read_sql = mocker.patch( + "pandas.io.sql.read_sql_query", + return_value=pd.DataFrame({"id": [1]}), + ) + adapter.fetchdf(f"SELECT id FROM {long_tmp} WHERE is_active IS TRUE", quote_identifiers=True) + assert ( + mock_read_sql.call_args[0][0] + == f"SELECT `id` FROM `db`.`{short_tmp}` WHERE COALESCE(`is_active`, FALSE) = TRUE" + ) + + +def test_doris_time_transform_inequalities_unaliased_ctas_and_sync_mv_db_restore( + make_mocked_engine_adapter: t.Callable, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter, database="analytics") + + # 1. Direct time-transform inequalities (`>=`, `<=`, `!=`) in `delete_from` are rewritten into + # bare-column predicates (and `!=` is split into two OR-free DELETE statements). + adapter.delete_from( + "db.events", + "DATE_TRUNC(ds, 'day') >= '2024-01-15' AND DATE_TRUNC(ds, 'day') <= '2024-01-20'", + ) + adapter.delete_from("db.events", "DATE_TRUNC(ds, 'day') != '2024-01-15'") + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-21'", + "DELETE FROM `db`.`events` WHERE `ds` < '2024-01-15'", + "DELETE FROM `db`.`events` WHERE `ds` >= '2024-01-16'", + ] + + # 2. `DELETE ... USING` with an extra `OR` condition (`id IN (SELECT id FROM db.src) AND (status = 'a' OR status IS NULL)`) + # expands `extra_conditions` into separate OR-free `DELETE ... USING` statements. + adapter.cursor.execute.reset_mock() + adapter.delete_from( + "db.target", + where=parse_one( + "id IN (SELECT id FROM db.src) AND (status = 'a' OR status IS NULL)", + read="doris", + ), + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id` FROM `db`.`src`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`status` = 'a'", + "DELETE FROM `db`.`target` AS `_t1` USING (SELECT `id` FROM `db`.`src`) AS `_t2` WHERE `_t1`.`id` = `_t2`.`id` AND `_t1`.`status` IS NULL", + ] + + # 3. Unannotated CTAS with unaliased `AVG(score)` or division `a / b` at column 0 promotes `id` to column 0. + adapter.cursor.execute.reset_mock() + adapter.ctas( + "db.unaliased_ctas", + parse_one("SELECT AVG(score), id FROM db.src GROUP BY 2", read="doris"), + ) + adapter.ctas( + "db.div_ctas", + parse_one("SELECT a / b AS ratio, id FROM db.src", read="doris"), + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`unaliased_ctas` AS SELECT `id`, AVG(`score`) FROM `db`.`src` GROUP BY 1", + "CREATE TABLE IF NOT EXISTS `db`.`div_ctas` AS SELECT `id`, `a` / `b` AS `ratio` FROM `db`.`src`", + ] + + # 4. Sync MV creation and drop in another schema (`other_db`) restores the configured session database (`analytics`). + adapter.cursor.execute.reset_mock() + adapter.create_view( + "other_db.rollup_mv", + parse_one("SELECT id, SUM(val) AS s FROM other_db.fact GROUP BY id", read="doris"), + replace=False, + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + ) + adapter.drop_view("other_db.rollup_mv", materialized=True) + assert to_sql_calls(adapter) == [ + "USE `other_db`", + "CREATE MATERIALIZED VIEW `rollup_mv` AS SELECT `id` AS `mv_id`, SUM(`val`) AS `s` FROM `other_db`.`fact` GROUP BY `id`", + "USE `analytics`", + "USE `other_db`", + "DROP MATERIALIZED VIEW IF EXISTS `rollup_mv` ON `other_db`.`fact`", + "USE `analytics`", + ] + + +def test_doris_insert_append_scd2_create_table_like_and_alter_ops( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `insert_append` reorders OLAP columns (moving valid key `id` ahead of `score DOUBLE`) + # and resolves table columns when `target_columns_to_types` contains `UNKNOWN`. + adapter.insert_append( + "db.metrics", + parse_one("SELECT score, id FROM db.src", read="doris"), + target_columns_to_types={ + "score": exp.DataType.build("DOUBLE"), + "id": exp.DataType.build("INT"), + }, + ) + mocker.patch.object( + adapter, + "columns", + return_value={ + "id": exp.DataType.build("INT"), + "score": exp.DataType.build("DOUBLE"), + }, + ) + adapter.insert_append( + "db.metrics", + parse_one("SELECT score, id FROM db.src", read="doris"), + target_columns_to_types={ + "score": exp.DataType.build("UNKNOWN"), + "id": exp.DataType.build("UNKNOWN"), + }, + ) + assert to_sql_calls(adapter) == [ + "INSERT INTO `db`.`metrics` (`id`, `score`) SELECT `id`, `score` FROM (SELECT `score`, `id` FROM `db`.`src`) AS `_subquery`", + "INSERT INTO `db`.`metrics` (`id`, `score`) SELECT `id`, `score` FROM (SELECT `score`, `id` FROM `db`.`src`) AS `_subquery`", + ] + + # 2. `_create_table_like` on an Iceberg catalog table falls back to `create_table` with `self.columns(source_table_name)` + # because Doris rejects `CREATE TABLE ... LIKE` on external catalog tables. + adapter.cursor.execute.reset_mock() + adapter._create_table_like("iceberg_cat.db.target_tbl", "iceberg_cat.db.source_tbl") + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `iceberg_cat`.`db`.`target_tbl` (`id` INT, `score` DOUBLE) ENGINE=iceberg" + ] + + # 3. `_describe_columns_with_keys` and `get_alter_operations` normalize `STRUCT` (unquoting field names + # and stripping `NULL`) and `AGG_STATE`, and suppress key-role-change type modifications. + mocker.stopall() + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + adapter.cursor.fetchall.side_effect = [ + [ + ("id", "INT(11)", "No", "true", None, "NONE"), + ("k2", "INT(11)", "No", "true", None, "NONE"), + ( + "info", + "STRUCT<`uid`:BIGINT(20) NULL, `tag`:VARCHAR(32) NULL>", + "Yes", + "false", + None, + "NONE", + ), + ("st", "AGG_STATE", "No", "false", None, "GENERIC"), + ], + [ + ("id", "INT(11)", "No", "true", None, "NONE"), + ("k2", "BIGINT(20)", "Yes", "false", None, "NONE"), + ("info", "STRUCT", "Yes", "false", None, "NONE"), + ("st", "AGG_STATE", "No", "false", None, "GENERIC"), + ], + ] + ops = adapter.get_alter_operations("db.curr_tbl", "db.targ_tbl") + assert ops == [] + + # 4. `_build_partitioned_by_exp` normalizes `TRUNCATE(col, 10)` and `BUCKET(col, 8)` into Doris Iceberg syntax, + # and `_build_create_table_exp` does not force `NOT NULL` when `auto_partition = true` is paired with `VALUES IN`. + adapter.cursor.execute.reset_mock() + adapter.create_table( + "iceberg_cat.db.part_ice", + {"id": exp.DataType.build("INT"), "name": exp.DataType.build("STRING")}, + partitioned_by=[ + parse_one("TRUNCATE(name, 10)", read="doris"), + parse_one("BUCKET(id, 8)", read="doris"), + ], + ) + adapter.create_table( + "db.auto_list_olap", + {"region": exp.DataType.build("VARCHAR(32)"), "id": exp.DataType.build("INT")}, + partitioned_by=[exp.to_column("region")], + table_properties={ + "auto_partition": exp.Literal.string("true"), + "partitions": exp.Literal.string("PARTITION p_us VALUES IN ('US')"), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `iceberg_cat`.`db`.`part_ice` (`id` INT, `name` STRING) ENGINE=iceberg PARTITION BY LIST (TRUNCATE(10, `name`), BUCKET(8, `id`)) ()", + "CREATE TABLE IF NOT EXISTS `db`.`auto_list_olap` (`region` VARCHAR(32), `id` INT) AUTO PARTITION BY LIST (`region`) (PARTITION p_us VALUES IN ('US'))", + ] + + +def test_doris_not_pushdown_week_quarter_and_candidate_extraction( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. De Morgan `NOT (a = 1 AND b = 2)` and `NOT (a = 1 OR b = 2)`, `1 != 1`, aligned & non-aligned `DATE_TRUNC(ds, 'week')` / + # `DATE_TRUNC(ts, 'quarter')`, and `DATE_TRUNC(ds, 'day') IS [NOT] NULL` in `delete_from`. + adapter.delete_from("db.t", "1 != 1 AND a = 1") + adapter.delete_from("db.t", "NOT (a = 1 OR b = 2)") + adapter.delete_from("db.t", "NOT (a = 1 AND b = 2)") + adapter.delete_from( + "db.t", + "DATE_TRUNC(ds, 'week') = '2024-01-15' AND DATE_TRUNC(ts, 'quarter') = '2024-04-01' AND DATE_TRUNC(ds, 'day') IS NOT NULL", + ) + adapter.delete_from("db.t", "DATE_TRUNC(ds, 'week') = '2024-01-17' AND a = 1") + adapter.delete_from("db.t", "DATE_TRUNC(ds, 'week') IN ('2024-01-16', '2024-01-17')") + adapter.delete_from("db.t", "DATE_TRUNC(ds, 'week') IN ('2024-01-15', '2024-01-17')") + adapter.delete_from( + "db.t", + "DATE_TRUNC(ds, 'week') >= '2024-01-17' AND DATE_TRUNC(ds, 'week') <= '2024-01-24'", + ) + adapter.delete_from("db.t", "DATE_TRUNC(ds, 'week') != '2024-01-17' AND a = 1") + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`t` WHERE `a` <> 1 AND `b` <> 2", + "DELETE FROM `db`.`t` WHERE `a` <> 1", + "DELETE FROM `db`.`t` WHERE `b` <> 2", + "DELETE FROM `db`.`t` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-22' AND `ts` >= '2024-04-01' AND `ts` < '2024-07-01' AND NOT `ds` IS NULL", + "DELETE FROM `db`.`t` WHERE `ds` >= '2024-01-15' AND `ds` < '2024-01-22'", + "DELETE FROM `db`.`t` WHERE `ds` >= '2024-01-22' AND `ds` < '2024-01-29'", + "DELETE FROM `db`.`t` WHERE NOT `ds` IS NULL AND `a` = 1", + ] + + # 2. Candidate loops in `_extract_or_key_rows_from_where` and `_extract_subquery_from_where` + # find the valid key-row OR / subquery IN conjunct even when preceded by another OR / IN conjunct. + adapter.cursor.execute.reset_mock() + adapter.delete_from( + "db.t", + "(ds < '2024-01-01' OR ds >= '2024-01-02') AND ((id = 1 AND k = 'a') OR (id = 2 AND k = 'b'))", + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `db`.`t` WHERE `id` = 1 AND `k` IN ('a') AND `ds` < '2024-01-01'", + "DELETE FROM `db`.`t` WHERE `id` = 1 AND `k` IN ('a') AND `ds` >= '2024-01-02'", + "DELETE FROM `db`.`t` WHERE `id` = 2 AND `k` IN ('b') AND `ds` < '2024-01-01'", + "DELETE FROM `db`.`t` WHERE `id` = 2 AND `k` IN ('b') AND `ds` >= '2024-01-02'", + ] + + # 3. Sync MV aliases that collide with base table column names (`SUM(amount) AS amount`) are prefixed with `mv_`, + # and `partitioned_by` / `clustered_by` kwargs are ignored for `SYNC` MVs. + adapter.cursor.execute.reset_mock() + adapter.create_view( + "db.sync_agg_mv", + parse_one("SELECT id, SUM(amount) AS amount FROM db.sales GROUP BY id", read="doris"), + replace=False, + materialized=True, + view_properties={"materialized_type": exp.Literal.string("SYNC")}, + partitioned_by=[exp.to_column("id")], + clustered_by=[exp.to_column("id")], + ) + assert to_sql_calls(adapter) == [ + "USE `db`", + "CREATE MATERIALIZED VIEW `sync_agg_mv` AS SELECT `id` AS `mv_id`, SUM(`amount`) AS `mv_amount` FROM `db`.`sales` GROUP BY `id`", + ] + + # 4. `_sanitize_key_value` decodes bytes and unwraps NumPy scalars. + import numpy as np + + assert adapter._sanitize_key_value(b"abc") == "abc" + assert adapter._sanitize_key_value(np.int64(42)) == 42 + assert type(adapter._sanitize_key_value(np.int64(42))) is int + + +def test_doris_4_1_native_merge_into_and_timestamptz( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + target_cols = { + "id": exp.DataType.build("INT", dialect="doris"), + "name": exp.DataType.build("VARCHAR(64)", dialect="doris"), + "val": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + } + + # 1. Native MERGE INTO when merge_filter is provided (queries Doris version and SHOW CREATE TABLE when table_properties is omitted) + adapter.cursor.fetchone.side_effect = [ + ("Doris version doris-4.1.5-rc02", "5.7.99"), + ("target", "CREATE TABLE `target` (`id` INT) ENGINE=OLAP UNIQUE KEY(`id`)"), + ] + adapter.merge( + target_table="db.target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + merge_filter=parse_one("__MERGE_SOURCE__.ds > __MERGE_TARGET__.ds", read="doris"), + ) + assert to_sql_calls(adapter) == [ + "SELECT @@version_comment, VERSION()", + "SHOW CREATE TABLE `db`.`target`", + "MERGE INTO `db`.`target` `__MERGE_TARGET__` USING (SELECT `id`, `name`, `val`, `ds` FROM `db`.`src`) AS `__MERGE_SOURCE__` ON `__MERGE_SOURCE__`.`ds` > `__MERGE_TARGET__`.`ds` AND `__MERGE_TARGET__`.`id` = `__MERGE_SOURCE__`.`id` WHEN MATCHED THEN UPDATE SET `__MERGE_TARGET__`.`name` = `__MERGE_SOURCE__`.`name`, `__MERGE_TARGET__`.`val` = `__MERGE_SOURCE__`.`val`, `__MERGE_TARGET__`.`ds` = `__MERGE_SOURCE__`.`ds` WHEN NOT MATCHED THEN INSERT (`id`, `name`, `val`, `ds`) VALUES (`__MERGE_SOURCE__`.`id`, `__MERGE_SOURCE__`.`name`, `__MERGE_SOURCE__`.`val`, `__MERGE_SOURCE__`.`ds`)", + ] + + # 2. Native MERGE INTO when when_matched is provided with unique_key in table_properties + # (uses cached _doris_version and skips SHOW CREATE TABLE, stripping key column assignments in UPDATE SET) + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchone.side_effect = None + when_matched = parse_one( + "WHEN MATCHED THEN UPDATE SET __MERGE_TARGET__.id = __MERGE_SOURCE__.id, __MERGE_TARGET__.val = __MERGE_SOURCE__.val", + into=exp.Whens, + ) + adapter.merge( + target_table="db.target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + table_properties={"unique_key": exp.to_column("id")}, + ) + assert to_sql_calls(adapter) == [ + "MERGE INTO `db`.`target` `__MERGE_TARGET__` USING (SELECT `id`, `name`, `val`, `ds` FROM `db`.`src`) AS `__MERGE_SOURCE__` ON `__MERGE_TARGET__`.`id` = `__MERGE_SOURCE__`.`id` WHEN MATCHED THEN UPDATE SET `__MERGE_TARGET__`.`val` = `__MERGE_SOURCE__`.`val` WHEN NOT MATCHED THEN INSERT (`id`, `name`, `val`, `ds`) VALUES (`__MERGE_SOURCE__`.`id`, `__MERGE_SOURCE__`.`name`, `__MERGE_SOURCE__`.`val`, `__MERGE_SOURCE__`.`ds`)" + ] + + # 2b. Guard against Doris < 4.1 or non-UNIQUE KEY targets (DUPLICATE KEY, AGGREGATE KEY, Iceberg) + with pytest.raises(SQLMeshError, match="configured as DUPLICATE KEY"): + adapter.merge( + target_table="db.dup_target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + table_properties={"duplicate_key": exp.to_column("id")}, + ) + with pytest.raises(SQLMeshError, match="configured as AGGREGATE KEY"): + adapter.merge( + target_table="db.agg_target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + table_properties={"aggregate_key": exp.to_column("id")}, + ) + with pytest.raises(SQLMeshError, match="only supported on internal OLAP UNIQUE KEY tables"): + adapter.merge( + target_table="iceberg_cat.db.target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + table_format="iceberg", + ) + adapter.cursor.fetchone.return_value = ( + "dup_target", + "CREATE TABLE `dup_target` (`id` INT) ENGINE=OLAP DUPLICATE KEY(`id`)", + ) + with pytest.raises(SQLMeshError, match="is not a UNIQUE KEY table"): + adapter.merge( + target_table="db.dup_target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + ) + adapter._doris_version = None + adapter.cursor.fetchone.return_value = ("Doris version doris-2.1.7-rc01", "5.7.99") + with pytest.raises( + SQLMeshError, match="requires Apache Doris 4.1\\+ \\(detected Doris 2\\.1\\)" + ): + adapter.merge( + target_table="db.target", + source_table=parse_one("SELECT id, name, val, ds FROM db.src", read="doris"), + target_columns_to_types=target_cols, + unique_key=[exp.to_column("id")], + when_matched=when_matched, + table_properties={"unique_key": exp.to_column("id")}, + ) + + # 3. AtTimeZone rewrites to CONVERT_TZ(..., @@time_zone, '+00:00') + norm_expr = adapter.normalize_value( + exp.to_column("ts"), + exp.DataType.build("TIMESTAMPTZ"), + decimal_precision=3, + timestamp_precision=6, + ) + assert ( + adapter._to_sql(norm_expr) + == "CAST(DATE_FORMAT(CONVERT_TZ(`ts`, @@time_zone, '+00:00'), '%Y-%m-%d %T.%f') AS VARCHAR)" + ) + + # 4. TIMESTAMPTZ maps to DATETIME (for Doris 2.1+ compatibility and MySQL protocol datetime return), + # defaults to precision 0, and supports same-type precision widening (TIMESTAMPTZ -> TIMESTAMPTZ(0) + # -> TIMESTAMPTZ(6), DATETIME(2) -> DATETIME(6)), while cross-type TIMESTAMPTZ <-> DATETIME is + # treated as destructive. + assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "DATETIME" + assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "DATETIME(6)" + from sqlmesh.core.schema_diff import TableAlterChangeColumnTypeOperation + + ops_default = adapter.schema_differ.compare_columns( + "t", + {"ts": exp.DataType.build("TIMESTAMPTZ", dialect="doris")}, + {"ts": exp.DataType.build("TIMESTAMPTZ(0)", dialect="doris")}, + ) + assert ( + len(ops_default) == 1 + and isinstance(ops_default[0], TableAlterChangeColumnTypeOperation) + and ops_default[0].is_part_of_destructive_change is False + ) + ops = adapter.schema_differ.compare_columns( + "t", + {"ts": exp.DataType.build("TIMESTAMPTZ(2)", dialect="doris")}, + {"ts": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris")}, + ) + assert ( + len(ops) == 1 + and isinstance(ops[0], TableAlterChangeColumnTypeOperation) + and ops[0].is_part_of_destructive_change is False + ) + ops_dt = adapter.schema_differ.compare_columns( + "t", + {"dt": exp.DataType.build("DATETIME(2)", dialect="doris")}, + {"dt": exp.DataType.build("DATETIME(6)", dialect="doris")}, + ) + assert ( + len(ops_dt) == 1 + and isinstance(ops_dt[0], TableAlterChangeColumnTypeOperation) + and ops_dt[0].is_part_of_destructive_change is False + ) + ops_cross = adapter.schema_differ.compare_columns( + "t", + {"ts": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris")}, + {"ts": exp.DataType.build("DATETIME(6)", dialect="doris")}, + ignore_destructive=True, + ) + assert ops_cross == [] + + # 5. alter_table retries when Doris raises `state(SCHEMA_CHANGE) is not NORMAL` + call_count = 0 + + def _flaky_execute(sql: str, **kwargs: t.Any) -> None: + nonlocal call_count + call_count += 1 + if call_count == 2: + raise RuntimeError( + "errCode = 2, detailMessage = Table[t]'s state(SCHEMA_CHANGE) is not NORMAL. Do not allow doing ALTER ops" + ) + + adapter.cursor.execute.reset_mock() + adapter.cursor.execute.side_effect = _flaky_execute + adapter.alter_table( + [ + t.cast( + exp.Alter, parse_one("ALTER TABLE `db`.`t` MODIFY COLUMN `a` BIGINT", read="doris") + ), + t.cast( + exp.Alter, + parse_one("ALTER TABLE `db`.`t` MODIFY COLUMN `b` DATETIME(6)", read="doris"), + ), + ] + ) + assert call_count == 3 + adapter.cursor.execute.side_effect = None + + +def test_lance_table_reading_and_write_guard( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.core.dialect import find_tables + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Reading from a Doris Lance catalog table extracts upstream table dependency cleanly + lance_select = parse_one( + "SELECT id, L2_DISTANCE(vec, [1.0, 2.0, 3.0]) AS dist FROM lance_cat.ml_db.embeddings ORDER BY dist LIMIT 5", + read="doris", + ) + assert ( + lance_select.sql("doris") + == "SELECT id, L2_DISTANCE(vec, ARRAY(1.0, 2.0, 3.0)) AS dist FROM lance_cat.ml_db.embeddings ORDER BY dist LIMIT 5" + ) + assert find_tables(lance_select, default_catalog=None, dialect="doris") == { + '"lance_cat"."ml_db"."embeddings"' + } + + # 2. Reading from a Doris Lance table-valued function (TVF) + lance_tvf = parse_one( + "SELECT id, vec FROM s3('uri' = 's3://bucket/data.lance', 'format' = 'lance')", + read="doris", + ) + assert ( + lance_tvf.sql("doris") + == "SELECT id, vec FROM S3('uri' = 's3://bucket/data.lance', 'format' = 'lance')" + ) + + # 3. Attempting to materialize / write a Lance table raises a clear SQLMeshError + with pytest.raises(SQLMeshError, match="Doris Lance tables are read-only"): + adapter.create_table( + "lance_cat.ml_db.out_tbl", + target_columns_to_types={"id": exp.DataType.build("INT", dialect="doris")}, + table_format="lance", + ) + + with pytest.raises(SQLMeshError, match="Doris Lance tables are read-only"): + adapter.ctas( + "lance_cat.ml_db.out_tbl", + parse_one("SELECT 1 AS id", read="doris"), + table_properties={"engine": exp.Literal.string("lance")}, + ) + + with pytest.raises(SQLMeshError, match="Doris Hudi tables are read-only"): + adapter.create_table( + "hudi_cat.ml_db.out_tbl", + target_columns_to_types={"id": exp.DataType.build("INT", dialect="doris")}, + table_format="hudi", + ) + + # 4. External Paimon catalog table creation emits ENGINE=paimon, 'file.format', and 'primary-key' + adapter.cursor.execute.reset_mock() + adapter.create_table( + "paimon_cat.silver.orders", + target_columns_to_types={ + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + "ds": exp.DataType.build("DATE", dialect="doris"), + }, + table_format="paimon", + storage_format="parquet", + partitioned_by=[exp.to_column("ds")], + unique_key=[exp.to_column("id")], + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `paimon_cat`.`silver`.`orders` (`payload` STRING, `id` INT, `ds` DATE) ENGINE=paimon PARTITION BY LIST (`ds`) () PROPERTIES ('file.format'='parquet', 'primary-key'='id,ds')" + ] + + # 5. External Hive catalog table creation emits ENGINE=hive, 'file_format', and skips OLAP key reordering + adapter.cursor.execute.reset_mock() + adapter.create_table( + "hive_cat.raw_db.events", + target_columns_to_types={ + "payload": exp.DataType.build("STRING", dialect="doris"), + "id": exp.DataType.build("INT", dialect="doris"), + }, + table_format="hive", + storage_format="orc", + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `hive_cat`.`raw_db`.`events` (`payload` STRING, `id` INT) ENGINE=hive PROPERTIES ('file_format'='orc')" + ] + + +def test_iceberg_reading_time_travel_branch_and_compaction_ddl( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], +): + from sqlmesh.core.dialect import find_tables + + _ = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Iceberg time travel (FOR TIME AS OF, FOR VERSION AS OF) and branch/tag (@branch, @tag) + for sql, expected in [ + ( + "SELECT * FROM ice_cat.db.tbl FOR TIME AS OF '2026-01-01 00:00:00' AS t", + "SELECT * FROM ice_cat.db.tbl FOR TIME AS OF '2026-01-01 00:00:00' AS t", + ), + ( + "SELECT * FROM ice_cat.db.tbl FOR VERSION AS OF 123456789 AS t", + "SELECT * FROM ice_cat.db.tbl FOR VERSION AS OF 123456789 AS t", + ), + ( + "SELECT * FROM ice_cat.db.tbl@branch(audit_branch) AS t", + "SELECT * FROM ice_cat.db.tbl @branch(audit_branch) AS t", + ), + ( + "SELECT * FROM ice_cat.db.tbl @tag(v1_release) AS t", + "SELECT * FROM ice_cat.db.tbl @tag(v1_release) AS t", + ), + ]: + parsed = parse_one(sql, read="doris") + assert parsed.sql("doris") == expected + # Upstream table dependency extraction preserves `ice_cat.db.tbl` without `@branch(...)` + tables = list(parsed.find_all(exp.Table)) + assert [(tbl.catalog, tbl.db, tbl.name) for tbl in tables] == [("ice_cat", "db", "tbl")] + assert find_tables(parsed, default_catalog=None, dialect="doris") == { + '"ice_cat"."db"."tbl"' + } + + # 2. Iceberg compaction (rewrite_data_files), snapshot expiration, and branch/tag DDL parse as exp.Alter + for ddl in [ + "ALTER TABLE ice_cat.db.tbl EXECUTE rewrite_data_files('target-file-size-bytes' = '134217728')", + "ALTER TABLE ice_cat.db.tbl EXECUTE expire_snapshots('older_than' = '2026-01-01 00:00:00')", + "ALTER TABLE ice_cat.db.tbl EXECUTE remove_orphan_files('older_than' = '2026-01-01 00:00:00')", + "ALTER TABLE ice_cat.db.tbl CREATE BRANCH audit_branch", + "ALTER TABLE ice_cat.db.tbl CREATE TAG v1_tag", + "ALTER TABLE ice_cat.db.tbl DROP BRANCH IF EXISTS audit_branch", + ]: + parsed_ddl = parse_one(ddl, read="doris") + assert isinstance(parsed_ddl, exp.Alter) + assert parsed_ddl.this.sql("doris") == "ice_cat.db.tbl" + assert parsed_ddl.sql("doris") == ddl + + # 3. Full SQLMesh model loading & @IF post-statement rendering with @this_model resolution + from sqlmesh.core.dialect import parse + from sqlmesh.core.macros import RuntimeStage + from sqlmesh.core.model import load_sql_based_model + + model_sql = """ + MODEL ( + name ice_cat.silver.curated_events, + kind FULL, + dialect doris, + table_format iceberg + ); + SELECT + e.id, + e.msg + FROM ice_cat.bronze.events FOR VERSION AS OF 123456 AS e + JOIN ice_cat.bronze.tags@tag(v1) AS t ON e.id = t.id + WHERE e.msg MATCH_PHRASE 'hello world' AND SEARCH('msg:hello'); + + @IF( + @runtime_stage = 'evaluating', + ALTER TABLE @this_model EXECUTE rewrite_data_files('target-file-size-bytes' = '134217728') + ); + @IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_msg (msg) USING INVERTED PROPERTIES ('parser' = 'english') + ); + @IF( + @runtime_stage = 'evaluating', + BUILD INDEX idx_msg ON @this_model PARTITION (p1) + ); + """ + model = load_sql_based_model( + parse(model_sql, default_dialect="doris"), default_catalog="internal", dialect="doris" + ) + assert model.depends_on == {'"ice_cat"."bronze"."events"', '"ice_cat"."bronze"."tags"'} + assert [ + ps.sql("doris") for ps in model.render_post_statements(runtime_stage=RuntimeStage.CREATING) + ] == [ + "ALTER TABLE `ice_cat`.`silver`.`curated_events` ADD INDEX `idx_msg` (`msg`) USING INVERTED PROPERTIES ('parser'='english')" + ] + assert [ + ps.sql("doris") + for ps in model.render_post_statements(runtime_stage=RuntimeStage.EVALUATING) + ] == [ + "ALTER TABLE `ice_cat`.`silver`.`curated_events` EXECUTE rewrite_data_files('target-file-size-bytes' = '134217728')", + "BUILD INDEX `idx_msg` ON `ice_cat`.`silver`.`curated_events` PARTITION (`p1`)", + ] + + +def test_doris_4_1_search_indexes_compaction_and_sequence_col( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + mocker: MockerFixture, +): + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. Full-text search (SEARCH, MATCH_ANY, MATCH_PHRASE, etc.) and vector distance functions + ft_query = parse_one( + "SELECT id, L1_DISTANCE(v1, v2) AS l1, INNER_PRODUCT(v1, v2) AS ip " + "FROM db.docs WHERE title MATCH_PHRASE 'apache doris' AND SEARCH('title:doris AND body:sqlmesh')", + read="doris", + ) + assert ( + ft_query.sql("doris") + == "SELECT id, L1_DISTANCE(v1, v2) AS l1, INNER_PRODUCT(v1, v2) AS ip " + "FROM db.docs WHERE title MATCH_PHRASE 'apache doris' AND SEARCH('title:doris AND body:sqlmesh')" + ) + + # 2. Index DDL: create_index with USING INVERTED / ANN, PROPERTIES, COMMENT, and BUILD INDEX + adapter.create_index( + "db.docs", + "idx_title_ft", + [exp.to_column("title")], + index_type="INVERTED", + index_properties={"parser": "english", "support_phrase": "true"}, + comment="fulltext index", + ) + adapter.create_index( + "db.docs", + "idx_vec_ann", + [exp.to_column("embedding")], + index_type="ANN", + index_properties={"index_type": "hnsw", "metric_type": "l2_distance", "dim": "128"}, + ) + assert to_sql_calls(adapter) == [ + "CREATE INDEX IF NOT EXISTS `idx_title_ft` ON `db`.`docs`(`title`) USING INVERTED PROPERTIES ('parser'='english', 'support_phrase'='true') COMMENT 'fulltext index'", + "CREATE INDEX IF NOT EXISTS `idx_vec_ann` ON `db`.`docs`(`embedding`) USING ANN PROPERTIES ('index_type'='hnsw', 'metric_type'='l2_distance', 'dim'='128')", + ] + + build_idx = parse_one("BUILD INDEX idx_title_ft ON db.docs PARTITIONS (p1, p2)", read="doris") + assert isinstance(build_idx, exp.Alter) + assert build_idx.this.sql("doris") == "db.docs" + assert build_idx.sql("doris") == "BUILD INDEX idx_title_ft ON db.docs PARTITIONS (p1, p2)" + + build_idx_singular = parse_one( + "BUILD INDEX idx_title_ft ON db.docs PARTITION (p1, p2)", read="doris" + ) + assert isinstance(build_idx_singular, exp.Alter) + assert build_idx_singular.this.sql("doris") == "db.docs" + assert ( + build_idx_singular.sql("doris") == "BUILD INDEX idx_title_ft ON db.docs PARTITION (p1, p2)" + ) + + alter_add_idx = parse_one( + "ALTER TABLE db.docs ADD INDEX idx_title (title) USING INVERTED PROPERTIES ('parser' = 'english') COMMENT 'ft'", + read="doris", + ) + assert isinstance(alter_add_idx, exp.Alter) + assert ( + alter_add_idx.sql("doris") + == "ALTER TABLE db.docs ADD INDEX idx_title (title) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'ft'" + ) + alter_drop_idx = parse_one("ALTER TABLE db.docs DROP INDEX idx_title", read="doris") + assert isinstance(alter_drop_idx, exp.Alter) + assert alter_drop_idx.sql("doris") == "ALTER TABLE db.docs DROP INDEX idx_title" + + # 3. OLAP compaction_policy validation and function_column.sequence_col support + adapter.cursor.execute.reset_mock() + adapter.create_table( + "db.ts_metrics", + target_columns_to_types={ + "id": exp.DataType.build("BIGINT", dialect="doris"), + "updated_at": exp.DataType.build("DATETIME", dialect="doris"), + "val": exp.DataType.build("DOUBLE", dialect="doris"), + }, + table_properties={ + "unique_key": exp.to_column("id"), + "function_column.sequence_col": exp.Literal.string("updated_at"), + "compaction_policy": exp.Literal.string("time_series"), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`ts_metrics` (`id` BIGINT, `updated_at` DATETIME, `val` DOUBLE) UNIQUE KEY (`id`) DISTRIBUTED BY HASH (`id`) BUCKETS 10 PROPERTIES ('function_column.sequence_col'='updated_at', 'compaction_policy'='time_series')" + ] + + with pytest.raises(SQLMeshError, match="Invalid Doris compaction_policy"): + adapter.create_table( + "db.bad_compaction", + target_columns_to_types={"id": exp.DataType.build("INT", dialect="doris")}, + table_properties={"compaction_policy": exp.Literal.string("invalid_policy")}, + ) + + # 4. _replace_by_key skips pre-DELETE when function_column.sequence_col is configured so sequence comparison works + adapter.cursor.execute.reset_mock() + adapter._replace_by_key( + target_table="db.ts_metrics", + source_table=parse_one("SELECT id, updated_at, val FROM db.src", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("BIGINT", dialect="doris"), + "updated_at": exp.DataType.build("DATETIME", dialect="doris"), + "val": exp.DataType.build("DOUBLE", dialect="doris"), + }, + key=[exp.to_column("id")], + is_unique_key=True, + table_properties={"function_column.sequence_col": exp.Literal.string("updated_at")}, + ) + sql_calls = to_sql_calls(adapter) + assert not any(call.startswith("DELETE FROM") for call in sql_calls) + assert any( + "PROPERTIES ('function_column.sequence_col'='updated_at')" in call for call in sql_calls + ) + + # 5. Self-referencing replace_query with aliased, default-catalog-qualified table reference preserves alias + adapter._default_catalog = "internal" + adapter.cursor.execute.reset_mock() + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcd1234") + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", schema="db", name="tbl", type=DataObjectType.TABLE + ), + ) + adapter.replace_query( + "db.tbl", + parse_one("SELECT t.id, t.val FROM internal.db.tbl AS t", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "val": exp.DataType.build("INT", dialect="doris"), + }, + ) + self_ref_calls = to_sql_calls(adapter) + assert any( + "CREATE TABLE IF NOT EXISTS `internal`.`db`.`__temp_tbl_abcd1234`" in c + for c in self_ref_calls + ) + assert any("FROM `internal`.`db`.`__temp_tbl_abcd1234` AS `t`" in c for c in self_ref_calls) + + # 6. Null-safe IS TRUE / IS FALSE / IS NOT TRUE / IS NOT FALSE rewrites outside DELETE + assert ( + adapter._to_sql( + parse_one( + "SELECT a IS TRUE, b IS FALSE, c IS NOT TRUE, d IS NOT FALSE FROM db.t", + read="doris", + ) + ) + == "SELECT COALESCE(`a`, FALSE) = TRUE, COALESCE(`b`, TRUE) = FALSE, NOT (COALESCE(`c`, FALSE) = TRUE), NOT (COALESCE(`d`, TRUE) = FALSE) FROM `db`.`t`" + ) + + # 7. _describe_columns_with_keys returns (cols, key_cols, has_explicit_keys) 3-tuple directly + adapter.cursor.fetchall.return_value = [ + ("id", "BIGINT", "No", "true", None, ""), + ("val", "VARCHAR(32)", "Yes", "false", None, ""), + ] + desc_cols, desc_keys, desc_explicit = adapter._describe_columns_with_keys("db.tbl") + assert list(desc_cols) == ["id", "val"] + assert desc_keys == {"id"} + assert desc_explicit is True + + # 8. _delete_by_time_transform_keys raises SQLMeshError on SELECT * or mismatched projection count + with pytest.raises(SQLMeshError, match="expected a SELECT query with 1 explicit projection"): + adapter.delete_from( + "db.events", + where=parse_one( + "DATE_TRUNC(ds, 'day') IN (SELECT * FROM db.staging)", + read="doris", + ), + ) + with pytest.raises( + SQLMeshError, match="expected 2 projection\\(s\\) matching key expressions, got 1" + ): + adapter.delete_from( + "db.events", + where=parse_one( + "(id, DATE_TRUNC(ds, 'day')) IN (SELECT id FROM db.staging)", + read="doris", + ), + ) + + # 9. Negated boolean DELETE predicates (`IS NOT TRUE` and `IS NOT FALSE`) preserve NULL semantics + # by splitting into two OR-free DELETE statements (`col = FALSE/TRUE` and `col IS NULL`). + adapter.cursor.execute.reset_mock() + adapter.delete_from( + "sqlmesh._intervals", + where=exp.and_( + exp.column("is_dev").is_(exp.true()).not_(), + exp.column("name").eq(exp.Literal.string("db.m1")), + ), + ) + adapter.delete_from( + "sqlmesh._intervals", + where=exp.and_( + exp.column("is_dev").is_(exp.false()).not_(), + exp.column("name").eq(exp.Literal.string("db.m1")), + ), + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` = FALSE AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` IS NULL AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` = TRUE AND `name` = 'db.m1'", + "DELETE FROM `sqlmesh`.`_intervals` WHERE `is_dev` IS NULL AND `name` = 'db.m1'", + ] + + # 10. Partitioned Iceberg FULL refresh (`replace_query`) clears all rows before inserting + # (including when a previously populated partition is omitted or when the result is empty), + # while Hive external tables continue using INSERT OVERWRITE TABLE. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="iceberg_catalog", + schema="silver", + name="curated_events", + type=DataObjectType.TABLE, + ), + ) + curated_cols = { + "user_id": exp.DataType.build("BIGINT", dialect="doris"), + "event_type": exp.DataType.build("STRING", dialect="doris"), + "event_ts": exp.DataType.build("DATETIME", dialect="doris"), + } + curated_partitions = [ + parse_one("day(event_ts)", read="doris"), + parse_one("bucket(16, user_id)", read="doris"), + ] + # 10a. Refresh with a query omitting a previously populated partition clears all partitions first + adapter.replace_query( + "iceberg_catalog.silver.curated_events", + parse_one( + "SELECT user_id, event_type, event_ts FROM bronze.raw_events WHERE event_ts >= '2026-01-02'", + read="doris", + ), + target_columns_to_types=curated_cols, + table_format="iceberg", + partitioned_by=curated_partitions, + ) + # 10b. Refresh with a SQL query returning an empty result (`WHERE 1 = 0`) or empty `source_queries` + # clears all partitions via full-table DELETE instead of no-oping like Doris INSERT OVERWRITE + adapter.replace_query( + "iceberg_catalog.silver.curated_events", + parse_one( + "SELECT user_id, event_type, event_ts FROM bronze.raw_events WHERE 1 = 0", + read="doris", + ), + target_columns_to_types=curated_cols, + table_format="iceberg", + partitioned_by=curated_partitions, + ) + adapter._insert_overwrite_by_condition( + "iceberg_catalog.silver.curated_events", + source_queries=[], + target_columns_to_types=curated_cols, + table_format="iceberg", + partitioned_by=curated_partitions, + ) + # 10c. Hive external table continues using INSERT OVERWRITE TABLE + adapter.replace_query( + "hive_catalog.silver.curated_events", + parse_one("SELECT user_id, event_type, event_ts FROM bronze.raw_events", read="doris"), + target_columns_to_types=curated_cols, + table_format="hive", + ) + assert to_sql_calls(adapter) == [ + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE NOT `user_id` IS NULL", + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE `user_id` IS NULL", + "INSERT INTO `iceberg_catalog`.`silver`.`curated_events` (`user_id`, `event_type`, `event_ts`) SELECT `user_id`, `event_type`, `event_ts` FROM `bronze`.`raw_events` WHERE `event_ts` >= '2026-01-02'", + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE NOT `user_id` IS NULL", + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE `user_id` IS NULL", + "INSERT INTO `iceberg_catalog`.`silver`.`curated_events` (`user_id`, `event_type`, `event_ts`) SELECT `user_id`, `event_type`, `event_ts` FROM `bronze`.`raw_events` WHERE 1 = 0", + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE NOT `user_id` IS NULL", + "DELETE FROM `iceberg_catalog`.`silver`.`curated_events` WHERE `user_id` IS NULL", + "INSERT OVERWRITE TABLE `hive_catalog`.`silver`.`curated_events` (`user_id`, `event_type`, `event_ts`) SELECT `user_id`, `event_type`, `event_ts` FROM `bronze`.`raw_events`", + ] + + # 10d. Truncate hints are stored in thread-local state (`_get_thread_state`) so concurrent threads + # cannot leak Iceberg truncate hints into an OLAP `_truncate_table` call. + import threading + + def _set_other_thread_hints() -> None: + other_state = adapter._get_thread_state() + other_state.truncate_columns_hint = curated_cols + other_state.truncate_is_iceberg_hint = True + + worker = threading.Thread(target=_set_other_thread_hints) + worker.start() + worker.join() + + adapter.cursor.execute.reset_mock() + adapter._truncate_table("analytics.olap_tbl") + assert to_sql_calls(adapter) == ["TRUNCATE TABLE `analytics`.`olap_tbl`"] + + # 11. `ctas` with `exists=True` and a self-referencing query on an OLAP table uses CTAS (`CREATE TABLE IF NOT EXISTS ... AS SELECT`) + # instead of `INSERT INTO` so targeting an existing table does not duplicate rows. + adapter.cursor.execute.reset_mock() + adapter.ctas( + "analytics.self_ref_olap", + parse_one("SELECT id, val FROM analytics.self_ref_olap", read="doris"), + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "val": exp.DataType.build("INT", dialect="doris"), + }, + exists=True, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `analytics`.`self_ref_olap` AS SELECT CAST(`id` AS INT) AS `id`, CAST(`val` AS INT) AS `val` FROM (SELECT `id`, `val` FROM `analytics`.`self_ref_olap`) AS `_subquery`" + ] + + # 12. `ctas` with `exists=True` on an already-existing internal Iceberg table skips the `INSERT INTO` loop + # so preexisting rows are not duplicated. + adapter.cursor.execute.reset_mock() + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject( + catalog="internal", + schema="analytics", + name="existing_iceberg_tbl", + type=DataObjectType.TABLE, + ), + ) + adapter.ctas( + "analytics.existing_iceberg_tbl", + parse_one("SELECT user_id, event_type, event_ts FROM bronze.raw_events", read="doris"), + target_columns_to_types=curated_cols, + exists=True, + table_format="iceberg", + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `analytics`.`existing_iceberg_tbl` (`user_id` BIGINT, `event_type` STRING, `event_ts` DATETIME) ENGINE=iceberg" + ] + + # 13. First async MV creation (`replace=True`) skips full base-table `SHOW FULL TABLES` + `DESC ... ALL` scans + # when absent from `SHOW ALTER TABLE MATERIALIZED VIEW`, while still detecting and dropping an existing + # synchronous MV when `get_data_object` returns `None`. + adapter.cursor.execute.reset_mock() + mocker.patch.object(adapter, "get_data_object", return_value=None) + mocker.patch.object( + adapter, + "fetchall", + side_effect=[ + # 13a. Brand-new async MV: `SHOW ALTER TABLE MATERIALIZED VIEW` is empty -> does NOT call `SHOW FULL TABLES` or `DESC ... ALL` + [], + # 13b. Existing sync MV replaced by async MV: `get_data_object` returned None, but `SHOW ALTER TABLE MATERIALIZED VIEW` + `DESC` finds it + [ + ( + 1001, + "raw_events", + "2026-01-01", + "2026-01-01", + "raw_events", + "sync_to_async_mv", + 2001, + 3001, + "FINISHED", + "", + "100%", + 3600, + ) + ], + [ + ("raw_events", "DUP_KEYS", "user_id", "BIGINT", "Yes", "true", None, ""), + ("sync_to_async_mv", "AGG_KEYS", "mv_user_id", "BIGINT", "Yes", "true", None, ""), + ], + ], + ) + adapter.create_view( + "analytics.new_async_mv", + parse_one("SELECT user_id FROM bronze.raw_events", read="doris"), + replace=True, + materialized=True, + ) + adapter.create_view( + "analytics.sync_to_async_mv", + parse_one("SELECT user_id FROM bronze.raw_events", read="doris"), + replace=True, + materialized=True, + ) + assert to_sql_calls(adapter) == [ + "DROP MATERIALIZED VIEW IF EXISTS `analytics`.`new_async_mv`", + "CREATE MATERIALIZED VIEW `analytics`.`new_async_mv` AS SELECT `user_id` FROM `bronze`.`raw_events`", + "USE `analytics`", + "DROP MATERIALIZED VIEW IF EXISTS `sync_to_async_mv` ON `analytics`.`raw_events`", + "CREATE MATERIALIZED VIEW `analytics`.`sync_to_async_mv` AS SELECT `user_id` FROM `bronze`.`raw_events`", + ] + + # 14. `_wait_for_schema_change` polls `SHOW ALTER TABLE COLUMN` until jobs reach `FINISHED` / `CANCELLED`. + adapter.cursor.execute.reset_mock() + adapter.cursor.fetchall.side_effect = [ + [(1, "docs", "2026-01-01", "2026-01-01", "docs", 10, 20, 30, 0, "RUNNING", "", 60)], + [(1, "docs", "2026-01-01", "2026-01-01", "docs", 10, 20, 30, 0, "FINISHED", "", 60)], + ] + DorisEngineAdapter._wait_for_schema_change(adapter, "db.docs", timeout=1.0) + assert to_sql_calls(adapter) == [ + "SHOW ALTER TABLE COLUMN FROM `db` WHERE TableName = 'docs'", + "SHOW ALTER TABLE COLUMN FROM `db` WHERE TableName = 'docs'", + ] + adapter.cursor.fetchall.side_effect = None + + # 15. `not_null_columns` in `physical_properties` marks columns NOT NULL on CREATE TABLE and wraps CTAS projections in NON_NULLABLE(...) + adapter.cursor.execute.reset_mock() + adapter.create_table( + "analytics.searchable_logs", + { + "log_id": exp.DataType.build("BIGINT", dialect="doris"), + "message": exp.DataType.build("STRING", dialect="doris"), + "embedding": exp.DataType.build("ARRAY", dialect="doris"), + }, + table_properties={ + "unique_key": exp.Tuple(expressions=[exp.column("log_id")]), + "not_null_columns": exp.Tuple(expressions=[exp.column("embedding")]), + "enable_unique_key_merge_on_write": exp.Literal.string("true"), + }, + ) + adapter.ctas( + "analytics.searchable_logs_ctas", + parse_one( + "SELECT log_id, message, embedding FROM raw.logs", + read="doris", + ), + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.column("log_id")]), + "not_null_columns": exp.Tuple(expressions=[exp.column("embedding")]), + }, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs` (`log_id` BIGINT, `message` STRING, `embedding` ARRAY NOT NULL) UNIQUE KEY (`log_id`) DISTRIBUTED BY HASH (`log_id`) BUCKETS 10 PROPERTIES ('enable_unique_key_merge_on_write'='true')", + "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs_ctas` DUPLICATE KEY (`log_id`) AS SELECT `log_id`, `message`, NON_NULLABLE(`embedding`) AS `embedding` FROM `raw`.`logs`", + ] + with pytest.raises(SQLMeshError, match="not_null_columns"): + adapter.create_table( + "analytics.bad_not_null", + {"log_id": exp.DataType.build("BIGINT", dialect="doris")}, + table_properties={ + "not_null_columns": exp.Tuple(expressions=[exp.column("missing_col")]), + }, + ) diff --git a/tests/core/engine_adapter/test_starrocks.py b/tests/core/engine_adapter/test_starrocks.py index db0b1cc4ae..700c297ded 100644 --- a/tests/core/engine_adapter/test_starrocks.py +++ b/tests/core/engine_adapter/test_starrocks.py @@ -42,10 +42,10 @@ pytestmark = [pytest.mark.starrocks, pytest.mark.engine] -def _load_sql_model(model_sql: str) -> SqlModel: +def _load_sql_model(model_sql: str, default_catalog: t.Optional[str] = None) -> SqlModel: """Parse StarRocks MODEL SQL into a SqlModel instance.""" expressions = parse(model_sql, default_dialect="starrocks") - return t.cast(SqlModel, load_sql_based_model(expressions)) + return t.cast(SqlModel, load_sql_based_model(expressions, default_catalog=default_catalog)) def _columns(model: SqlModel) -> t.Dict[str, exp.DataType]: @@ -2486,3 +2486,91 @@ def test_non_starrocks_engine_is_not_affected(self) -> None: "excluded_refresh_tables" in StarRocksEngineAdapter.RESOLVE_TABLE_REFS_IN_PHYSICAL_PROPERTIES ) + + def test_tuple_and_default_catalog_refs_are_resolved( + self, + make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter], + ) -> None: + """Tuple values and 2-part refs against 3-part default_catalog snapshots resolve to physical tables.""" + base_model = _load_sql_model( + """ + MODEL ( + name starrocks.managed_model, + kind FULL, + dialect starrocks, + columns (a INT) + ); + SELECT 1 AS a; + """, + ) + base_snapshot = self._make_snapshot(base_model) + physical_name = exp.to_table(base_snapshot.table_name()) + expected_physical = f"{physical_name.db}.{physical_name.name}" + + mv_model = _load_sql_model( + """ + MODEL ( + name starrocks.test_mv_tuple, + kind VIEW (materialized true), + dialect starrocks, + columns (a INT), + physical_properties ( + refresh_scheme = ASYNC, + excluded_trigger_tables = (starrocks.managed_model, external_db.raw_table) + ) + ); + SELECT 1 AS a; + """, + ) + + adapter = make_mocked_engine_adapter(StarRocksEngineAdapter) + snapshots = {base_snapshot.name: base_snapshot} + rendered = mv_model.render_physical_properties( + snapshots=snapshots, + engine_adapter=adapter, + ) + assert rendered["excluded_trigger_tables"] == exp.Literal.string( + f"{expected_physical},external_db.raw_table" + ) + + # Also verify 2-part reference resolution when snapshots are normalized with a 3-part default_catalog + catalog_base_model = _load_sql_model( + """ + MODEL ( + name starrocks.managed_model, + kind FULL, + dialect starrocks, + columns (a INT) + ); + SELECT 1 AS a; + """, + default_catalog="default_catalog", + ) + catalog_base_snapshot = self._make_snapshot(catalog_base_model) + catalog_physical_name = exp.to_table(catalog_base_snapshot.table_name()) + expected_catalog_physical = f"{catalog_physical_name.catalog}.{catalog_physical_name.db}.{catalog_physical_name.name}" + + catalog_mv_model = _load_sql_model( + """ + MODEL ( + name starrocks.test_mv_tuple, + kind VIEW (materialized true), + dialect starrocks, + columns (a INT), + physical_properties ( + refresh_scheme = ASYNC, + excluded_trigger_tables = (starrocks.managed_model, external_db.raw_table) + ) + ); + SELECT 1 AS a; + """, + default_catalog="default_catalog", + ) + + rendered_with_catalog = catalog_mv_model.render_physical_properties( + snapshots={catalog_base_snapshot.name: catalog_base_snapshot}, + engine_adapter=adapter, + ) + assert rendered_with_catalog["excluded_trigger_tables"] == exp.Literal.string( + f"{expected_catalog_physical},external_db.raw_table" + ) diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index c506d401d9..b529eeac38 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -15,6 +15,7 @@ ClickhouseConnectionConfig, ConnectionConfig, DatabricksConnectionConfig, + DorisConnectionConfig, DuckDBAttachOptions, DuckDBConnectionConfig, FabricConnectionConfig, @@ -2479,3 +2480,174 @@ def test_starrocks(make_config): assert advanced_config.charset == "utf8mb4" assert advanced_config.ssl_disabled is True assert advanced_config.concurrent_tasks == 10 + + +def test_doris(make_config): + """Test DorisConnectionConfig basic functionality and multi-catalog/session options""" + # Basic configuration + config = make_config( + type="doris", + host="localhost", + user="root", + password="password", + port=9030, + database="demo", + check_import=False, + ) + assert isinstance(config, DorisConnectionConfig) + assert config.type_ == "doris" + assert config.host == "localhost" + assert config.user == "root" + assert config.password == "password" + assert config.port == 9030 + assert config.database == "demo" + assert config.catalog is None + assert config.get_catalog() is None + assert config.DIALECT == "doris" + assert config.DISPLAY_NAME == "Apache Doris" + assert config.DISPLAY_ORDER == 20 + assert config.is_recommended_for_state_sync is False + assert config.is_forbidden_for_state_sync is True + assert config._cursor_init is None + + # Test with minimal configuration (using default port and optional password) + minimal_config = make_config( + type="doris", + host="fe.doris.cluster", + user="doris_user", + check_import=False, + ) + assert isinstance(minimal_config, DorisConnectionConfig) + assert minimal_config.port == 9030 # Default Doris FE port + assert minimal_config.host == "fe.doris.cluster" + assert minimal_config.user == "doris_user" + assert minimal_config.password is None + + # Test with Iceberg catalog, session variables, and TLS options + advanced_config = make_config( + type="doris", + host="doris-fe", + user="admin", + password="admin123", + port=9030, + database="analytics", + catalog="iceberg_cat", + charset="utf8mb4", + ssl_ca="/path/to/ca.pem", + ssl_verify_cert=True, + session_variables={"query_timeout": 300, "workload_group": "etl_group"}, + concurrent_tasks=10, + check_import=False, + ) + assert isinstance(advanced_config, DorisConnectionConfig) + assert advanced_config.charset == "utf8mb4" + assert advanced_config.catalog == "iceberg_cat" + assert advanced_config.get_catalog() == "iceberg_cat" + assert advanced_config.concurrent_tasks == 10 + assert "database" not in advanced_config._static_connection_kwargs + assert advanced_config._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "verify_mode": True, + } + assert advanced_config._extra_engine_config == { + "database": "analytics", + "catalog": "iceberg_cat", + } + + mock_cursor = MagicMock() + assert advanced_config._cursor_init is not None + advanced_config._cursor_init(mock_cursor) + assert [call.args[0] for call in mock_cursor.execute.call_args_list] == [ + "SET `query_timeout` = 300", + "SET `workload_group` = 'etl_group'", + "SWITCH `iceberg_cat`", + "USE `analytics`", + ] + + ssl_bool_config = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl=True, + check_import=False, + ) + assert ssl_bool_config._static_connection_kwargs["ssl"] == {"check_hostname": False} + + ssl_false_config = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl=False, + ssl_ca="/path/to/ca.pem", + check_import=False, + ) + assert "ssl" not in ssl_false_config._static_connection_kwargs + + ssl_empty_dict_config = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={}, + check_import=False, + ) + assert ssl_empty_dict_config._static_connection_kwargs["ssl"] == {"check_hostname": False} + + ssl_verify_cert_false_identity_true = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl_ca="/path/to/ca.pem", + ssl_verify_cert=False, + ssl_verify_identity=True, + check_import=False, + ) + assert ssl_verify_cert_false_identity_true._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "verify_mode": False, + "check_hostname": False, + } + + ssl_dict_verify_mode_false = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"ca": "/path/to/ca.pem", "verify_mode": False, "check_hostname": True}, + check_import=False, + ) + assert ssl_dict_verify_mode_false._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "verify_mode": False, + "check_hostname": False, + } + + ssl_dict_verify_mode_none_str = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"ca": "/path/to/ca.pem", "verify_mode": "none"}, + ssl_verify_identity=True, + check_import=False, + ) + assert ssl_dict_verify_mode_none_str._static_connection_kwargs["ssl"] == { + "ca": "/path/to/ca.pem", + "verify_mode": "none", + "check_hostname": False, + } + + with pytest.raises(ConfigError, match="Invalid Doris session variable name"): + make_config( + type="doris", + host="doris-fe", + user="admin", + session_variables={"bad key; DROP": 1}, + check_import=False, + ) + + with pytest.raises(ConfigError, match="Invalid Doris session variable value"): + make_config( + type="doris", + host="doris-fe", + user="admin", + session_variables={"timeout": float("nan")}, + check_import=False, + ) From ca25fda7fc5fc101877b6ff41f32464f6490ff28 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:06:33 -0800 Subject: [PATCH 02/54] Docs: Clarify optional Doris connection database and explicit VARCHAR(n) key lengths Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 2532b17bf8..dff769f501 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -84,7 +84,9 @@ pip install "sqlmesh[doris,postgres]" port: 9030 user: root password: "" - database: analytics + # Optional: omit on a fresh Doris instance so SQLMesh can create model schemas automatically, + # or create the database first (CREATE DATABASE IF NOT EXISTS analytics) before setting it here: + # database: analytics state_connection: type: duckdb database: state.db @@ -108,6 +110,7 @@ pip install "sqlmesh[doris,postgres]" port: 9030 user: doris_user password: {{ env_var('DORIS_PASSWORD') | tojson }} + # Must already exist in Doris if specified (or omit to let SQLMesh qualify schemas directly) database: analytics catalog: internal connect_timeout: 30 @@ -909,4 +912,4 @@ If you are migrating StarRocks SQLMesh models or existing Doris DDL conventions, | `errCode = 2, replication num 3 is larger than available backends 1` | Doris defaults to `replication_num = 3`, which fails on single-BE local/CI clusters. | Set `replication_num: 1` under `model_defaults.physical_properties` in `config.yaml` for local dev. | | `PARTITION BY` omitted on OLAP table (logged as a warning) | Doris OLAP tables with `PARTITION BY (...) ()` and no partition creation strategy have zero partitions and reject all `INSERT`s. | Add `auto_partition = 'true'`, `'dynamic_partition.enable' = 'true'`, or explicit `partitions = '...'` to `physical_properties`, or set `partition_by_time_column false` in `INCREMENTAL_BY_TIME_RANGE` if you do not want OLAP partitioning. | | `Doris only supports creating views in the 'internal' catalog` | External Iceberg catalogs do not support Doris `VIEW` or `MATERIALIZED VIEW` objects. | Set `virtual_environment_mode: dev_only` and `environment_catalog_mapping: {"^(?!prod$).*": "internal"}` in `config.yaml`. | -| `Key column cannot be of type STRING / DOUBLE / ARRAY` | Doris forbids `STRING`/`TEXT`, floating-point (`FLOAT`/`DOUBLE`), and complex types as OLAP key columns. | SQLMesh automatically reorders valid key columns first and coerces `STRING`/`TEXT` key columns to `VARCHAR(255)`. For explicit keys on `DOUBLE`/`FLOAT`, use `DECIMAL` or an integer/date key column. | +| `Key column cannot be of type STRING / DOUBLE / ARRAY` | Doris forbids `STRING`/`TEXT`, floating-point (`FLOAT`/`DOUBLE`), and complex types as OLAP key columns. | SQLMesh automatically reorders valid key columns first and coerces untyped `STRING`/`TEXT` key columns to `VARCHAR(255)` (for keys longer than 255 bytes, cast or declare an explicit `VARCHAR(n)` up to `VARCHAR(65533)`). For explicit keys on `DOUBLE`/`FLOAT`, use `DECIMAL` or an integer/date key column. | From 89409ecafa657c4849788df8de74e304f86896ca Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:51:55 -0800 Subject: [PATCH 03/54] Fix: Harden Doris catalog tracking, async DDL sync, and NOT NULL materialization Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 85 +- sqlmesh/core/dialect.py | 7 +- sqlmesh/core/engine_adapter/doris.py | 928 ++++++++++++++---- .../integration/test_integration.py | 2 +- .../integration/test_integration_doris.py | 307 ++++++ tests/core/engine_adapter/test_doris.py | 285 +++++- tests/core/test_dialect.py | 24 + 7 files changed, 1412 insertions(+), 226 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index dff769f501..f4f0d55083 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -212,6 +212,13 @@ MODEL ( target.updated_at = source.updated_at ) ); + +SELECT + order_id::BIGINT AS order_id, + status::VARCHAR(32) AS status, + updated_at::DATETIME AS updated_at +FROM raw.orders_changelog +WHERE updated_at BETWEEN @start_ts AND @end_ts; ``` ### AGGREGATE KEY model @@ -240,7 +247,7 @@ GROUP BY user_id, event_type; ## Distribution & Bucketing -Doris requires a `DISTRIBUTED BY` clause on every OLAP table. SQLMesh supports three ways to configure distribution: +Doris OLAP tables distribute tablets across backends via `DISTRIBUTED BY HASH(...)` or `DISTRIBUTED BY RANDOM` (on Doris 2.1+, omitting `DISTRIBUTED BY` on `DUPLICATE KEY` or `AGGREGATE KEY` tables defaults to `DISTRIBUTED BY RANDOM BUCKETS AUTO`, whereas `UNIQUE KEY` tables and Doris 2.0 require explicit hash distribution). SQLMesh supports three ways to configure distribution: 1. **SQLMesh `clustered_by` + optional `buckets`** (recommended for cross-engine portability): ```sql @@ -252,6 +259,11 @@ Doris requires a `DISTRIBUTED BY` clause on every OLAP table. SQLMesh supports t buckets = 16 ) ); + + SELECT + user_id::BIGINT AS user_id, + event_type::VARCHAR(64) AS event_type + FROM raw.events; ``` 2. **Shorthand string `distributed_by`**: ```sql @@ -281,7 +293,7 @@ If none of `clustered_by` or `distributed_by` is provided: ## Partitioning -Doris OLAP tables support **Multi-Range Batch Partitioning**, **Explicit Partition Definitions**, **Dynamic Partitioning**, and **Auto Partitioning** (Doris 2.1+). You can specify the partition expression via the top-level `partitioned_by` model property or via `partitioned_by` / `partition_by` inside `physical_properties`. +Doris OLAP tables support **Multi-Range Batch Partitioning**, **Explicit Partition Definitions**, **Dynamic Partitioning**, and **Auto Partitioning** (Doris 2.1+). You can specify the partition expression via the top-level `partitioned_by` model property or via `partition_by` (or quoted `'partitioned_by'`) inside `physical_properties`. !!! note "OLAP Partitioning Requirements" - On Doris OLAP tables, `PARTITION BY` requires at least one of `partitions`, `auto_partition = 'true'`, or `'dynamic_partition.enable' = 'true'` in `physical_properties`. If none is provided, SQLMesh omits the `PARTITION BY` clause so Doris does not create a zero-partition table that rejects inserts. @@ -300,6 +312,12 @@ MODEL ( partitions = 'FROM ("2023-01-01") TO ("2027-01-01") INTERVAL 1 MONTH' ) ); + +SELECT + event_date::DATE AS event_date, + user_id::BIGINT AS user_id +FROM raw.events +WHERE event_date BETWEEN @start_ds AND @end_ds; ``` ### Auto partitioning (`auto_partition = true`) @@ -317,6 +335,12 @@ MODEL ( auto_partition = 'true' ) ); + +SELECT + event_time::DATETIME AS event_time, + user_id::BIGINT AS user_id +FROM raw.events +WHERE event_time BETWEEN @start_ts AND @end_ts; ``` ### Dynamic partitioning (`dynamic_partition.enable = 'true'`) @@ -342,6 +366,12 @@ MODEL ( 'dynamic_partition.buckets' = '16' ) ); + +SELECT + event_date::DATE AS event_date, + user_id::BIGINT AS user_id +FROM raw.events +WHERE event_date BETWEEN @start_ds AND @end_ds; ``` --- @@ -358,7 +388,8 @@ Because `DorisEngineAdapter` supports multi-catalog 3-part naming (`catalog.db.t MODEL ( name silver.user_sessions, kind INCREMENTAL_BY_TIME_RANGE ( - time_column session_date + time_column session_date, + partition_by_time_column false ) ); @@ -466,7 +497,7 @@ SQLMesh's Doris dialect natively parses and transpiles Apache Iceberg snapshot t FROM iceberg_catalog.bronze.clickstream@tag(v1_release) AS c; ``` - **Iceberg Compaction (`rewrite_data_files`) & Branch Management via Post-Statements**: - You can run Iceberg file compaction (`rewrite_data_files`, requires Doris 4.0.2+), snapshot expiration (`expire_snapshots`, requires Doris 4.1.0+), or branch/tag management (requires Doris 3.1+) in model post-statements. Because Doris does not support Iceberg orphan-file cleanup (`remove_orphan_files`), run orphan-file cleanup through a supported external Iceberg maintenance tool (such as Apache Spark's `remove_orphan_files` procedure): + You can run Iceberg file compaction (`rewrite_data_files`, requires Doris 4.0.2+), snapshot expiration (`expire_snapshots`, requires Doris 4.1.0+ with ISO-8601 `yyyy-MM-ddTHH:mm:ss` or epoch-millisecond `older_than`), or branch/tag management (requires Doris 3.1+) in model post-statements. Because Doris does not support Iceberg orphan-file cleanup (`remove_orphan_files`), run orphan-file cleanup through a supported external Iceberg maintenance tool (such as Apache Spark's `remove_orphan_files` procedure): ```sql MODEL ( name iceberg_catalog.silver.curated_events, @@ -682,6 +713,11 @@ MODEL ( file_cache_ttl_seconds = '86400' ) ); + +SELECT + event_id::BIGINT AS event_id, + payload::STRING AS payload +FROM raw.cloud_events; ``` --- @@ -727,7 +763,8 @@ Because SQLMesh `VIEW (materialized true)` models are promoted via a virtual vie MODEL ( name analytics.user_events, kind INCREMENTAL_BY_TIME_RANGE ( - time_column event_time + time_column event_time, + partition_by_time_column false ), grain (user_id, event_time) ); @@ -737,7 +774,7 @@ SELECT event_type::VARCHAR(64) AS event_type, event_time::DATETIME AS event_time FROM raw.user_events -WHERE event_time BETWEEN @start_ds AND @end_ds; +WHERE event_time BETWEEN @start_ts AND @end_ts; -- Note: For INCREMENTAL_BY_TIME_RANGE models, every synchronous MV (rollup) on the -- base table must include the time_column (event_time) in its SELECT and GROUP BY @@ -757,10 +794,12 @@ WHERE event_time BETWEEN @start_ds AND @end_ds; ### Inverted & Vector Indexes, Full-Text Search & Compaction (Doris 2.1–5.0) -You can attach Doris inverted indexes (`USING INVERTED` with tokenizer `PROPERTIES`), bloom filter indexes (`USING NGRAM_BF`), or Doris 4.x+ ANN vector indexes (`USING ANN` with `hnsw` or `ivf` index types; ANN indexes on `UNIQUE KEY` tables require Doris 4.1.4+ with Merge-on-Write enabled, whereas earlier Doris 4.x versions only support ANN indexes on `DUPLICATE KEY` tables) using post-statements. Because Doris requires ANN vector index columns to be `ARRAY NOT NULL` at table creation time (and Doris does not allow `ALTER TABLE ... MODIFY COLUMN` to change an existing nullable column to `NOT NULL`), declare `not_null_columns = ('embedding')` in `physical_properties` so SQLMesh defines the vector column as `ARRAY NOT NULL` during `CREATE TABLE` (or wraps it in `NON_NULLABLE(...)` during `CTAS`). For `INCREMENTAL_*` models, SQLMesh executes `@runtime_stage = 'creating'` post-statements on the newly created empty table before inserting rows during `@runtime_stage = 'evaluating'`, so all inserted rows are indexed automatically on write in both compute-storage integrated and decoupled (`storage_vault_name`) deployments. For `FULL` models—where initial table creation populates rows via `CTAS` before `@runtime_stage = 'creating'` post-statements run—or when adding an index to a table with existing rows: +You can attach Doris inverted indexes (`USING INVERTED` with tokenizer `PROPERTIES`), bloom filter indexes (`USING NGRAM_BF`), or Doris 4.x+ ANN vector indexes (`USING ANN` with `hnsw` or `ivf` index types; ANN indexes on `UNIQUE KEY` tables require Doris 4.1.4+ with Merge-on-Write enabled, whereas earlier Doris 4.x versions only support ANN indexes on `DUPLICATE KEY` tables) using post-statements. Because Doris requires ANN vector index columns to be `ARRAY NOT NULL` at table creation time (and Doris ignores `NON_NULLABLE(...)` on non-`AUTO PARTITION` CTAS columns while forbidding `ALTER TABLE ... MODIFY COLUMN` from changing an existing nullable column to `NOT NULL`), declare `not_null_columns = ('embedding')` in `physical_properties` so SQLMesh materializes the table schema with `embedding ARRAY NOT NULL` before inserting rows. + +Because SQLMesh evaluates and inserts the initial batch of rows into a new snapshot table **before** executing `@IF(@runtime_stage = 'creating', ...)` post-statements, Doris `ALTER TABLE ... ADD INDEX` alone only registers the index for subsequent writes: -- **Compute-storage integrated deployments**: Run `BUILD INDEX ON @this_model` in `@runtime_stage = 'creating'` (after `ADD INDEX`) to backfill the inverted index on rows written prior to `ADD INDEX`. -- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail evaluation); `ADD INDEX` only indexes newly written rows, so existing rows require a table rewrite or compaction to populate the index. +- **Compute-storage integrated deployments**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` to backfill the index on the initial batch before downstream models run (`DorisEngineAdapter` automatically polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` until `FINISHED`). +- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes, and run `sqlmesh plan --restate-model ` if you need the initial batch rewritten through the newly registered index. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: @@ -793,24 +832,27 @@ SELECT created_at::DATETIME(6) AS created_at FROM raw.logs; --- DorisEngineAdapter automatically polls SHOW ALTER TABLE COLUMN after each ALTER TABLE --- statement so asynchronous column/index schema-change jobs finish before the next statement runs. +-- DorisEngineAdapter automatically polls SHOW ALTER TABLE COLUMN and SHOW BUILD INDEX +-- so asynchronous index creation and backfill jobs finish before downstream models run. @IF( @runtime_stage = 'creating', ALTER TABLE @this_model ADD INDEX idx_message (message) USING INVERTED PROPERTIES ("parser" = "english", "support_phrase" = "true") COMMENT 'Full-text inverted index' ); +@IF( + @runtime_stage = 'creating', + BUILD INDEX idx_message ON @this_model +); + @IF( @runtime_stage = 'creating', ALTER TABLE @this_model ADD INDEX idx_embedding (embedding) USING ANN PROPERTIES ("index_type" = "hnsw", "metric_type" = "l2_distance", "dim" = "3") COMMENT 'ANN vector index' ); --- Only for FULL models (where CTAS writes rows before @runtime_stage = 'creating') in --- compute-storage integrated deployments; do NOT run BUILD INDEX in compute-storage decoupled mode: --- @IF( --- @runtime_stage = 'creating', --- BUILD INDEX idx_message ON @this_model --- ); +@IF( + @runtime_stage = 'creating', + BUILD INDEX idx_embedding ON @this_model +); ``` !!! note "Recovering from Failed Index Post-Statements & Forward-Only Migrations" @@ -864,6 +906,13 @@ MODEL ( number_of_rows(threshold := 1) ) ); + +SELECT + order_id::BIGINT AS order_id, + customer_id::BIGINT AS customer_id, + order_date::DATE AS order_date +FROM raw.orders +WHERE order_date BETWEEN @start_ds AND @end_ds; ``` - **Constant folding in `LIMIT` clauses**: Doris requires `LIMIT` arguments to be integer literals rather than arithmetic expressions (such as `LIMIT 10 + 1` emitted by `number_of_rows`). `DorisEngineAdapter` automatically folds constant arithmetic in `LIMIT` clauses before execution. @@ -901,7 +950,7 @@ If you are migrating StarRocks SQLMesh models or existing Doris DDL conventions, | `table_type = 'PRIMARY' \| 'UNIQUE' \| 'DUPLICATE' \| 'AGGREGATE'`, `keys = ('c1', 'c2')` | Maps directly to the corresponding Doris key model (`PRIMARY` maps to Doris `UNIQUE KEY`) | | `clustered_by (c1)` + `physical_properties (buckets = 10)` | Emits `DISTRIBUTED BY HASH(c1) BUCKETS 10` | | `physical_properties (partition_by = ...)` | Supported alias for top-level `partitioned_by` | -| `physical_properties (not_null_columns = ('c1', 'c2'))` | Defines specified OLAP columns as `NOT NULL` on `CREATE TABLE` (and wraps `CTAS` projections in `NON_NULLABLE(...)`) | +| `physical_properties (not_null_columns = ('c1', 'c2'))` | Defines specified OLAP or Iceberg columns as `NOT NULL` at `CREATE TABLE` time (materializing OLAP `CTAS` via schema DDL + `INSERT INTO`, and preserving `NOT NULL` across `ALTER TABLE ... MODIFY COLUMN` widenings) | --- diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 0a673a421d..766595f2da 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -497,7 +497,9 @@ def _parse_types( start = self._curr udt_name = ( self._curr.text.upper() - if self._curr and self._curr.token_type == TokenType.USERDEFINED + if isinstance(self, Doris.Parser) + and self._curr + and self._curr.token_type == TokenType.USERDEFINED else None ) if udt_name == "AGG_STATE" and not check_func: @@ -536,6 +538,7 @@ def _parse_types( if ( udt_name + and udt_name != "USER-DEFINED" and isinstance(parsed_type, exp.DataType) and parsed_type.this == exp.DataType.Type.USERDEFINED and not parsed_type.args.get("kind") @@ -1906,12 +1909,14 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore _override(Snowflake.Parser, _parse_table_parts) _override(TSQL.Parser, _parse_alter_table_alter) _override(TSQL.Generator, altercolumn_sql) + _override(Doris.Parser, _parse_alter_table_alter) _override(Doris.Parser, _doris_parse_range) _override(Doris.Parser, _doris_parse_table_parts) _override(Doris.Parser, _doris_parse_version) _override(Doris.Parser, _doris_parse_table) _override(Doris.Parser, _doris_parse_index_constraint) _override(Doris.Parser, _doris_parse_index_params) + _override(Doris.Generator, altercolumn_sql) _override(Doris.Generator, columndef_sql) _override(Doris.Generator, _doris_indexconstraintoption_sql) _override(Doris.Generator, _doris_indexparameters_sql) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index bc4af509ff..c6db5bb2df 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections import defaultdict +from dataclasses import dataclass import datetime from decimal import Decimal import hashlib @@ -38,7 +39,9 @@ SourceQuery, set_catalog, ) +from sqlmesh.core.schema_diff import TableAlterChangeColumnTypeOperation from sqlmesh.utils import columns_to_types_all_known, get_source_columns_to_types, random_id +from sqlmesh.utils.connection_pool import ConnectionPool from sqlmesh.utils.errors import SQLMeshError if t.TYPE_CHECKING: @@ -52,6 +55,19 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class _DorisNotNullChangeColumnTypeOperation(TableAlterChangeColumnTypeOperation): + @property + def _alter_actions(self) -> t.List[exp.Expr]: + return [ + exp.AlterColumn( + this=self.column, + dtype=self.column_type, + allow_null=False, + ) + ] + + @set_catalog() class DorisEngineAdapter( EngineAdapterWithIndexSupport, @@ -239,6 +255,14 @@ def _active_catalog(self) -> t.Optional[str]: def _active_catalog(self, value: t.Optional[str]) -> None: self._get_thread_state().active_catalog = value + @property + def _active_catalog_conn_token(self) -> t.Optional[t.Tuple[int, int]]: + return getattr(self._get_thread_state(), "active_catalog_conn_token", None) + + @_active_catalog_conn_token.setter + def _active_catalog_conn_token(self, value: t.Optional[t.Tuple[int, int]]) -> None: + self._get_thread_state().active_catalog_conn_token = value + @property def _needs_catalog_restore(self) -> bool: return bool(getattr(self._get_thread_state(), "needs_catalog_restore", False)) @@ -255,6 +279,22 @@ def _last_described_key_cols(self) -> t.Set[str]: def _last_described_key_cols(self, value: t.Set[str]) -> None: self._get_thread_state().last_described_key_cols = set(value) + @property + def _last_described_has_explicit_keys(self) -> bool: + return bool(getattr(self._get_thread_state(), "last_described_has_explicit_keys", False)) + + @_last_described_has_explicit_keys.setter + def _last_described_has_explicit_keys(self, value: bool) -> None: + self._get_thread_state().last_described_has_explicit_keys = bool(value) + + @property + def _last_described_not_null_cols(self) -> t.Set[str]: + return getattr(self._get_thread_state(), "last_described_not_null_cols", set()) + + @_last_described_not_null_cols.setter + def _last_described_not_null_cols(self, value: t.Set[str]) -> None: + self._get_thread_state().last_described_not_null_cols = set(value) + @property def _in_iceberg_replace_by_key(self) -> bool: return bool(getattr(self._get_thread_state(), "in_iceberg_replace_by_key", False)) @@ -263,6 +303,91 @@ def _in_iceberg_replace_by_key(self) -> bool: def _in_iceberg_replace_by_key(self, value: bool) -> None: self._get_thread_state().in_iceberg_replace_by_key = bool(value) + @property + def _pool_generation(self) -> int: + gen = getattr(self._connection_pool, "_doris_pool_generation", None) + if not isinstance(gen, int): + gen = self._extra_config.get("_doris_pool_generation") + if not isinstance(gen, int): + gen = 0 + return gen + + @_pool_generation.setter + def _pool_generation(self, value: int) -> None: + if isinstance(getattr(self, "_extra_config", None), dict): + self._extra_config["_doris_pool_generation"] = value + try: + setattr(self._connection_pool, "_doris_pool_generation", value) + except Exception: + pass + + def _current_conn_token(self) -> t.Tuple[int, int]: + return (self._pool_generation, id(self._connection_pool.get())) + + def _mark_catalog_synced(self, catalog: t.Optional[str] = None) -> None: + if catalog is not None: + self._active_catalog = catalog + self._needs_catalog_restore = False + token = self._current_conn_token() + self._active_catalog_conn_token = token + if isinstance(self._connection_pool, ConnectionPool): + self._connection_pool.set_attribute("_doris_catalog_token", token) + + def _should_restore_catalog(self) -> bool: + if self._needs_catalog_restore: + return True + if self._active_catalog is None: + return False + expected_token = self._active_catalog_conn_token + if expected_token is None: + return True + current_token = self._current_conn_token() + if current_token != expected_token: + return True + if isinstance(self._connection_pool, ConnectionPool): + return self._connection_pool.get_attribute("_doris_catalog_token") != expected_token + return False + + def close(self) -> t.Any: + self._pool_generation = self._pool_generation + 1 + self._active_catalog_conn_token = None + return super().close() + + def recycle(self) -> None: + calling_token = self._active_catalog_conn_token + super().recycle() + new_gen = self._pool_generation + 1 + self._pool_generation = new_gen + if calling_token is not None and self._active_catalog is not None: + updated_token = (new_gen, calling_token[1]) + self._active_catalog_conn_token = updated_token + if isinstance(self._connection_pool, ConnectionPool): + self._connection_pool.set_attribute("_doris_catalog_token", updated_token) + + _SWITCH_SQL_PREFIX_RE = re.compile( + r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*SWITCH\s+(.+?)\s*;?\s*\Z", + re.IGNORECASE | re.DOTALL, + ) + + @classmethod + def _extract_switch_catalog(cls, sql: t.Optional[str]) -> t.Optional[str]: + if not sql: + return None + match = cls._SWITCH_SQL_PREFIX_RE.match(sql) + if not match: + return None + raw_target = match.group(1).strip() + if not raw_target or ";" in raw_target: + return None + try: + ident = parse_one(raw_target, into=exp.Identifier, read=cls.DIALECT) + if isinstance(ident, exp.Identifier) and ident.name: + return ident.name + except Exception: + pass + cleaned = raw_target.strip("`").strip('"').strip("'").strip() + return cleaned or None + @staticmethod def _get_table_properties(kwargs: t.Dict[str, t.Any]) -> t.Optional[t.Dict[str, t.Any]]: return kwargs.get("table_properties") or kwargs.get("physical_properties") @@ -307,8 +432,7 @@ def get_current_catalog(self) -> t.Optional[str]: def set_current_catalog(self, catalog: str) -> None: catalog_sql = self._to_sql(exp.to_identifier(catalog, quoted=True)) self.execute(exp.Command(this="SWITCH", expression=catalog_sql)) - self._active_catalog = catalog - self._needs_catalog_restore = False + self._mark_catalog_synced(catalog) database = self._extra_config.get("database") default_cat = self.default_catalog or "internal" if database and catalog.lower() == str(default_cat).lower(): @@ -321,10 +445,13 @@ def _restore_active_catalog_on_reconnect(self, sql: t.Optional[str] = None) -> N if not active_cat and not database: self._needs_catalog_restore = False return - if sql and re.match( - r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*SWITCH\b", - sql, - flags=re.IGNORECASE | re.DOTALL, + if self._extract_switch_catalog(sql) is not None or ( + sql + and re.match( + r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*SWITCH\b", + sql, + flags=re.IGNORECASE | re.DOTALL, + ) ): self._needs_catalog_restore = False return @@ -335,7 +462,7 @@ def _restore_active_catalog_on_reconnect(self, sql: t.Optional[str] = None) -> N if database and (not active_cat or active_cat.lower() == str(default_cat).lower()): db_sql = self._to_sql(exp.to_identifier(str(database), quoted=True)) self.cursor.execute(f"USE {db_sql}") - self._needs_catalog_restore = False + self._mark_catalog_synced() _READ_ONLY_EXTERNAL_FORMATS: t.ClassVar[t.Dict[str, str]] = { "lance": "Lance", @@ -536,6 +663,58 @@ def normalize_value( timestamp_precision=timestamp_precision, ) + @staticmethod + def _extract_disconnect_info( + exc: BaseException, + ) -> t.Tuple[bool, bool, t.Optional[str], t.Any]: + matched_disconnect: t.Optional[t.Tuple[str, t.Any]] = None + is_timeout = False + curr: t.Optional[BaseException] = exc + seen: t.Set[int] = set() + while curr is not None and id(curr) not in seen: + seen.add(id(curr)) + if isinstance(curr, TimeoutError) or "timed out" in str(curr).lower(): + is_timeout = True + err_code = curr.args[0] if getattr(curr, "args", None) else None + err_name = curr.__class__.__name__ + if matched_disconnect is None and ( + (err_code in (2006, 2013, 2055) and err_name == "OperationalError") + or (err_code == 0 and err_name == "InterfaceError") + ): + matched_disconnect = (err_name, err_code) + curr = getattr(curr, "__cause__", None) or getattr(curr, "__context__", None) + if matched_disconnect is not None: + return True, is_timeout, matched_disconnect[0], matched_disconnect[1] + return False, is_timeout, None, None + + def _ensure_catalog_restored(self, sql: t.Optional[str] = None) -> None: + if not self._should_restore_catalog(): + return + try: + self._restore_active_catalog_on_reconnect(sql) + except Exception as e: + is_disconnect, is_timeout, err_name, err_code = self._extract_disconnect_info(e) + if is_disconnect and not is_timeout and not self._connection_pool.is_transaction_active: + logger.warning( + "Doris connection lost (%s: %s) while restoring active catalog, reconnecting and retrying...", + err_name, + err_code, + ) + self._connection_pool.close() + self._needs_catalog_restore = True + try: + self._connection_pool.get_cursor() + self._restore_active_catalog_on_reconnect(sql) + except Exception: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + else: + if is_disconnect and not self._connection_pool.is_transaction_active: + self._connection_pool.close() + self._needs_catalog_restore = True + raise + def _fetch_native_df( self, query: t.Union[exp.Expr, str], quote_identifiers: bool = False ) -> DF: @@ -553,6 +732,9 @@ def _fetch_native_df( query = self._to_sql(parsed_q, quote=quote_identifiers) except Exception: pass + sql_str = ( + self._to_sql(query, quote=quote_identifiers) if isinstance(query, exp.Expr) else query + ) if self._pre_ping: try: self.ping() @@ -562,28 +744,19 @@ def _fetch_native_df( self._needs_catalog_restore = True try: self._connection_pool.get_cursor() - if getattr(self, "_needs_catalog_restore", False): - self._restore_active_catalog_on_reconnect() - return super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + self._ensure_catalog_restored(sql_str) + df = super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + switched_cat = self._extract_switch_catalog(sql_str) + if switched_cat: + self._mark_catalog_synced(switched_cat) + return df except Exception as e: - err_code = e.args[0] if getattr(e, "args", None) else None - err_name = e.__class__.__name__ - is_disconnect = (err_code in (2006, 2013, 2055) and err_name == "OperationalError") or ( - err_code == 0 and err_name == "InterfaceError" - ) - is_timeout = ( - isinstance(getattr(e, "__context__", None), TimeoutError) - or "timed out" in str(e).lower() - ) + is_disconnect, is_timeout, err_name, err_code = self._extract_disconnect_info(e) if ( is_disconnect and not is_timeout and not self._connection_pool.is_transaction_active - and self._is_safe_to_retry_sql( - self._to_sql(query, quote=quote_identifiers) - if isinstance(query, exp.Expr) - else query - ) + and self._is_safe_to_retry_sql(sql_str) ): logger.warning( "Doris connection lost (%s: %s) during fetchdf, reconnecting and retrying once...", @@ -593,8 +766,12 @@ def _fetch_native_df( self._connection_pool.close() try: self._connection_pool.get_cursor() - self._restore_active_catalog_on_reconnect() - return super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + self._restore_active_catalog_on_reconnect(sql_str) + df = super()._fetch_native_df(query, quote_identifiers=quote_identifiers) + switched_cat = self._extract_switch_catalog(sql_str) + if switched_cat: + self._mark_catalog_synced(switched_cat) + return df except Exception: self._connection_pool.close() self._needs_catalog_restore = True @@ -651,6 +828,57 @@ def _normalize_schema_tmp_identifiers(cls, expression: exp.Expr) -> exp.Expr: ident.set("this", cls._shorten_schema_tmp_name(ident.name)) return expression + def _wait_on_table_state_error( + self, + err: BaseException, + statement: t.Optional[t.Union[exp.Expr, str]] = None, + ) -> bool: + err_text = str(err) + err_lower = err_text.lower() + is_schema_change = "state(schema_change) is not normal" in err_lower + is_rollup = "state(rollup) is not normal" in err_lower + is_build_index = "state(build_index)" in err_lower or ( + "build index" in err_lower and "not normal" in err_lower + ) + if not is_schema_change and not is_rollup and not is_build_index: + return False + + target_tbl: t.Optional[exp.Table] = None + m = re.search(r"Table\[([^\]]+)\]", err_text, flags=re.IGNORECASE) + err_tbl_name = m.group(1).strip() if m else "" + inferred_db: t.Optional[str] = None + + if isinstance(statement, exp.Expr): + if isinstance(statement.this, exp.Table): + target_tbl = statement.this.copy() + if target_tbl.db: + inferred_db = target_tbl.db + for tbl_node in statement.find_all(exp.Table): + if err_tbl_name and tbl_node.name.lower() == err_tbl_name.lower() and tbl_node.db: + inferred_db = tbl_node.db + break + if not inferred_db and tbl_node.db: + inferred_db = tbl_node.db + + if not inferred_db and self._extra_config.get("database"): + inferred_db = str(self._extra_config["database"]) + + if err_tbl_name: + target_tbl = exp.table_(err_tbl_name, db=inferred_db) + elif target_tbl is not None and not target_tbl.db and inferred_db: + target_tbl.set("db", exp.to_identifier(inferred_db)) + + if target_tbl is not None: + if is_schema_change: + self._wait_for_schema_change(target_tbl, timeout=5.0) + self._wait_for_build_index(target_tbl, timeout=5.0) + elif is_build_index: + self._wait_for_build_index(target_tbl, timeout=5.0) + if is_rollup: + self._wait_for_rollup(target_tbl, timeout=5.0) + time.sleep(0.1) + return True + def execute( self, expressions: t.Union[str, exp.Expr, t.Sequence[exp.Expr]], @@ -660,13 +888,33 @@ def execute( **kwargs: t.Any, ) -> None: if isinstance(expressions, str): - super().execute( + for attempt in range(300): + try: + super().execute( + expressions, + ignore_unsupported_errors=ignore_unsupported_errors, + quote_identifiers=quote_identifiers, + track_rows_processed=track_rows_processed, + **kwargs, + ) + break + except Exception as err: + if attempt < 299 and self._wait_on_table_state_error(err, expressions): + continue + raise + if re.match( + r"\A(?:\s|/\*.*?\*/|--[^\r\n]*(?:[\r\n]|\Z))*BUILD\s+INDEX\b", expressions, - ignore_unsupported_errors=ignore_unsupported_errors, - quote_identifiers=quote_identifiers, - track_rows_processed=track_rows_processed, - **kwargs, - ) + flags=re.IGNORECASE | re.DOTALL, + ): + try: + parsed_build = parse_one(expressions, read=self.dialect) + if isinstance(parsed_build, exp.Alter) and isinstance( + parsed_build.this, exp.Table + ): + self._wait_for_build_index(parsed_build.this) + except Exception: + pass return processed_expressions: t.List[exp.Expr] = [] @@ -691,11 +939,14 @@ def execute( processed_expressions.append(e) if any( - isinstance(e, exp.Alter) and isinstance(e.this, exp.Table) + (isinstance(e, exp.Alter) and isinstance(e.this, exp.Table)) + or isinstance(e, (exp.Create, exp.Drop)) for e in processed_expressions ): for e in processed_expressions: - if isinstance(e, exp.Alter) and isinstance(e.this, exp.Table): + if (isinstance(e, exp.Alter) and isinstance(e.this, exp.Table)) or isinstance( + e, (exp.Create, exp.Drop) + ): for attempt in range(300): try: super().execute( @@ -707,15 +958,44 @@ def execute( ) break except Exception as err: - if ( - "state(schema_change) is not normal" in str(err).lower() - and attempt < 299 - ): - self._wait_for_schema_change(e.this, timeout=5.0) - time.sleep(0.1) + if attempt < 299 and self._wait_on_table_state_error(err, e): continue raise - self._wait_for_schema_change(e.this) + if isinstance(e, exp.Alter) and isinstance(e.this, exp.Table): + if e.text("kind") == "BUILD_INDEX": + self._wait_for_build_index(e.this) + else: + self._wait_for_schema_change(e.this) + elif ( + isinstance(e, exp.Create) + and isinstance(e.this, exp.Table) + and not e.this.db + and isinstance(e.expression, exp.Query) + and ( + "MATERIALIZED" in (e.text("kind") or "").upper() + or any( + isinstance(p, exp.MaterializedProperty) + for p in ( + e.args["properties"].expressions + if isinstance(e.args.get("properties"), exp.Properties) + else [] + ) + ) + ) + ): + cte_names = { + cte.alias.lower() for cte in e.expression.find_all(exp.CTE) if cte.alias + } + src_tbl = next( + ( + t_node + for t_node in e.expression.find_all(exp.Table) + if t_node.name and t_node.name.lower() not in cte_names + ), + None, + ) + if src_tbl is not None: + self._wait_for_rollup(src_tbl) else: super().execute( e, @@ -742,20 +1022,14 @@ def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any logger.info("Connection to the database was lost. Reconnecting...") self._connection_pool.close() self._needs_catalog_restore = True + self._ensure_catalog_restored(sql) try: - if getattr(self, "_needs_catalog_restore", False): - self._restore_active_catalog_on_reconnect(sql) super()._execute(sql, track_rows_processed=track_rows_processed, **kwargs) + switched_cat = self._extract_switch_catalog(sql) + if switched_cat: + self._mark_catalog_synced(switched_cat) except Exception as e: - err_code = e.args[0] if getattr(e, "args", None) else None - err_name = e.__class__.__name__ - is_disconnect = (err_code in (2006, 2013, 2055) and err_name == "OperationalError") or ( - err_code == 0 and err_name == "InterfaceError" - ) - is_timeout = ( - isinstance(getattr(e, "__context__", None), TimeoutError) - or "timed out" in str(e).lower() - ) + is_disconnect, is_timeout, err_name, err_code = self._extract_disconnect_info(e) if ( is_disconnect and not is_timeout @@ -776,6 +1050,9 @@ def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any self._connection_pool.close() self._restore_active_catalog_on_reconnect(sql) super()._execute(sql, track_rows_processed=track_rows_processed, **kwargs) + switched_cat = self._extract_switch_catalog(sql) + if switched_cat: + self._mark_catalog_synced(switched_cat) except Exception: self._connection_pool.close() self._needs_catalog_restore = True @@ -791,8 +1068,8 @@ def columns( ) -> t.Dict[str, exp.DataType]: """Fetches column names and types for the target table, normalizing Doris type aliases.""" cols, key_cols, has_explicit_keys = self._describe_columns_with_keys(table_name) - self._last_described_key_cols: t.Set[str] = key_cols - self._last_described_has_explicit_keys: bool = has_explicit_keys + self._last_described_key_cols = key_cols + self._last_described_has_explicit_keys = has_explicit_keys return cols def _describe_columns_with_keys( @@ -802,6 +1079,7 @@ def _describe_columns_with_keys( describe_output = self.cursor.fetchall() cols: t.Dict[str, exp.DataType] = {} key_cols: t.Set[str] = set() + not_null_cols: t.Set[str] = set() has_explicit_keys = False for row in itertools.takewhile( lambda r: not str(r[0]).startswith("#"), @@ -809,11 +1087,18 @@ def _describe_columns_with_keys( ): column_name = row[0] if len(row) > 0 else None column_type = row[1] if len(row) > 1 else None + null_flag = row[2] if len(row) > 2 else None is_key = row[3] if len(row) > 3 else None col_name_str = _decoded_str(column_name) if column_name else "" raw_type = _decoded_str(column_type) if column_type else "" if not col_name_str.strip() or not raw_type.strip(): continue + if null_flag is not None and _decoded_str(null_flag).strip().lower() in ( + "no", + "false", + "0", + ): + not_null_cols.add(col_name_str.lower()) if is_key is not None: has_explicit_keys = True if _decoded_str(is_key).strip().lower() in ( @@ -879,6 +1164,7 @@ def _describe_columns_with_keys( if cols and not key_cols and not has_explicit_keys: first_col = next(iter(cols)) key_cols.add(first_col.lower()) + self._last_described_not_null_cols = not_null_cols return cols, key_cols, has_explicit_keys def get_alter_operations( @@ -917,17 +1203,21 @@ def get_alter_operations( self._last_described_key_cols = set() self._last_described_has_explicit_keys = False + self._last_described_not_null_cols = set() current_cols = self.columns(current_table_name) - current_key_cols = set(getattr(self, "_last_described_key_cols", set())) - current_has_explicit_keys = bool(getattr(self, "_last_described_has_explicit_keys", False)) + current_key_cols = set(self._last_described_key_cols) + current_has_explicit_keys = bool(self._last_described_has_explicit_keys) + current_not_null_cols = set(self._last_described_not_null_cols) if current_cols and not current_key_cols and not current_has_explicit_keys: current_key_cols = {next(iter(current_cols)).lower()} self._last_described_key_cols = set() self._last_described_has_explicit_keys = False + self._last_described_not_null_cols = set() target_cols = self.columns(target_table_name) - target_key_cols = set(getattr(self, "_last_described_key_cols", set())) - target_has_explicit_keys = bool(getattr(self, "_last_described_has_explicit_keys", False)) + target_key_cols = set(self._last_described_key_cols) + target_has_explicit_keys = bool(self._last_described_has_explicit_keys) + target_not_null_cols = set(self._last_described_not_null_cols) if target_cols and not target_key_cols and not target_has_explicit_keys: target_key_cols = {next(iter(target_cols)).lower()} @@ -968,28 +1258,50 @@ def get_alter_operations( and op.column.name.lower() in current_key_cols } protected_key_cols = invalid_target_key_cols | dropped_key_cols | key_role_changed_cols - return [ - op - for op in operations - if not ( - ( - isinstance(op, TableAlterChangeColumnTypeOperation) - and ( - self._to_sql(op.current_type).upper() - == self._to_sql(op.column_type).upper() - or ( - op.column.name.lower() in current_key_cols - and not self._is_valid_olap_key_type(op.column_type) - ) - or op.column.name.lower() in key_role_changed_cols - ) - ) + preserved_not_null_cols = current_not_null_cols & target_not_null_cols + filtered_ops: t.List[TableAlterOperation] = [] + for op in operations: + if isinstance(op, TableAlterChangeColumnTypeOperation) and ( + self._to_sql(op.current_type).upper() == self._to_sql(op.column_type).upper() or ( - isinstance(op, (TableAlterDropColumnOperation, TableAlterAddColumnOperation)) - and op.column.name.lower() in protected_key_cols + op.column.name.lower() in current_key_cols + and not self._is_valid_olap_key_type(op.column_type) ) - ) - ] + or op.column.name.lower() in key_role_changed_cols + ): + continue + if ( + isinstance(op, (TableAlterDropColumnOperation, TableAlterAddColumnOperation)) + and op.column.name.lower() in protected_key_cols + ): + continue + if ( + isinstance(op, TableAlterChangeColumnTypeOperation) + and op.column.name.lower() in preserved_not_null_cols + ): + op = _DorisNotNullChangeColumnTypeOperation( + target_table=op.target_table, + column_parts=op.column_parts, + expected_table_struct=op.expected_table_struct, + array_element_selector=op.array_element_selector, + column_type=op.column_type, + current_type=op.current_type, + is_part_of_destructive_change=op.is_part_of_destructive_change, + ) + filtered_ops.append(op) + return filtered_ops + + def _resolve_alter_state_index(self, default_index: int) -> int: + desc = getattr(self.cursor, "description", None) + if isinstance(desc, (list, tuple)): + for idx, col_meta in enumerate(desc): + if ( + isinstance(col_meta, (list, tuple)) + and col_meta + and _decoded_str(col_meta[0]).strip().lower() == "state" + ): + return idx + return default_index def _wait_for_schema_change(self, table_name: TableName, timeout: float = 30.0) -> None: """Wait for any active asynchronous OLAP SCHEMA_CHANGE job on ``table_name`` to complete.""" @@ -997,7 +1309,10 @@ def _wait_for_schema_change(self, table_name: TableName, timeout: float = 30.0) if not table.name or self._is_iceberg_table(catalog_name=table.catalog): return escaped_tbl = table.name.replace("'", "''") - from_clause = f" FROM `{table.db.replace('`', '``')}`" if table.db else "" + db_name = table.db or ( + str(self._extra_config["database"]) if self._extra_config.get("database") else "" + ) + from_clause = f" FROM `{db_name.replace('`', '``')}`" if db_name else "" show_sql = f"SHOW ALTER TABLE COLUMN{from_clause} WHERE TableName = '{escaped_tbl}'" deadline = time.monotonic() + timeout while time.monotonic() < deadline: @@ -1008,9 +1323,70 @@ def _wait_for_schema_change(self, table_name: TableName, timeout: float = 30.0) return if not rows or not isinstance(rows, (list, tuple)): return + state_idx = self._resolve_alter_state_index(9) + if all( + len(row) <= state_idx + or str(_decoded_str(row[state_idx])).strip().upper() in ("FINISHED", "CANCELLED") + for row in rows + ): + return + time.sleep(0.05) + + def _wait_for_rollup(self, table_name: TableName, timeout: float = 30.0) -> None: + """Wait for any active asynchronous OLAP ROLLUP (synchronous MV) job on ``table_name`` to complete.""" + table = exp.to_table(table_name) + if not table.name or self._is_iceberg_table(catalog_name=table.catalog): + return + escaped_tbl = table.name.replace("'", "''") + db_name = table.db or ( + str(self._extra_config["database"]) if self._extra_config.get("database") else "" + ) + from_clause = f" FROM `{db_name.replace('`', '``')}`" if db_name else "" + show_sql = ( + f"SHOW ALTER TABLE MATERIALIZED VIEW{from_clause} WHERE TableName = '{escaped_tbl}'" + ) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + self.cursor.execute(show_sql) + rows = self.cursor.fetchall() + except Exception: + return + if not rows or not isinstance(rows, (list, tuple)): + return + state_idx = self._resolve_alter_state_index(8) if all( - len(row) <= 9 - or str(_decoded_str(row[9])).strip().upper() in ("FINISHED", "CANCELLED") + len(row) <= state_idx + or str(_decoded_str(row[state_idx])).strip().upper() in ("FINISHED", "CANCELLED") + for row in rows + ): + return + time.sleep(0.05) + + def _wait_for_build_index(self, table_name: TableName, timeout: float = 60.0) -> None: + """Wait for any active asynchronous OLAP BUILD INDEX job on ``table_name`` to complete.""" + table = exp.to_table(table_name) + if not table.name or self._is_iceberg_table(catalog_name=table.catalog): + return + escaped_tbl = table.name.replace("'", "''") + db_name = table.db or ( + str(self._extra_config["database"]) if self._extra_config.get("database") else "" + ) + from_clause = f" FROM `{db_name.replace('`', '``')}`" if db_name else "" + show_sql = f"SHOW BUILD INDEX{from_clause} WHERE TableName = '{escaped_tbl}'" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + self.cursor.execute(show_sql) + rows = self.cursor.fetchall() + except Exception: + return + if not rows or not isinstance(rows, (list, tuple)): + return + state_idx = self._resolve_alter_state_index(7) + if all( + len(row) <= state_idx + or str(_decoded_str(row[state_idx])).strip().upper() in ("FINISHED", "CANCELLED") for row in rows ): return @@ -1501,6 +1877,10 @@ def _create_materialized_view( registry[f"{sync_source_tbl.db.lower()}.{target_table.name.lower()}"] = ( sync_source_tbl.copy() ) + rollup_wait_tbl = sync_source_tbl.copy() + if not rollup_wait_tbl.db and sync_db: + rollup_wait_tbl.set("db", exp.to_identifier(sync_db)) + self._wait_for_rollup(rollup_wait_tbl) elif not is_sync_mv: self._pop_sync_mv_source_table(target_table) raw_build = ( @@ -4545,15 +4925,145 @@ def _matches_target_table(t_node: exp.Table) -> bool: ) precreated_by_replace_query = bool(self_referencing and from_replace_query and not replace) - if (is_internal_iceberg or precreated_by_replace_query) and target_columns_to_types_known: + table_props = self._get_table_properties(kwargs) or {} + raw_not_null_cols = kwargs.get("not_null_columns") or table_props.get("not_null_columns") + explicit_not_null_list = self._extract_column_names(raw_not_null_cols) + has_explicit_not_null_cols = bool(explicit_not_null_list) + requires_schema_ddl_ctas = is_internal_iceberg or ( + has_explicit_not_null_cols and not self_referencing + ) + + target_already_existed = bool( + requires_schema_ddl_ctas + and exists + and not replace + and not from_replace_query + and self.get_data_object(table) is not None + ) + + # 3. Doris's `CreateTableAsSelectCommand` ignores `NON_NULLABLE(...)` for non-`AUTO PARTITION` + # columns and forbids `ALTER TABLE ... MODIFY COLUMN ... NOT NULL`. Therefore, when + # `not_null_columns` is requested (for example `ARRAY NOT NULL` for `USING ANN` + # vector indexes or `NOT NULL` Iceberg columns), resolve column types if needed and + # materialize via schema DDL (`_create_table_from_columns`) + `INSERT INTO`. + if ( + has_explicit_not_null_cols + and not self_referencing + and not target_columns_to_types_known + and not target_already_existed + and first_query is not None + ): + curr_query_node: t.Optional[exp.Expr] = first_query + while isinstance(curr_query_node, (exp.SetOperation, exp.Subquery)): + curr_query_node = curr_query_node.this + if ( + isinstance(curr_query_node, exp.Select) + and curr_query_node.expressions + and not any( + isinstance(proj, exp.Star) or isinstance(proj.this, exp.Star) + for proj in curr_query_node.expressions + ) + ): + proj_names_lower = { + proj.alias_or_name.lower() + for proj in curr_query_node.expressions + if proj.alias_or_name + } + if proj_names_lower: + missing_not_null = sorted( + {c.lower() for c in explicit_not_null_list} - proj_names_lower + ) + if missing_not_null: + raise SQLMeshError( + f"not_null_columns {missing_not_null} not found in table columns {sorted(proj_names_lower)}." + ) + known_by_lower = { + c.lower(): dtype + for c, dtype in (target_columns_to_types or {}).items() + if dtype.this != exp.DataType.Type.UNKNOWN + } + inferred_cols: t.Dict[str, exp.DataType] = {} + all_inferred = True + for proj in curr_query_node.expressions: + col_alias = proj.alias_or_name + if not col_alias: + all_inferred = False + break + existing_dtype = known_by_lower.get(col_alias.lower()) + if existing_dtype is not None: + inferred_cols[col_alias] = existing_dtype.copy() + continue + unaliased = proj.unalias() + if ( + isinstance(unaliased, exp.Cast) + and isinstance(unaliased.to, exp.DataType) + and unaliased.to.this != exp.DataType.Type.UNKNOWN + ): + inferred_cols[col_alias] = unaliased.to.copy() + else: + all_inferred = False + break + if all_inferred and inferred_cols: + target_columns_to_types = ( + self._reorder_olap_columns(inferred_cols, kwargs) + if not is_iceberg + else inferred_cols + ) + target_columns_to_types_known = True + + if not target_columns_to_types_known and not is_iceberg: + probe_table = self._get_temp_table(table) + probe_query = ( + exp.select( + exp.cast(exp.Literal.number(0), to="TINYINT").as_( + "__sqlmesh_probe_key", quoted=True + ), + exp.Column(this=exp.Star(), table=exp.to_identifier("_q", quoted=True)), + ) + .from_(first_query.copy().subquery(alias="_q")) + .where(exp.false()) + .limit(0) + ) + probe_props: t.Dict[str, t.Any] = { + "duplicate_key": exp.Tuple( + expressions=[exp.column("__sqlmesh_probe_key", quoted=True)] + ), + "buckets": exp.Literal.number(1), + } + for prop_name in ( + "replication_num", + "replication_allocation", + "storage_vault_name", + ): + if prop_name in table_props: + probe_props[prop_name] = table_props[prop_name] + super()._create_table_from_source_queries( + probe_table, + [SourceQuery(query_factory=lambda: probe_query)], + exists=True, + replace=False, + table_properties=probe_props, + track_rows_processed=False, + ) + try: + probed_cols = { + col_k: col_v + for col_k, col_v in self.columns(probe_table).items() + if col_k.lower() != "__sqlmesh_probe_key" + } + finally: + self.drop_table(probe_table, exists=True) + if probed_cols and columns_to_types_all_known(probed_cols): + target_columns_to_types = self._reorder_olap_columns(probed_cols, kwargs) + target_columns_to_types_known = True + + if target_already_existed and not target_columns_to_types_known: + return + + if ( + requires_schema_ddl_ctas or precreated_by_replace_query + ) and target_columns_to_types_known: assert target_columns_to_types is not None - target_already_existed = bool( - is_internal_iceberg - and exists - and not replace - and not from_replace_query - and self.get_data_object(table) is not None - ) if not precreated_by_replace_query: if replace: self.drop_table(table_name, exists=True) @@ -4791,43 +5301,39 @@ def _build_create_table_exp( # Doris AUTO PARTITION BY RANGE requires the partition column(s) to be defined as NOT NULL, # and users may also specify physical_properties (not_null_columns = (...)) for columns - # that must be created as NOT NULL (e.g. ARRAY NOT NULL for ANN vector indexes, since - # Doris does not allow ALTER TABLE ... MODIFY COLUMN to change NULL to NOT NULL). - if not is_iceberg: - table_props = self._get_table_properties(kwargs) or {} - raw_not_null_cols = kwargs.get("not_null_columns") or table_props.get( - "not_null_columns" - ) - explicit_not_null_cols = { - c.lower() for c in self._extract_column_names(raw_not_null_cols) - } - if explicit_not_null_cols: - known_cols_lower: t.Set[str] = set() - if target_columns_to_types: - known_cols_lower = {c.lower() for c in target_columns_to_types} - elif isinstance(table_name_or_schema, exp.Schema): - known_cols_lower = { - expr.name.lower() - for expr in table_name_or_schema.expressions - if isinstance(expr, exp.ColumnDef) - } - elif isinstance(expression, exp.Select) and not any( - isinstance(proj, exp.Star) or isinstance(proj.this, exp.Star) + # that must be created as NOT NULL (e.g. ARRAY NOT NULL for ANN vector indexes, or + # required columns on OLAP and Iceberg tables). + table_props = self._get_table_properties(kwargs) or {} + raw_not_null_cols = kwargs.get("not_null_columns") or table_props.get("not_null_columns") + explicit_not_null_cols = {c.lower() for c in self._extract_column_names(raw_not_null_cols)} + if explicit_not_null_cols: + known_cols_lower: t.Set[str] = set() + if target_columns_to_types: + known_cols_lower = {c.lower() for c in target_columns_to_types} + elif isinstance(table_name_or_schema, exp.Schema): + known_cols_lower = { + expr.name.lower() + for expr in table_name_or_schema.expressions + if isinstance(expr, exp.ColumnDef) + } + elif isinstance(expression, exp.Select) and not any( + isinstance(proj, exp.Star) or isinstance(proj.this, exp.Star) + for proj in expression.expressions + ): + known_cols_lower = { + proj.alias_or_name.lower() for proj in expression.expressions - ): - known_cols_lower = { - proj.alias_or_name.lower() - for proj in expression.expressions - if proj.alias_or_name - } - if known_cols_lower: - missing_not_null = sorted(explicit_not_null_cols - known_cols_lower) - if missing_not_null: - raise SQLMeshError( - f"not_null_columns {missing_not_null} not found in table columns {sorted(known_cols_lower)}." - ) + if proj.alias_or_name + } + if known_cols_lower: + missing_not_null = sorted(explicit_not_null_cols - known_cols_lower) + if missing_not_null: + raise SQLMeshError( + f"not_null_columns {missing_not_null} not found in table columns {sorted(known_cols_lower)}." + ) - not_null_cols: t.Set[str] = set(explicit_not_null_cols) + not_null_cols: t.Set[str] = set(explicit_not_null_cols) + if not is_iceberg: auto_partition = table_props.get("auto_partition") is_auto = bool( auto_partition is not None @@ -4862,86 +5368,88 @@ def _build_create_table_exp( auto_range_cols = set() not_null_cols |= auto_range_cols - if not_null_cols and isinstance(table_name_or_schema, exp.Schema): - table_name_or_schema = table_name_or_schema.copy() - updated_exprs: t.List[exp.Expr] = [] - for expr in table_name_or_schema.expressions: - if isinstance(expr, exp.ColumnDef) and expr.name.lower() in not_null_cols: - existing_constraints = list(expr.args.get("constraints") or []) - has_not_null = any( - isinstance(c, exp.NotNullColumnConstraint) - or isinstance(getattr(c, "kind", None), exp.NotNullColumnConstraint) - for c in existing_constraints + if not_null_cols and isinstance(table_name_or_schema, exp.Schema): + table_name_or_schema = table_name_or_schema.copy() + updated_exprs: t.List[exp.Expr] = [] + for expr in table_name_or_schema.expressions: + if isinstance(expr, exp.ColumnDef) and expr.name.lower() in not_null_cols: + existing_constraints = list(expr.args.get("constraints") or []) + has_not_null = any( + isinstance(c, exp.NotNullColumnConstraint) + or isinstance(getattr(c, "kind", None), exp.NotNullColumnConstraint) + for c in existing_constraints + ) + if not has_not_null: + expr = expr.copy() + expr.set( + "constraints", + [exp.ColumnConstraint(kind=exp.NotNullColumnConstraint())] + + existing_constraints, ) - if not has_not_null: - expr = expr.copy() - expr.set( - "constraints", - [exp.ColumnConstraint(kind=exp.NotNullColumnConstraint())] - + existing_constraints, - ) - updated_exprs.append(expr) - table_name_or_schema.set("expressions", updated_exprs) - elif not_null_cols and isinstance(expression, (exp.Select, exp.SetOperation)): - expression = expression.copy() - auto_select_branches: t.List[exp.Select] = [] - - def _collect_auto_branches(node: exp.Expr) -> None: - if isinstance(node, exp.Subquery): - _collect_auto_branches(node.this) - elif isinstance(node, exp.SetOperation): - _collect_auto_branches(node.this) - _collect_auto_branches(node.expression) - elif isinstance(node, exp.Select): - auto_select_branches.append(node) - - _collect_auto_branches(expression) - if auto_select_branches: - target_indices = { - idx - for idx, proj in enumerate(auto_select_branches[0].expressions) - if proj.alias_or_name.lower() in not_null_cols - } - for branch_idx, branch in enumerate(auto_select_branches): - new_select_exprs: t.List[exp.Expr] = [] - for idx, proj in enumerate(branch.expressions): - if proj.alias_or_name.lower() in not_null_cols or idx in target_indices: - if isinstance(proj, exp.Alias): - inner = proj.this - if not ( - isinstance(inner, exp.Anonymous) - and inner.name.upper() == "NON_NULLABLE" - ): - proj = proj.copy() - proj.set( - "this", - exp.Anonymous( - this="NON_NULLABLE", - expressions=[inner.copy()], - ), - ) - elif isinstance(proj, exp.Column): - wrapped = exp.Anonymous( - this="NON_NULLABLE", expressions=[proj.copy()] - ) - proj = ( - exp.alias_( - wrapped, - proj.name, - quoted=bool(getattr(proj.this, "quoted", True)), - ) - if branch_idx == 0 - else wrapped - ) - elif not ( - isinstance(proj, exp.Anonymous) - and proj.name.upper() == "NON_NULLABLE" + updated_exprs.append(expr) + table_name_or_schema.set("expressions", updated_exprs) + elif ( + not is_iceberg + and not_null_cols + and isinstance(expression, (exp.Select, exp.SetOperation)) + ): + expression = expression.copy() + auto_select_branches: t.List[exp.Select] = [] + + def _collect_auto_branches(node: exp.Expr) -> None: + if isinstance(node, exp.Subquery): + _collect_auto_branches(node.this) + elif isinstance(node, exp.SetOperation): + _collect_auto_branches(node.this) + _collect_auto_branches(node.expression) + elif isinstance(node, exp.Select): + auto_select_branches.append(node) + + _collect_auto_branches(expression) + if auto_select_branches: + target_indices = { + idx + for idx, proj in enumerate(auto_select_branches[0].expressions) + if proj.alias_or_name.lower() in not_null_cols + } + for branch_idx, branch in enumerate(auto_select_branches): + new_select_exprs: t.List[exp.Expr] = [] + for idx, proj in enumerate(branch.expressions): + if proj.alias_or_name.lower() in not_null_cols or idx in target_indices: + if isinstance(proj, exp.Alias): + inner = proj.this + if not ( + isinstance(inner, exp.Anonymous) + and inner.name.upper() == "NON_NULLABLE" ): - proj = exp.Anonymous( - this="NON_NULLABLE", expressions=[proj.copy()] + proj = proj.copy() + proj.set( + "this", + exp.Anonymous( + this="NON_NULLABLE", + expressions=[inner.copy()], + ), + ) + elif isinstance(proj, exp.Column): + wrapped = exp.Anonymous( + this="NON_NULLABLE", expressions=[proj.copy()] + ) + proj = ( + exp.alias_( + wrapped, + proj.name, + quoted=bool(getattr(proj.this, "quoted", True)), ) - new_select_exprs.append(proj) - branch.set("expressions", new_select_exprs) + if branch_idx == 0 + else wrapped + ) + elif not ( + isinstance(proj, exp.Anonymous) + and proj.name.upper() == "NON_NULLABLE" + ): + proj = exp.Anonymous(this="NON_NULLABLE", expressions=[proj.copy()]) + new_select_exprs.append(proj) + branch.set("expressions", new_select_exprs) return super()._build_create_table_exp( table_name_or_schema, @@ -5731,8 +6239,12 @@ def _parse_partition_expressions( flattened_partitioned_by: t.List[exp.Expr] = [] for item in partitioned_by: + if isinstance(item, exp.Paren): + item = item.unnest() if isinstance(item, (exp.Tuple, exp.Array)): - flattened_partitioned_by.extend(item.expressions) + flattened_partitioned_by.extend( + e.unnest() if isinstance(e, exp.Paren) else e for e in item.expressions + ) else: flattened_partitioned_by.append(item) @@ -5798,6 +6310,8 @@ def _parse_partition_expressions( @staticmethod def _to_raw_partition_sql(expr: t.Union[exp.Literal, exp.Var, str, t.Any]) -> exp.Var: + while isinstance(expr, exp.Paren): + expr = expr.this if isinstance(expr, exp.Literal): raw_text = str(expr.this) elif isinstance(expr, exp.Var): @@ -5805,7 +6319,7 @@ def _to_raw_partition_sql(expr: t.Union[exp.Literal, exp.Var, str, t.Any]) -> ex elif isinstance(expr, str): raw_text = expr else: - return expr + raise SQLMeshError(f"Invalid partition definition expression: {expr}") if ";" in raw_text or "--" in raw_text or "/*" in raw_text: raise SQLMeshError(f"Invalid partition definition expression: {raw_text}") return exp.Var(this=raw_text, quoted=False) @@ -5909,8 +6423,10 @@ def _normalize_iceberg_part_node(node: exp.Expr) -> exp.Expr: create_expressions: t.Optional[t.List[exp.Expr]] = None if partitions: - if isinstance(partitions, exp.Tuple): + if isinstance(partitions, (exp.Tuple, exp.Array)): create_expressions = [self._to_raw_partition_sql(e) for e in partitions.expressions] + elif isinstance(partitions, (list, tuple)): + create_expressions = [self._to_raw_partition_sql(e) for e in partitions] else: create_expressions = [self._to_raw_partition_sql(partitions)] elif is_auto or dynamic_partition_enabled: @@ -6703,6 +7219,10 @@ def _build_distributed_by_exp( default_buckets_expr = self._resolve_buckets(top_level_buckets) if distributed_by is not None: + while isinstance(distributed_by, exp.Paren) and not isinstance( + distributed_by.this, exp.EQ + ): + distributed_by = distributed_by.this distributed_info: t.Dict[str, t.Any] = {} # Case 1: Function expression such as HASH(a, b) or RANDOM() diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index 6098824096..7111239641 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -3258,7 +3258,7 @@ def test_value_normalization( if ctx.dialect == "trino" and ctx.engine_adapter.current_catalog_type == "hive": pytest.skip("Trino on Hive doesn't support TIMESTAMP WITH TIME ZONE fields") if ctx.dialect == "fabric": - pytest.skip(f"{ctx.dialect} doesn't support TIMESTAMP WITH TIME ZONE fields") + pytest.skip("Fabric doesn't support TIMESTAMP WITH TIME ZONE fields") if not isinstance(ctx.engine_adapter, RowDiffMixin): pytest.skip( diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 8a4e682a03..af7cfe6643 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -944,3 +944,310 @@ def _build_index() -> bool: ) assert len(tz_check_rows) == 2 assert tz_check_rows[0][0] == tz_check_rows[1][0] == "2024-06-01 04:00:00.123456" + + +def test_doris_sqlmesh_plan_ann_inverted_not_null_and_widen( + ctx: TestContext, tmp_path: t.Any +) -> None: + from pathlib import Path + + project_dir = Path(tmp_path) / "ann_project" + models_dir = project_dir / "models" + models_dir.mkdir(parents=True) + + schema_name = ctx.schema("ann_e2e") + ctx.engine_adapter.create_schema(schema_name) + raw_tbl = exp.table_("raw_logs", db=schema_name) + raw_tbl_sql = ctx.engine_adapter._to_sql(raw_tbl) + ctx.engine_adapter.execute( + f""" + CREATE TABLE {raw_tbl_sql} ( + `log_id` BIGINT NOT NULL, + `tag` VARCHAR(32) NOT NULL, + `message` STRING, + `embedding` ARRAY NOT NULL, + `updated_at` DATETIME + ) DUPLICATE KEY(`log_id`) + DISTRIBUTED BY HASH(`log_id`) BUCKETS 1 + PROPERTIES ('replication_num' = '1') + """ + ) + ctx.engine_adapter.execute( + f"INSERT INTO {raw_tbl_sql} VALUES " + f"(1, 'net', 'connection timeout while querying upstream', [0.1, 0.2, 0.9], '2026-01-01 00:00:00'), " + f"(2, 'ok', 'healthy heartbeat response', [0.9, 0.1, 0.0], '2026-01-01 00:01:00')" + ) + + uk_model_file = models_dir / "searchable_logs.sql" + uk_model_file.write_text( + f""" + MODEL ( + name {schema_name}.searchable_logs, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key log_id + ), + physical_properties ( + enable_unique_key_merge_on_write = 'true', + not_null_columns = ('embedding', 'tag'), + storage_format = 'V3', + compaction_policy = 'size_based', + 'function_column.sequence_col' = 'updated_at', + replication_num = 1 + ) + ); + + SELECT + log_id::BIGINT AS log_id, + tag::VARCHAR(32) AS tag, + message::STRING AS message, + embedding::ARRAY AS embedding, + updated_at::DATETIME AS updated_at + FROM {schema_name}.raw_logs; + + @IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_message (message) USING INVERTED PROPERTIES ("parser" = "english", "support_phrase" = "true") + ); + + @IF( + @runtime_stage = 'creating', + BUILD INDEX idx_message ON @this_model + ); + + @IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_embedding (embedding) USING ANN PROPERTIES ("index_type" = "hnsw", "metric_type" = "l2_distance", "dim" = "3") + ); + + @IF( + @runtime_stage = 'creating', + BUILD INDEX idx_embedding ON @this_model + ); + """, + encoding="utf-8", + ) + + # Also test an unannotated FULL model (no `::` casts) with `not_null_columns = ('embedding')` + `USING ANN` + full_model_file = models_dir / "searchable_logs_full.sql" + full_model_file.write_text( + f""" + MODEL ( + name {schema_name}.searchable_logs_full, + kind FULL, + physical_properties ( + duplicate_key = ('log_id'), + not_null_columns = ('embedding'), + replication_num = 1 + ) + ); + + SELECT + log_id, + message, + embedding + FROM {schema_name}.raw_logs; + + @IF( + @runtime_stage = 'creating', + ALTER TABLE @this_model ADD INDEX idx_embedding_full (embedding) USING ANN PROPERTIES ("index_type" = "hnsw", "metric_type" = "l2_distance", "dim" = "3") + ); + + @IF( + @runtime_stage = 'creating', + BUILD INDEX idx_embedding_full ON @this_model + ); + """, + encoding="utf-8", + ) + + sm_ctx = ctx.create_context( + config_mutator=lambda _, cfg: setattr(cfg.model_defaults, "dialect", "doris"), + path=project_dir, + ) + sm_ctx.plan(auto_apply=True, no_prompts=True) + + search_rows = ctx.engine_adapter.fetchall( + parse_one( + f"SELECT log_id, L2_DISTANCE_APPROXIMATE(embedding, [0.1, 0.2, 0.9]) AS dist " + f"FROM {schema_name}.searchable_logs " + f"WHERE message MATCH_PHRASE 'connection timeout' AND SEARCH('message:timeout') " + f"ORDER BY dist ASC LIMIT 5", + read="doris", + ) + ) + assert [r[0] for r in search_rows] == [1] + + full_rows = ctx.engine_adapter.fetchall( + parse_one( + f"SELECT log_id FROM {schema_name}.searchable_logs_full " + f"ORDER BY L2_DISTANCE_APPROXIMATE(embedding, [0.1, 0.2, 0.9]) ASC LIMIT 1", + read="doris", + ) + ) + assert [r[0] for r in full_rows] == [1] + + # Widen `tag` from VARCHAR(32) to VARCHAR(128) via forward-only migration and verify `NOT NULL` is preserved + uk_model_file.write_text( + uk_model_file.read_text(encoding="utf-8").replace( + "tag::VARCHAR(32) AS tag", "tag::VARCHAR(128) AS tag" + ), + encoding="utf-8", + ) + sm_ctx.load() + sm_ctx.plan(forward_only=True, auto_apply=True, no_prompts=True) + + uk_snap = sm_ctx.get_snapshot(f"{schema_name}.searchable_logs") + assert uk_snap is not None + phys_tbl = uk_snap.table_name() + desc_rows = ctx.engine_adapter.fetchall(f"DESCRIBE {phys_tbl}", quote_identifiers=False) + desc_by_col = {str(r[0]): (str(r[1]).upper(), str(r[2])) for r in desc_rows} + assert "128" in desc_by_col["tag"][0] + assert desc_by_col["tag"][1].lower() == "no" + assert desc_by_col["embedding"][1].lower() == "no" + + +def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( + ctx: TestContext, tmp_path: t.Any +) -> None: + import datetime as dt + from pathlib import Path + + project_dir = Path(tmp_path) / "dyn_sync_project" + models_dir = project_dir / "models" + models_dir.mkdir(parents=True) + + schema_name = ctx.schema("dyn_sync_e2e") + ctx.engine_adapter.create_schema(schema_name) + + today_utc = dt.datetime.now(dt.timezone.utc).date() + d1 = (today_utc - dt.timedelta(days=2)).isoformat() + d2 = (today_utc - dt.timedelta(days=1)).isoformat() + + raw_tbl = exp.table_("raw_user_events", db=schema_name) + raw_tbl_sql = ctx.engine_adapter._to_sql(raw_tbl) + ctx.engine_adapter.execute( + f""" + CREATE TABLE {raw_tbl_sql} ( + `user_id` BIGINT NOT NULL, + `event_type` VARCHAR(64), + `event_date` DATE NOT NULL, + `event_time` DATETIME NOT NULL + ) DUPLICATE KEY(`user_id`) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 + PROPERTIES ('replication_num' = '1') + """ + ) + ctx.engine_adapter.execute( + f"INSERT INTO {raw_tbl_sql} VALUES " + f"(10, 'click', '{d1}', '{d1} 10:00:00'), " + f"(10, 'buy', '{d1}', '{d1} 10:00:00'), " + f"(20, 'view', '{d2}', '{d2} 12:00:00')" + ) + + (models_dir / "rolling_daily_events.sql").write_text( + f""" + MODEL ( + name {schema_name}.rolling_daily_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_date + ), + start '{d1}', + partitioned_by RANGE(event_date), + physical_properties ( + 'dynamic_partition.enable' = 'true', + 'dynamic_partition.time_unit' = 'DAY', + 'dynamic_partition.time_zone' = 'UTC', + 'dynamic_partition.start' = '-7', + 'dynamic_partition.end' = '3', + 'dynamic_partition.create_history_partition' = 'true', + 'dynamic_partition.prefix' = 'p', + 'dynamic_partition.buckets' = '1', + 'dynamic_partition.replication_num' = '1', + replication_num = 1 + ) + ); + + SELECT + event_date::DATE AS event_date, + user_id::BIGINT AS user_id + FROM {schema_name}.raw_user_events + WHERE event_date BETWEEN @start_ds AND @end_ds; + """, + encoding="utf-8", + ) + + # Create TWO back-to-back synchronous MVs in `@runtime_stage = 'creating'` to verify + # ROLLUP state retry/synchronization, plus `@start_ts` / `@end_ts` restatement. + (models_dir / "user_events.sql").write_text( + f""" + MODEL ( + name {schema_name}.user_events, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column event_time, + partition_by_time_column false + ), + start '{d1}', + grain (user_id, event_time), + physical_properties ( + replication_num = 1 + ) + ); + + SELECT + user_id::BIGINT AS user_id, + event_type::VARCHAR(64) AS event_type, + event_time::DATETIME AS event_time + FROM {schema_name}.raw_user_events + WHERE event_time BETWEEN @start_ts AND @end_ts; + + @IF( + @runtime_stage = 'creating', + CREATE MATERIALIZED VIEW user_events_sync_mv1 AS + SELECT + user_id AS mv_user_id, + event_time AS mv_event_time, + COUNT(event_type) AS event_count + FROM @this_model + GROUP BY user_id, event_time + ); + + @IF( + @runtime_stage = 'creating', + CREATE MATERIALIZED VIEW user_events_sync_mv2 AS + SELECT + event_time AS mv2_event_time, + user_id AS mv2_user_id, + MAX(event_type) AS max_event_type + FROM @this_model + GROUP BY event_time, user_id + ); + """, + encoding="utf-8", + ) + + sm_ctx = ctx.create_context( + config_mutator=lambda _, cfg: setattr(cfg.model_defaults, "dialect", "doris"), + path=project_dir, + ) + sm_ctx.plan(auto_apply=True, no_prompts=True) + + dyn_rows = ctx.engine_adapter.fetchall( + f"SELECT user_id, COUNT(*) FROM {schema_name}.rolling_daily_events GROUP BY user_id ORDER BY user_id", + quote_identifiers=False, + ) + assert list(dyn_rows) == [(10, 2), (20, 1)] + + # Insert an additional source row in d1 and restate `user_events` to verify DELETE + INSERT works with active sync MVs + ctx.engine_adapter.execute( + f"INSERT INTO {raw_tbl_sql} VALUES (10, 'checkout', '{d1}', '{d1} 10:00:00')" + ) + sm_ctx.plan( + restate_models=[f"{schema_name}.user_events"], + auto_apply=True, + no_prompts=True, + ) + restated_rows = ctx.engine_adapter.fetchall( + f"SELECT user_id, COUNT(event_type) FROM {schema_name}.user_events GROUP BY user_id, event_time ORDER BY user_id", + quote_identifiers=False, + ) + assert list(restated_rows) == [(10, 3), (20, 1)] diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 11fba0677d..59bebc7fda 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -28,6 +28,8 @@ def _make_function( adapter = make_mocked_engine_adapter(klass, *args, **kwargs) if isinstance(adapter, DorisEngineAdapter): setattr(adapter, "_wait_for_schema_change", mocker.MagicMock()) + setattr(adapter, "_wait_for_rollup", mocker.MagicMock()) + setattr(adapter, "_wait_for_build_index", mocker.MagicMock()) orig_resolve = adapter._resolve_sync_mv_source_from_metadata bound_default_fetchall = DorisEngineAdapter.fetchall.__get__( adapter, DorisEngineAdapter @@ -5733,7 +5735,8 @@ def _set_other_thread_hints() -> None: ] adapter.cursor.fetchall.side_effect = None - # 15. `not_null_columns` in `physical_properties` marks columns NOT NULL on CREATE TABLE and wraps CTAS projections in NON_NULLABLE(...) + # 15. `not_null_columns` in `physical_properties` marks columns NOT NULL on CREATE TABLE and materializes + # OLAP CTAS via schema DDL + INSERT INTO (since Doris ignores NON_NULLABLE(...) on non-AUTO PARTITION CTAS columns). adapter.cursor.execute.reset_mock() adapter.create_table( "analytics.searchable_logs", @@ -5748,8 +5751,30 @@ def _set_other_thread_hints() -> None: "enable_unique_key_merge_on_write": exp.Literal.string("true"), }, ) + # 15a. CTAS with explicit casts resolves types directly without a probe table adapter.ctas( "analytics.searchable_logs_ctas", + parse_one( + "SELECT CAST(log_id AS BIGINT) AS log_id, CAST(message AS STRING) AS message, CAST(embedding AS ARRAY) AS embedding FROM raw.logs", + read="doris", + ), + table_properties={ + "duplicate_key": exp.Tuple(expressions=[exp.column("log_id")]), + "not_null_columns": exp.Tuple(expressions=[exp.column("embedding")]), + }, + ) + # 15b. Unannotated CTAS without explicit casts probes types via a 0-row temp table, then creates NOT NULL schema + INSERT INTO + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="probe123") + adapter.cursor.fetchall.side_effect = [ + [ + ("__sqlmesh_probe_key", "TINYINT", "Yes", "true", None, ""), + ("log_id", "BIGINT", "Yes", "false", None, ""), + ("message", "TEXT", "Yes", "false", None, ""), + ("embedding", "ARRAY", "Yes", "false", None, ""), + ] + ] + adapter.ctas( + "analytics.searchable_logs_unannotated", parse_one( "SELECT log_id, message, embedding FROM raw.logs", read="doris", @@ -5759,9 +5784,16 @@ def _set_other_thread_hints() -> None: "not_null_columns": exp.Tuple(expressions=[exp.column("embedding")]), }, ) + adapter.cursor.fetchall.side_effect = None assert to_sql_calls(adapter) == [ "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs` (`log_id` BIGINT, `message` STRING, `embedding` ARRAY NOT NULL) UNIQUE KEY (`log_id`) DISTRIBUTED BY HASH (`log_id`) BUCKETS 10 PROPERTIES ('enable_unique_key_merge_on_write'='true')", - "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs_ctas` DUPLICATE KEY (`log_id`) AS SELECT `log_id`, `message`, NON_NULLABLE(`embedding`) AS `embedding` FROM `raw`.`logs`", + "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs_ctas` (`log_id` BIGINT, `message` STRING, `embedding` ARRAY NOT NULL) DUPLICATE KEY (`log_id`)", + "INSERT INTO `analytics`.`searchable_logs_ctas` (`log_id`, `message`, `embedding`) SELECT CAST(`log_id` AS BIGINT) AS `log_id`, CAST(`message` AS STRING) AS `message`, CAST(`embedding` AS ARRAY) AS `embedding` FROM (SELECT CAST(`log_id` AS BIGINT) AS `log_id`, CAST(`message` AS STRING) AS `message`, CAST(`embedding` AS ARRAY) AS `embedding` FROM `raw`.`logs`) AS `_subquery`", + "CREATE TABLE IF NOT EXISTS `analytics`.`__temp_searchable_logs_unannotated_probe123` DUPLICATE KEY (`__sqlmesh_probe_key`) DISTRIBUTED BY HASH (`__sqlmesh_probe_key`) BUCKETS 1 AS SELECT CAST(0 AS TINYINT) AS `__sqlmesh_probe_key`, `_q`.* FROM (SELECT `log_id`, `message`, `embedding` FROM `raw`.`logs`) AS `_q` WHERE FALSE LIMIT 0", + "DESCRIBE `analytics`.`__temp_searchable_logs_unannotated_probe123`", + "DROP TABLE IF EXISTS `analytics`.`__temp_searchable_logs_unannotated_probe123`", + "CREATE TABLE IF NOT EXISTS `analytics`.`searchable_logs_unannotated` (`log_id` BIGINT, `message` STRING, `embedding` ARRAY NOT NULL) DUPLICATE KEY (`log_id`)", + "INSERT INTO `analytics`.`searchable_logs_unannotated` (`log_id`, `message`, `embedding`) SELECT CAST(`log_id` AS BIGINT) AS `log_id`, CAST(`message` AS STRING) AS `message`, CAST(`embedding` AS ARRAY) AS `embedding` FROM (SELECT `log_id`, `message`, `embedding` FROM `raw`.`logs`) AS `_subquery`", ] with pytest.raises(SQLMeshError, match="not_null_columns"): adapter.create_table( @@ -5771,3 +5803,252 @@ def _set_other_thread_hints() -> None: "not_null_columns": exp.Tuple(expressions=[exp.column("missing_col")]), }, ) + + +def test_doris_audit_fixes_and_docs_models( + make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], + mocker: MockerFixture, +): + from pathlib import Path + import re + import threading + from pandas.errors import DatabaseError + from sqlmesh.core import dialect as d + from sqlmesh.core.model import load_sql_based_model + from sqlmesh.utils.connection_pool import ThreadLocalConnectionPool + + adapter = make_mocked_engine_adapter(DorisEngineAdapter) + + # 1. `get_alter_operations` preserves `NOT NULL` on `MODIFY COLUMN` iff column is `NOT NULL` in both + # `current` and `target`, relaxes when `target` is nullable, and never forces `NOT NULL` on `ADD COLUMN`. + adapter.cursor.fetchall.side_effect = [ + [ + ("id", "BIGINT", "No", "true", None, ""), + ("msg_keep_nn", "VARCHAR(64)", "No", "false", None, ""), + ("msg_relax_null", "VARCHAR(64)", "No", "false", None, ""), + ], + [ + ("id", "BIGINT", "No", "true", None, ""), + ("msg_keep_nn", "VARCHAR(128)", "No", "false", None, ""), + ("msg_relax_null", "VARCHAR(128)", "Yes", "false", None, ""), + ("added_col", "VARCHAR(32)", "No", "false", None, ""), + ], + ] + ops = adapter.get_alter_operations("db.curr_tbl", "db.tgt_tbl") + adapter.cursor.fetchall.side_effect = None + assert [op.expression.sql("doris", identify=True) for op in ops] == [ + "ALTER TABLE `db`.`curr_tbl` ADD COLUMN `added_col` VARCHAR(32)", + "ALTER TABLE `db`.`curr_tbl` MODIFY COLUMN `msg_keep_nn` VARCHAR(128) NOT NULL", + "ALTER TABLE `db`.`curr_tbl` MODIFY COLUMN `msg_relax_null` VARCHAR(128)", + ] + + # 2. `_fetch_native_df` unwraps `pandas.errors.DatabaseError` wrapping PyMySQL `OperationalError(2006)` + # and reconnects + restores active catalog once. + class OperationalError(Exception): + pass + + adapter.set_current_catalog("hive_cat") + adapter.cursor.execute.reset_mock() + raw_disconnect = OperationalError(2006, "MySQL server has gone away") + wrapped_db_err = DatabaseError("Execution failed on sql") + wrapped_db_err.__cause__ = raw_disconnect + expected_df = pd.DataFrame({"id": [1]}) + super_fetch = mocker.patch( + "sqlmesh.core.engine_adapter.mixins.PandasNativeFetchDFSupportMixin._fetch_native_df", + side_effect=[wrapped_db_err, expected_df], + ) + out_df = adapter._fetch_native_df("SELECT 1 AS id") + pd.testing.assert_frame_equal(t.cast(pd.DataFrame, out_df), expected_df) + assert super_fetch.call_count == 2 + assert to_sql_calls(adapter) == ["SWITCH `hive_cat`"] + + # 3. Catalog state tracking across `close()`, cross-thread `recycle()`, direct `SWITCH`, + # and dead-socket catalog restore before a write statement (`INSERT INTO`). + created_cursors: t.List[t.Any] = [] + + def _conn_factory() -> t.Any: + conn = mocker.MagicMock() + cur = mocker.MagicMock() + created_cursors.append(cur) + conn.cursor.return_value = cur + return conn + + pooled_adapter = DorisEngineAdapter(_conn_factory, multithreaded=True) + assert isinstance(pooled_adapter._connection_pool, ThreadLocalConnectionPool) + pooled_adapter.execute("SWITCH `iceberg_cat`") + assert pooled_adapter._active_catalog == "iceberg_cat" + + # After `close()`, next statement on the new connection restores `SWITCH `iceberg_cat`` automatically + pooled_adapter.close() + pooled_adapter.execute("SELECT 1") + assert [c[0][0] for c in created_cursors[-1].execute.call_args_list] == [ + "SWITCH `iceberg_cat`", + "SELECT 1", + ] + + # Cross-thread `recycle()` closes worker thread's connection; worker's next query restores its catalog + def _other_thread_recycle() -> None: + pooled_adapter.recycle() + + recycler = threading.Thread(target=_other_thread_recycle) + recycler.start() + recycler.join() + pooled_adapter.execute("SELECT 2") + assert [c[0][0] for c in created_cursors[-1].execute.call_args_list] == [ + "SWITCH `iceberg_cat`", + "SELECT 2", + ] + + # Dead socket during `_ensure_catalog_restored` before a non-idempotent write (`INSERT INTO`) + # reconnects cleanly during `SWITCH` and then runs the `INSERT INTO` once on the fresh connection. + pooled_adapter._needs_catalog_restore = True + created_cursors[-1].execute.side_effect = OperationalError(2006, "MySQL server has gone away") + pooled_adapter.execute("INSERT INTO `db`.`t` VALUES (1)") + assert [c[0][0] for c in created_cursors[-1].execute.call_args_list] == [ + "SWITCH `iceberg_cat`", + "INSERT INTO `db`.`t` VALUES (1)", + ] + + # 4. `_wait_for_rollup` polls `SHOW ALTER TABLE MATERIALIZED VIEW` and `execute()` retries when + # Doris reports `Table[...]'s state(ROLLUP) is not NORMAL`. + rollup_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + setattr( + rollup_adapter, + "_wait_for_rollup", + DorisEngineAdapter._wait_for_rollup.__get__(rollup_adapter, DorisEngineAdapter), + ) + rollup_adapter.cursor.fetchall.side_effect = [ + [(10, "events", "2026-01-01", None, "events", "mv1", 20, 30, "RUNNING", "", "50%", 3600)], + [ + ( + 10, + "events", + "2026-01-01", + "2026-01-01", + "events", + "mv1", + 20, + 30, + "FINISHED", + "", + "100%", + 3600, + ) + ], + ] + call_idx = {"n": 0} + + def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: + if sql_val.startswith("ALTER TABLE") and call_idx["n"] == 0: + call_idx["n"] += 1 + raise OperationalError( + 1105, + "errCode = 2, detailMessage = Table[events]'s state(ROLLUP) is not NORMAL. Do not allow doing ALTER ops", + ) + + rollup_adapter.cursor.execute.side_effect = _exec_side_effect + rollup_adapter.execute(parse_one("ALTER TABLE db.events ADD COLUMN c2 INT", read="doris")) + assert to_sql_calls(rollup_adapter) == [ + "ALTER TABLE `db`.`events` ADD COLUMN `c2` INT", + "SHOW ALTER TABLE MATERIALIZED VIEW FROM `db` WHERE TableName = 'events'", + "SHOW ALTER TABLE MATERIALIZED VIEW FROM `db` WHERE TableName = 'events'", + "ALTER TABLE `db`.`events` ADD COLUMN `c2` INT", + ] + + # 5. `exp.Paren` and `exp.Array` handling in `distributed_by`, `partitions`, and `partition_by`, + # plus `dynamic_partition` properties and injection validation inside `exp.Paren`. + ddl_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + ddl_adapter.create_table( + "analytics.paren_props_tbl", + { + "event_date": exp.DataType.build("DATE", dialect="doris"), + "user_id": exp.DataType.build("BIGINT", dialect="doris"), + }, + table_properties={ + "partition_by": exp.Paren(this=exp.Literal.string("event_date")), + "partitions": exp.Paren( + this=exp.Literal.string('PARTITION p2026 VALUES LESS THAN ("2027-01-01")') + ), + "distributed_by": exp.Paren(this=exp.Literal.string("user_id")), + "buckets": exp.Literal.number(8), + }, + ) + ddl_adapter.create_table( + "analytics.dynamic_part_tbl", + { + "event_date": exp.DataType.build("DATE", dialect="doris"), + "user_id": exp.DataType.build("BIGINT", dialect="doris"), + }, + partitioned_by=[parse_one("RANGE(event_date)", read="doris")], + table_properties={ + "dynamic_partition.enable": exp.Literal.string("true"), + "dynamic_partition.time_unit": exp.Literal.string("DAY"), + "dynamic_partition.time_zone": exp.Literal.string("UTC"), + "dynamic_partition.start": exp.Literal.string("-32"), + "dynamic_partition.end": exp.Literal.string("3"), + "dynamic_partition.create_history_partition": exp.Literal.string("true"), + "dynamic_partition.prefix": exp.Literal.string("p"), + "dynamic_partition.buckets": exp.Literal.string("4"), + }, + ) + assert to_sql_calls(ddl_adapter) == [ + 'CREATE TABLE IF NOT EXISTS `analytics`.`paren_props_tbl` (`event_date` DATE, `user_id` BIGINT) PARTITION BY RANGE (`event_date`) (PARTITION p2026 VALUES LESS THAN ("2027-01-01")) DISTRIBUTED BY HASH (`user_id`) BUCKETS 8', + "CREATE TABLE IF NOT EXISTS `analytics`.`dynamic_part_tbl` (`event_date` DATE, `user_id` BIGINT) PARTITION BY RANGE (`event_date`) () PROPERTIES ('dynamic_partition.enable'='true', 'dynamic_partition.time_unit'='DAY', 'dynamic_partition.time_zone'='UTC', 'dynamic_partition.start'='-32', 'dynamic_partition.end'='3', 'dynamic_partition.create_history_partition'='true', 'dynamic_partition.prefix'='p', 'dynamic_partition.buckets'='4')", + ] + with pytest.raises(SQLMeshError, match="Invalid partition definition expression"): + ddl_adapter.create_table( + "analytics.bad_paren_part", + {"event_date": exp.DataType.build("DATE", dialect="doris")}, + table_properties={ + "partition_by": exp.Paren(this=exp.Literal.string("event_date")), + "partitions": exp.Paren( + this=exp.Literal.string( + "PARTITION p1 VALUES LESS THAN MAXVALUE); DROP TABLE x --" + ) + ), + }, + ) + + # 6. Every `MODEL (...)` SQL block in `docs/integrations/engines/doris.md` parses cleanly as a SQLMesh model. + doris_doc_path = ( + Path(__file__).resolve().parents[3] / "docs" / "integrations" / "engines" / "doris.md" + ) + doc_text = doris_doc_path.read_text(encoding="utf-8") + sql_blocks = re.findall(r"```sql\n(.*?)```", doc_text, flags=re.DOTALL) + model_blocks = [ + blk for blk in sql_blocks if re.search(r"^\s*MODEL\s*\(", blk, flags=re.MULTILINE) + ] + assert len(model_blocks) >= 17 + for blk in model_blocks: + parsed_model = load_sql_based_model(d.parse(blk, default_dialect="doris"), dialect="doris") + assert parsed_model.name is not None + + # 7. `BUILD INDEX` synchronization polls `SHOW BUILD INDEX` (for both AST and string `execute()` calls). + build_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + wait_build_mock = mocker.patch.object(build_adapter, "_wait_for_build_index") + wait_schema_mock = mocker.patch.object(build_adapter, "_wait_for_schema_change") + build_adapter.execute(parse_one("BUILD INDEX idx_msg ON analytics.logs", read="doris")) + build_adapter.execute("/* SQLMESH_PLAN: 123 */ BUILD INDEX idx_emb ON analytics.logs") + assert wait_build_mock.call_count == 2 + assert wait_schema_mock.call_count == 0 + + raw_build_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + raw_build_adapter.cursor.description = [ + ("JobId",), + ("TableName",), + ("PartitionName",), + ("AlterInvertedIndexes",), + ("CreateTime",), + ("FinishTime",), + ("TransactionId",), + ("State",), + ("Msg",), + ("Progress",), + ] + raw_build_adapter.cursor.fetchall.side_effect = [ + [("1", "logs", "logs", "idx_msg", "t0", None, "10", "WAITING_TXN", "", None)], + [("1", "logs", "logs", "idx_msg", "t0", "t1", "10", "FINISHED", "", None)], + ] + mocker.patch("sqlmesh.core.engine_adapter.doris.time.sleep") + DorisEngineAdapter._wait_for_build_index(raw_build_adapter, "analytics.logs", timeout=5.0) + assert raw_build_adapter.cursor.fetchall.call_count == 2 diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 8d19ffee36..6a8cf059f6 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1605,3 +1605,27 @@ def test_extend_sqlglot_is_idempotent(): assert parse_one("SELECT CAST(1 AS INT)").sql() == "SELECT CAST(1 AS INT)" # The class-level registries must not grow on repeated calls. assert Generator.UNWRAPPED_INTERVAL_VALUES == before + + +def test_doris_userdefined_isolation_and_modify_column_not_null(): + # Non-Doris dialects must keep standard INFORMATION_SCHEMA `USER-DEFINED` behavior (`kind=None`) + for other_dialect in ("postgres", "mysql", "snowflake"): + other_udt = exp.DataType.build("USER-DEFINED", dialect=other_dialect) + assert other_udt.this == exp.DataType.Type.USERDEFINED + assert other_udt.args.get("kind") is None + + # Doris dialect preserves native analytical UDT names and parameterized AGG_STATE + for doris_udt_sql in ("BITMAP", "QUANTILE_STATE", "AGG_STATE"): + doris_udt = exp.DataType.build(doris_udt_sql, dialect="doris", udt=True) + assert doris_udt.sql("doris") == doris_udt_sql + + # Doris `ALTER TABLE ... MODIFY COLUMN ... NOT NULL` parses as structured `exp.Alter` + # (enabling `@this_model` macro resolution) and preserves `NOT NULL` on generation. + alter_ast = d.parse_one( + "ALTER TABLE @this_model MODIFY COLUMN msg VARCHAR(128) NOT NULL", + dialect="doris", + ) + assert isinstance(alter_ast, exp.Alter) + assert ( + alter_ast.sql("doris") == "ALTER TABLE @this_model MODIFY COLUMN msg VARCHAR(128) NOT NULL" + ) From fd497b5311535f92fc748bccf581a64969c09281 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:16:16 -0800 Subject: [PATCH 04/54] Fix: Reject unresolved not_null_columns CTAS and clarify Doris index/distribution docs Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 12 ++++++------ sqlmesh/core/engine_adapter/doris.py | 12 ++++++++++++ tests/core/engine_adapter/test_doris.py | 13 +++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index f4f0d55083..749fa5fdf4 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -247,7 +247,7 @@ GROUP BY user_id, event_type; ## Distribution & Bucketing -Doris OLAP tables distribute tablets across backends via `DISTRIBUTED BY HASH(...)` or `DISTRIBUTED BY RANDOM` (on Doris 2.1+, omitting `DISTRIBUTED BY` on `DUPLICATE KEY` or `AGGREGATE KEY` tables defaults to `DISTRIBUTED BY RANDOM BUCKETS AUTO`, whereas `UNIQUE KEY` tables and Doris 2.0 require explicit hash distribution). SQLMesh supports three ways to configure distribution: +Doris OLAP tables distribute tablets across backends via `DISTRIBUTED BY HASH(...)` or `DISTRIBUTED BY RANDOM` (on Doris 2.1+, omitting `DISTRIBUTED BY` defaults to `DISTRIBUTED BY RANDOM BUCKETS AUTO` for `DUPLICATE KEY` tables and `AGGREGATE KEY` tables without `REPLACE` or `REPLACE_IF_NOT_NULL` value columns; `AGGREGATE KEY` tables with `REPLACE` or `REPLACE_IF_NOT_NULL` columns, `UNIQUE KEY` tables, and Doris 2.0 reject random distribution and require explicit `DISTRIBUTED BY HASH(...)` in native DDL). SQLMesh supports three ways to configure distribution: 1. **SQLMesh `clustered_by` + optional `buckets`** (recommended for cross-engine portability): ```sql @@ -796,10 +796,10 @@ WHERE event_time BETWEEN @start_ts AND @end_ts; You can attach Doris inverted indexes (`USING INVERTED` with tokenizer `PROPERTIES`), bloom filter indexes (`USING NGRAM_BF`), or Doris 4.x+ ANN vector indexes (`USING ANN` with `hnsw` or `ivf` index types; ANN indexes on `UNIQUE KEY` tables require Doris 4.1.4+ with Merge-on-Write enabled, whereas earlier Doris 4.x versions only support ANN indexes on `DUPLICATE KEY` tables) using post-statements. Because Doris requires ANN vector index columns to be `ARRAY NOT NULL` at table creation time (and Doris ignores `NON_NULLABLE(...)` on non-`AUTO PARTITION` CTAS columns while forbidding `ALTER TABLE ... MODIFY COLUMN` from changing an existing nullable column to `NOT NULL`), declare `not_null_columns = ('embedding')` in `physical_properties` so SQLMesh materializes the table schema with `embedding ARRAY NOT NULL` before inserting rows. -Because SQLMesh evaluates and inserts the initial batch of rows into a new snapshot table **before** executing `@IF(@runtime_stage = 'creating', ...)` post-statements, Doris `ALTER TABLE ... ADD INDEX` alone only registers the index for subsequent writes: +During normal snapshot creation (when column types are fully known ahead of evaluation, such as models with explicit `columns` or fully type-annotated projections), SQLMesh creates an empty table and runs `@IF(@runtime_stage = 'creating', ...)` post-statements **before** loading data, so `ALTER TABLE ... ADD INDEX` already indexes the initial load. During **lazy** snapshot creation (when column types are not fully resolved ahead of evaluation), SQLMesh may create the table and insert the initial batch **before** running `@IF(@runtime_stage = 'creating', ...)` post-statements, in which case `ALTER TABLE ... ADD INDEX` alone only registers the index for subsequent writes: -- **Compute-storage integrated deployments**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` to backfill the index on the initial batch before downstream models run (`DorisEngineAdapter` automatically polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` until `FINISHED`). -- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes, and run `sqlmesh plan --restate-model ` if you need the initial batch rewritten through the newly registered index. +- **Compute-storage integrated deployments**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` so any pre-existing rows from lazy creation are backfilled (on an empty table during normal snapshot creation, `BUILD INDEX` is a fast no-op). `DorisEngineAdapter` polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` for up to 60 seconds, but returns early on timeout, `CANCELLED` jobs, or polling errors—verify index readiness via `SHOW ALTER TABLE COLUMN` / `SHOW BUILD INDEX` when downstream models depend on index builds completing first. +- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes. Snapshots created through the normal empty-table path already index the initial load; only when a snapshot uses lazy creation (initial batch loaded before `ADD INDEX`) do you need `sqlmesh plan --restate-model ` to rewrite that initial batch through the newly registered index. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: @@ -832,8 +832,8 @@ SELECT created_at::DATETIME(6) AS created_at FROM raw.logs; --- DorisEngineAdapter automatically polls SHOW ALTER TABLE COLUMN and SHOW BUILD INDEX --- so asynchronous index creation and backfill jobs finish before downstream models run. +-- DorisEngineAdapter polls SHOW ALTER TABLE COLUMN and SHOW BUILD INDEX for up to 60s +-- (verify status via SHOW ALTER TABLE COLUMN / SHOW BUILD INDEX when index readiness matters). @IF( @runtime_stage = 'creating', ALTER TABLE @this_model ADD INDEX idx_message (message) USING INVERTED PROPERTIES ("parser" = "english", "support_phrase" = "true") COMMENT 'Full-text inverted index' diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index c6db5bb2df..5cec12c3b7 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -5060,6 +5060,18 @@ def _matches_target_table(t_node: exp.Table) -> bool: if target_already_existed and not target_columns_to_types_known: return + if ( + has_explicit_not_null_cols + and not self_referencing + and first_query is not None + and not target_columns_to_types_known + ): + raise SQLMeshError( + f"Doris cannot apply not_null_columns {sorted(explicit_not_null_list)} to CTAS on " + f"'{table_name}' because the column types could not be resolved. Specify explicit " + "column types or CAST each projection." + ) + if ( requires_schema_ddl_ctas or precreated_by_replace_query ) and target_columns_to_types_known: diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 59bebc7fda..7a2907892b 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6052,3 +6052,16 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: mocker.patch("sqlmesh.core.engine_adapter.doris.time.sleep") DorisEngineAdapter._wait_for_build_index(raw_build_adapter, "analytics.logs", timeout=5.0) assert raw_build_adapter.cursor.fetchall.call_count == 2 + + # 8. External Iceberg CTAS with `not_null_columns` rejects unresolved column types instead of silently creating nullable columns. + iceberg_adapter = make_mocked_engine_adapter( + DorisEngineAdapter, catalog_types={"iceberg_cat": "iceberg"} + ) + with pytest.raises(SQLMeshError, match="cannot apply not_null_columns"): + iceberg_adapter.ctas( + "iceberg_cat.db.tbl", + parse_one("SELECT id, payload FROM raw.events", read="doris"), + table_properties={ + "not_null_columns": exp.Tuple(expressions=[exp.column("payload")]), + }, + ) From 9866565c2b1465ea6a3e00191adc4cc251aca6b4 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:36:57 -0800 Subject: [PATCH 05/54] Docs: Clarify Doris 2.1.0 distribution, NGRAM_BF index builds, and decoupled backfill Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 749fa5fdf4..05c415e352 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -247,7 +247,7 @@ GROUP BY user_id, event_type; ## Distribution & Bucketing -Doris OLAP tables distribute tablets across backends via `DISTRIBUTED BY HASH(...)` or `DISTRIBUTED BY RANDOM` (on Doris 2.1+, omitting `DISTRIBUTED BY` defaults to `DISTRIBUTED BY RANDOM BUCKETS AUTO` for `DUPLICATE KEY` tables and `AGGREGATE KEY` tables without `REPLACE` or `REPLACE_IF_NOT_NULL` value columns; `AGGREGATE KEY` tables with `REPLACE` or `REPLACE_IF_NOT_NULL` columns, `UNIQUE KEY` tables, and Doris 2.0 reject random distribution and require explicit `DISTRIBUTED BY HASH(...)` in native DDL). SQLMesh supports three ways to configure distribution: +Doris OLAP tables distribute tablets across backends via `DISTRIBUTED BY HASH(...)` or `DISTRIBUTED BY RANDOM`. Later Doris 2.1.x+ and 3.x/4.x releases allow omitting `DISTRIBUTED BY`—defaulting to `DISTRIBUTED BY RANDOM BUCKETS AUTO` for `DUPLICATE KEY` tables and `AGGREGATE KEY` tables without `REPLACE` or `REPLACE_IF_NOT_NULL` value columns—whereas Doris 2.0 and 2.1.0 require an explicit `DISTRIBUTED BY` clause on schema-defined OLAP tables (so on Doris 2.0 / 2.1.0, always configure distribution explicitly via `clustered_by` or `distributed_by` on non-`UNIQUE KEY` models), and `AGGREGATE KEY` tables with `REPLACE` or `REPLACE_IF_NOT_NULL` columns as well as `UNIQUE KEY` tables reject random distribution and require `DISTRIBUTED BY HASH(...)` in native DDL. SQLMesh supports three ways to configure distribution: 1. **SQLMesh `clustered_by` + optional `buckets`** (recommended for cross-engine portability): ```sql @@ -796,10 +796,15 @@ WHERE event_time BETWEEN @start_ts AND @end_ts; You can attach Doris inverted indexes (`USING INVERTED` with tokenizer `PROPERTIES`), bloom filter indexes (`USING NGRAM_BF`), or Doris 4.x+ ANN vector indexes (`USING ANN` with `hnsw` or `ivf` index types; ANN indexes on `UNIQUE KEY` tables require Doris 4.1.4+ with Merge-on-Write enabled, whereas earlier Doris 4.x versions only support ANN indexes on `DUPLICATE KEY` tables) using post-statements. Because Doris requires ANN vector index columns to be `ARRAY NOT NULL` at table creation time (and Doris ignores `NON_NULLABLE(...)` on non-`AUTO PARTITION` CTAS columns while forbidding `ALTER TABLE ... MODIFY COLUMN` from changing an existing nullable column to `NOT NULL`), declare `not_null_columns = ('embedding')` in `physical_properties` so SQLMesh materializes the table schema with `embedding ARRAY NOT NULL` before inserting rows. -During normal snapshot creation (when column types are fully known ahead of evaluation, such as models with explicit `columns` or fully type-annotated projections), SQLMesh creates an empty table and runs `@IF(@runtime_stage = 'creating', ...)` post-statements **before** loading data, so `ALTER TABLE ... ADD INDEX` already indexes the initial load. During **lazy** snapshot creation (when column types are not fully resolved ahead of evaluation), SQLMesh may create the table and insert the initial batch **before** running `@IF(@runtime_stage = 'creating', ...)` post-statements, in which case `ALTER TABLE ... ADD INDEX` alone only registers the index for subsequent writes: +When SQLMesh pre-creates an empty physical snapshot table during the snapshot-creation step (`create_snapshot`), `@IF(@runtime_stage = 'creating', ...)` post-statements run before data is loaded, so `ALTER TABLE ... ADD INDEX` already indexes the initial load. However, when a missing target table is created **during evaluation**—whether via lazy creation (unresolved column types) or evaluation-time table creation on a missing target table (even with known column types)—SQLMesh inserts the first batch **before** running `@IF(@runtime_stage = 'creating', ...)` post-statements, so `ALTER TABLE ... ADD INDEX` for `INVERTED` or `ANN` indexes only registers the index for subsequent writes: -- **Compute-storage integrated deployments**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` so any pre-existing rows from lazy creation are backfilled (on an empty table during normal snapshot creation, `BUILD INDEX` is a fast no-op). `DorisEngineAdapter` polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` for up to 60 seconds, but returns early on timeout, `CANCELLED` jobs, or polling errors—verify index readiness via `SHOW ALTER TABLE COLUMN` / `SHOW BUILD INDEX` when downstream models depend on index builds completing first. -- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes. Snapshots created through the normal empty-table path already index the initial load; only when a snapshot uses lazy creation (initial batch loaded before `ADD INDEX`) do you need `sqlmesh plan --restate-model ` to rewrite that initial batch through the newly registered index. +- **Compute-storage integrated deployments**: + - **`USING INVERTED` and `USING ANN` indexes**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` so any pre-existing rows inserted before the post-statement are backfilled (on an empty table, `BUILD INDEX` is a fast no-op). `DorisEngineAdapter` polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` for up to 60 seconds, but returns early on timeout, `CANCELLED` jobs, or polling errors—verify index readiness via `SHOW ALTER TABLE COLUMN` / `SHOW BUILD INDEX` when downstream models depend on index builds completing first. + - **`USING NGRAM_BF` bloom filter indexes**: Do **not** run `BUILD INDEX`—Doris rejects `BUILD INDEX` for `NGRAM_BF` indexes. Instead, `ALTER TABLE @this_model ADD INDEX () USING NGRAM_BF PROPERTIES(...)` triggers an asynchronous Doris `SCHEMA_CHANGE` job directly that builds the filter across existing and new rows (monitor progress via `SHOW ALTER TABLE COLUMN`). +- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes. If the initial batch was inserted before `ADD INDEX` ran during evaluation-time creation (for `INVERTED` or `ANN` indexes), choose the backfill strategy matching the model kind: + - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. + - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED` (for example, `INSERT OVERWRITE TABLE SELECT * FROM `). You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 4a180d597e64e57eb63054f24bc95c445cb4e044 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:54:35 -0800 Subject: [PATCH 06/54] Docs: Qualify Doris enable_add_index_for_new_data modes and INSERT OVERWRITE writer pause Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 05c415e352..fa2b9049b4 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -800,11 +800,11 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Compute-storage integrated deployments**: - **`USING INVERTED` and `USING ANN` indexes**: Follow `ALTER TABLE @this_model ADD INDEX ...` with `BUILD INDEX ON @this_model` inside `@runtime_stage = 'creating'` so any pre-existing rows inserted before the post-statement are backfilled (on an empty table, `BUILD INDEX` is a fast no-op). `DorisEngineAdapter` polls `SHOW ALTER TABLE COLUMN` and `SHOW BUILD INDEX` for up to 60 seconds, but returns early on timeout, `CANCELLED` jobs, or polling errors—verify index readiness via `SHOW ALTER TABLE COLUMN` / `SHOW BUILD INDEX` when downstream models depend on index builds completing first. - - **`USING NGRAM_BF` bloom filter indexes**: Do **not** run `BUILD INDEX`—Doris rejects `BUILD INDEX` for `NGRAM_BF` indexes. Instead, `ALTER TABLE @this_model ADD INDEX () USING NGRAM_BF PROPERTIES(...)` triggers an asynchronous Doris `SCHEMA_CHANGE` job directly that builds the filter across existing and new rows (monitor progress via `SHOW ALTER TABLE COLUMN`). -- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX` so `ADD INDEX` indexes subsequent writes. If the initial batch was inserted before `ADD INDEX` ran during evaluation-time creation (for `INVERTED` or `ANN` indexes), choose the backfill strategy matching the model kind: + - **`USING NGRAM_BF` bloom filter indexes**: Do **not** run `BUILD INDEX`—Doris rejects `BUILD INDEX` for `NGRAM_BF` indexes. When `enable_add_index_for_new_data = false` (so Doris uses the heavy `SCHEMA_CHANGE` path rather than light index change), `ALTER TABLE @this_model ADD INDEX () USING NGRAM_BF PROPERTIES(...)` rebuilds the filter across existing and new rows via an asynchronous `SCHEMA_CHANGE` job (monitor via `SHOW ALTER TABLE COLUMN`). If `enable_add_index_for_new_data = true` with light index changes enabled, `ADD INDEX` registers the filter only for new writes, so `SHOW ALTER TABLE COLUMN` reporting `FINISHED` alone does not confirm historical-row backfill—use the heavy `SCHEMA_CHANGE` path (`enable_add_index_for_new_data = false`) when existing rows must be indexed. +- **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX`. Because Doris 4.1.5 may rebuild existing data through a `SCHEMA_CHANGE` job when `enable_add_index_for_new_data` is `false`, check the active index-change mode (`enable_add_index_for_new_data`) and `SHOW ALTER TABLE COLUMN` job status before running any manual rewrite or restatement so you avoid unnecessary reprocessing. Only if existing rows from evaluation-time creation remain unindexed, choose the backfill strategy matching the model kind: - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`. - - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED` (for example, `INSERT OVERWRITE TABLE SELECT * FROM `). + - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED` (for example, `INSERT OVERWRITE TABLE SELECT * FROM `), and keep all writers stopped until the overwrite completes so concurrent appends are not lost. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 5d43536fc168fdfb34ca4f8b289ead7c4f081d0b Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:13:09 -0800 Subject: [PATCH 07/54] Docs: Preserve __DORIS_SEQUENCE_COL__ during manual UNIQUE KEY INSERT OVERWRITE Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index fa2b9049b4..796860ea8e 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -804,7 +804,7 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX`. Because Doris 4.1.5 may rebuild existing data through a `SCHEMA_CHANGE` job when `enable_add_index_for_new_data` is `false`, check the active index-change mode (`enable_add_index_for_new_data`) and `SHOW ALTER TABLE COLUMN` job status before running any manual rewrite or restatement so you avoid unnecessary reprocessing. Only if existing rows from evaluation-time creation remain unindexed, choose the backfill strategy matching the model kind: - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED` (for example, `INSERT OVERWRITE TABLE SELECT * FROM `), and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From e3ec857d1c7d146e6f6d248a0d91abd3d3f4dfac Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:33:44 -0800 Subject: [PATCH 08/54] Docs: Warn against self-INSERT OVERWRITE when UNIQUE KEY sequence tombstones must be retained Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 796860ea8e..c6d8b2e9c3 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -804,7 +804,7 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX`. Because Doris 4.1.5 may rebuild existing data through a `SCHEMA_CHANGE` job when `enable_add_index_for_new_data` is `false`, check the active index-change mode (`enable_add_index_for_new_data`) and `SHOW ALTER TABLE COLUMN` job status before running any manual rewrite or restatement so you avoid unnecessary reprocessing. Only if existing rows from evaluation-time creation remain unindexed, choose the backfill strategy matching the model kind: - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve history without truncating, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. Do **not** use this self-`INSERT OVERWRITE` when `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained against late out-of-order writes, because a standard `SELECT` excludes deleted rows and discards their tombstones (allowing an older out-of-order write to resurrect a deleted key); in that case, build the index via the heavy `SCHEMA_CHANGE` path (`enable_add_index_for_new_data = false`) or reload from an authoritative upstream source instead. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From ac5980ffa694ad696c1f7110845814cd82223ce8 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:16:42 -0800 Subject: [PATCH 09/54] Docs: Detail Doris decoupled index rebuild waits and CDC tombstone replay Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index c6d8b2e9c3..438a7c0a2b 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -804,7 +804,11 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX`. Because Doris 4.1.5 may rebuild existing data through a `SCHEMA_CHANGE` job when `enable_add_index_for_new_data` is `false`, check the active index-change mode (`enable_add_index_for_new_data`) and `SHOW ALTER TABLE COLUMN` job status before running any manual rewrite or restatement so you avoid unnecessary reprocessing. Only if existing rows from evaluation-time creation remain unindexed, choose the backfill strategy matching the model kind: - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. Do **not** use this self-`INSERT OVERWRITE` when `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained against late out-of-order writes, because a standard `SELECT` excludes deleted rows and discards their tombstones (allowing an older out-of-order write to resurrect a deleted key); in that case, build the index via the heavy `SCHEMA_CHANGE` path (`enable_add_index_for_new_data = false`) or reload from an authoritative upstream source instead. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). Choose the backfill path based on whether `UNIQUE KEY` delete/sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained against late out-of-order writes: + - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained**: Do **not** use self-`INSERT OVERWRITE` (a standard `SELECT` excludes deleted rows and discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Instead, either: + 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and tombstones)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). + 2. **Truncate and replay from an authoritative changelog/CDC source with tombstones and sequence values** (prefer option 1 when possible, and use `TRUNCATE TABLE` only after verifying the upstream changelog/CDC source retains the complete history of live rows, deleted-key tombstones, and sequence values): After `SHOW ALTER TABLE COLUMN` reports `FINISHED` and `SHOW INDEX FROM ` lists ``, stop all writers to the table before truncating, run `TRUNCATE TABLE `, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) so all rewritten segments are indexed without losing delete watermarks, and keep all writers stopped until the replay completes—reloading live rows alone discards deleted keys' tombstones and is not sufficient. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From c6f3ada63a7aee8687efb37088356e505cbb4fdf Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:20:56 -0800 Subject: [PATCH 10/54] Docs: Guard Doris overwrite delete-sign filter, compaction tombstones, and reader isolation Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 438a7c0a2b..19d4870913 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -804,11 +804,11 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Compute-storage decoupled deployments (`storage_vault_name`)**: Doris 4.x does not support `BUILD INDEX` in compute-storage decoupled mode (executing `BUILD INDEX` will fail); omit `BUILD INDEX`. Because Doris 4.1.5 may rebuild existing data through a `SCHEMA_CHANGE` job when `enable_add_index_for_new_data` is `false`, check the active index-change mode (`enable_add_index_for_new_data`) and `SHOW ALTER TABLE COLUMN` job status before running any manual rewrite or restatement so you avoid unnecessary reprocessing. Only if existing rows from evaluation-time creation remain unindexed, choose the backfill strategy matching the model kind: - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). Choose the backfill path based on whether `UNIQUE KEY` delete/sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained against late out-of-order writes: - - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, rewrite the existing rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must be retained**: Do **not** use self-`INSERT OVERWRITE` (a standard `SELECT` excludes deleted rows and discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Instead, either: - 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and tombstones)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Truncate and replay from an authoritative changelog/CDC source with tombstones and sequence values** (prefer option 1 when possible, and use `TRUNCATE TABLE` only after verifying the upstream changelog/CDC source retains the complete history of live rows, deleted-key tombstones, and sequence values): After `SHOW ALTER TABLE COLUMN` reports `FINISHED` and `SHOW INDEX FROM ` lists ``, stop all writers to the table before truncating, run `TRUNCATE TABLE `, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) so all rewritten segments are indexed without losing delete watermarks, and keep all writers stopped until the replay completes—reloading live rows alone discards deleted keys' tombstones and is not sufficient. + - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). Choose the backfill path based on whether currently stored `UNIQUE KEY` delete/sequence tombstones (`__DORIS_DELETE_SIGN__`) must be preserved across the backfill operation: + - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: + 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). + 2. **Replay from an authoritative changelog/CDC source with reader/writer isolation and failure recovery** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): To avoid exposing an empty or partially replayed table to readers or leaving the live physical table incomplete if replay fails, stage into a separate physical table—for example, deploy a new snapshot in a `dev` environment (`stamp` bump + `sqlmesh plan ` with `enable_add_index_for_new_data: false` in `session_variables`, or an isolated staging table) while the `prod` virtual view keeps serving readers from the untouched existing physical table—coordinate/pause writers, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) into the staged table (reloading live rows alone discards tombstones and is not sufficient), validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop the failed staged snapshot rather than promoting), and only then atomically repoint/promote `prod` and resume writers. If you cannot stage into a separate table and must run `TRUNCATE TABLE ` in place, enforce a full **reader and writer** maintenance window for the entire operation: create and verify a recoverable full copy of `` (preserving hidden delete-sign and sequence columns) before truncating, stop all readers and writers, run `TRUNCATE TABLE `, replay live rows plus `__DORIS_DELETE_SIGN__ = 1` tombstones and sequence values, and if replay fails restore `` from that verified recovery copy before allowing any readers or writers back onto the table. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 722964ab1eb900c5195a7dbce7dd5c1b9c886c76 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:47:46 -0800 Subject: [PATCH 11/54] Docs: Distinguish deployable vs forward-only Doris cutover and CDC writer redirection Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 19d4870913..c9f7c24213 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,7 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay from an authoritative changelog/CDC source with reader/writer isolation and failure recovery** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): To avoid exposing an empty or partially replayed table to readers or leaving the live physical table incomplete if replay fails, stage into a separate physical table—for example, deploy a new snapshot in a `dev` environment (`stamp` bump + `sqlmesh plan ` with `enable_add_index_for_new_data: false` in `session_variables`, or an isolated staging table) while the `prod` virtual view keeps serving readers from the untouched existing physical table—coordinate/pause writers, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) into the staged table (reloading live rows alone discards tombstones and is not sufficient), validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop the failed staged snapshot rather than promoting), and only then atomically repoint/promote `prod` and resume writers. If you cannot stage into a separate table and must run `TRUNCATE TABLE ` in place, enforce a full **reader and writer** maintenance window for the entire operation: create and verify a recoverable full copy of `` (preserving hidden delete-sign and sequence columns) before truncating, stop all readers and writers, run `TRUNCATE TABLE `, replay live rows plus `__DORIS_DELETE_SIGN__ = 1` tombstones and sequence values, and if replay fails restore `` from that verified recovery copy before allowing any readers or writers back onto the table. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a breaking/non-breaking `stamp` bump rather than `--forward-only` or a metadata-only plan), deploy in `` (`sqlmesh plan ` with `enable_add_index_for_new_data: false` in `session_variables`), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from the current `prod` physical table (`SHOW CREATE VIEW `)** while `` keeps serving readers from the untouched old physical table. Pause all external writers targeting the old physical table, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote the plan to `prod`, **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table and verify their destination before resuming writes** (if you later remove `forward_only false` to restore default `forward_only = true`, re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so `prod` promotes to the staged table and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and pause all external writers first, create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access; after a successful in-place replay, likewise verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) before resuming readers and writers. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From ac3a2c5763b2917f472fb9097a811cf996d7bc97 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:12:47 -0800 Subject: [PATCH 12/54] Docs: Pause SQLMesh runs and guard prod backfill during Doris CDC cutover Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index c9f7c24213..9c856d4d94 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a breaking/non-breaking `stamp` bump rather than `--forward-only` or a metadata-only plan), deploy in `` (`sqlmesh plan ` with `enable_add_index_for_new_data: false` in `session_variables`), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from the current `prod` physical table (`SHOW CREATE VIEW `)** while `` keeps serving readers from the untouched old physical table. Pause all external writers targeting the old physical table, replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote the plan to `prod`, **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table and verify their destination before resuming writes** (if you later remove `forward_only false` to restore default `forward_only = true`, re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so `prod` promotes to the staged table and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and pause all external writers first, create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access; after a successful in-place replay, likewise verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) before resuming readers and writers. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). Before staging or replaying, catch up any upstream SQLMesh models through a cron- and partition-aligned `` (at least one full model interval past `start` and accounting for any model `lookback`), then **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs as well as all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (with `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) through `` into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state so post-cutover `sqlmesh run` jobs only process new intervals/partitions after ``), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table and verify their destination before resuming external writers and scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs as well as all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access; after a successful in-place replay, likewise verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) before resuming readers, external writers, and scheduled `sqlmesh run` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From e0a2b95695622f8a6f182cb991f332b7733bbc7c Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:32:01 -0800 Subject: [PATCH 13/54] Docs: Align Doris CDC replay offsets and reconcile in-place intervals Co-authored-by: xinge.ji Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 9c856d4d94..70afbbf4a3 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). Before staging or replaying, catch up any upstream SQLMesh models through a cron- and partition-aligned `` (at least one full model interval past `start` and accounting for any model `lookback`), then **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs as well as all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (with `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) through `` into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state so post-cutover `sqlmesh run` jobs only process new intervals/partitions after ``), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table and verify their destination before resuming external writers and scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs as well as all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access; after a successful in-place replay, likewise verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) before resuming readers, external writers, and scheduled `sqlmesh run` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to the latest cron- and partition-aligned timestamp **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up any upstream SQLMesh models through `` before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones, either drain writers at a cron/partition boundary so `` aligns with `` or keep `sqlmesh run` paused until external CDC writers advance past that open partition). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (with `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that exact ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that exact same `` and verify both their destination table and `` resume position before resuming external writers and scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the exact same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals lag behind the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to the latest cron- and partition-aligned timestamp **fully covered** by `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up upstream SQLMesh models through `` first; if the replay covers less than one full model interval from `start`, keep `sqlmesh run` disabled until a full interval is covered, and if `` includes partial-interval CDC events beyond `` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones, either drain writers at a cron/partition boundary or keep `sqlmesh run` paused until external CDC writers advance past that open partition) and record those covered intervals without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (including a `stamp` bump on `` if needed so `sqlmesh plan` has a change to apply rather than returning early as a no-op). Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and scheduled `sqlmesh run` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 248ab1c78f00d6c32501bf991dcaa7ed7001033e Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:57:30 -0800 Subject: [PATCH 14/54] docs(doris): clarify explicit cutover timestamps, open-partition reconciliation, and staged snapshot cleanup Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 70afbbf4a3..8aba0d5b2c 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to the latest cron- and partition-aligned timestamp **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up any upstream SQLMesh models through `` before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones, either drain writers at a cron/partition boundary so `` aligns with `` or keep `sqlmesh run` paused until external CDC writers advance past that open partition). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (with `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that exact ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table (if replay fails, `prod` remains untouched—abandon/drop `` without promoting). Promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that exact same `` and verify both their destination table and `` resume position before resuming external writers and scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the exact same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals lag behind the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to the latest cron- and partition-aligned timestamp **fully covered** by `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up upstream SQLMesh models through `` first; if the replay covers less than one full model interval from `start`, keep `sqlmesh run` disabled until a full interval is covered, and if `` includes partial-interval CDC events beyond `` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones, either drain writers at a cron/partition boundary or keep `sqlmesh run` paused until external CDC writers advance past that open partition) and record those covered intervals without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (including a `stamp` bump on `` if needed so `sqlmesh plan` has a change to apply rather than returning early as a no-op). Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and scheduled `sqlmesh run` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up any upstream SQLMesh models through `` before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep `sqlmesh run` paused until external CDC writers advance past explicit timestamp `` with time and timezone, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **before running another `prod` plan** require either bumping `stamp` to create a new snapshot version or reverting the cutover `stamp`/`forward_only` edit **and** verified cleanup of the failed snapshot, its recorded intervals, and its incomplete staged table (`sqlmesh invalidate --sync` followed by `sqlmesh janitor --ignore-ttl`, confirming the incomplete staged table and its snapshot/interval state are removed) so promotion cannot reuse the incomplete table as already processed. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up upstream SQLMesh models through `` first; if the replay covers less than one full model interval from `start`, keep `sqlmesh run` disabled until a full interval is covered and reconciled), and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep `sqlmesh run` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 9e7c109c86aa2fc9f12fab6a61b91c99c8d3edb6 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:15:27 -0800 Subject: [PATCH 15/54] docs(doris): guard upstream catch-up, pre-existing intervals, and janitor --ignore-ttl impact Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 8aba0d5b2c..ce9c5d82db 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up any upstream SQLMesh models through `` before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep `sqlmesh run` paused until external CDC writers advance past explicit timestamp `` with time and timezone, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **before running another `prod` plan** require either bumping `stamp` to create a new snapshot version or reverting the cutover `stamp`/`forward_only` edit **and** verified cleanup of the failed snapshot, its recorded intervals, and its incomplete staged table (`sqlmesh invalidate --sync` followed by `sqlmesh janitor --ignore-ttl`, confirming the incomplete staged table and its snapshot/interval state are removed) so promotion cannot reuse the incomplete table as already processed. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation: stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up upstream SQLMesh models through `` first; if the replay covers less than one full model interval from `start`, keep `sqlmesh run` disabled until a full interval is covered and reconciled), and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep `sqlmesh run` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up and verifying any upstream SQLMesh models through ``—without selecting ``, passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, since `--backfill-model ` includes upstream dependencies and `--empty-backfill` records their intervals without producing data—before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, catch up and verify all upstream SQLMesh models through `` first via `sqlmesh run --select-model --end ` without selecting ``—passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `prod` plan), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up and verifying all upstream SQLMesh models through `` first without selecting ``, passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, since `--backfill-model ` includes upstream dependencies and `--empty-backfill` records their intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run` disabled until a full interval is covered and reconciled)—and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), catch up and verify all upstream SQLMesh models through `` first via `sqlmesh run --select-model --end ` without selecting `` (passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data), verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From d2ee31960579cf93ca9f65c9a52efbe764acf3d2 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:15:29 -0800 Subject: [PATCH 16/54] docs(doris): use --no-signals and state verification for upstream interval checks Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index ce9c5d82db..9fba0a5169 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up and verifying any upstream SQLMesh models through ``—without selecting ``, passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, since `--backfill-model ` includes upstream dependencies and `--empty-backfill` records their intervals without producing data—before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, catch up and verify all upstream SQLMesh models through `` first via `sqlmesh run --select-model --end ` without selecting ``—passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `prod` plan), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up and verifying all upstream SQLMesh models through `` first without selecting ``, passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, since `--backfill-model ` includes upstream dependencies and `--empty-backfill` records their intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run` disabled until a full interval is covered and reconciled)—and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), catch up and verify all upstream SQLMesh models through `` first via `sqlmesh run --select-model --end ` without selecting `` (passing `--ignore-cron` if needed and confirming via `sqlmesh check_intervals` that no upstream intervals remain missing through ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data), verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up and verifying any upstream SQLMesh models through ``—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in SQLMesh state (for example, via `sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data—before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, catch up and verify all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end `, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `prod` plan), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up and verifying all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run` disabled until a full interval is covered and reconciled)—and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), catch up and verify all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end `, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data— verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 3dc05d23a27adce2d753617f90831aea0fd71816 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:21:42 -0800 Subject: [PATCH 17/54] docs(doris): harden prod targeting, upstream checks, and state export safety in cutover docs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 9fba0a5169..2159729039 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`, catching up and verifying any upstream SQLMesh models through ``—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in SQLMesh state (for example, via `sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data—before running `--empty-backfill`; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, catch up and verify all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end `, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in SQLMesh state via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `prod` plan), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `--skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback` and catching up and verifying all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run` disabled until a full interval is covered and reconciled)—and record those covered intervals without running model SQL batches via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), catch up and verify all upstream SQLMesh models through `` first—running `sqlmesh run --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals --no-signals --select-model --end `, and independently verifying in SQLMesh state (`sqlmesh state export -o `) that each upstream model's recorded intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data— verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval via `sqlmesh plan --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 936a99c8064dc102da49bf4d2e5e02c9d17e0287 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:40:14 -0800 Subject: [PATCH 18/54] docs(doris): guard auto-restatements and upstream end bounds in cutover docs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 2159729039..e6c4086ab6 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron`, no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table and re-verifying `SHOW CREATE VIEW `, before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same `` and verify both their destination table and `` resume position before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals first and **require the replay up to `` / `` to cover every previously recorded interval on ``**, or else use item 2 with `forward_only false` and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary, or its configured `end` if earlier, regardless of model `cron`, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets `` and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no upstream `evaluatable` model whose configured `end` precedes `` or ``—use option 1 instead if any upstream `evaluatable` model ends before the cutover/reconciliation boundary, because `--backfill-model --empty-backfill` records the plan's full interval on every selected `evaluatable` upstream snapshot without capping at each model's metadata-only `end`—no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if one does, so `--empty-backfill` never records intervals past an upstream model's `end`—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if one does, because `--empty-backfill` would otherwise record intervals past that model's `end`—and `` keeps `disable_restatement true`, inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **no upstream `evaluatable` model may have a configured `end` earlier than `` or ``**—use option 1 instead if one does, because `--backfill-model --empty-backfill` does not cap recorded upstream intervals at each model's metadata-only `end`—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—if one does, do not run `--empty-backfill`, which would falsely record intervals beyond that model's `end`—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and no upstream `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` if any upstream `evaluatable` model ends before ``), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 8d3988caf7c3834f2812c19e4f26b539d9c86f78 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:03:21 -0800 Subject: [PATCH 19/54] docs(doris): require non-breaking cutover categorization and upstream start bounds Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index e6c4086ab6..f589b597d7 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no upstream `evaluatable` model whose configured `end` precedes `` or ``—use option 1 instead if any upstream `evaluatable` model ends before the cutover/reconciliation boundary, because `--backfill-model --empty-backfill` records the plan's full interval on every selected `evaluatable` upstream snapshot without capping at each model's metadata-only `end`—no non-deployable upstream snapshots, and a `NON_BREAKING` or `BREAKING` `stamp` bump rather than `--forward-only` or a metadata-only plan; prefer `NON_BREAKING` when downstream schemas/semantics are unchanged so downstream models remain `INDIRECT_NON_BREAKING`). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if one does, so `--empty-backfill` never records intervals past an upstream model's `end`—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if one does, because `--empty-backfill` would otherwise record intervals past that model's `end`—and `` keeps `disable_restatement true`, inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --select-model --backfill-model --empty-backfill --execution-time ` (passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (verify with `--explain` first that both physical-layer updates and model batches report `SKIP`—`--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` now targets the newly promoted physical snapshot table**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `stamp` bump under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **no upstream `evaluatable` model may have a configured `end` earlier than `` or ``**—use option 1 instead if one does, because `--backfill-model --empty-backfill` does not cap recorded upstream intervals at each model's metadata-only `end`—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm no upstream `evaluatable` model has a configured `end` earlier than ``—if one does, do not run `--empty-backfill`, which would falsely record intervals beyond that model's `end`—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and no upstream `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` if any upstream `evaluatable` model ends before ``), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no `` or selected `evaluatable` upstream model whose configured `start` is later than the plan's common `plan.start` or whose configured `end` precedes `` or ``—use option 1 instead whenever the plan's common `[plan.start, plan.end)` range would precede ``'s or any selected `evaluatable` upstream model's configured `start` or extend past its configured `end`, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` on every selected `evaluatable` snapshot without clamping to per-model metadata-only `start` or `end` bounds—no non-deployable upstream snapshots, and a **`NON_BREAKING` `stamp` bump** rather than `BREAKING`, `--forward-only`, or `METADATA`: pass `--no-auto-categorization` when staging in `` and explicitly choose `NON_BREAKING` for ``, because default `full` auto-categorization classifies a `stamp` bump as `BREAKING` and creates unbackfilled `INDIRECT_BREAKING` downstream physical tables on `prod` promotion; if any accompanying schema/semantic change requires `BREAKING`, you must stage, backfill, and validate every affected downstream physical table before promoting rather than restricting `--backfill-model` to ``). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm that the empty-backfill plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if either `start` or `end` bound would be exceeded, so `--empty-backfill` never records unevaluated pre-`start` or post-`end` history—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—excluding `--empty-backfill` and using option 1 before cutover if either `start` or `end` bound cannot be guaranteed, because `--empty-backfill` would otherwise record unevaluated pre-`start` or post-`end` intervals—inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --no-auto-categorization --select-model --backfill-model --empty-backfill --execution-time ` (choosing `NON_BREAKING` for `` so all downstream models remain `INDIRECT_NON_BREAKING` without downstream rebuilds, passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING`, that every affected downstream physical table was already staged, backfilled, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output; `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views still target their populated physical tables**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `NON_BREAKING` `stamp` bump via `--no-auto-categorization` under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without downstream rebuilds or overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **neither the reconciliation plan's common `plan.start` may precede ``'s or any selected `evaluatable` upstream model's configured `start`, nor may any selected `evaluatable` model have a configured `end` earlier than `` or ``**—use option 1 instead if either `start` or `end` bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `NON_BREAKING` `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—if either `start` or `end` bound would be exceeded, do not run `--empty-backfill`, which would falsely record unevaluated pre-`start` or post-`end` history—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` before cutover and using option 1 if either `start` or `end` bound cannot be guaranteed), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From a29656a8d33e4601631e99a5d8ec2eeeec0c6b4c Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:24:17 -0800 Subject: [PATCH 20/54] docs(doris): clarify unpartitioned insert_overwrite CDC replay write constraints Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index f589b597d7..869cd1e5f4 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,8 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no `` or selected `evaluatable` upstream model whose configured `start` is later than the plan's common `plan.start` or whose configured `end` precedes `` or ``—use option 1 instead whenever the plan's common `[plan.start, plan.end)` range would precede ``'s or any selected `evaluatable` upstream model's configured `start` or extend past its configured `end`, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` on every selected `evaluatable` snapshot without clamping to per-model metadata-only `start` or `end` bounds—no non-deployable upstream snapshots, and a **`NON_BREAKING` `stamp` bump** rather than `BREAKING`, `--forward-only`, or `METADATA`: pass `--no-auto-categorization` when staging in `` and explicitly choose `NON_BREAKING` for ``, because default `full` auto-categorization classifies a `stamp` bump as `BREAKING` and creates unbackfilled `INDIRECT_BREAKING` downstream physical tables on `prod` promotion; if any accompanying schema/semantic change requires `BREAKING`, you must stage, backfill, and validate every affected downstream physical table before promoting rather than restricting `--backfill-model` to ``). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm that the empty-backfill plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if either `start` or `end` bound would be exceeded, so `--empty-backfill` never records unevaluated pre-`start` or post-`end` history—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—excluding `--empty-backfill` and using option 1 before cutover if either `start` or `end` bound cannot be guaranteed, because `--empty-backfill` would otherwise record unevaluated pre-`start` or post-`end` intervals—inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --no-auto-categorization --select-model --backfill-model --empty-backfill --execution-time ` (choosing `NON_BREAKING` for `` so all downstream models remain `INDIRECT_NON_BREAKING` without downstream rebuilds, passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so the first post-cutover `sqlmesh run prod` does not treat the table as a first insert and replace it), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING`, that every affected downstream physical table was already staged, backfilled, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output; `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views still target their populated physical tables**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `NON_BREAKING` `stamp` bump via `--no-auto-categorization` under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without downstream rebuilds or overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **neither the reconciliation plan's common `plan.start` may precede ``'s or any selected `evaluatable` upstream model's configured `start`, nor may any selected `evaluatable` model have a configured `end` earlier than `` or ``**—use option 1 instead if either `start` or `end` bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `NON_BREAKING` `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—if either `start` or `end` bound would be exceeded, do not run `--empty-backfill`, which would falsely record unevaluated pre-`start` or post-`end` history—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on an `insert_overwrite true` model whose SQLMesh query does not emit tombstones or other CDC-only rows, waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state—either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` before cutover and using option 1 if either `start` or `end` bound cannot be guaranteed), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers, external writers, and (once any open partition is reconciled) scheduled `sqlmesh run prod` jobs. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values, and note that on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true`—where `partitioned_by` is unset—`insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query; limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no `` or selected `evaluatable` upstream model whose configured `start` is later than the plan's common `plan.start` or whose configured `end` precedes `` or ``—use option 1 instead whenever the plan's common `[plan.start, plan.end)` range would precede ``'s or any selected `evaluatable` upstream model's configured `start` or extend past its configured `end`, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` on every selected `evaluatable` snapshot without clamping to per-model metadata-only `start` or `end` bounds—no non-deployable upstream snapshots, and a **`NON_BREAKING` `stamp` bump** rather than `BREAKING`, `--forward-only`, or `METADATA`: pass `--no-auto-categorization` when staging in `` and explicitly choose `NON_BREAKING` for ``, because default `full` auto-categorization classifies a `stamp` bump as `BREAKING` and creates unbackfilled `INDIRECT_BREAKING` downstream physical tables on `prod` promotion; if any accompanying schema/semantic change requires `BREAKING`, you must stage, backfill, and validate every affected downstream physical table before promoting rather than restricting `--backfill-model` to ``). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm that the empty-backfill plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if either `start` or `end` bound would be exceeded, so `--empty-backfill` never records unevaluated pre-`start` or post-`end` history—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on a **partitioned** `insert_overwrite true` model where ``'s SQLMesh query does not reproduce that open partition's CDC rows or tombstones—and only when post-`` SQLMesh runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves (never resume `sqlmesh run` on an unpartitioned `insert_overwrite true` model or any model whose future SQLMesh runs would overwrite partitions or keys without preserving replayed/CDC rows, tombstones, and sequence values; keep SQLMesh writes for `` permanently disabled instead)—note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state: either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—excluding `--empty-backfill` and using option 1 before cutover if either `start` or `end` bound cannot be guaranteed, because `--empty-backfill` would otherwise record unevaluated pre-`start` or post-`end` intervals—inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --no-auto-categorization --select-model --backfill-model --empty-backfill --execution-time ` (choosing `NON_BREAKING` for `` so all downstream models remain `INDIRECT_NON_BREAKING` without downstream rebuilds, passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so, on models whose future SQLMesh writes preserve replayed rows, tombstones, and sequence values, the first post-cutover `sqlmesh run prod` does not treat the table as a first insert (`is_first_insert = true`) and replace it—whereas unpartitioned `insert_overwrite true` models still replace the entire table via `replace_query` even when `is_first_insert = false` and must keep SQLMesh writes disabled unless ``'s query reproduces the full table including tombstones and sequence values), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING`, that every affected downstream physical table was already staged, backfilled, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output; `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions—or replace the entire table when unpartitioned—in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views still target their populated physical tables**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers—and only resume scheduled or ad-hoc `sqlmesh run prod` evaluations targeting `` (once any open partition is reconciled) when ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values; otherwise (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones) keep SQLMesh writes for `` permanently disabled** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values, and note that on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true`—where `partitioned_by` is unset—`insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query; limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**): Either temporarily make the cutover snapshot deployable (`forward_only false` + `NON_BREAKING` `stamp` bump via `--no-auto-categorization` under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without downstream rebuilds or overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **neither the reconciliation plan's common `plan.start` may precede ``'s or any selected `evaluatable` upstream model's configured `start`, nor may any selected `evaluatable` model have a configured `end` earlier than `` or ``**—use option 1 instead if either `start` or `end` bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `NON_BREAKING` `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, if ``'s future SQLMesh writes do not preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values (including unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones, because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`), **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**; otherwise, when ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—if either `start` or `end` bound would be exceeded, do not run `--empty-backfill`, which would falsely record unevaluated pre-`start` or post-`end` history—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on a **partitioned** `insert_overwrite true` model where ``'s SQLMesh query does not reproduce that open partition's CDC rows or tombstones—and only when post-`` SQLMesh runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves (never resume `sqlmesh run` on an unpartitioned `insert_overwrite true` model or any model whose future SQLMesh runs would overwrite partitions or keys without preserving replayed/CDC rows, tombstones, and sequence values; keep SQLMesh writes for `` permanently disabled instead)—waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state: either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` before cutover and using option 1 if either `start` or `end` bound cannot be guaranteed), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers and external writers—and only resume scheduled or ad-hoc `sqlmesh run prod` evaluations targeting `` (once any open partition is reconciled) when ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values; otherwise (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones) keep SQLMesh writes for `` permanently disabled. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From 351317287e8c19fa715e4ddb6c592fab065f9e9b Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:49:12 -0800 Subject: [PATCH 21/54] docs(doris): structure CDC replay cutover steps and recovery checks Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 47 ++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 869cd1e5f4..3b12cb851c 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -808,8 +808,51 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). - 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values, and note that on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true`—where `partitioned_by` is unset—`insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query; limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**): `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table—configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full` (with no pinned `physical_version`, no `auto_restatement_cron` on ``, no `` or selected `evaluatable` upstream model whose configured `start` is later than the plan's common `plan.start` or whose configured `end` precedes `` or ``—use option 1 instead whenever the plan's common `[plan.start, plan.end)` range would precede ``'s or any selected `evaluatable` upstream model's configured `start` or extend past its configured `end`, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` on every selected `evaluatable` snapshot without clamping to per-model metadata-only `start` or `end` bounds—no non-deployable upstream snapshots, and a **`NON_BREAKING` `stamp` bump** rather than `BREAKING`, `--forward-only`, or `METADATA`: pass `--no-auto-categorization` when staging in `` and explicitly choose `NON_BREAKING` for ``, because default `full` auto-categorization classifies a `stamp` bump as `BREAKING` and creates unbackfilled `INDIRECT_BREAKING` downstream physical tables on `prod` promotion; if any accompanying schema/semantic change requires `BREAKING`, you must stage, backfill, and validate every affected downstream physical table before promoting rather than restricting `--backfill-model` to ``). First, **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion**. After all writers are drained, **capture a durable upstream changelog/CDC source offset (``)** so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old `prod` physical table after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot propagate automatic restatements onto ``, verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, confirm that the empty-backfill plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—use option 1 instead if either `start` or `end` bound would be exceeded, so `--empty-backfill` never records unevaluated pre-`start` or post-`end` history—and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill` by running `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, checking `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verifying in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if `` includes partial-interval CDC events beyond `` inside `[, )` on a **partitioned** `insert_overwrite true` model where ``'s SQLMesh query does not reproduce that open partition's CDC rows or tombstones—and only when post-`` SQLMesh runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves (never resume `sqlmesh run` on an unpartitioned `insert_overwrite true` model or any model whose future SQLMesh runs would overwrite partitions or keys without preserving replayed/CDC rows, tombstones, and sequence values; keep SQLMesh writes for `` permanently disabled instead)—note that waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state: either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after promoting and resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` with time and timezone, confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—excluding `--empty-backfill` and using option 1 before cutover if either `start` or `end` bound cannot be guaranteed, because `--empty-backfill` would otherwise record unevaluated pre-`start` or post-`end` intervals—inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``'s old or promoted `prod` physical table, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals`—especially none extending beyond ``—so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval in `prod` SQLMesh state via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run`, bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table, re-verifying `SHOW CREATE VIEW `, and confirming `` still has no `pending_restatement_intervals` beyond `` before resuming `sqlmesh run prod`). Stage the empty deployable table without running model SQL batches via `sqlmesh plan --no-auto-categorization --select-model --backfill-model --empty-backfill --execution-time ` (choosing `NON_BREAKING` for `` so all downstream models remain `INDIRECT_NON_BREAKING` without downstream rebuilds, passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`; `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so, on models whose future SQLMesh writes preserve replayed rows, tombstones, and sequence values, the first post-cutover `sqlmesh run prod` does not treat the table as a first insert (`is_first_insert = true`) and replace it—whereas unpartitioned `insert_overwrite true` models still replace the entire table via `replace_query` even when `is_first_insert = false` and must keep SQLMesh writes disabled unless ``'s query reproduces the full table including tombstones and sequence values), and **verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `** while `` keeps serving readers from the untouched old physical table. Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to that ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. If replay fails, `prod` remains untouched, **do not promote that staged snapshot**—because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync` (letting normal TTL janitor cleanup remove the unreferenced failed snapshot later); if you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run` (before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING`, that every affected downstream physical table was already staged, backfilled, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output; `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions—or replace the entire table when unpartitioned—in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state), **verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views still target their populated physical tables**, and—because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly—**explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from that same ``, verify both their destination table and `` resume position, and verify `` has no `pending_restatement_intervals` beyond the reconciled boundary before resuming external writers—and only resume scheduled or ad-hoc `sqlmesh run prod` evaluations targeting `` (once any open partition is reconciled) when ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values; otherwise (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones) keep SQLMesh writes for `` permanently disabled** (if you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table). - 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values, and note that on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true`—where `partitioned_by` is unset—`insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query; limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**): Either temporarily make the cutover snapshot deployable (`forward_only false` + `NON_BREAKING` `stamp` bump via `--no-auto-categorization` under `virtual_environment_mode: full`) and follow item 2 above so all SQLMesh and external writers stay paused, replay and writer resumption share the captured ``, `prod` promotes via `sqlmesh plan prod ... --skip-backfill` without downstream rebuilds or overwriting staged data, and external writers are redirected and verified before resuming, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation for the entire operation (note before starting that `` must keep default `disable_restatement true`—do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows—that **neither the reconciliation plan's common `plan.start` may precede ``'s or any selected `evaluatable` upstream model's configured `start`, nor may any selected `evaluatable` model have a configured `end` earlier than `` or ``**—use option 1 instead if either `start` or `end` bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds—and that `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot—verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first and **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary**, or else use item 2 with `forward_only false`, `disable_restatement true`, and a new `NON_BREAKING` `stamp` value so a new snapshot version starts with empty intervals rather than leaving uncovered post-`` intervals marked complete after `TRUNCATE TABLE`): stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout the entire cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and writer resumption use the same boundary. Create and verify a recoverable full copy of `` (preserving hidden `__DORIS_DELETE_SIGN__` tombstones and sequence values) before truncating, run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. If replay fails, restore `` in place (including tombstones and sequence values) from that verified recovery copy—or, if in-place restore fails, promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted physical table at ``—and validate counts, indexes, `` (when virtual views are used), and external CDC writer target destinations before reopening reader or writer access. After a successful in-place replay, if ``'s future SQLMesh writes do not preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values (including unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones, because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`), **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**; otherwise, when ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values, **reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs**—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals): set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay—never intervals beyond ``, accounting for any model `lookback`; before running any upstream catch-up jobs or cutover plans, **inspect project `before_all` and `after_all` environment statements—`--select-model` does not suppress environment statements—along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate `` after `` is captured**, confirm `` keeps `disable_restatement true` so upstream `sqlmesh run prod` calls cannot record `pending_restatement_intervals` on ``, confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than ``—if either `start` or `end` bound would be exceeded, do not run `--empty-backfill`, which would falsely record unevaluated pre-`start` or post-`end` history—verify separately that any `SEED` dependencies are already loaded and available—materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs—and that any `EXTERNAL` or `EMBEDDED` sources are ready, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm`, because state exports include model SQL and environment statements, and removing `/state.json` immediately after verification—confirming that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` beyond ``, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled)—and record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). If `` includes partial-interval CDC events beyond `` inside `[, )` on a **partitioned** `insert_overwrite true` model where ``'s SQLMesh query does not reproduce that open partition's CDC rows or tombstones—and only when post-`` SQLMesh runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves (never resume `sqlmesh run` on an unpartitioned `insert_overwrite true` model or any model whose future SQLMesh runs would overwrite partitions or keys without preserving replayed/CDC rows, tombstones, and sequence values; keep SQLMesh writes for `` permanently disabled instead)—waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state: either drain writers at a cron- and partition-aligned boundary so `` aligns with ``, or after resuming external CDC writers keep scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone), confirm `` keeps `disable_restatement true` and confirm that the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and that no selected `evaluatable` model has a configured `end` earlier than `` (excluding `--empty-backfill` before cutover and using option 1 if either `start` or `end` bound cannot be guaranteed), inspect project `before_all` and `after_all` statements (`--select-model` does not suppress environment statements) and upstream model side effects first to confirm they cannot write to or mutate ``, check seed availability and external sources separately while catching up and verifying all upstream SQLMesh models that acquire recurring evaluation intervals—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with restrictive `0600` permissions under `umask 077`, deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` extending beyond ``, so `--backfill-model --empty-backfill` cannot mark uncaught upstream intervals complete without producing data, verify that completed interval's live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed) **before** resuming `sqlmesh run prod` so the next run cannot evaluate that interval and replace rows missing from the model query. Finally, verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` beyond the reconciled boundary, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) and is positioned at `` before resuming readers and external writers—and only resume scheduled or ad-hoc `sqlmesh run prod` evaluations targeting `` (once any open partition is reconciled) when ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values; otherwise (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones) keep SQLMesh writes for `` permanently disabled. + 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): + 1. **Verify prerequisites and post-cutover write compatibility**: `INCREMENTAL_UNMANAGED` models default to `forward_only = true`, under which a `stamp` bump in `sqlmesh plan ` evaluates into a separate non-deployable `__temp` preview table while `prod` retains its existing physical table on promotion (`prod` will **not** serve that `dev` table). Use this `dev`-snapshot promotion workflow **only** when the snapshot is deployable so `prod` will target the staged physical table: + - Configure `kind INCREMENTAL_UNMANAGED (forward_only false)` while keeping default `disable_restatement true` (do **not** set `disable_restatement false` on ``, because `sqlmesh run prod` processes automatic restatements across all `prod` snapshots before applying `--select-model`, which can record `pending_restatement_intervals` on `` even when `` is excluded from `--select-model`) under `virtual_environment_mode: full`, with no pinned `physical_version`, no `auto_restatement_cron` on ``, and no non-deployable upstream snapshots. + - Confirm no `` or selected `evaluatable` upstream model has a configured `start` later than the plan's common `plan.start` or a configured `end` earlier than `` (or `` when reconciling an open partition)—use option 1 instead whenever `[plan.start, plan.end)` would precede ``'s or any selected `evaluatable` upstream model's configured `start` or extend past its configured `end`, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` on every selected `evaluatable` snapshot without clamping to per-model metadata-only `start` or `end` bounds. + - Use a **`NON_BREAKING` `stamp` bump** rather than `BREAKING`, `--forward-only`, or `METADATA`: pass `--no-auto-categorization` when staging in `` and explicitly choose `NON_BREAKING` for ``, because default `full` auto-categorization classifies a `stamp` bump as `BREAKING` and creates unbackfilled `INDIRECT_BREAKING` downstream physical tables on `prod` promotion. Note that even when `` is `NON_BREAKING`, SQLMesh still categorizes any indirectly modified downstream **materialized view** (`VIEW (materialized true)`) whose change is not forward-only as `INDIRECT_BREAKING`—if any accompanying change requires `BREAKING` **or** any downstream materialized view becomes `INDIRECT_BREAKING`, you must stage, backfill or refresh, and validate every affected downstream physical table and materialized view before promoting to `prod` rather than restricting `--backfill-model` to ``. + - Inspect project `before_all` and `after_all` environment statements (`--select-model` does not suppress environment statements) along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``'s old or promoted `prod` physical table. + - Determine post-cutover SQLMesh write eligibility: on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true` (`partitioned_by` unset), `insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query. Limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**. + 2. **Drain writers, capture ``, and catch up upstream intervals**: + - Pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external CDC/ingestion writers, keeping every SQLMesh and external writer paused throughout staging, replay, promotion, and writer redirection—no writes to the old physical table may occur during replay or promotion. + - After all writers are drained, capture a durable upstream changelog/CDC source offset (``) so no CDC events committed before writers finished pausing can be omitted, and set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`, which `--empty-backfill` can treat as inclusive of that full day and record through the next day when no prior non-seed interval ends exist) equal to the latest cron- and partition-aligned boundary **fully covered** by `` (at least one full model interval past `start` and accounting for any model `lookback`). + - Verify separately that any `SEED` dependencies are already loaded and available (materializing any unloaded `SEED` first, since loaded `SEED` models do not acquire new intervals on later runs) and that any `EXTERNAL` or `EMBEDDED` sources are ready; re-confirm `disable_restatement true` on `` and re-confirm the `start` and `end` bounds from step 1 against ``. + - For every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits, which do not have evaluation intervals—catch up and verify `prod` interval coverage through `` before running `--empty-backfill`: run `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, check `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independently verify in `prod` SQLMesh state—creating the export file inside a `0700` private directory with restrictive `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm` (because state exports include model SQL and environment statements) and removing `/state.json` immediately after verification—that each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and that `` has no `pending_restatement_intervals` in `prod`, since `check_intervals` applies model `cron` while `--backfill-model --empty-backfill` ignores `cron` and records upstream dependency intervals without producing data. + 3. **Stage the empty deployable snapshot and replay CDC data**: + - Stage the empty deployable table without running model SQL batches via `sqlmesh plan --no-auto-categorization --select-model --backfill-model --empty-backfill --execution-time `, choosing `NON_BREAKING` for `` so non-materialized-view downstream models remain `INDIRECT_NON_BREAKING` without downstream rebuilds (and staging, backfilling or refreshing, and validating any affected `BREAKING` or `INDIRECT_BREAKING` downstream physical tables or materialized views as noted in step 1), while passing `` with an explicit time and timezone and `enable_add_index_for_new_data: false` in `session_variables`. `--empty-backfill` creates the empty deployable table and runs `@runtime_stage = 'creating'` index post-statements without executing model batches while recording deployable intervals only through `` so, on models whose future SQLMesh writes preserve replayed rows, tombstones, and sequence values, the first post-cutover `sqlmesh run prod` does not treat the table as a first insert (`is_first_insert = true`) and replace it—whereas unpartitioned `insert_overwrite true` models still replace the entire table via `replace_query` even when `is_first_insert = false` and must keep SQLMesh writes disabled unless ``'s query reproduces the full table including tombstones and sequence values. + - Verify before replaying that `SHOW CREATE VIEW ` points to a new deployable physical snapshot table (without a `__temp` suffix) distinct from `SHOW CREATE VIEW `, while `` keeps serving readers from the untouched old physical table. + - Replay **both** live rows and deleted keys' tombstones (`__DORIS_DELETE_SIGN__ = 1`) together with their sequence values (`__DORIS_SEQUENCE_COL__` or the configured `function_column.sequence_col` column) from the upstream source **up to ``** into the staged deployable physical table (reloading live rows alone discards tombstones and is not sufficient), and validate row/tombstone counts and `SHOW INDEX` on the staged table. + 4. **Recover if replay fails (skip if step 3 succeeded)**: `prod` remains untouched; **do not promote that staged snapshot**. Because `--empty-backfill` already recorded deployable intervals on that snapshot version in state and dropping `` alone does **not** clear the deployable snapshot, its recorded intervals, or its incomplete physical table, **prefer recovering by bumping `stamp` to a new value** (so the next plan creates a fresh snapshot version and physical table rather than reusing the incomplete table as already processed) and invalidating `` via `sqlmesh invalidate --sync`, letting normal TTL janitor cleanup remove the unreferenced failed snapshot later. If you instead revert the cutover edit or need immediate state/table removal via `sqlmesh janitor --ignore-ttl` after `sqlmesh invalidate --sync` (verifying the failed snapshot, its recorded intervals, and its incomplete staged table are removed before running another `sqlmesh plan prod`), **assess its global impact first**—`sqlmesh janitor --ignore-ttl` purges **all** unreferenced snapshots across the entire project and may drop unrelated physical tables before their configured retention TTL expires. + 5. **Promote to `prod` and verify views and external writer positioning before first resume**: + - Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run`. Before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING` or any downstream materialized view became `INDIRECT_BREAKING`, that every affected downstream physical table and materialized view was already staged, backfilled or refreshed, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables or materialized views will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output. `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions—or replace the entire table when unpartitioned—in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state. + - Verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views or materialized views target their populated objects. + - Because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly, explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from ``, and **verify before resuming external writers** that their destination table is the newly promoted `prod` physical table, their starting offset is ``, and `` has no `pending_restatement_intervals` in `prod`. + 6. **Resume external CDC writers, reconcile any partial open partition, and handle post-cutover `sqlmesh run` jobs**: + - **If SQLMesh writes for `` must remain permanently disabled** (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones, or any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): after verifying `` in step 5, resume external CDC writers from `` and keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled. + - **If ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and `` aligned with `` (no partial open partition)**: after verifying `` in step 5, resume external CDC writers from `` and resume scheduled `sqlmesh run prod` jobs. + - **If `` is a partitioned `insert_overwrite true` model whose future post-`` runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves, and `` included partial-interval CDC events inside `[, )` that ``'s query does not reproduce for that open partition** (if writers were not drained at an aligned boundary): waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state. First (after verifying destination table and `` in step 5), resume external CDC writers from `` while keeping all scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (with time and timezone). Once external writers advance past ``, confirm `` keeps `disable_restatement true`, confirm the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and no selected `evaluatable` model has a configured `end` earlier than `` (using option 1 before cutover if either bound cannot be guaranteed), re-check project `before_all` and `after_all` statements and upstream side effects, verify seed and external source readiness while catching up and verifying all recurring upstream SQLMesh models—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with `0600` permissions under `umask 077` (deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`), verify `[, )` live rows, tombstones, and sequence values on the promoted `prod` table, and reconcile that completed interval via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed so the forward-only update applies on the existing `prod` physical table version without creating a new table). Re-verify `SHOW CREATE VIEW `, confirm `` still has no `pending_restatement_intervals` in `prod`, and **verify external CDC writers still target the promoted `prod` physical table at their advanced post-`` position** (do **not** reset them to ``) before resuming `sqlmesh run prod`. + - If you later remove `forward_only false` to restore default `forward_only = true`, apply that metadata-only plan to `prod` via `sqlmesh plan prod --select-model ` without `--run` and re-verify `SHOW CREATE VIEW ` still targets the promoted physical table. + 3. **Replay on a forward-only `INCREMENTAL_UNMANAGED` model (`forward_only = true`, the default) or non-deployable environment** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): Either temporarily make the cutover snapshot deployable (`forward_only false` + `NON_BREAKING` `stamp` bump via `--no-auto-categorization` under `virtual_environment_mode: full`) and execute item 2's ordered steps above, **or**—when keeping `forward_only = true` in place or when `prod` cannot promote a separate deployable snapshot table (note that Doris `ALTER TABLE ... REPLACE WITH TABLE` is not supported in compute-storage decoupled mode)—execute an **in-place maintenance-window cutover** directly on the retained `prod` `` with full **reader and writer** isolation using these ordered steps: + 1. **Verify prerequisites, post-cutover write compatibility, and a viable recovery path before truncation**: + - Confirm `` keeps default `disable_restatement true` (do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows), and inspect project `before_all` and `after_all` environment statements (`--select-model` does not suppress environment statements) along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``. + - Determine post-cutover SQLMesh write eligibility: on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true` (`partitioned_by` unset), `insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query. Limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**. + - When post-cutover `sqlmesh run` will be resumed (requiring `--empty-backfill` interval reconciliation), confirm neither the reconciliation plan's common `plan.start` precedes ``'s or any selected `evaluatable` upstream model's configured `start` nor any selected `evaluatable` model has a configured `end` earlier than `` (or `` when reconciling an open partition)—use option 1 instead if either bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds. Also verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first: `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot, so **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary** (if it cannot, use item 2 when deployable snapshot promotion is available so a new snapshot version starts with empty intervals, or use option 1 instead—do **not** truncate `` in place). + - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (and any `@runtime_stage = 'creating'` index DDL) and verifying on a scratch table that dropping, recreating from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical counts and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). + 2. **Isolate readers and writers, capture ``, back up ``, truncate, and replay CDC data**: + - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and initial writer resumption share that boundary. + - Capture `SHOW CREATE TABLE ` and create and verify the full recovery copy of `` (preserving live rows, hidden `__DORIS_DELETE_SIGN__` tombstones, and sequence values) before truncating. + - Run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. + 3. **Recover if replay fails (skip if step 2 succeeded)**: Restore `` in place (including `__DORIS_DELETE_SIGN__` tombstones and sequence values) from the verified recovery copy created in step 2. If in-place restoration into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, drop and recreate `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`) and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy (only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted table at ``). Validate row/tombstone counts, `SHOW INDEX`, `` (when virtual views are used), and external CDC writer target destinations and `` position before reopening reader or writer access. + 4. **Reconcile `prod` intervals (when resuming `sqlmesh run`) and verify pre-resume state**: + - **If SQLMesh writes for `` will remain permanently disabled** (required for unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones—because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`—and for any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): skip `--empty-backfill` reconciliation on `` and proceed to the pre-resume checks at the end of this step. + - **If ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values and you plan to resume `sqlmesh run prod`**: reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals). Set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay, accounting for any model `lookback`; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled). Re-confirm the `before_all` / `after_all`, `disable_restatement true`, and `start` / `end` bound checks from step 1, verify `SEED` availability (materializing any unloaded `SEED` first) and `EXTERNAL` / `EMBEDDED` readiness separately, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification inside a `0700` private directory with `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm` (removing `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`). Then record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). + - **Pre-resume check (all paths before first writer resume)**: Verify `` (when virtual views are used) targets ``, verify `` has no `pending_restatement_intervals` in `prod`, and verify all external CDC writers' destination points to `` (redirecting and verifying any writer whose target was changed during maintenance, never leaving writers pointed at a backup or superseded table) **and is positioned at `` before resuming external CDC writers**. + 5. **Resume external CDC writers and readers, reconcile any partial open partition, and handle post-cutover `sqlmesh run` jobs**: + - **If SQLMesh writes for `` remain permanently disabled** (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones, or any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): after verifying `` in step 4, resume readers and external CDC writers from ``, and keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled. + - **If ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and `` aligned with `` (no partial open partition)**: after verifying `` in step 4, resume readers, external CDC writers from ``, and scheduled `sqlmesh run prod` jobs. + - **If `` is a partitioned `insert_overwrite true` model whose future post-`` runs solely overwrite post-`` partitions whose rows, tombstones, and sequence values ``'s query preserves, and `` included partial-interval CDC events inside `[, )` that ``'s query does not reproduce for that open partition** (if writers were not drained at an aligned boundary): waiting for resumed CDC writers to advance past that open partition does **not** record `[, )` in SQLMesh state. First (after verifying destination table and `` in step 4), resume external CDC writers from `` (and reopen reader access once `` and `` are verified) while keeping all scheduled and ad-hoc `sqlmesh run` jobs for `` paused until external CDC writers advance past explicit timestamp `` (including time and timezone). Once external writers advance past ``, confirm `` keeps `disable_restatement true`, confirm the reconciliation plan's common `plan.start` does not precede ``'s or any selected `evaluatable` upstream model's configured `start` and no selected `evaluatable` model has a configured `end` earlier than `` (using option 1 before cutover if either bound cannot be guaranteed), re-check project `before_all` and `after_all` statements and upstream side effects, verify seed and external source readiness while catching up and verifying all recurring upstream SQLMesh models—excluding loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—through `` via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end `, and independent `prod` state verification via `sqlmesh state export --environment prod -o /state.json --no-confirm` in a `0700` private directory with `0600` permissions under `umask 077` (deleting `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`), verify `[, )` live rows, tombstones, and sequence values on ``, and record that completed interval in `prod` via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (bumping `stamp` if needed). Re-verify `` (when virtual views are used), confirm `` still has no `pending_restatement_intervals` in `prod`, and **verify external CDC writers still target `` at their advanced post-`` position** (do **not** reset them to ``) before resuming `sqlmesh run prod`. You can also configure OLAP `compaction_policy` (`'time_series'` or `'size_based'`) and `function_column.sequence_col` in `physical_properties`: From e871be569d19babb4286fc52ebe9e8439fd81748 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:03:26 -0800 Subject: [PATCH 22/54] docs(doris): clear partial replay rows before recovery reload Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 3b12cb851c..d9242ba09e 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -839,12 +839,12 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - Confirm `` keeps default `disable_restatement true` (do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows), and inspect project `before_all` and `after_all` environment statements (`--select-model` does not suppress environment statements) along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``. - Determine post-cutover SQLMesh write eligibility: on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true` (`partitioned_by` unset), `insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query. Limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**. - When post-cutover `sqlmesh run` will be resumed (requiring `--empty-backfill` interval reconciliation), confirm neither the reconciliation plan's common `plan.start` precedes ``'s or any selected `evaluatable` upstream model's configured `start` nor any selected `evaluatable` model has a configured `end` earlier than `` (or `` when reconciling an open partition)—use option 1 instead if either bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds. Also verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first: `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot, so **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary** (if it cannot, use item 2 when deployable snapshot promotion is available so a new snapshot version starts with empty intervals, or use option 1 instead—do **not** truncate `` in place). - - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (and any `@runtime_stage = 'creating'` index DDL) and verifying on a scratch table that dropping, recreating from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical counts and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). + - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and testing full-replacement in-place restoration on a scratch table (`TRUNCATE TABLE` first before reloading from the recovery copy so no leftover partial-replay keys or higher sequence values can survive), and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (and any `@runtime_stage = 'creating'` index DDL) and verifying on a scratch table that dropping, recreating an empty table from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical keys, tombstones, sequence values, and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). 2. **Isolate readers and writers, capture ``, back up ``, truncate, and replay CDC data**: - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and initial writer resumption share that boundary. - Capture `SHOW CREATE TABLE ` and create and verify the full recovery copy of `` (preserving live rows, hidden `__DORIS_DELETE_SIGN__` tombstones, and sequence values) before truncating. - Run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. - 3. **Recover if replay fails (skip if step 2 succeeded)**: Restore `` in place (including `__DORIS_DELETE_SIGN__` tombstones and sequence values) from the verified recovery copy created in step 2. If in-place restoration into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, drop and recreate `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`) and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy (only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively promote a deployable recovery snapshot via item 2 and redirect external CDC writers to that promoted table at ``). Validate row/tombstone counts, `SHOW INDEX`, `` (when virtual views are used), and external CDC writer target destinations and `` position before reopening reader or writer access. + 3. **Recover if replay fails (skip if step 2 succeeded)**: Because a failed replay can leave partial rows in ``—and loading a backup over leftover `UNIQUE KEY` rows can retain extra keys or higher sequence values even when row counts match—**every recovery path must clear or fully replace partial replay data before loading the verified recovery copy**: for in-place restoration, run `TRUNCATE TABLE ` first to remove all partial replay data and then reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy created in step 2 (never `INSERT INTO` over un-truncated partial replay rows). If in-place `TRUNCATE TABLE` and reload into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, `DROP TABLE `, recreate an empty `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`), and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table; only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively bump `stamp` to stage a fresh **empty** deployable recovery snapshot table via item 2 (never reusing an incomplete or partially populated table), load live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table, promote it to `prod`, and redirect external CDC writers to that promoted table at ``. Across whichever recovery path is used, **validate live keys, `__DORIS_DELETE_SIGN__` tombstones, and sequence values against the verified recovery copy** (not row counts alone), along with `SHOW INDEX`, `` (when virtual views are used), and external CDC writer target destinations and `` position before reopening reader or writer access. 4. **Reconcile `prod` intervals (when resuming `sqlmesh run`) and verify pre-resume state**: - **If SQLMesh writes for `` will remain permanently disabled** (required for unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones—because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`—and for any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): skip `--empty-backfill` reconciliation on `` and proceed to the pre-resume checks at the end of this step. - **If ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values and you plan to resume `sqlmesh run prod`**: reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals). Set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay, accounting for any model `lookback`; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled). Re-confirm the `before_all` / `after_all`, `disable_restatement true`, and `start` / `end` bound checks from step 1, verify `SEED` availability (materializing any unloaded `SEED` first) and `EXTERNAL` / `EMBEDDED` readiness separately, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification inside a `0700` private directory with `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm` (removing `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`). Then record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). From 9169ca2b186b9c22c4bd896ee0aef06b155e8bb0 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:17:42 -0800 Subject: [PATCH 23/54] docs(doris): recreate cancelled Routine Load jobs on drop-and-recreate recovery Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index d9242ba09e..71d28b7e27 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -839,12 +839,12 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - Confirm `` keeps default `disable_restatement true` (do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows), and inspect project `before_all` and `after_all` environment statements (`--select-model` does not suppress environment statements) along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``. - Determine post-cutover SQLMesh write eligibility: on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true` (`partitioned_by` unset), `insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query. Limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**. - When post-cutover `sqlmesh run` will be resumed (requiring `--empty-backfill` interval reconciliation), confirm neither the reconciliation plan's common `plan.start` precedes ``'s or any selected `evaluatable` upstream model's configured `start` nor any selected `evaluatable` model has a configured `end` earlier than `` (or `` when reconciling an open partition)—use option 1 instead if either bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds. Also verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first: `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot, so **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary** (if it cannot, use item 2 when deployable snapshot promotion is available so a new snapshot version starts with empty intervals, or use option 1 instead—do **not** truncate `` in place). - - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and testing full-replacement in-place restoration on a scratch table (`TRUNCATE TABLE` first before reloading from the recovery copy so no leftover partial-replay keys or higher sequence values can survive), and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (and any `@runtime_stage = 'creating'` index DDL) and verifying on a scratch table that dropping, recreating an empty table from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical keys, tombstones, sequence values, and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). + - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and testing full-replacement in-place restoration on a scratch table (`TRUNCATE TABLE` first before reloading from the recovery copy so no leftover partial-replay keys or higher sequence values can survive), and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (plus any `@runtime_stage = 'creating'` index DDL, and—if any Doris Routine Load jobs write to ``—capturing each job's configuration and partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD`, because `DROP TABLE` cancels Routine Load jobs bound to the dropped table and recreating the table does not restart a cancelled job) and verifying on a scratch table that dropping, recreating an empty table from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical keys, tombstones, sequence values, and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). 2. **Isolate readers and writers, capture ``, back up ``, truncate, and replay CDC data**: - - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)** so replay and initial writer resumption share that boundary. + - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)**—including each paused Doris Routine Load job's configuration and partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD`—so replay, recovery, and initial writer resumption share that boundary. - Capture `SHOW CREATE TABLE ` and create and verify the full recovery copy of `` (preserving live rows, hidden `__DORIS_DELETE_SIGN__` tombstones, and sequence values) before truncating. - Run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. - 3. **Recover if replay fails (skip if step 2 succeeded)**: Because a failed replay can leave partial rows in ``—and loading a backup over leftover `UNIQUE KEY` rows can retain extra keys or higher sequence values even when row counts match—**every recovery path must clear or fully replace partial replay data before loading the verified recovery copy**: for in-place restoration, run `TRUNCATE TABLE ` first to remove all partial replay data and then reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy created in step 2 (never `INSERT INTO` over un-truncated partial replay rows). If in-place `TRUNCATE TABLE` and reload into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, `DROP TABLE `, recreate an empty `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`), and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table; only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively bump `stamp` to stage a fresh **empty** deployable recovery snapshot table via item 2 (never reusing an incomplete or partially populated table), load live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table, promote it to `prod`, and redirect external CDC writers to that promoted table at ``. Across whichever recovery path is used, **validate live keys, `__DORIS_DELETE_SIGN__` tombstones, and sequence values against the verified recovery copy** (not row counts alone), along with `SHOW INDEX`, `` (when virtual views are used), and external CDC writer target destinations and `` position before reopening reader or writer access. + 3. **Recover if replay fails (skip if step 2 succeeded)**: Because a failed replay can leave partial rows in ``—and loading a backup over leftover `UNIQUE KEY` rows can retain extra keys or higher sequence values even when row counts match—**every recovery path must clear or fully replace partial replay data before loading the verified recovery copy**: for in-place restoration, run `TRUNCATE TABLE ` first to remove all partial replay data and then reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy created in step 2 (never `INSERT INTO` over un-truncated partial replay rows). If in-place `TRUNCATE TABLE` and reload into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, `DROP TABLE `, recreate an empty `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`), and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table; only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively bump `stamp` to stage a fresh **empty** deployable recovery snapshot table via item 2 (never reusing an incomplete or partially populated table), load live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table, and promote it to `prod`. **Before starting or resuming any writers**, validate live keys, `__DORIS_DELETE_SIGN__` tombstones, and sequence values on the restored table against the verified recovery copy (not row counts alone), along with `SHOW INDEX`, `` (when virtual views are used), and paused external CDC writers' target destinations and `` starting positions; then—if `` was dropped and recreated (which permanently cancels Doris Routine Load jobs bound to the old table) or replaced via item 2—recreate each affected Doris Routine Load job from its captured configuration targeting the restored `prod` table at the verified `` partition offsets and confirm via `SHOW ROUTINE LOAD` that it is `RUNNING` (resuming any other paused external writers from ``) before reopening reader access. 4. **Reconcile `prod` intervals (when resuming `sqlmesh run`) and verify pre-resume state**: - **If SQLMesh writes for `` will remain permanently disabled** (required for unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones—because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`—and for any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): skip `--empty-backfill` reconciliation on `` and proceed to the pre-resume checks at the end of this step. - **If ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values and you plan to resume `sqlmesh run prod`**: reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals). Set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay, accounting for any model `lookback`; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled). Re-confirm the `before_all` / `after_all`, `disable_restatement true`, and `start` / `end` bound checks from step 1, verify `SEED` availability (materializing any unloaded `SEED` first) and `EXTERNAL` / `EMBEDDED` readiness separately, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification inside a `0700` private directory with `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm` (removing `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`). Then record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). From 300db1f68040be44fdbfb5421a8a5d104d69f1cf Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:33:37 -0800 Subject: [PATCH 24/54] docs(doris): stop cutover after failed replay restore and note Routine Load credentials Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 71d28b7e27..e2ce8b0e1c 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -828,7 +828,7 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr 5. **Promote to `prod` and verify views and external writer positioning before first resume**: - Once replay validation succeeds, promote to `prod` using `sqlmesh plan prod --select-model --skip-backfill --execution-time ` without `--run`. Before applying, verify directly in the plan diff/summary that `` is `NON_BREAKING` and every downstream model is `INDIRECT_NON_BREAKING` with no downstream rebuilds—or, if any change was `BREAKING` or any downstream materialized view became `INDIRECT_BREAKING`, that every affected downstream physical table and materialized view was already staged, backfilled or refreshed, and validated before promotion—and do **not** rely on `SKIP` in `--explain` as proof that downstream physical tables or materialized views will not be created or promoted, since restricting `--backfill-model` in `` can hide downstream physical updates from `SKIP` output. `--skip-backfill` prevents a `prod` plan backfill from evaluating any interval that became due after ``, which with `insert_overwrite true` would delete and replace matching partitions—or replace the entire table when unpartitioned—in the validated staged table and with `insert_overwrite false` would append model-query rows onto the replay, while the deployable intervals recorded through `` during `--empty-backfill` remain in state. - Verify `SHOW CREATE VIEW ` targets the newly promoted physical snapshot table and that any downstream `prod` views or materialized views target their populated objects. - - Because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly, explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from ``, and **verify before resuming external writers** that their destination table is the newly promoted `prod` physical table, their starting offset is ``, and `` has no `pending_restatement_intervals` in `prod`. + - Because repointing `` does **not** change the destination of external CDC/ingestion writers that target a physical snapshot table directly, explicitly reconfigure/redirect all external writers from the old physical table to the newly promoted `prod` physical table starting from `` (for Doris Routine Load jobs—whose old job remains `PAUSED` on the retained old table, and whose `SHOW CREATE ROUTINE LOAD` output masks sensitive properties—first durably preserve each job's configuration, separately verified authentication credentials, and `` partition offsets, then either create the replacement Routine Load job targeting the promoted table under a new name or run irreversible `STOP ROUTINE LOAD` on the old paused job before recreating it under the original name), and **verify before resuming external writers** that their destination table is the newly promoted `prod` physical table, their starting offset is ``, and `` has no `pending_restatement_intervals` in `prod`. 6. **Resume external CDC writers, reconcile any partial open partition, and handle post-cutover `sqlmesh run` jobs**: - **If SQLMesh writes for `` must remain permanently disabled** (including unpartitioned `insert_overwrite true` models whose query omits replayed CDC rows or tombstones, or any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): after verifying `` in step 5, resume external CDC writers from `` and keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled. - **If ``'s future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and `` aligned with `` (no partial open partition)**: after verifying `` in step 5, resume external CDC writers from `` and resume scheduled `sqlmesh run prod` jobs. @@ -839,12 +839,12 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - Confirm `` keeps default `disable_restatement true` (do **not** set `disable_restatement false`, because `sqlmesh run prod` evaluates automatic restatements across all `prod` snapshots even when `--select-model` excludes `` and can record `pending_restatement_intervals` beyond the reconciliation boundary that later overwrite replayed CDC rows), and inspect project `before_all` and `after_all` environment statements (`--select-model` does not suppress environment statements) along with upstream model pre/post-statements and side effects to confirm none of those actions can write to or mutate ``. - Determine post-cutover SQLMesh write eligibility: on an **unpartitioned** `INCREMENTAL_UNMANAGED` model with `insert_overwrite true` (`partitioned_by` unset), `insert_overwrite_by_partition` delegates to `replace_query` and **replaces the entire table on every subsequent `sqlmesh run` evaluation even after interval reconciliation (`is_first_insert = false`)**, discarding all replayed CDC rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values absent from ``'s query. Limit post-cutover `sqlmesh run` resumption strictly to models whose future SQLMesh writes preserve replayed rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values—such as `insert_overwrite false` appends that do not clobber replayed keys, tombstones, or sequence values, or **partitioned** `insert_overwrite true` models whose future runs only overwrite post-cutover partitions for which ``'s query produces all required rows, tombstones, and sequence values—or else **keep SQLMesh writes (`sqlmesh run` evaluations targeting ``) permanently disabled**. - When post-cutover `sqlmesh run` will be resumed (requiring `--empty-backfill` interval reconciliation), confirm neither the reconciliation plan's common `plan.start` precedes ``'s or any selected `evaluatable` upstream model's configured `start` nor any selected `evaluatable` model has a configured `end` earlier than `` (or `` when reconciling an open partition)—use option 1 instead if either bound cannot be satisfied, because `--backfill-model --empty-backfill` records `[plan.start, plan.end)` across all selected `evaluatable` snapshots without clamping to per-model metadata-only `start` or `end` bounds. Also verify ``'s currently recorded `prod` intervals and `pending_restatement_intervals` first: `sqlmesh plan prod ... --empty-backfill` only adds intervals and **does not clear or shrink** existing interval records on ``'s `prod` snapshot, so **require the replay up to `` / `` to cover every previously recorded interval on `` with no `pending_restatement_intervals` extending beyond the reconciliation boundary** (if it cannot, use item 2 when deployable snapshot promotion is available so a new snapshot version starts with empty intervals, or use option 1 instead—do **not** truncate `` in place). - - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and testing full-replacement in-place restoration on a scratch table (`TRUNCATE TABLE` first before reloading from the recovery copy so no leftover partial-replay keys or higher sequence values can survive), and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (plus any `@runtime_stage = 'creating'` index DDL, and—if any Doris Routine Load jobs write to ``—capturing each job's configuration and partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD`, because `DROP TABLE` cancels Routine Load jobs bound to the dropped table and recreating the table does not restart a cancelled job) and verifying on a scratch table that dropping, recreating an empty table from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical keys, tombstones, sequence values, and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). + - **Require a verified recovery path viable under the active environment constraint before truncating**: When `prod` **cannot** promote a separate deployable snapshot table (for example, `virtual_environment_mode: dev_only` or a pinned `physical_version`), snapshot promotion via item 2 is unavailable and cannot serve as a fallback. Before maintenance, verify end-to-end in the same cluster mode both (a) creating a full recovery copy of `` preserving live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values, and testing full-replacement in-place restoration on a scratch table (`TRUNCATE TABLE` first before reloading from the recovery copy so no leftover partial-replay keys or higher sequence values can survive), and (b) a secondary non-promotion recovery procedure in case in-place restore into `` fails—specifically, capturing `SHOW CREATE TABLE ` (plus any `@runtime_stage = 'creating'` index DDL, and—if any Doris Routine Load jobs write to ``—capturing each job's configuration and partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD` **and separately capturing and verifying access to any authentication credentials required to recreate those jobs**, since `SHOW CREATE ROUTINE LOAD` may mask sensitive properties, `DROP TABLE` cancels Routine Load jobs bound to the dropped table, and recreating the table does not restart a cancelled job) and verifying on a scratch table that dropping, recreating an empty table from that DDL with `enable_add_index_for_new_data = false`, and reloading live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the recovery copy restores identical keys, tombstones, sequence values, and indexes (if no viable recovery path can be verified under this constraint before maintenance, **do not truncate ``—use option 1 instead**). 2. **Isolate readers and writers, capture ``, back up ``, truncate, and replay CDC data**: - - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)**—including each paused Doris Routine Load job's configuration and partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD`—so replay, recovery, and initial writer resumption share that boundary. + - Stop all readers and **pause and drain all scheduled or ad-hoc `sqlmesh run` / `sqlmesh plan` jobs and all external writers** first (keeping every SQLMesh and external writer paused throughout cutover), then **capture the drained external writers' durable upstream changelog/CDC source offset (``)**—including each paused Doris Routine Load job's configuration, partition offsets via `SHOW CREATE ROUTINE LOAD` and `SHOW ROUTINE LOAD`, and separately verified unmasked credentials—so replay, recovery, and initial writer resumption share that boundary. - Capture `SHOW CREATE TABLE ` and create and verify the full recovery copy of `` (preserving live rows, hidden `__DORIS_DELETE_SIGN__` tombstones, and sequence values) before truncating. - Run `TRUNCATE TABLE `, replay **both** live rows and `__DORIS_DELETE_SIGN__ = 1` tombstones together with their sequence values from the upstream source **up to ``** into ``, and validate row/tombstone counts and `SHOW INDEX` on ``. - 3. **Recover if replay fails (skip if step 2 succeeded)**: Because a failed replay can leave partial rows in ``—and loading a backup over leftover `UNIQUE KEY` rows can retain extra keys or higher sequence values even when row counts match—**every recovery path must clear or fully replace partial replay data before loading the verified recovery copy**: for in-place restoration, run `TRUNCATE TABLE ` first to remove all partial replay data and then reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy created in step 2 (never `INSERT INTO` over un-truncated partial replay rows). If in-place `TRUNCATE TABLE` and reload into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, `DROP TABLE `, recreate an empty `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`), and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table; only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively bump `stamp` to stage a fresh **empty** deployable recovery snapshot table via item 2 (never reusing an incomplete or partially populated table), load live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table, and promote it to `prod`. **Before starting or resuming any writers**, validate live keys, `__DORIS_DELETE_SIGN__` tombstones, and sequence values on the restored table against the verified recovery copy (not row counts alone), along with `SHOW INDEX`, `` (when virtual views are used), and paused external CDC writers' target destinations and `` starting positions; then—if `` was dropped and recreated (which permanently cancels Doris Routine Load jobs bound to the old table) or replaced via item 2—recreate each affected Doris Routine Load job from its captured configuration targeting the restored `prod` table at the verified `` partition offsets and confirm via `SHOW ROUTINE LOAD` that it is `RUNNING` (resuming any other paused external writers from ``) before reopening reader access. + 3. **Recover if replay fails (skip if step 2 succeeded)**: Because a failed replay can leave partial rows in ``—and loading a backup over leftover `UNIQUE KEY` rows can retain extra keys or higher sequence values even when row counts match—**every recovery path must clear or fully replace partial replay data before loading the verified recovery copy**: for in-place restoration, run `TRUNCATE TABLE ` first to remove all partial replay data and then reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy created in step 2 (never `INSERT INTO` over un-truncated partial replay rows). If in-place `TRUNCATE TABLE` and reload into `` fails, execute the secondary recovery path verified in step 1 that matches your environment: when deployable snapshot promotion **is** unavailable, `DROP TABLE `, recreate an empty `` at the same table name from the captured `SHOW CREATE TABLE ` DDL (with `enable_add_index_for_new_data = false`), and reload live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table; only when deployable snapshot promotion **is** available under `virtual_environment_mode: full` without a pinned `physical_version` may you alternatively bump `stamp` to stage a fresh **empty** deployable recovery snapshot table via item 2 (never reusing an incomplete or partially populated table), load live rows, `__DORIS_DELETE_SIGN__` tombstones, and sequence values from the verified recovery copy into that empty table, and promote it to `prod` (note that promoting via item 2 leaves any old Doris Routine Load job `PAUSED` on the old physical table, so whenever Routine Load is later recreated in a subsequent attempt or rollback, either assign the replacement job a new name or run irreversible `STOP ROUTINE LOAD` on the paused job **only after** durably preserving its configuration, separately verified credentials, and `` partition offsets). Validate live keys, `__DORIS_DELETE_SIGN__` tombstones, and sequence values on the restored table against the verified recovery copy (not row counts alone), along with `SHOW INDEX`, `` (when virtual views are used), and paused external CDC writers' target destinations and `` positions—then **stop this cutover attempt here: do not proceed to steps 4 or 5, and do not resume writers, recreate Routine Load jobs, or reopen reader access as part of this failed attempt**; resolve the root cause and start a new cutover attempt from step 1 before proceeding with replay or writer resumption. 4. **Reconcile `prod` intervals (when resuming `sqlmesh run`) and verify pre-resume state**: - **If SQLMesh writes for `` will remain permanently disabled** (required for unpartitioned `insert_overwrite true` models whose query omits historical CDC rows or tombstones—because `insert_overwrite_by_partition` calls `replace_query` and replaces the entire table on every `sqlmesh run` even when `is_first_insert = false`—and for any model whose future SQLMesh writes do not preserve replayed/CDC rows, tombstones, and sequence values): skip `--empty-backfill` reconciliation on `` and proceed to the pre-resume checks at the end of this step. - **If ``'s future SQLMesh writes preserve replayed rows, tombstones, and sequence values and you plan to resume `sqlmesh run prod`**: reconcile SQLMesh's recorded `prod` snapshot intervals with the replay boundary before resuming any SQLMesh jobs—otherwise, if `` was initially created with `--skip-backfill` (leaving recorded intervals empty) or its recorded intervals do not yet cover the replay, the next `sqlmesh run prod` can treat the populated table as a first insert (`is_first_insert = true`) and truncate/replace it (or re-evaluate already-replayed intervals). Set `` to an **explicit timestamp including time and timezone** (for example, `'2025-01-10 00:00:00+00:00'`—**never** a date-only value such as `'2025-01-10'`) equal to the latest cron- and partition-aligned boundary **fully covered** by ``—which **must also be at or past the end of every interval previously recorded on `` in `prod`**, because `--empty-backfill` does not clear existing interval records beyond `` (record **only** intervals completely covered by the replay, accounting for any model `lookback`; if the replay covers less than one full model interval from `start` on a snapshot with no prior recorded intervals, keep `sqlmesh run prod` disabled until a full interval is covered and reconciled). Re-confirm the `before_all` / `after_all`, `disable_restatement true`, and `start` / `end` bound checks from step 1, verify `SEED` availability (materializing any unloaded `SEED` first) and `EXTERNAL` / `EMBEDDED` readiness separately, and for every upstream SQLMesh model that acquires recurring evaluation intervals—excluding already-loaded `SEED` models and `EXTERNAL` or `EMBEDDED` models without audits—catch up and verify `prod` interval coverage through `` first via `sqlmesh run prod --select-model --end --ignore-cron` without selecting ``, `sqlmesh check_intervals prod --no-signals --select-model --end ` so unready signals cannot hide gaps, and independent `prod` state verification inside a `0700` private directory with `0600` permissions under `umask 077` via `sqlmesh state export --environment prod -o /state.json --no-confirm` (removing `/state.json` immediately after confirming each applicable upstream model's recorded `prod` intervals cover through its `interval_unit`-floored `` boundary regardless of model `cron` and `` has no `pending_restatement_intervals` in `prod`). Then record those covered intervals in `prod` without running model SQL batches via `sqlmesh plan prod --forward-only --select-model --backfill-model --empty-backfill --execution-time ` without `--run` (passing `` with an explicit time and timezone, and including a `stamp` bump on `` if needed so `sqlmesh plan prod` has a forward-only change to apply rather than returning early as a no-op). From 77537dd8e7731ab5455c0146cb57effc064eecbb Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:20:15 -0800 Subject: [PATCH 25/54] fix(doris): harden TLS verification, cursor init pooling, and CTAS CTE/version matching Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- sqlmesh/core/config/connection.py | 34 ++++++++---- sqlmesh/core/engine_adapter/base.py | 35 +++++++++++- sqlmesh/utils/connection_pool.py | 18 +++++-- .../integration/test_integration_doris.py | 4 +- tests/core/engine_adapter/test_base.py | 53 +++++++++++++++++++ tests/core/test_connection_config.py | 49 ++++++++++++++--- 7 files changed, 168 insertions(+), 27 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index e2ce8b0e1c..7a7aef0e43 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -984,7 +984,7 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; | `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`, `DELETE ... USING` + `INSERT` or native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | | `INCREMENTAL_BY_PARTITION` | Supported (Column & time-truncation range `DELETE` + `INSERT`) | Supported (Partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | | `SCD_TYPE_2_BY_TIME` / `SCD_TYPE_2_BY_COLUMN` | Supported | Supported (Requires Doris 4.1.0+ & Iceberg v2 for row-level `DELETE`) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | -| `VIEW` | Supported (`CREATE OR REPLACE VIEW`) | Requires `internal` catalog (`environment_catalog_mapping`) | Requires `internal` catalog (`environment_catalog_mapping`) | Requires `internal` catalog (`environment_catalog_mapping`) | +| `VIEW` | Supported (`CREATE OR REPLACE VIEW`; can query external catalog tables) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | | `VIEW (materialized true)` | Supported (Asynchronous MTMVs; synchronous MVs are base-table rollup indexes created separately on the base table) | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | | `SEED` / Python DataFrame Models | Supported (Automatic `VARCHAR(255)` OLAP key coercion) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Paimon: Read-Only in Doris 2.1–4.1.5; Lance/Hudi/Delta/LakeSoul: Write Guarded | | `CUSTOM ('doris_ddl_only')` | Supported (DDL + `ALTER TABLE` schema evolution only) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Not supported (Read-Only) | diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 4250138ca0..14fb89b453 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2827,6 +2827,11 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: isinstance(verify_mode, str) and verify_mode.strip().lower() in ("false", "0", "no", "none") ): + if self.ssl_verify_identity is True: + raise ConfigError( + "Doris `ssl_verify_identity=True` cannot be combined with " + "`ssl_verify_cert=False` or disabled `ssl.verify_mode`." + ) ssl_config["check_hostname"] = False if ssl_config: kwargs["ssl"] = ssl_config @@ -2893,17 +2898,24 @@ def _cursor_init(self) -> t.Optional[t.Callable[[t.Any], None]]: return None def init(cursor: t.Any) -> None: - if self.session_variables: - self._validate_session_variables(self.session_variables) - for k, v in self.session_variables.items(): - val_sql = exp.convert(v).sql("doris", comments=False) - cursor.execute(f"SET `{k}` = {val_sql}") - if self.catalog: - catalog_sql = exp.to_identifier(self.catalog, quoted=True).sql("doris") - cursor.execute(f"SWITCH {catalog_sql}") - if self.database: - db_sql = exp.to_identifier(self.database, quoted=True).sql("doris") - cursor.execute(f"USE {db_sql}") + try: + if self.session_variables: + self._validate_session_variables(self.session_variables) + for k, v in self.session_variables.items(): + val_sql = exp.convert(v).sql("doris", comments=False) + cursor.execute(f"SET `{k}` = {val_sql}") + if self.catalog: + catalog_sql = exp.to_identifier(self.catalog, quoted=True).sql("doris") + cursor.execute(f"SWITCH {catalog_sql}") + if self.database: + db_sql = exp.to_identifier(self.database, quoted=True).sql("doris") + cursor.execute(f"USE {db_sql}") + except Exception: + try: + cursor.close() + except Exception: + pass + raise return init diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index dd94904993..d129aa1675 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -533,8 +533,13 @@ def replace_query( def _table_parts( table: exp.Table, - ) -> t.Tuple[t.Optional[exp.Expr], t.Optional[exp.Expr], t.Optional[exp.Expr]]: - return (table.args.get("catalog"), table.args.get("db"), table.this) + ) -> t.Tuple[t.Optional[exp.Expr], ...]: + return ( + table.args.get("catalog"), + table.args.get("db"), + table.this, + table.args.get("version"), + ) def _qualified(table: exp.Table, catalog: str) -> exp.Table: table = table.copy() @@ -542,11 +547,37 @@ def _qualified(table: exp.Table, catalog: str) -> exp.Table: table.set("catalog", exp.parse_identifier(catalog)) return quote_identifiers(table) + def _is_cte_reference(table: exp.Table, quoted_table: exp.Table) -> bool: + if quoted_table.args.get("db") or quoted_table.args.get("catalog"): + return False + curr: t.Optional[exp.Expr] = table + child_cte: t.Optional[exp.CTE] = None + while curr is not None: + if isinstance(curr, exp.CTE): + child_cte = curr + with_clause = curr.args.get("with_") + if isinstance(with_clause, exp.With): + is_recursive = bool(with_clause.args.get("recursive")) + for cte in with_clause.expressions: + if not is_recursive and cte is child_cte: + break + if ( + isinstance(cte, exp.CTE) + and cte.alias + and quote_identifiers(exp.to_identifier(cte.alias)) == quoted_table.this + ): + return True + child_cte = None + curr = curr.parent + return False + def _is_target_table(node: exp.Expr) -> bool: nonlocal default_catalog, default_catalog_loaded, qualified_target if not isinstance(node, exp.Table): return False quoted_node = quote_identifiers(node.copy()) + if _is_cte_reference(node, quoted_node): + return False if _table_parts(quoted_node) == _table_parts(quoted_target): return True if quoted_node.this != quoted_target.this or quoted_node.args.get( diff --git a/sqlmesh/utils/connection_pool.py b/sqlmesh/utils/connection_pool.py index 9a70db6885..2a3d5a40ef 100644 --- a/sqlmesh/utils/connection_pool.py +++ b/sqlmesh/utils/connection_pool.py @@ -140,9 +140,14 @@ def get_cursor(self) -> t.Any: thread_id = get_ident() with self._thread_cursors_lock: if thread_id not in self._thread_cursors: - self._thread_cursors[thread_id] = self.get().cursor() + cursor = self.get().cursor() if self._cursor_init: - self._cursor_init(self._thread_cursors[thread_id]) + try: + self._cursor_init(cursor) + except Exception: + _try_close(cursor, "cursor") + raise + self._thread_cursors[thread_id] = cursor return self._thread_cursors[thread_id] def get_attribute(self, key: str) -> t.Optional[t.Any]: @@ -286,9 +291,14 @@ def __init__( def get_cursor(self) -> t.Any: if not self._cursor: - self._cursor = self.get().cursor() + cursor = self.get().cursor() if self._cursor_init: - self._cursor_init(self._cursor) + try: + self._cursor_init(cursor) + except Exception: + _try_close(cursor, "cursor") + raise + self._cursor = cursor return self._cursor def get(self) -> t.Any: diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index af7cfe6643..152c685caf 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -957,7 +957,7 @@ def test_doris_sqlmesh_plan_ann_inverted_not_null_and_widen( schema_name = ctx.schema("ann_e2e") ctx.engine_adapter.create_schema(schema_name) - raw_tbl = exp.table_("raw_logs", db=schema_name) + raw_tbl = exp.to_table(f"{schema_name}.raw_logs", dialect="doris") raw_tbl_sql = ctx.engine_adapter._to_sql(raw_tbl) ctx.engine_adapter.execute( f""" @@ -1123,7 +1123,7 @@ def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( d1 = (today_utc - dt.timedelta(days=2)).isoformat() d2 = (today_utc - dt.timedelta(days=1)).isoformat() - raw_tbl = exp.table_("raw_user_events", db=schema_name) + raw_tbl = exp.to_table(f"{schema_name}.raw_user_events", dialect="doris") raw_tbl_sql = ctx.engine_adapter._to_sql(raw_tbl) ctx.engine_adapter.execute( f""" diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index d7bd74aa45..95b4ca161a 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3325,6 +3325,59 @@ def test_replace_query_self_referencing_two_part_and_aliased( ] +def test_replace_query_cte_and_versioned_table_not_self_referencing( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(EngineAdapter) + adapter.SUPPORTS_REPLACE_TABLE = False + adapter.INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.DELETE_INSERT + + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject(schema="db", name="test", type="table"), + ) + mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="abcdefgh") + + adapter.replace_query( + "test", + parse_one("WITH test AS (SELECT 2 AS a) SELECT t.a FROM test AS t"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + assert to_sql_calls(adapter) == [ + 'DELETE FROM "test" WHERE TRUE', + 'INSERT INTO "test" ("a") WITH "test" AS (SELECT 2 AS "a") SELECT "t"."a" FROM "test" AS "t"', + ] + + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "test", + parse_one("WITH test AS (SELECT a FROM test) SELECT t.a FROM test AS t"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "test" ("a" INT)', + 'CREATE TABLE IF NOT EXISTS "__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "test") AS "_subquery"', + 'DELETE FROM "test" WHERE TRUE', + 'INSERT INTO "test" ("a") WITH "test" AS (SELECT "a" FROM "__temp_test_abcdefgh") SELECT "t"."a" FROM "test" AS "t"', + 'DROP TABLE IF EXISTS "__temp_test_abcdefgh"', + ] + + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM db.test FOR VERSION AS OF 1", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + assert to_sql_calls(adapter) == [ + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "db"."test" FOR VERSION AS OF 1', + ] + + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index b529eeac38..010349c30b 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2544,7 +2544,8 @@ def test_doris(make_config): assert advanced_config.catalog == "iceberg_cat" assert advanced_config.get_catalog() == "iceberg_cat" assert advanced_config.concurrent_tasks == 10 - assert "database" not in advanced_config._static_connection_kwargs + assert "database" not in advanced_config._connection_kwargs_keys + assert "database" not in advanced_config._connection_factory_with_kwargs.keywords assert advanced_config._static_connection_kwargs["ssl"] == { "ca": "/path/to/ca.pem", "verify_mode": True, @@ -2601,11 +2602,8 @@ def test_doris(make_config): ssl_verify_identity=True, check_import=False, ) - assert ssl_verify_cert_false_identity_true._static_connection_kwargs["ssl"] == { - "ca": "/path/to/ca.pem", - "verify_mode": False, - "check_hostname": False, - } + with pytest.raises(ConfigError, match="ssl_verify_identity=True` cannot be combined"): + _ = ssl_verify_cert_false_identity_true._static_connection_kwargs ssl_dict_verify_mode_false = make_config( type="doris", @@ -2628,12 +2626,49 @@ def test_doris(make_config): ssl_verify_identity=True, check_import=False, ) - assert ssl_dict_verify_mode_none_str._static_connection_kwargs["ssl"] == { + with pytest.raises(ConfigError, match="ssl_verify_identity=True` cannot be combined"): + _ = ssl_dict_verify_mode_none_str._static_connection_kwargs + + ssl_dict_verify_mode_none_str_no_identity = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"ca": "/path/to/ca.pem", "verify_mode": "none", "check_hostname": True}, + check_import=False, + ) + assert ssl_dict_verify_mode_none_str_no_identity._static_connection_kwargs["ssl"] == { "ca": "/path/to/ca.pem", "verify_mode": "none", "check_hostname": False, } + from sqlmesh.utils.connection_pool import SingletonConnectionPool, ThreadLocalConnectionPool + + for pool_cls in (SingletonConnectionPool, ThreadLocalConnectionPool): + failing_cursor = MagicMock() + failing_cursor.execute.side_effect = [ + None, + None, + RuntimeError("SWITCH failed"), + ] + healthy_cursor = MagicMock() + mock_conn = MagicMock() + mock_conn.cursor.side_effect = [failing_cursor, healthy_cursor] + pool = pool_cls(lambda: mock_conn, cursor_init=advanced_config._cursor_init) + + with pytest.raises(RuntimeError, match="SWITCH failed"): + pool.get_cursor() + failing_cursor.close.assert_called() + + recovered_cursor = pool.get_cursor() + assert recovered_cursor is healthy_cursor + assert [call.args[0] for call in healthy_cursor.execute.call_args_list] == [ + "SET `query_timeout` = 300", + "SET `workload_group` = 'etl_group'", + "SWITCH `iceberg_cat`", + "USE `analytics`", + ] + with pytest.raises(ConfigError, match="Invalid Doris session variable name"): make_config( type="doris", From 0627ce3a3092aa3f8d80d281e66f33697f7c516d Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:27:15 -0800 Subject: [PATCH 26/54] fix(doris): validate ssl_disabled conflicts and resolve active catalog in CTAS matching Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- sqlmesh/core/config/connection.py | 22 +++++++++++++++ sqlmesh/core/engine_adapter/base.py | 37 +++++++++++++++++++------- tests/core/engine_adapter/test_base.py | 32 ++++++++++++++++++++++ tests/core/test_connection_config.py | 24 +++++++++++++++++ 5 files changed, 106 insertions(+), 11 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 7a7aef0e43..19688d70e1 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -925,7 +925,7 @@ FROM raw.logs; ``` 3. Backfill the index for existing rows: - **Compute-storage integrated deployments**: Run `BUILD INDEX idx_message ON sqlmesh__analytics.analytics__searchable_logs__;` and wait for `SHOW BUILD INDEX` to finish. - - **Compute-storage decoupled (`storage_vault_name`) deployments**: Because Doris does not support `BUILD INDEX` in compute-storage decoupled mode, run a standard non-forward-only `sqlmesh plan` (or `sqlmesh plan --restate-model analytics.searchable_logs` after recreating the table) so all rows are written into the newly indexed table. + - **Compute-storage decoupled (`storage_vault_name`) deployments**: Because Doris does not support `BUILD INDEX` in compute-storage decoupled mode, either build the index via a heavy `SCHEMA_CHANGE` job (`SET enable_add_index_for_new_data = false;` before `ALTER TABLE ... ADD INDEX`) or trigger an explicit table rebuild—by running `sqlmesh plan --restate-model analytics.searchable_logs` or bumping `stamp` in `MODEL (...)` on a non-forward-only `sqlmesh plan` (since a plain `sqlmesh plan` with no model changes and completed intervals will not re-evaluate rows), provided the upstream source still retains all data needed for the rebuild (see [Inverted & Vector Indexes, Full-Text Search & Compaction](#inverted-vector-indexes-full-text-search-compaction-doris-2150) above for model-kind-specific backfill and `INCREMENTAL_UNMANAGED` procedures). Downstream SQLMesh models can query full-text and vector indexes using Doris's native search operators (`MATCH`, `MATCH_ANY`, `MATCH_ALL`, `MATCH_PHRASE`, `MATCH_PHRASE_PREFIX`, `MATCH_REGEXP`, `MATCH_PHRASE_EDGE`, `SEARCH(...)`) and vector distance functions (`L2_DISTANCE`, `COSINE_DISTANCE`, `INNER_PRODUCT`, `L1_DISTANCE`, and their `_APPROXIMATE` counterparts): diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 14fb89b453..8fb274cc2e 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2797,6 +2797,28 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: "charset": self.charset or "utf8mb4", "local_infile": False, } + ssl_dict_requests_verification = isinstance(self.ssl, dict) and ( + bool(self.ssl.get("check_hostname")) + or ( + self.ssl.get("verify_mode") is not None + and self.ssl.get("verify_mode") is not False + and not ( + isinstance(self.ssl.get("verify_mode"), str) + and str(self.ssl.get("verify_mode")).strip().lower() + in ("false", "0", "no", "none") + ) + ) + ) + if (self.ssl_disabled or self.ssl is False) and ( + self.ssl_verify_cert is True + or self.ssl_verify_identity is True + or ssl_dict_requests_verification + ): + raise ConfigError( + "Doris `ssl_disabled=True` or `ssl=False` cannot be combined with " + "certificate or hostname verification (`ssl_verify_cert=True`, " + "`ssl_verify_identity=True`, or `ssl` verification settings)." + ) if not self.ssl_disabled and self.ssl is not False: ssl_config: t.Dict[str, t.Any] = ( dict(self.ssl) if isinstance(self.ssl, dict) else {} # type: ignore[arg-type] diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index d129aa1675..aed0900aac 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -580,21 +580,38 @@ def _is_target_table(node: exp.Expr) -> bool: return False if _table_parts(quoted_node) == _table_parts(quoted_target): return True - if quoted_node.this != quoted_target.this or quoted_node.args.get( - "db" - ) != quoted_target.args.get("db"): + if ( + quoted_node.this != quoted_target.this + or quoted_node.args.get("db") != quoted_target.args.get("db") + or quoted_node.args.get("version") != quoted_target.args.get("version") + or not quoted_node.args.get("db") + or (quoted_node.args.get("catalog") and quoted_target.args.get("catalog")) + or self.catalog_support.is_unsupported + ): return False if not default_catalog_loaded: - default_catalog = self.default_catalog default_catalog_loaded = True + default_catalog = getattr(self, "_active_catalog", None) + if not default_catalog: + try: + default_catalog = self.default_catalog + except MissingDefaultCatalogError: + default_catalog = None + if not default_catalog: + try: + current_cat = self.get_current_catalog() + if isinstance(current_cat, str) and current_cat: + default_catalog = current_cat + except Exception: + pass if default_catalog: qualified_target = _qualified(target_table, default_catalog) - return ( - default_catalog is not None - and qualified_target is not None - and _table_parts(_qualified(node, default_catalog)) - == _table_parts(qualified_target) - ) + if default_catalog is None or qualified_target is None: + raise SQLMeshError( + f"Cannot disambiguate unqualified catalog reference '{node.sql(dialect=self.dialect)}' " + f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default catalog." + ) + return _table_parts(_qualified(node, default_catalog)) == _table_parts(qualified_target) def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: if not _is_target_table(node): diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index 95b4ca161a..bebb53b788 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3377,6 +3377,38 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "db"."test" FOR VERSION AS OF 1', ] + mocker.patch.object( + EngineAdapter, + "catalog_support", + new_callable=mocker.PropertyMock, + return_value=CatalogSupport.FULL_SUPPORT, + ) + adapter.cursor.execute.reset_mock() + mocker.patch.object(adapter, "get_current_catalog", return_value="internal") + adapter.replace_query( + "internal.db.test", + parse_one("SELECT a FROM db.test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "internal"."db"."test" ("a" INT)', + 'CREATE SCHEMA IF NOT EXISTS "internal"."db"', + 'CREATE TABLE IF NOT EXISTS "internal"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "internal"."db"."test") AS "_subquery"', + 'DELETE FROM "internal"."db"."test" WHERE TRUE', + 'INSERT INTO "internal"."db"."test" ("a") SELECT "a" FROM "internal"."db"."__temp_test_abcdefgh"', + 'DROP TABLE IF EXISTS "internal"."db"."__temp_test_abcdefgh"', + ] + + adapter.cursor.execute.reset_mock() + mocker.patch.object(adapter, "get_current_catalog", return_value=None) + with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified catalog reference"): + adapter.replace_query( + "internal.db.test", + parse_one("SELECT a FROM db.test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [] + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 010349c30b..01f82adbe9 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2642,6 +2642,30 @@ def test_doris(make_config): "check_hostname": False, } + ssl_disabled_verify_identity = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl_disabled=True, + ssl_ca="/path/to/ca.pem", + ssl_verify_identity=True, + check_import=False, + ) + with pytest.raises(ConfigError, match="ssl_disabled=True` or `ssl=False` cannot be combined"): + _ = ssl_disabled_verify_identity._static_connection_kwargs + + ssl_false_verify_cert = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl=False, + ssl_ca="/path/to/ca.pem", + ssl_verify_cert=True, + check_import=False, + ) + with pytest.raises(ConfigError, match="ssl_disabled=True` or `ssl=False` cannot be combined"): + _ = ssl_false_verify_cert._static_connection_kwargs + from sqlmesh.utils.connection_pool import SingletonConnectionPool, ThreadLocalConnectionPool for pool_cls in (SingletonConnectionPool, ThreadLocalConnectionPool): From e916b6eeaeb857dfafc46b1d11a54a7b2a5df375 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:08:26 -0800 Subject: [PATCH 27/54] fix(doris): address CodeRabbit review feedback on dialect, SSL, startup, and catalog resolution Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/wait-for-db.sh | 30 +++++++++++++++++-- docs/reference/configuration.md | 2 +- sqlmesh/core/config/connection.py | 16 +++++----- sqlmesh/core/dialect.py | 5 ++-- sqlmesh/core/engine_adapter/base.py | 9 +++++- .../integration/docker/compose.doris.yaml | 8 ++--- .../integration/test_integration_doris.py | 2 +- tests/core/engine_adapter/test_base.py | 30 +++++++++++++++++++ tests/core/engine_adapter/test_doris.py | 10 +++---- tests/core/test_connection_config.py | 10 +++++++ tests/core/test_dialect.py | 28 +++++++++++++++++ 11 files changed, 127 insertions(+), 23 deletions(-) diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index 489cdd4583..efbae44620 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -105,6 +105,11 @@ db2_ready() { } doris_ready() { + local timeout_secs="${DORIS_STARTUP_TIMEOUT_SECS:-300}" + local deadline=$((SECONDS + timeout_secs)) + local backend_registered=0 + local reg_out="" + probe_port 9030 echo "Waiting for Doris FE to accept SQL connections..." @@ -113,13 +118,25 @@ doris_ready() { echo "Doris FE is alive" break fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE to become alive" >&2 + exit 1 + fi sleep 2 done - docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>/dev/null || true - echo "Checking for 1 alive Doris backends..." while true; do + if [ "$backend_registered" -eq 0 ]; then + if reg_out=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then + backend_registered=1 + elif echo "$reg_out" | grep -Eqi "already exists|already registered"; then + backend_registered=1 + else + echo "WARN: Failed to register Doris backend (will retry): $reg_out" >&2 + fi + fi + echo "Checking Doris backends..." ALIVE_BACKENDS=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true) @@ -136,6 +153,15 @@ doris_ready() { break fi + if [ "$SECONDS" -ge "$deadline" ]; then + if [ "$backend_registered" -eq 0 ] && [ -n "$reg_out" ]; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2 + else + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend to become alive" >&2 + fi + exit 1 + fi + echo "Waiting for more backends to become alive..." sleep 3 done diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index dcdc65511d..dbc595eb1d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -186,7 +186,7 @@ Some connections use default values if not specified: - The state connection defaults to `connection` if omitted. - The test connection defaults to `connection` if omitted. -NOTE: Spark and Trino engines may not be used for the state connection. +NOTE: Doris, Spark, and Trino engines may not be used for the state connection. | Option | Description | Type | Required | |--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------:|:----------------------------------------------------------------------:| diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 8fb274cc2e..aef8323646 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2835,14 +2835,16 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: and "check_hostname" not in ssl_config ): ssl_config["check_hostname"] = False + hostname_requested = self.ssl_verify_identity is True or ( + self.ssl_verify_identity is None and bool(ssl_config.get("check_hostname")) + ) + if hostname_requested and not (ssl_config.get("ca") or ssl_config.get("capath")): + raise ConfigError( + "Doris hostname verification (`ssl_verify_identity` or `ssl.check_hostname`) " + "requires `ssl_ca` (or `ssl.ca` / `ssl.capath`). " + "PyMySQL disables hostname verification when no CA is configured." + ) if self.ssl_verify_identity is not None: - if self.ssl_verify_identity and not ( - ssl_config.get("ca") or ssl_config.get("capath") - ): - raise ConfigError( - "Doris `ssl_verify_identity` requires `ssl_ca` (or `ssl.ca` / `ssl.capath`). " - "PyMySQL disables hostname verification when no CA is configured." - ) ssl_config["check_hostname"] = bool(self.ssl_verify_identity) verify_mode = ssl_config.get("verify_mode") if verify_mode is False or ( diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 766595f2da..cccc459ea6 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -925,10 +925,10 @@ def _doris_parse_alter_add(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], return self._parse_alter_table_add() -def _doris_parse_alter_drop(self: Parser) -> t.Optional[exp.Expr]: +def _doris_parse_alter_drop(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], None]: if self._match_texts(("BRANCH", "TAG"), advance=False): return _doris_parse_alter_raw_action(self) - return self._parse_drop() + return self._parse_alter_table_drop() def _doris_alter_sql(self: Generator, expression: exp.Alter) -> str: @@ -1796,6 +1796,7 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore exp.DataType.Type.IPV4: "IPV4", exp.DataType.Type.IPV6: "IPV6", exp.DataType.Type.JSONB: "JSONB", + exp.DataType.Type.TIMESTAMPTZ: "TIMESTAMPTZ", exp.DataType.Type.VARIANT: "VARIANT", } ) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index aed0900aac..fd27802c97 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -592,12 +592,19 @@ def _is_target_table(node: exp.Expr) -> bool: if not default_catalog_loaded: default_catalog_loaded = True default_catalog = getattr(self, "_active_catalog", None) + if not default_catalog and self.dialect == "trino": + try: + current_cat = self.get_current_catalog() + if isinstance(current_cat, str) and current_cat: + default_catalog = current_cat + except Exception: + pass if not default_catalog: try: default_catalog = self.default_catalog except MissingDefaultCatalogError: default_catalog = None - if not default_catalog: + if not default_catalog and self.dialect != "trino": try: current_cat = self.get_current_catalog() if isinstance(current_cat, str) and current_cat: diff --git a/tests/core/engine_adapter/integration/docker/compose.doris.yaml b/tests/core/engine_adapter/integration/docker/compose.doris.yaml index 4077193995..aa1b3067ef 100644 --- a/tests/core/engine_adapter/integration/docker/compose.doris.yaml +++ b/tests/core/engine_adapter/integration/docker/compose.doris.yaml @@ -14,8 +14,8 @@ services: echo "force_olap_table_replication_num = 1" >> /opt/apache-doris/fe/conf/fe.conf exec bash /opt/apache-doris/fe/bin/start_fe.sh --console ports: - - "8030:8030" - - "9030:9030" + - "${DORIS_BIND_ADDRESS:-127.0.0.1}:8030:8030" + - "${DORIS_BIND_ADDRESS:-127.0.0.1}:9030:9030" volumes: - iceberg-warehouse:/tmp/doris_iceberg_warehouse networks: @@ -42,8 +42,8 @@ services: echo "storage_flood_stage_left_capacity_bytes = 104857600" >> /opt/apache-doris/be/conf/be.conf exec bash /opt/apache-doris/be/bin/start_be.sh --console ports: - - "8040:8040" - - "9050:9050" + - "${DORIS_BIND_ADDRESS:-127.0.0.1}:8040:8040" + - "${DORIS_BIND_ADDRESS:-127.0.0.1}:9050:9050" volumes: - iceberg-warehouse:/tmp/doris_iceberg_warehouse networks: diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 152c685caf..8d15b93d3d 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -659,7 +659,7 @@ def test_doris_complex_types_and_alter_table_schema_evolution(ctx: TestContext): assert evolved_cols["notes"] == exp.DataType.build("TEXT", dialect="doris") assert evolved_cols["cnt"] == exp.DataType.build("BIGINT", dialect="doris") assert evolved_cols["dt"] == exp.DataType.build("DATETIME(6)", dialect="doris") - assert evolved_cols["tz"] == exp.DataType.build("DATETIME(6)", dialect="doris") + assert evolved_cols["tz"] == exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris") # 3. Direct time-transform inequality delete on OLAP table ev_tbl = ctx.table("test_time_ineq_del") diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index bebb53b788..25b5fd6664 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3409,6 +3409,36 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( ) assert to_sql_calls(adapter) == [] + # Trino resolves the session catalog (get_current_catalog) before _default_catalog + adapter.dialect = "trino" + adapter._default_catalog = "dev" + mocker.patch.object(adapter, "get_current_catalog", return_value="prod") + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM dev.db.test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."test"', + ] + + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM prod.db.test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', + 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', + 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh"', + 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', + ] + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 7a2907892b..c6a70a845c 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -5077,12 +5077,12 @@ def test_doris_4_1_native_merge_into_and_timestamptz( == "CAST(DATE_FORMAT(CONVERT_TZ(`ts`, @@time_zone, '+00:00'), '%Y-%m-%d %T.%f') AS VARCHAR)" ) - # 4. TIMESTAMPTZ maps to DATETIME (for Doris 2.1+ compatibility and MySQL protocol datetime return), - # defaults to precision 0, and supports same-type precision widening (TIMESTAMPTZ -> TIMESTAMPTZ(0) - # -> TIMESTAMPTZ(6), DATETIME(2) -> DATETIME(6)), while cross-type TIMESTAMPTZ <-> DATETIME is + # 4. TIMESTAMPTZ renders natively as TIMESTAMPTZ, defaults to precision 0, and supports + # same-type precision widening (TIMESTAMPTZ -> TIMESTAMPTZ(0) -> TIMESTAMPTZ(6), + # DATETIME(2) -> DATETIME(6)), while cross-type TIMESTAMPTZ <-> DATETIME is # treated as destructive. - assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "DATETIME" - assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "DATETIME(6)" + assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" + assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" from sqlmesh.core.schema_diff import TableAlterChangeColumnTypeOperation ops_default = adapter.schema_differ.compare_columns( diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 01f82adbe9..2e3474ceb0 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2666,6 +2666,16 @@ def test_doris(make_config): with pytest.raises(ConfigError, match="ssl_disabled=True` or `ssl=False` cannot be combined"): _ = ssl_false_verify_cert._static_connection_kwargs + ssl_dict_check_hostname_no_ca = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"check_hostname": True}, + check_import=False, + ) + with pytest.raises(ConfigError, match="requires `ssl_ca`"): + _ = ssl_dict_check_hostname_no_ca._static_connection_kwargs + from sqlmesh.utils.connection_pool import SingletonConnectionPool, ThreadLocalConnectionPool for pool_cls in (SingletonConnectionPool, ThreadLocalConnectionPool): diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 6a8cf059f6..6870cc613c 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1629,3 +1629,31 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): assert ( alter_ast.sql("doris") == "ALTER TABLE @this_model MODIFY COLUMN msg VARCHAR(128) NOT NULL" ) + + # Doris `ALTER TABLE ... DROP` preserves base `_parse_alter_table_drop` support for + # `DROP PARTITION` and comma-separated drop actions while keeping raw action parsing for + # `BRANCH` and `TAG`. + drop_part_ast = d.parse_one( + "ALTER TABLE @this_model DROP IF EXISTS PARTITION (p202401)", + dialect="doris", + ) + assert isinstance(drop_part_ast, exp.Alter) + assert isinstance(drop_part_ast.args["actions"][0], exp.DropPartition) + + multi_drop_ast = d.parse_one( + "ALTER TABLE @this_model DROP COLUMN a, DROP COLUMN b", + dialect="doris", + ) + assert isinstance(multi_drop_ast, exp.Alter) + assert len(multi_drop_ast.args["actions"]) == 2 + assert multi_drop_ast.sql("doris") == "ALTER TABLE @this_model DROP COLUMN a, DROP COLUMN b" + + drop_branch_ast = d.parse_one( + "ALTER TABLE @this_model DROP BRANCH IF EXISTS b1", + dialect="doris", + ) + assert isinstance(drop_branch_ast, exp.Alter) + assert drop_branch_ast.sql("doris") == "ALTER TABLE @this_model DROP BRANCH IF EXISTS b1" + + assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" + assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" From 74ea9694294de86364c6ee3097bff733a184e629 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:36 -0800 Subject: [PATCH 28/54] fix(doris): address CodeRabbit review feedback on dialect, self-ref staging, Makefile, and docs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- Makefile | 14 +++++- docs/integrations/engines/doris.md | 2 +- sqlmesh/core/dialect.py | 22 +++++---- sqlmesh/core/engine_adapter/base.py | 67 +++++++++++++++++++++----- tests/core/engine_adapter/test_base.py | 25 ++++++++++ tests/core/test_dialect.py | 21 ++++++++ 6 files changed, 126 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 9c804d43a3..b234ff1bf9 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,7 @@ engine-%-install: $(PIP) install -e ".[dev,web,slack,lsp,${*}]" ./examples/custom_materializations engine-docker-%-up: + @if [ "${*}" = "doris" ]; then $(MAKE) doris-host-setup; fi docker compose -f ./tests/core/engine_adapter/integration/docker/compose.${*}.yaml up -d ./.github/scripts/wait-for-db.sh ${*} @@ -242,7 +243,18 @@ starrocks-test: engine-starrocks-up db2-test: engine-db2-up pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml -doris-test: engine-doris-up +doris-host-setup: + @if [ "$$(uname -s)" = "Linux" ]; then \ + current=$$(sysctl -n vm.max_map_count 2>/dev/null || echo 0); \ + if [ "$$current" -lt 2000000 ]; then \ + sudo -n sysctl -w vm.max_map_count=2000000 >/dev/null 2>&1 || { \ + echo "ERROR: vm.max_map_count is $$current (< 2000000); run 'sudo sysctl -w vm.max_map_count=2000000' before starting Doris."; \ + exit 1; \ + }; \ + fi; \ + fi + +doris-test: doris-host-setup engine-doris-up pytest -n auto -m "doris" --reruns 3 --junitxml=test-results/junit-doris.xml ################# diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 19688d70e1..9dc328cc80 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -750,7 +750,7 @@ GROUP BY user_id, CAST(event_time AS DATE); !!! note "Automatic Physical Table Resolution & Non-Destructive Evaluation" - Doris materialized views are only created on initial deployment or when a rebuild is required (`RECREATE_MATERIALIZED_VIEW_ON_EVALUATION = False`), preserving MTMV refresh state across routine `sqlmesh run` invocations. - - When `build = 'DEFERRED'` is used on an asynchronous MTMV that also defines `audits (...)`, SQLMesh automatically submits `REFRESH MATERIALIZED VIEW AUTO` after creation. Because Doris executes MTMV refresh tasks asynchronously without blocking `REFRESH MATERIALIZED VIEW`, audits may run before the background refresh task finishes or fails; use `build = 'IMMEDIATE'` when audits require the initial refresh to complete before auditing. + - When `build = 'DEFERRED'` is used on an asynchronous MTMV that also defines `audits (...)`, SQLMesh automatically submits `REFRESH MATERIALIZED VIEW AUTO` after creation. Because Doris schedules and executes initial MTMV refresh tasks asynchronously for both `BUILD IMMEDIATE` and `REFRESH MATERIALIZED VIEW`, SQLMesh does not wait for the background refresh task to finish; audits that depend on populated MTMV data can therefore run against an empty view unless a refresh-status check (via `mv_infos()` or `tasks("type" = "mv")`) confirms completion before auditing. - When `excluded_trigger_tables` (supports a single string, comma-separated string, or tuple `('db.t1', 'db.t2')` on async MTMVs) references a SQLMesh-managed model, SQLMesh automatically resolves the model name to its underlying physical snapshot table name. ### Synchronous Materialized Views (Base-Table Rollup Indexes) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index cccc459ea6..10409e6f83 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -658,7 +658,7 @@ def columndef_sql(self: Generator, expression: exp.ColumnDef, sep: str = " ") -> def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Optional[exp.Expr]: - this = self.__parse_range(this) # type: ignore + this = this or self._parse_bitwise() if this is not None: idx = self._index negated = self._match(TokenType.NOT) @@ -666,13 +666,12 @@ def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Opt op_name = self._curr.text.upper() self._advance() rhs = self._parse_bitwise() - expr: exp.Expr = self.expression(exp.Anonymous(this=op_name, expressions=[this, rhs])) + this = self.expression(exp.Anonymous(this=op_name, expressions=[this, rhs])) if negated: - expr = self.expression(exp.Not(this=expr)) - return expr - if negated: + this = self.expression(exp.Not(this=this)) + elif negated: self._retreat(idx) - return this + return self.__parse_range(this) # type: ignore _doris_parse_range.__name__ = "_parse_range" @@ -962,9 +961,9 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: def _doris_struct_sql(self: Generator, expression: exp.Struct) -> str: - if any(isinstance(e, exp.PropertyEQ) for e in expression.expressions): + if any(isinstance(e, (exp.PropertyEQ, exp.Alias)) for e in expression.expressions): args: t.List[str] = [] - for e in expression.expressions: + for idx, e in enumerate(expression.expressions): if isinstance(e, exp.PropertyEQ): key_sql = ( self.sql(exp.Literal.string(e.this.name)) @@ -973,11 +972,14 @@ def _doris_struct_sql(self: Generator, expression: exp.Struct) -> str: ) args.append(key_sql) args.append(self.sql(e.expression)) + elif isinstance(e, exp.Alias): + args.append(self.sql(exp.Literal.string(e.alias))) + args.append(self.sql(e.this)) else: + args.append(self.sql(exp.Literal.string(f"col{idx + 1}"))) args.append(self.sql(e)) return self.func("NAMED_STRUCT", *args) - values = [e.this if isinstance(e, exp.Alias) else e for e in expression.expressions] - return self.func("STRUCT", *values) + return self.func("STRUCT", *expression.expressions) def _parse_if(self: Parser) -> t.Optional[exp.Expr]: diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index fd27802c97..191637edd1 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -529,7 +529,8 @@ def replace_query( quoted_target = quote_identifiers(target_table.copy()) default_catalog: t.Optional[str] = None default_catalog_loaded = False - qualified_target: t.Optional[exp.Table] = None + default_db: t.Optional[str] = None + default_db_loaded = False def _table_parts( table: exp.Table, @@ -572,20 +573,57 @@ def _is_cte_reference(table: exp.Table, quoted_table: exp.Table) -> bool: return False def _is_target_table(node: exp.Expr) -> bool: - nonlocal default_catalog, default_catalog_loaded, qualified_target + nonlocal default_catalog, default_catalog_loaded, default_db, default_db_loaded if not isinstance(node, exp.Table): return False quoted_node = quote_identifiers(node.copy()) if _is_cte_reference(node, quoted_node): return False - if _table_parts(quoted_node) == _table_parts(quoted_target): + target_to_check = quoted_target + if _table_parts(quoted_node) == _table_parts(target_to_check): return True + if quoted_node.this != target_to_check.this or quoted_node.args.get( + "version" + ) != target_to_check.args.get("version"): + return False + if bool(quoted_node.args.get("db")) != bool(target_to_check.args.get("db")): + if (not quoted_node.args.get("db") and quoted_node.args.get("catalog")) or ( + not target_to_check.args.get("db") and target_to_check.args.get("catalog") + ): + return False + if not default_db_loaded: + default_db_loaded = True + default_db = getattr(self, "_active_database", None) + if not default_db and callable(getattr(self, "get_current_database", None)): + try: + current_db = getattr(self, "get_current_database")() + if isinstance(current_db, str) and current_db: + default_db = current_db + except Exception: + pass + if not default_db: + for key in ("database", "schema"): + val = self._extra_config.get(key) + if isinstance(val, str) and val: + default_db = val + break + if not default_db: + raise SQLMeshError( + f"Cannot disambiguate unqualified schema reference '{node.sql(dialect=self.dialect)}' " + f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default database." + ) + quoted_default_db = quote_identifiers(exp.parse_identifier(default_db)) + if not quoted_node.args.get("db"): + quoted_node.set("db", quoted_default_db) + else: + target_to_check = target_to_check.copy() + target_to_check.set("db", quoted_default_db.copy()) + if _table_parts(quoted_node) == _table_parts(target_to_check): + return True if ( - quoted_node.this != quoted_target.this - or quoted_node.args.get("db") != quoted_target.args.get("db") - or quoted_node.args.get("version") != quoted_target.args.get("version") + quoted_node.args.get("db") != target_to_check.args.get("db") or not quoted_node.args.get("db") - or (quoted_node.args.get("catalog") and quoted_target.args.get("catalog")) + or (quoted_node.args.get("catalog") and target_to_check.args.get("catalog")) or self.catalog_support.is_unsupported ): return False @@ -611,21 +649,24 @@ def _is_target_table(node: exp.Expr) -> bool: default_catalog = current_cat except Exception: pass - if default_catalog: - qualified_target = _qualified(target_table, default_catalog) - if default_catalog is None or qualified_target is None: + if default_catalog is None: raise SQLMeshError( f"Cannot disambiguate unqualified catalog reference '{node.sql(dialect=self.dialect)}' " f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default catalog." ) - return _table_parts(_qualified(node, default_catalog)) == _table_parts(qualified_target) + return _table_parts(_qualified(quoted_node, default_catalog)) == _table_parts( + _qualified(target_to_check, default_catalog) + ) def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: if not _is_target_table(node): return node replacement = replacement_table.copy() - if isinstance(node, exp.Table) and node.args.get("alias") is not None: - replacement.set("alias", node.args["alias"].copy()) + if isinstance(node, exp.Table): + if node.args.get("alias") is not None: + replacement.set("alias", node.args["alias"].copy()) + if node.args.get("sample") is not None: + replacement.set("sample", node.args["sample"].copy()) return replacement self_referencing = any(_is_target_table(table) for table in query.find_all(exp.Table)) diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index 25b5fd6664..b26a83873b 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3439,6 +3439,31 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', ] + # Database-less table reference that matches target table name raises when no default database is known + with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + + # Database-less table reference is detected as self-referencing when active/default database matches target_table.db + adapter._extra_config["schema"] = "db" + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test TABLESAMPLE BERNOULLI (10)", read="trino"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', + 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', + 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh" TABLESAMPLE BERNOULLI (10)', + 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', + ] + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 6870cc613c..0b4815ad8c 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1657,3 +1657,24 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" + + negated_match_ast = d.parse_one( + "SELECT * FROM docs WHERE body NOT MATCH_ANY 'error warning'", + dialect="doris", + ) + assert ( + negated_match_ast.sql("doris") + == "SELECT * FROM docs WHERE NOT body MATCH_ANY 'error warning'" + ) + + aliased_struct_ast = d.parse_one( + "SELECT STRUCT(1 AS id, 'a' AS name)", + dialect="doris", + ) + assert aliased_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'name', 'a')" + + mixed_struct_ast = d.parse_one( + "SELECT STRUCT(1 AS id, 'a')", + dialect="doris", + ) + assert mixed_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'col2', 'a')" From c4fac90da75cbc033bd25c9dc7a7d09aa423ea9d Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:14:12 -0800 Subject: [PATCH 29/54] fix(doris): address CodeRabbit review feedback on session db, partition staging, TIMESTAMPTZ test, and docs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/wait-for-db.sh | 16 ++- docs/integrations/engines/doris.md | 41 +++++-- sqlmesh/core/engine_adapter/base.py | 104 ++++++++++++++++-- sqlmesh/core/engine_adapter/doris.py | 29 ++--- .../integration/test_integration.py | 13 ++- .../integration/test_integration_doris.py | 4 + tests/core/engine_adapter/test_base.py | 34 ++++++ 7 files changed, 206 insertions(+), 35 deletions(-) diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index efbae44620..c43272ba7f 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -24,8 +24,15 @@ function_exists() { probe_port() { HOSTNAME=${DOCKER_HOSTNAME:-localhost} - echo "Probing '$HOSTNAME' on port $1" - while ! nc -z $HOSTNAME $1; do + local port="$1" + local deadline="$2" + local timeout_secs="$3" + echo "Probing '$HOSTNAME' on port $port" + while ! nc -z "$HOSTNAME" "$port"; do + if [ -n "$deadline" ] && [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: Timed out after ${timeout_secs:-0}s waiting for '$HOSTNAME' on port $port" >&2 + return 1 + fi sleep 1 done } @@ -110,7 +117,10 @@ doris_ready() { local backend_registered=0 local reg_out="" - probe_port 9030 + if ! probe_port 9030 "$deadline" "$timeout_secs"; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE port 9030 to open" >&2 + exit 1 + fi echo "Waiting for Doris FE to accept SQL connections..." while true; do diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 9dc328cc80..253827563e 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -214,11 +214,22 @@ MODEL ( ); SELECT - order_id::BIGINT AS order_id, - status::VARCHAR(32) AS status, - updated_at::DATETIME AS updated_at -FROM raw.orders_changelog -WHERE updated_at BETWEEN @start_ts AND @end_ts; + order_id, + status, + updated_at +FROM ( + SELECT + order_id::BIGINT AS order_id, + status::VARCHAR(32) AS status, + updated_at::DATETIME AS updated_at, + ROW_NUMBER() OVER ( + PARTITION BY order_id + ORDER BY updated_at DESC, changelog_id DESC + ) AS _rn + FROM raw.orders_changelog + WHERE updated_at BETWEEN @start_ts AND @end_ts +) AS latest_changes +WHERE _rn = 1; ``` ### AGGREGATE KEY model @@ -432,10 +443,24 @@ SQLMesh can also create, partition, and overwrite Apache Iceberg tables directly - **Concurrent reader visibility**: Readers querying the Iceberg table between the `DELETE` snapshot(s) and the `INSERT INTO` snapshot may see an empty or partially cleared table. - **Insert failure risk**: If the `INSERT INTO` statement fails after the `DELETE` statements commit, the production table remains empty until recovered. - **Verified recovery procedures** (quiesce concurrent writers before starting recovery): - 1. **Replace or roll back to the prior Iceberg snapshot (`FOR VERSION AS OF`)**: Because `DorisEngineAdapter` clears Iceberg tables using two separate `DELETE` statements (`IS NOT NULL` and `IS NULL`), a failure after the first `DELETE` can leave residual rows in the table. Do not blindly `INSERT` the prior snapshot on top of a partially cleared table (which would duplicate surviving rows), and do not select the recovery snapshot by `committed_at` timestamp order alone (since `$snapshots` includes commits from other branches). Instead, quiesce writers, trace the `parent_id` chain from the production `main` branch head in `$refs` back through the failed refresh's `delete` snapshot(s) in `$snapshots` to find the first failed-refresh `delete` snapshot's `parent_id` (``), verify that `` belongs to the production table's lineage, and either roll back the table metadata to `` in your Iceberg catalog or clear any remaining rows before restoring that snapshot: + 1. **Replace or roll back to the prior Iceberg snapshot (`FOR VERSION AS OF`)**: Because `DorisEngineAdapter` clears Iceberg tables using two separate `DELETE` statements (`IS NOT NULL` and `IS NULL`), a failure after the first `DELETE` can leave residual rows in the table. Do not blindly `INSERT` the prior snapshot on top of a partially cleared table (which would duplicate surviving rows), and do not select the recovery snapshot by `committed_at` timestamp order alone (since `$snapshots` / `iceberg_meta(..., "query_type" = "snapshots")` includes commits from other branches). Instead, quiesce writers, trace the `parent_id` chain from the production `main` branch head in `refs` back through the failed refresh's `delete` snapshot(s) in `snapshots` to find the first failed-refresh `delete` snapshot's `parent_id` (``), verify that `` belongs to the production table's lineage, and either roll back the table metadata to `` in your Iceberg catalog or clear any remaining rows before restoring that snapshot: ```sql - -- 1. Inspect the production 'main' branch head and follow parent_id through the failed refresh's - -- 'delete' snapshot(s) to identify the first failed-refresh delete snapshot's parent_id (): + -- 1a. On Doris 4.1.0–4.1.3, inspect the 'main' branch head and snapshot parent_id chain + -- using the iceberg_meta() table-valued function: + SELECT name, type, snapshot_id + FROM iceberg_meta( + "table" = "iceberg_catalog.silver.curated_events", + "query_type" = "refs" + ) + WHERE name = 'main'; + + SELECT snapshot_id, parent_id, committed_at, operation + FROM iceberg_meta( + "table" = "iceberg_catalog.silver.curated_events", + "query_type" = "snapshots" + ); + + -- 1b. On Doris 4.1.4+, you can also query the $refs and $snapshots metadata tables directly: SELECT name, type, snapshot_id FROM iceberg_catalog.silver.curated_events$refs WHERE name = 'main'; diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 191637edd1..27c6398822 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -593,19 +593,37 @@ def _is_target_table(node: exp.Expr) -> bool: return False if not default_db_loaded: default_db_loaded = True + session_db_resolved = False default_db = getattr(self, "_active_database", None) - if not default_db and callable(getattr(self, "get_current_database", None)): + if isinstance(default_db, str) and default_db.strip(): + default_db = default_db.strip() + session_db_resolved = True + else: + default_db = None + if not session_db_resolved and callable( + getattr(self, "get_current_database", None) + ): try: current_db = getattr(self, "get_current_database")() - if isinstance(current_db, str) and current_db: - default_db = current_db + if isinstance(current_db, str): + session_db_resolved = True + default_db = current_db.strip() or None except Exception: pass - if not default_db: + if not session_db_resolved and self.dialect in ("doris", "mysql", "starrocks"): + try: + db_row = self.fetchone("SELECT DATABASE()") + if isinstance(db_row, (list, tuple)) and db_row: + session_db_resolved = True + if isinstance(db_row[0], str) and db_row[0].strip(): + default_db = db_row[0].strip() + except Exception: + pass + if not session_db_resolved and not default_db: for key in ("database", "schema"): val = self._extra_config.get(key) - if isinstance(val, str) and val: - default_db = val + if isinstance(val, str) and val.strip(): + default_db = val.strip() break if not default_db: raise SQLMeshError( @@ -658,6 +676,50 @@ def _is_target_table(node: exp.Expr) -> bool: _qualified(target_to_check, default_catalog) ) + temp_table_partitions: t.Optional[t.Set[str]] = None + staged_partition_sql: t.Optional[str] = None + + def _partition_names(partition_exp: exp.Expr) -> t.Set[str]: + names: t.Set[str] = set() + for expr in partition_exp.expressions: + name = expr.name if isinstance(expr, (exp.Identifier, exp.Var, exp.Column)) else "" + if not name and isinstance(expr, exp.Partition): + for sub in expr.expressions: + if sub.name: + names.add(sub.name.lower()) + elif name: + names.add(name.lower()) + return names + + def _get_temp_table_partitions(temp_table: exp.Table) -> t.Set[str]: + nonlocal temp_table_partitions + if temp_table_partitions is not None: + return temp_table_partitions + names: t.Set[str] = set() + getter = getattr(self, "_get_table_partition_names", None) + if callable(getter): + try: + res = getter(temp_table) + if isinstance(res, (set, list, tuple)): + names.update(str(x).lower() for x in res if x) + except Exception: + pass + if not names and self.dialect in ("doris", "starrocks"): + try: + rows = self.fetchall(f"SHOW PARTITIONS FROM {self._to_sql(temp_table)}") + if isinstance(rows, (list, tuple)): + for row in rows: + if ( + isinstance(row, (list, tuple)) + and len(row) > 1 + and isinstance(row[1], str) + ): + names.add(row[1].strip().lower()) + except Exception: + pass + temp_table_partitions = names + return names + def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: if not _is_target_table(node): return node @@ -667,6 +729,18 @@ def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.E replacement.set("alias", node.args["alias"].copy()) if node.args.get("sample") is not None: replacement.set("sample", node.args["sample"].copy()) + partition_arg = node.args.get("partition") + if partition_arg is not None: + req_names = _partition_names(partition_arg) + avail_names = _get_temp_table_partitions(replacement_table) + if req_names and req_names.issubset(avail_names): + replacement.set("partition", partition_arg.copy()) + elif staged_partition_sql != partition_arg.sql(dialect=self.dialect): + raise SQLMeshError( + f"Cannot apply partition selection '{partition_arg.sql(dialect=self.dialect)}' " + f"to temporary staging table '{replacement_table.sql(dialect=self.dialect)}' " + "because the staging table does not expose matching partitions." + ) return replacement self_referencing = any(_is_target_table(table) for table in query.find_all(exp.Table)) @@ -704,8 +778,24 @@ def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.E ) if self_referencing: assert target_columns_to_types is not None + self_ref_partition_clauses: t.Dict[str, t.Optional[exp.Expr]] = {} + for sq in source_queries: + sq_ast = sq.query_factory() + for tbl in sq_ast.find_all(exp.Table): + if _is_target_table(tbl): + part_expr = tbl.args.get("partition") + part_key = ( + part_expr.sql(dialect=self.dialect) if part_expr is not None else "" + ) + self_ref_partition_clauses[part_key] = part_expr + staging_target = target_table.copy() + if len(self_ref_partition_clauses) == 1: + only_key, only_part = next(iter(self_ref_partition_clauses.items())) + if only_key and only_part is not None: + staging_target.set("partition", only_part.copy()) + staged_partition_sql = only_key with self.temp_table( - self._select_columns(target_columns_to_types).from_(target_table), + self._select_columns(target_columns_to_types).from_(staging_target), name=target_table, target_columns_to_types=target_columns_to_types, **kwargs, diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 5cec12c3b7..07df706414 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -1928,27 +1928,28 @@ def _to_source_table(self, raw_src: t.Any) -> exp.Table: tbl.set("catalog", None) return tbl + def get_current_database(self) -> t.Optional[str]: + try: + db_row = self.fetchone("SELECT DATABASE()") + if isinstance(db_row, (list, tuple)) and db_row: + if isinstance(db_row[0], str) and db_row[0].strip(): + return db_row[0].strip() + return "" + except Exception: + pass + return None + def _resolve_sync_mv_source_from_metadata( self, target_table: exp.Table, is_explicit_sync: bool = False, skip_full_desc_scan: bool = False, ) -> t.Tuple[t.Optional[exp.Table], bool, bool]: - db_name: t.Optional[str] = target_table.db or ( - str(self._extra_config["database"]) if self._extra_config.get("database") else None + db_name: t.Optional[str] = ( + target_table.db + or self.get_current_database() + or (str(self._extra_config["database"]) if self._extra_config.get("database") else None) ) - if not db_name: - try: - db_row = self.fetchone("SELECT DATABASE()") - if ( - isinstance(db_row, (list, tuple)) - and db_row - and isinstance(db_row[0], str) - and db_row[0].strip() - ): - db_name = db_row[0].strip() - except Exception: - db_name = None if not db_name: return None, bool(is_explicit_sync), False diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index 7111239641..db76705571 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -2694,7 +2694,7 @@ def test_dialects(ctx: TestContext): "spark": pd.Timestamp("2020-01-01 00:00:00"), "databricks": pd.Timestamp("2020-01-01 00:00:00"), "starrocks": pd.Timestamp("2020-01-01 00:00:00"), - "doris": pd.Timestamp("2020-01-01 00:00:00"), + "doris": pd.Timestamp("2020-01-01 00:00:00+00:00"), }, ), ( @@ -2746,10 +2746,17 @@ def test_to_time_column( expected = result.get(ctx.dialect, result.get("default")) # Db2 (UPPERCASE strategy) returns column names in uppercase, same as Snowflake col_name = "THE_COL" if ctx.dialect in ("snowflake", "db2") else "the_col" + actual = df[col_name][0] + if ( + ctx.dialect == "doris" + and time_column_type.is_type(exp.DataType.Type.TIMESTAMPTZ) + and isinstance(actual, str) + ): + actual = pd.Timestamp(actual).tz_convert("UTC") if expected is pd.NaT or expected is None: - assert df[col_name][0] is expected + assert actual is expected else: - assert df[col_name][0] == expected + assert actual == expected def test_batch_size_on_incremental_by_unique_key_model(ctx: TestContext): diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 8d15b93d3d..b28bb87b64 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -440,6 +440,8 @@ def test_duplicate_key_delete_with_subquery_and_between(ctx: TestContext): [ {"id": 1, "ds": "2022-01-01"}, {"id": 2, "ds": "2022-01-02"}, + {"id": 2, "ds": "2022-01-03"}, + {"id": 3, "ds": "2022-01-02"}, {"id": 3, "ds": "2022-01-03"}, {"id": 4, "ds": "2022-01-04"}, ] @@ -464,6 +466,8 @@ def test_duplicate_key_delete_with_subquery_and_between(ctx: TestContext): target_table, pd.DataFrame( [ + {"id": 2, "ds": "2022-01-03"}, + {"id": 3, "ds": "2022-01-02"}, {"id": 3, "ds": "2022-01-03"}, {"id": 4, "ds": "2022-01-04"}, ] diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index b26a83873b..881501d142 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3464,6 +3464,40 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', ] + # When an active session database (e.g., via get_current_database / SELECT DATABASE()) differs from + # configured schema/database, bare table reference is not misclassified as target_table + adapter.get_current_database = lambda: "otherdb" # type: ignore[attr-defined] + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "test"', + ] + delattr(adapter, "get_current_database") + + # Partition selection on self-referencing table is preserved when temp table exposes matching partitions + adapter.dialect = "doris" + adapter._get_table_partition_names = lambda _tbl: {"p1"} # type: ignore[attr-defined] + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM db.test PARTITION (p1)", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`test` (`a` INT)", + "CREATE SCHEMA IF NOT EXISTS `dev`.`db`", + "CREATE TABLE IF NOT EXISTS `dev`.`db`.`__temp_test_abcdefgh` AS SELECT CAST(`a` AS INT) AS `a` FROM (SELECT `a` FROM `db`.`test` PARTITION(`p1`)) AS `_subquery`", + "DELETE FROM `db`.`test` WHERE TRUE", + "INSERT INTO `db`.`test` (`a`) SELECT `a` FROM `dev`.`db`.`__temp_test_abcdefgh` PARTITION(`p1`)", + "DROP TABLE IF EXISTS `dev`.`db`.`__temp_test_abcdefgh`", + ] + delattr(adapter, "_get_table_partition_names") + def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) From 722fb84b62ec93d813d5439e6d1e71962eae6e76 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:46:03 -0800 Subject: [PATCH 30/54] fix(doris): gate TIMESTAMPTZ at 4.1.0, pin pymysql>=1.2.0, and refine adapter hooks Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 7 +- pyproject.toml | 2 +- sqlmesh/core/dialect.py | 47 ++- sqlmesh/core/engine_adapter/base.py | 117 +++---- sqlmesh/core/engine_adapter/doris.py | 298 +++++++++++------- sqlmesh/core/engine_adapter/trino.py | 9 + .../integration/test_integration_doris.py | 27 +- tests/core/engine_adapter/test_base.py | 15 +- tests/core/engine_adapter/test_doris.py | 21 ++ tests/core/test_dialect.py | 14 + 10 files changed, 366 insertions(+), 191 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 253827563e..5bde21aa8c 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -148,7 +148,7 @@ Apache Doris supports three native OLAP table data models: **DUPLICATE KEY**, ** ### DUPLICATE KEY model (default) -By default, OLAP tables use the `DUPLICATE KEY` model. If no key is specified, Doris uses the first column as the duplicate key and hash distribution column. +By default, OLAP tables use the `DUPLICATE KEY` model. If no key or distribution is specified, Doris selects a prefix of leading columns as the duplicate key and uses random distribution (`DISTRIBUTED BY RANDOM BUCKETS AUTO`) on supported versions. ```sql MODEL ( @@ -1004,7 +1004,7 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; | SQLMesh Model Kind | Doris Internal OLAP Catalog (`internal` on Local Disk or S3/GCS Storage Vault) | External Iceberg Catalog (`table_format iceberg` on S3/GCS/HDFS) | External Hive Catalog (`table_format hive` on S3/GCS/HDFS) | External Read-Only Lakehouse Catalogs (`paimon`, `lance`, `hudi`, `deltalake`, `lakesoul`) | | --- | :---: | :---: | :---: | :---: | | Source / Upstream `SELECT` | Supported | Supported (incl. `FOR TIME/VERSION AS OF`, `@branch`, `@tag`) | Supported (Doris 2.1+) | Supported (Read-Only via corresponding External Catalog; Lance also supports `s3()` / `local()` TVFs) | -| `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `DELETE` + `INSERT`) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | +| `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `TRUNCATE TABLE` + `INSERT`; non-atomic, so a failed `INSERT` leaves the table empty) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | | `INCREMENTAL_BY_TIME_RANGE` | Supported (`DELETE WHERE time_col >= low AND time_col <= high` + `INSERT`) | Supported (`DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No row-level `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA row-level `DELETE`) | | `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`, `DELETE ... USING` + `INSERT` or native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | | `INCREMENTAL_BY_PARTITION` | Supported (Column & time-truncation range `DELETE` + `INSERT`) | Supported (Partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | @@ -1014,6 +1014,9 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; | `SEED` / Python DataFrame Models | Supported (Automatic `VARCHAR(255)` OLAP key coercion) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Paimon: Read-Only in Doris 2.1–4.1.5; Lance/Hudi/Delta/LakeSoul: Write Guarded | | `CUSTOM ('doris_ddl_only')` | Supported (DDL + `ALTER TABLE` schema evolution only) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Not supported (Read-Only) | +!!! note "Recovering from a Failed In-Place OLAP `FULL` Refresh" + Because Doris does not execute `TRUNCATE TABLE` and `INSERT INTO` inside a multi-statement transaction, an in-place OLAP `FULL` refresh that fails during `INSERT INTO` leaves the target table empty. To recover after fixing the failure, re-run the model (for example via `sqlmesh plan --restate-model ` or by re-applying the failed `sqlmesh plan`), since each `FULL` run rebuilds the table from its query. + --- ## Property Aliases & StarRocks Compatibility diff --git a/pyproject.toml b/pyproject.toml index efeb75f91e..d91cf75d20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ db2 = [ "ibm_db", "db2-sqlglot-dialect>=1.1.0;python_version>=\"3.10\"" ] -doris = ["pymysql"] +doris = ["pymysql>=1.2.0"] duckdb = [] fabric = ["pyodbc>=5.0.0"] fabric-mssql-python = ["mssql-python>=1.1.0;python_version>=\"3.10\""] diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 10409e6f83..4847c51382 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -930,6 +930,28 @@ def _doris_parse_alter_drop(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], return self._parse_alter_table_drop() +DORIS_TIMESTAMPTZ_MIN_VERSION: t.Tuple[int, int, int] = (4, 1, 0) + + +def doris_supports_timestamptz(version: t.Optional[t.Tuple[int, ...]] = None) -> bool: + if version is None: + return True + return (*version, 0, 0)[:3] >= DORIS_TIMESTAMPTZ_MIN_VERSION + + +def _doris_datatype_sql(self: Generator, expression: exp.DataType) -> str: + if expression.is_type(exp.DataType.Type.TIMESTAMPTZ) and not doris_supports_timestamptz( + getattr(self.dialect, "version", None) + ): + fallback = expression.copy() + fallback.set("this", exp.DataType.Type.DATETIME) + return self._datatype_sql(fallback) # type: ignore + return self._datatype_sql(expression) # type: ignore + + +_doris_datatype_sql.__name__ = "datatype_sql" + + def _doris_alter_sql(self: Generator, expression: exp.Alter) -> str: actions = expression.args.get("actions") or [] if expression.text("kind") == "BUILD_INDEX": @@ -1748,6 +1770,7 @@ def extend_sqlglot() -> None: if Doris.tokenizer_class.__name__ != "_DorisTokenizer": class _DorisTokenizer(Doris.tokenizer_class): # type: ignore + TIMESTAMPTZ_MIN_VERSION = DORIS_TIMESTAMPTZ_MIN_VERSION KEYWORDS = { **Doris.tokenizer_class.KEYWORDS, "AGG_STATE": TokenType.USERDEFINED, @@ -1763,10 +1786,25 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore "JSONB": TokenType.JSONB, "LARGEINT": TokenType.INT128, "QUANTILE_STATE": TokenType.USERDEFINED, - "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, + "TIMESTAMPTZ": ( + TokenType.TIMESTAMPTZ + if doris_supports_timestamptz(TIMESTAMPTZ_MIN_VERSION) + else TokenType.DATETIME + ), "VARIANT": TokenType.VARIANT, } + def tokenize(self, sql: str) -> t.List[Token]: + tokens = super().tokenize(sql) + if not doris_supports_timestamptz(getattr(self.dialect, "version", None)): + for token in tokens: + if ( + token.token_type == TokenType.TIMESTAMPTZ + and token.text.upper() == "TIMESTAMPTZ" + ): + token.token_type = TokenType.DATETIME + return tokens + Doris.Tokenizer = _DorisTokenizer Doris.tokenizer_class = _DorisTokenizer @@ -1798,7 +1836,11 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore exp.DataType.Type.IPV4: "IPV4", exp.DataType.Type.IPV6: "IPV6", exp.DataType.Type.JSONB: "JSONB", - exp.DataType.Type.TIMESTAMPTZ: "TIMESTAMPTZ", + exp.DataType.Type.TIMESTAMPTZ: ( + "TIMESTAMPTZ" + if doris_supports_timestamptz(DORIS_TIMESTAMPTZ_MIN_VERSION) + else "DATETIME" + ), exp.DataType.Type.VARIANT: "VARIANT", } ) @@ -1921,6 +1963,7 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore _override(Doris.Parser, _doris_parse_index_params) _override(Doris.Generator, altercolumn_sql) _override(Doris.Generator, columndef_sql) + _override(Doris.Generator, _doris_datatype_sql) _override(Doris.Generator, _doris_indexconstraintoption_sql) _override(Doris.Generator, _doris_indexparameters_sql) _override(Doris.Generator, _doris_alter_sql) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 27c6398822..6103fea671 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -295,6 +295,42 @@ def default_catalog(self) -> t.Optional[str]: ) return default_catalog + def get_current_database(self) -> t.Optional[str]: + """Returns the database/schema name of the current session, if supported.""" + return None + + def _default_schema_from_config(self) -> t.Optional[str]: + val = self._extra_config.get("schema") + if isinstance(val, str) and val.strip(): + return val.strip() + return None + + def _resolve_session_database(self) -> t.Optional[str]: + try: + current_db = self.get_current_database() + if isinstance(current_db, str): + return current_db.strip() or None + except Exception: + pass + return self._default_schema_from_config() + + def _resolve_session_catalog(self) -> t.Optional[str]: + try: + if self.default_catalog: + return self.default_catalog + except MissingDefaultCatalogError: + pass + try: + current_cat = self.get_current_catalog() + if isinstance(current_cat, str) and current_cat: + return current_cat + except Exception: + pass + return None + + def _get_table_partition_names(self, table: exp.Table) -> t.Set[str]: + return set() + @property def engine_run_mode(self) -> EngineRunMode: return EngineRunMode.SINGLE_MODE_ENGINE @@ -593,38 +629,7 @@ def _is_target_table(node: exp.Expr) -> bool: return False if not default_db_loaded: default_db_loaded = True - session_db_resolved = False - default_db = getattr(self, "_active_database", None) - if isinstance(default_db, str) and default_db.strip(): - default_db = default_db.strip() - session_db_resolved = True - else: - default_db = None - if not session_db_resolved and callable( - getattr(self, "get_current_database", None) - ): - try: - current_db = getattr(self, "get_current_database")() - if isinstance(current_db, str): - session_db_resolved = True - default_db = current_db.strip() or None - except Exception: - pass - if not session_db_resolved and self.dialect in ("doris", "mysql", "starrocks"): - try: - db_row = self.fetchone("SELECT DATABASE()") - if isinstance(db_row, (list, tuple)) and db_row: - session_db_resolved = True - if isinstance(db_row[0], str) and db_row[0].strip(): - default_db = db_row[0].strip() - except Exception: - pass - if not session_db_resolved and not default_db: - for key in ("database", "schema"): - val = self._extra_config.get(key) - if isinstance(val, str) and val.strip(): - default_db = val.strip() - break + default_db = self._resolve_session_database() if not default_db: raise SQLMeshError( f"Cannot disambiguate unqualified schema reference '{node.sql(dialect=self.dialect)}' " @@ -647,26 +652,7 @@ def _is_target_table(node: exp.Expr) -> bool: return False if not default_catalog_loaded: default_catalog_loaded = True - default_catalog = getattr(self, "_active_catalog", None) - if not default_catalog and self.dialect == "trino": - try: - current_cat = self.get_current_catalog() - if isinstance(current_cat, str) and current_cat: - default_catalog = current_cat - except Exception: - pass - if not default_catalog: - try: - default_catalog = self.default_catalog - except MissingDefaultCatalogError: - default_catalog = None - if not default_catalog and self.dialect != "trino": - try: - current_cat = self.get_current_catalog() - if isinstance(current_cat, str) and current_cat: - default_catalog = current_cat - except Exception: - pass + default_catalog = self._resolve_session_catalog() if default_catalog is None: raise SQLMeshError( f"Cannot disambiguate unqualified catalog reference '{node.sql(dialect=self.dialect)}' " @@ -696,27 +682,12 @@ def _get_temp_table_partitions(temp_table: exp.Table) -> t.Set[str]: if temp_table_partitions is not None: return temp_table_partitions names: t.Set[str] = set() - getter = getattr(self, "_get_table_partition_names", None) - if callable(getter): - try: - res = getter(temp_table) - if isinstance(res, (set, list, tuple)): - names.update(str(x).lower() for x in res if x) - except Exception: - pass - if not names and self.dialect in ("doris", "starrocks"): - try: - rows = self.fetchall(f"SHOW PARTITIONS FROM {self._to_sql(temp_table)}") - if isinstance(rows, (list, tuple)): - for row in rows: - if ( - isinstance(row, (list, tuple)) - and len(row) > 1 - and isinstance(row[1], str) - ): - names.add(row[1].strip().lower()) - except Exception: - pass + try: + for part_name in self._get_table_partition_names(temp_table): + if part_name: + names.add(str(part_name).lower()) + except Exception: + pass temp_table_partitions = names return names diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 07df706414..cecfd72fc6 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -17,7 +17,11 @@ from sqlglot.helper import ensure_list from sqlglot.optimizer.qualify_columns import quote_identifiers -from sqlmesh.core.dialect import to_schema +from sqlmesh.core.dialect import ( + DORIS_TIMESTAMPTZ_MIN_VERSION, + doris_supports_timestamptz, + to_schema, +) from sqlmesh.core.engine_adapter.base import ( EngineAdapterWithIndexSupport, InsertOverwriteStrategy, @@ -68,6 +72,135 @@ def _alter_actions(self) -> t.List[exp.Expr]: ] +def _build_doris_schema_differ_kwargs( + version: t.Optional[t.Tuple[int, ...]] = DORIS_TIMESTAMPTZ_MIN_VERSION, + dialect: str = "doris", +) -> t.Dict[str, t.Any]: + has_timestamptz = doris_supports_timestamptz(version) + parameterized_type_defaults: t.Dict[t.Any, t.List[t.Tuple[int, ...]]] = { + exp.DataType.build("DECIMAL", dialect=dialect).this: [(9, 0), (0,)], + exp.DataType.build("CHAR", dialect=dialect).this: [(1,), (255,)], + exp.DataType.build("VARCHAR", dialect=dialect).this: [(65533,)], + exp.DataType.build("DATETIME", dialect=dialect).this: [(0,)], + **( + {exp.DataType.build("TIMESTAMPTZ", dialect=dialect).this: [(0,)]} + if has_timestamptz + else {} + ), + exp.DataType.build("INT", dialect=dialect).this: [(11,)], + exp.DataType.build("BIGINT", dialect=dialect).this: [(20,)], + exp.DataType.build("SMALLINT", dialect=dialect).this: [(6,)], + exp.DataType.build("TINYINT", dialect=dialect).this: [(4,), (1,)], + } + compatible_types: t.Dict[exp.DataType, t.Set[exp.DataType]] = { + exp.DataType.build("BOOLEAN", dialect=dialect): { + exp.DataType.build("TINYINT", dialect=dialect), + exp.DataType.build("SMALLINT", dialect=dialect), + exp.DataType.build("INT", dialect=dialect), + exp.DataType.build("BIGINT", dialect=dialect), + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("TINYINT", dialect=dialect): { + exp.DataType.build("SMALLINT", dialect=dialect), + exp.DataType.build("INT", dialect=dialect), + exp.DataType.build("BIGINT", dialect=dialect), + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("DOUBLE", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("SMALLINT", dialect=dialect): { + exp.DataType.build("INT", dialect=dialect), + exp.DataType.build("BIGINT", dialect=dialect), + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("DOUBLE", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("INT", dialect=dialect): { + exp.DataType.build("BIGINT", dialect=dialect), + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("DOUBLE", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("BIGINT", dialect=dialect): { + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("INT128", dialect=dialect): { + exp.DataType.build("LARGEINT", dialect=dialect, udt=True), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("LARGEINT", dialect=dialect, udt=True): { + exp.DataType.build("INT128", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("FLOAT", dialect=dialect): { + exp.DataType.build("DOUBLE", dialect=dialect), + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("DOUBLE", dialect=dialect): { + exp.DataType.build("TEXT", dialect=dialect), + }, + exp.DataType.build("DATE", dialect=dialect): { + exp.DataType.build("DATETIME", dialect=dialect), + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), + }, + exp.DataType.build("DATETIME", dialect=dialect): { + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), + }, + exp.DataType.build("DATETIME(0)", dialect=dialect): { + exp.DataType.build("DATETIME", dialect=dialect), + *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(1, 7)), + }, + **{ + exp.DataType.build(f"DATETIME({p})", dialect="doris"): { + exp.DataType.build(f"DATETIME({p2})", dialect="doris") for p2 in range(p + 1, 7) + } + for p in range(1, 6) + }, + } + if has_timestamptz: + compatible_types.update( + { + exp.DataType.build("TIMESTAMPTZ", dialect=dialect): { + *( + exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") + for p in range(0, 7) + ), + }, + exp.DataType.build("TIMESTAMPTZ(0)", dialect=dialect): { + exp.DataType.build("TIMESTAMPTZ", dialect=dialect), + *( + exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") + for p in range(1, 7) + ), + }, + **{ + exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris"): { + exp.DataType.build(f"TIMESTAMPTZ({p2})", dialect="doris") + for p2 in range(p + 1, 7) + } + for p in range(1, 6) + }, + } + ) + return { + "parameterized_type_defaults": parameterized_type_defaults, + "types_with_unlimited_length": { + exp.DataType.build("TEXT", dialect=dialect).this: { + exp.DataType.build("VARCHAR", dialect=dialect).this, + exp.DataType.build("CHAR", dialect=dialect).this, + }, + exp.DataType.build("VARCHAR", dialect=dialect).this: { + exp.DataType.build("VARCHAR", dialect=dialect).this, + exp.DataType.build("CHAR", dialect=dialect).this, + exp.DataType.build("TEXT", dialect=dialect).this, + }, + }, + "compatible_types": compatible_types, + } + + @set_catalog() class DorisEngineAdapter( EngineAdapterWithIndexSupport, @@ -96,111 +229,10 @@ class DorisEngineAdapter( RESOLVE_TABLE_REFS_IN_PHYSICAL_PROPERTIES: t.FrozenSet[str] = frozenset( {"excluded_trigger_tables", "source_table"} ) - SCHEMA_DIFFER_KWARGS = { - "parameterized_type_defaults": { - exp.DataType.build("DECIMAL", dialect=DIALECT).this: [(9, 0), (0,)], - exp.DataType.build("CHAR", dialect=DIALECT).this: [(1,), (255,)], - exp.DataType.build("VARCHAR", dialect=DIALECT).this: [(65533,)], - exp.DataType.build("DATETIME", dialect=DIALECT).this: [(0,)], - exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT).this: [(0,)], - exp.DataType.build("INT", dialect=DIALECT).this: [(11,)], - exp.DataType.build("BIGINT", dialect=DIALECT).this: [(20,)], - exp.DataType.build("SMALLINT", dialect=DIALECT).this: [(6,)], - exp.DataType.build("TINYINT", dialect=DIALECT).this: [(4,), (1,)], - }, - "types_with_unlimited_length": { - exp.DataType.build("TEXT", dialect=DIALECT).this: { - exp.DataType.build("VARCHAR", dialect=DIALECT).this, - exp.DataType.build("CHAR", dialect=DIALECT).this, - }, - exp.DataType.build("VARCHAR", dialect=DIALECT).this: { - exp.DataType.build("VARCHAR", dialect=DIALECT).this, - exp.DataType.build("CHAR", dialect=DIALECT).this, - exp.DataType.build("TEXT", dialect=DIALECT).this, - }, - }, - "compatible_types": { - exp.DataType.build("BOOLEAN", dialect=DIALECT): { - exp.DataType.build("TINYINT", dialect=DIALECT), - exp.DataType.build("SMALLINT", dialect=DIALECT), - exp.DataType.build("INT", dialect=DIALECT), - exp.DataType.build("BIGINT", dialect=DIALECT), - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("TINYINT", dialect=DIALECT): { - exp.DataType.build("SMALLINT", dialect=DIALECT), - exp.DataType.build("INT", dialect=DIALECT), - exp.DataType.build("BIGINT", dialect=DIALECT), - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("DOUBLE", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("SMALLINT", dialect=DIALECT): { - exp.DataType.build("INT", dialect=DIALECT), - exp.DataType.build("BIGINT", dialect=DIALECT), - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("DOUBLE", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("INT", dialect=DIALECT): { - exp.DataType.build("BIGINT", dialect=DIALECT), - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("DOUBLE", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("BIGINT", dialect=DIALECT): { - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("INT128", dialect=DIALECT): { - exp.DataType.build("LARGEINT", dialect=DIALECT, udt=True), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("LARGEINT", dialect=DIALECT, udt=True): { - exp.DataType.build("INT128", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("FLOAT", dialect=DIALECT): { - exp.DataType.build("DOUBLE", dialect=DIALECT), - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("DOUBLE", dialect=DIALECT): { - exp.DataType.build("TEXT", dialect=DIALECT), - }, - exp.DataType.build("DATE", dialect=DIALECT): { - exp.DataType.build("DATETIME", dialect=DIALECT), - *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), - }, - exp.DataType.build("DATETIME", dialect=DIALECT): { - *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(0, 7)), - }, - exp.DataType.build("DATETIME(0)", dialect=DIALECT): { - exp.DataType.build("DATETIME", dialect=DIALECT), - *(exp.DataType.build(f"DATETIME({p})", dialect="doris") for p in range(1, 7)), - }, - exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT): { - *(exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") for p in range(0, 7)), - }, - exp.DataType.build("TIMESTAMPTZ(0)", dialect=DIALECT): { - exp.DataType.build("TIMESTAMPTZ", dialect=DIALECT), - *(exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris") for p in range(1, 7)), - }, - **{ - exp.DataType.build(f"DATETIME({p})", dialect="doris"): { - exp.DataType.build(f"DATETIME({p2})", dialect="doris") for p2 in range(p + 1, 7) - } - for p in range(1, 6) - }, - **{ - exp.DataType.build(f"TIMESTAMPTZ({p})", dialect="doris"): { - exp.DataType.build(f"TIMESTAMPTZ({p2})", dialect="doris") - for p2 in range(p + 1, 7) - } - for p in range(1, 6) - }, - }, - } + TIMESTAMPTZ_MIN_VERSION: t.ClassVar[t.Tuple[int, int, int]] = DORIS_TIMESTAMPTZ_MIN_VERSION + SCHEMA_DIFFER_KWARGS = _build_doris_schema_differ_kwargs( + DORIS_TIMESTAMPTZ_MIN_VERSION, dialect=DIALECT + ) INVALID_KEY_TYPES = { "FLOAT", @@ -1939,6 +1971,47 @@ def get_current_database(self) -> t.Optional[str]: pass return None + def _default_schema_from_config(self) -> t.Optional[str]: + for key in ("database", "schema"): + val = self._extra_config.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + return None + + def _resolve_session_database(self) -> t.Optional[str]: + active_db = getattr(self, "_active_database", None) + if isinstance(active_db, str) and active_db.strip(): + return active_db.strip() + return super()._resolve_session_database() + + def _resolve_session_catalog(self) -> t.Optional[str]: + if self._active_catalog: + return self._active_catalog + return super()._resolve_session_catalog() + + def _get_table_partition_names(self, table: exp.Table) -> t.Set[str]: + names: t.Set[str] = set() + try: + rows = self.fetchall(f"SHOW PARTITIONS FROM {self._to_sql(table)}") + if isinstance(rows, (list, tuple)): + for row in rows: + if ( + isinstance(row, (list, tuple)) + and len(row) > 1 + and isinstance(row[1], str) + and row[1].strip() + ): + names.add(row[1].strip().lower()) + except Exception: + pass + return names + + def _supports_timestamptz(self) -> bool: + cached_version = getattr(self, "_doris_version", None) + if cached_version is None: + return doris_supports_timestamptz(self.TIMESTAMPTZ_MIN_VERSION) + return doris_supports_timestamptz(cached_version) + def _resolve_sync_mv_source_from_metadata( self, target_table: exp.Table, @@ -7164,10 +7237,11 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) -> and ts_node.this.is_string and "." in ts_node.this.this ): + ts_type_name = "TIMESTAMPTZ" if self._supports_timestamptz() else "DATETIME" cast_node = exp.cast( ts_node.this.copy(), exp.DataType.build( - "TIMESTAMPTZ", + ts_type_name, expressions=[exp.DataTypeParam(this=exp.Literal.number(6))], ), ) @@ -7184,6 +7258,16 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) -> expression = replacement else: dtype.replace(replacement) + elif dtype.is_type("timestamptz") and not self._supports_timestamptz(): + replacement = exp.DataType.build( + "DATETIME", + expressions=list(dtype.expressions), + dialect=self.dialect, + ) + if dtype is expression: + expression = replacement + else: + dtype.replace(replacement) elif ( dtype.is_type("timestamptz") and not dtype.expressions diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index 00acddb26c..ee10a413c6 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -108,6 +108,15 @@ def set_current_catalog(self, catalog: str) -> None: """Sets the catalog name of the current connection.""" self.execute(exp.Use(this=schema_(db="information_schema", catalog=catalog))) + def _resolve_session_catalog(self) -> t.Optional[str]: + try: + current_cat = self.get_current_catalog() + if isinstance(current_cat, str) and current_cat: + return current_cat + except Exception: + pass + return super()._resolve_session_catalog() + @lru_cache() def get_catalog_type(self, catalog: t.Optional[str]) -> str: row: t.Tuple = tuple() diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index b28bb87b64..ed47fde441 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -837,13 +837,13 @@ def test_doris_4_1_native_merge_segment_v3_and_ann_index(ctx: TestContext): ) doris_adapter.execute( f"INSERT INTO {doris_adapter._to_sql(fts_tbl)} (`id`, `msg`) VALUES " - f"(1, 'apache doris vector search'), (2, 'iceberg table compaction')" + f"(1, 'apache doris search'), (2, 'iceberg table compaction'), (3, 'fast vector index')" ) fts_query = parse_one( f"SELECT id FROM {doris_adapter._to_sql(fts_tbl)} WHERE msg MATCH_ANY 'doris vector' ORDER BY id", read="doris", ) - assert [r[0] for r in doris_adapter.fetchall(fts_query)] == [1] + assert [r[0] for r in doris_adapter.fetchall(fts_query)] == [1, 3] # 6. UNIQUE KEY table with function_column.sequence_col and _replace_by_key sequence protection seq_tbl = ctx.table("test_doris_41_seq") @@ -1241,6 +1241,19 @@ def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( ) assert list(dyn_rows) == [(10, 2), (20, 1)] + # Verify both synchronous MV rollup jobs reached FINISHED before restatement + rollup_jobs = ctx.engine_adapter.fetchall( + f"SHOW ALTER TABLE MATERIALIZED VIEW FROM `{schema_name}` WHERE TableName = 'user_events'", + quote_identifiers=False, + ) + rollup_job_states = { + str(row[5]).strip(): str(row[8]).strip().upper() + for row in rollup_jobs + if len(row) > 8 and row[5] is not None and row[8] is not None + } + assert rollup_job_states.get("user_events_sync_mv1") == "FINISHED" + assert rollup_job_states.get("user_events_sync_mv2") == "FINISHED" + # Insert an additional source row in d1 and restate `user_events` to verify DELETE + INSERT works with active sync MVs ctx.engine_adapter.execute( f"INSERT INTO {raw_tbl_sql} VALUES (10, 'checkout', '{d1}', '{d1} 10:00:00')" @@ -1255,3 +1268,13 @@ def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( quote_identifiers=False, ) assert list(restated_rows) == [(10, 3), (20, 1)] + desc_all_rows = ctx.engine_adapter.fetchall( + f"DESC `{schema_name}`.`user_events` ALL", + quote_identifiers=False, + ) + rollup_index_names = { + str(row[0]).strip() + for row in desc_all_rows + if len(row) >= 1 and row[0] is not None and str(row[0]).strip() + } + assert {"user_events_sync_mv1", "user_events_sync_mv2"}.issubset(rollup_index_names) diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index 881501d142..dfb12353b6 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3410,9 +3410,16 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( assert to_sql_calls(adapter) == [] # Trino resolves the session catalog (get_current_catalog) before _default_catalog + from sqlmesh.core.engine_adapter.trino import TrinoEngineAdapter + adapter.dialect = "trino" adapter._default_catalog = "dev" mocker.patch.object(adapter, "get_current_catalog", return_value="prod") + mocker.patch.object( + adapter, + "_resolve_session_catalog", + TrinoEngineAdapter._resolve_session_catalog.__get__(adapter, TrinoEngineAdapter), + ) adapter.cursor.execute.reset_mock() adapter.replace_query( "db.test", @@ -3466,7 +3473,7 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( # When an active session database (e.g., via get_current_database / SELECT DATABASE()) differs from # configured schema/database, bare table reference is not misclassified as target_table - adapter.get_current_database = lambda: "otherdb" # type: ignore[attr-defined] + mocker.patch.object(adapter, "get_current_database", return_value="otherdb") adapter.cursor.execute.reset_mock() adapter.replace_query( "db.test", @@ -3477,11 +3484,11 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'DELETE FROM "db"."test" WHERE TRUE', 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "test"', ] - delattr(adapter, "get_current_database") + mocker.patch.object(adapter, "get_current_database", return_value=None) # Partition selection on self-referencing table is preserved when temp table exposes matching partitions adapter.dialect = "doris" - adapter._get_table_partition_names = lambda _tbl: {"p1"} # type: ignore[attr-defined] + mocker.patch.object(adapter, "_get_table_partition_names", return_value={"p1"}) adapter.cursor.execute.reset_mock() adapter.replace_query( "db.test", @@ -3496,7 +3503,7 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( "INSERT INTO `db`.`test` (`a`) SELECT `a` FROM `dev`.`db`.`__temp_test_abcdefgh` PARTITION(`p1`)", "DROP TABLE IF EXISTS `dev`.`db`.`__temp_test_abcdefgh`", ] - delattr(adapter, "_get_table_partition_names") + mocker.patch.object(adapter, "_get_table_partition_names", return_value=set()) def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index c6a70a845c..85b027d015 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6065,3 +6065,24 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: "not_null_columns": exp.Tuple(expressions=[exp.column("payload")]), }, ) + + # 9. Doris TIMESTAMPTZ support is gated at 4.1.0; Doris 2.1 falls back to DATETIME. + from sqlmesh.core.engine_adapter.doris import _build_doris_schema_differ_kwargs + + v21_kwargs = _build_doris_schema_differ_kwargs((2, 1, 0)) + v41_kwargs = _build_doris_schema_differ_kwargs((4, 1, 0)) + assert exp.DataType.Type.TIMESTAMPTZ not in v21_kwargs["parameterized_type_defaults"] + assert exp.DataType.Type.TIMESTAMPTZ in v41_kwargs["parameterized_type_defaults"] + + v21_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + v21_adapter._doris_version = (2, 1) + assert ( + v21_adapter._to_sql(parse_one("SELECT CAST(ts AS TIMESTAMPTZ(6))", read="doris")) + == "SELECT CAST(`ts` AS DATETIME(6))" + ) + v41_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + v41_adapter._doris_version = (4, 1) + assert ( + v41_adapter._to_sql(parse_one("SELECT CAST(ts AS TIMESTAMPTZ(6))", read="doris")) + == "SELECT CAST(`ts` AS TIMESTAMPTZ(6))" + ) diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 0b4815ad8c..a016ab71f6 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1657,6 +1657,20 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" + assert ( + exp.DataType.build("TIMESTAMPTZ", dialect="doris, version=4.1.0").sql( + "doris, version=4.1.0" + ) + == "TIMESTAMPTZ" + ) + assert ( + exp.DataType.build("TIMESTAMPTZ", dialect="doris, version=2.1").sql("doris, version=2.1") + == "DATETIME" + ) + assert ( + exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris, version=2.1") + == "DATETIME(6)" + ) negated_match_ast = d.parse_one( "SELECT * FROM docs WHERE body NOT MATCH_ANY 'error warning'", From 3a756fc65915b08cb7ab2910caf5cebbf385f260 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:21:58 -0800 Subject: [PATCH 31/54] Fix(doris): fail closed on session db/catalog errors and bound CI probes Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/wait-for-db.sh | 26 ++++++++++++++++--- docs/guides/connections.md | 2 +- sqlmesh/core/config/connection.py | 4 +-- sqlmesh/core/engine_adapter/base.py | 2 +- sqlmesh/core/engine_adapter/doris.py | 2 +- sqlmesh/core/engine_adapter/trino.py | 4 +-- .../integration/test_integration_doris.py | 9 +++++-- tests/core/engine_adapter/test_base.py | 22 ++++++++++++++++ tests/core/test_connection_config.py | 26 ++++++++++++++++--- 9 files changed, 82 insertions(+), 15 deletions(-) diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index c43272ba7f..40bce1a98d 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -114,6 +114,7 @@ db2_ready() { doris_ready() { local timeout_secs="${DORIS_STARTUP_TIMEOUT_SECS:-300}" local deadline=$((SECONDS + timeout_secs)) + local remaining=0 local backend_registered=0 local reg_out="" @@ -124,7 +125,12 @@ doris_ready() { echo "Waiting for Doris FE to accept SQL connections..." while true; do - if docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then + remaining=$((deadline - SECONDS)) + if [ "$remaining" -le 0 ]; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE to become alive" >&2 + exit 1 + fi + if timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then echo "Doris FE is alive" break fi @@ -138,7 +144,12 @@ doris_ready() { echo "Checking for 1 alive Doris backends..." while true; do if [ "$backend_registered" -eq 0 ]; then - if reg_out=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then + remaining=$((deadline - SECONDS)) + if [ "$remaining" -le 0 ]; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2 + exit 1 + fi + if reg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then backend_registered=1 elif echo "$reg_out" | grep -Eqi "already exists|already registered"; then backend_registered=1 @@ -148,7 +159,16 @@ doris_ready() { fi echo "Checking Doris backends..." - ALIVE_BACKENDS=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true) + remaining=$((deadline - SECONDS)) + if [ "$remaining" -le 0 ]; then + if [ "$backend_registered" -eq 0 ] && [ -n "$reg_out" ]; then + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2 + else + echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend to become alive" >&2 + fi + exit 1 + fi + ALIVE_BACKENDS=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true) # fallback value if failed to get number if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then diff --git a/docs/guides/connections.md b/docs/guides/connections.md index bf6d4811d6..702be1dcd1 100644 --- a/docs/guides/connections.md +++ b/docs/guides/connections.md @@ -33,7 +33,7 @@ gateways: database: state.db ``` -NOTE: Spark and Trino engines may not be used for the state connection. +NOTE: Spark, Trino, and Doris engines may not be used for the state connection; configure a separate `state_connection` (for example, DuckDB for local development or Postgres/MySQL in production) when using these engines. ## Default connection diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index aef8323646..cd69e1bbe1 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2851,9 +2851,9 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: isinstance(verify_mode, str) and verify_mode.strip().lower() in ("false", "0", "no", "none") ): - if self.ssl_verify_identity is True: + if hostname_requested: raise ConfigError( - "Doris `ssl_verify_identity=True` cannot be combined with " + "Doris `ssl.check_hostname=True` or `ssl_verify_identity=True` cannot be combined with " "`ssl_verify_cert=False` or disabled `ssl.verify_mode`." ) ssl_config["check_hostname"] = False diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 6103fea671..5ff1196755 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -311,7 +311,7 @@ def _resolve_session_database(self) -> t.Optional[str]: if isinstance(current_db, str): return current_db.strip() or None except Exception: - pass + return None return self._default_schema_from_config() def _resolve_session_catalog(self) -> t.Optional[str]: diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index cecfd72fc6..23e031a17c 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -1968,7 +1968,7 @@ def get_current_database(self) -> t.Optional[str]: return db_row[0].strip() return "" except Exception: - pass + return "" return None def _default_schema_from_config(self) -> t.Optional[str]: diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index ee10a413c6..3637174c8e 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -114,8 +114,8 @@ def _resolve_session_catalog(self) -> t.Optional[str]: if isinstance(current_cat, str) and current_cat: return current_cat except Exception: - pass - return super()._resolve_session_catalog() + return None + return None @lru_cache() def get_catalog_type(self, catalog: t.Optional[str]) -> str: diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index ed47fde441..98f42b63bd 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -1242,8 +1242,13 @@ def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( assert list(dyn_rows) == [(10, 2), (20, 1)] # Verify both synchronous MV rollup jobs reached FINISHED before restatement + user_events_snapshot = sm_ctx.get_snapshot(f"{schema_name}.user_events") + assert user_events_snapshot is not None + user_events_phys_table = exp.to_table(user_events_snapshot.table_name(), dialect="doris") + phys_db = user_events_phys_table.db + phys_name = user_events_phys_table.name rollup_jobs = ctx.engine_adapter.fetchall( - f"SHOW ALTER TABLE MATERIALIZED VIEW FROM `{schema_name}` WHERE TableName = 'user_events'", + f"SHOW ALTER TABLE MATERIALIZED VIEW FROM `{phys_db}` WHERE TableName = '{phys_name}'", quote_identifiers=False, ) rollup_job_states = { @@ -1269,7 +1274,7 @@ def test_doris_sqlmesh_plan_dynamic_partition_and_sync_mv_restatement( ) assert list(restated_rows) == [(10, 3), (20, 1)] desc_all_rows = ctx.engine_adapter.fetchall( - f"DESC `{schema_name}`.`user_events` ALL", + f"DESC `{phys_db}`.`{phys_name}` ALL", quote_identifiers=False, ) rollup_index_names = { diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index dfb12353b6..f651e42f7e 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3446,6 +3446,18 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', ] + # When Trino get_current_catalog fails, do not fall back to _default_catalog + mocker.patch.object(adapter, "get_current_catalog", side_effect=RuntimeError("catalog error")) + adapter.cursor.execute.reset_mock() + with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified catalog reference"): + adapter.replace_query( + "prod.db.test", + parse_one("SELECT a FROM db.test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [] + mocker.patch.object(adapter, "get_current_catalog", return_value="prod") + # Database-less table reference that matches target table name raises when no default database is known with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): adapter.replace_query( @@ -3484,6 +3496,16 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( 'DELETE FROM "db"."test" WHERE TRUE', 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "test"', ] + # When get_current_database fails, do not fall back to configured schema/database + mocker.patch.object(adapter, "get_current_database", side_effect=RuntimeError("db error")) + adapter.cursor.execute.reset_mock() + with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [] mocker.patch.object(adapter, "get_current_database", return_value=None) # Partition selection on self-referencing table is preserved when temp table exposes matching partitions diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 2e3474ceb0..2dea2cdaf4 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2602,16 +2602,26 @@ def test_doris(make_config): ssl_verify_identity=True, check_import=False, ) - with pytest.raises(ConfigError, match="ssl_verify_identity=True` cannot be combined"): + with pytest.raises(ConfigError, match="cannot be combined with"): _ = ssl_verify_cert_false_identity_true._static_connection_kwargs - ssl_dict_verify_mode_false = make_config( + ssl_dict_verify_mode_false_check_hostname_true = make_config( type="doris", host="doris-fe", user="admin", ssl={"ca": "/path/to/ca.pem", "verify_mode": False, "check_hostname": True}, check_import=False, ) + with pytest.raises(ConfigError, match="cannot be combined with"): + _ = ssl_dict_verify_mode_false_check_hostname_true._static_connection_kwargs + + ssl_dict_verify_mode_false = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"ca": "/path/to/ca.pem", "verify_mode": False}, + check_import=False, + ) assert ssl_dict_verify_mode_false._static_connection_kwargs["ssl"] == { "ca": "/path/to/ca.pem", "verify_mode": False, @@ -2629,13 +2639,23 @@ def test_doris(make_config): with pytest.raises(ConfigError, match="ssl_verify_identity=True` cannot be combined"): _ = ssl_dict_verify_mode_none_str._static_connection_kwargs - ssl_dict_verify_mode_none_str_no_identity = make_config( + ssl_dict_verify_mode_none_str_check_hostname_true = make_config( type="doris", host="doris-fe", user="admin", ssl={"ca": "/path/to/ca.pem", "verify_mode": "none", "check_hostname": True}, check_import=False, ) + with pytest.raises(ConfigError, match="cannot be combined with"): + _ = ssl_dict_verify_mode_none_str_check_hostname_true._static_connection_kwargs + + ssl_dict_verify_mode_none_str_no_identity = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"ca": "/path/to/ca.pem", "verify_mode": "none"}, + check_import=False, + ) assert ssl_dict_verify_mode_none_str_no_identity._static_connection_kwargs["ssl"] == { "ca": "/path/to/ca.pem", "verify_mode": "none", From e1a8106b2d89819ce5855fa067d505293c7b2b69 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:27:12 -0800 Subject: [PATCH 32/54] Fix(doris): align self-ref CTAS detection and Trino session schema lookup Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/wait-for-db.sh | 4 + docs/integrations/engines/doris.md | 4 +- sqlmesh/core/engine_adapter/doris.py | 125 +++++++++++++++++++----- sqlmesh/core/engine_adapter/trino.py | 11 +++ tests/core/engine_adapter/test_base.py | 24 +++++ tests/core/engine_adapter/test_doris.py | 17 ++++ 6 files changed, 161 insertions(+), 24 deletions(-) diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index 40bce1a98d..118ae5b053 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -112,6 +112,10 @@ db2_ready() { } doris_ready() { + if ! command -v timeout > /dev/null 2>&1; then + echo "ERROR: 'timeout' is required by wait-for-db.sh (GNU coreutils; on macOS install coreutils and add it to PATH)" >&2 + exit 1 + fi local timeout_secs="${DORIS_STARTUP_TIMEOUT_SECS:-300}" local deadline=$((SECONDS + timeout_secs)) local remaining=0 diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 5bde21aa8c..478d388c49 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -935,8 +935,8 @@ FROM raw.logs; !!! note "Recovering from Failed Index Post-Statements & Forward-Only Migrations" - **Recovering if an `@runtime_stage = 'creating'` index statement fails**: Because `SUPPORTS_TRANSACTIONS = False`, if `CREATE TABLE` (and earlier `ALTER TABLE` post-statements such as `ADD INDEX idx_message`) succeed but a later post-statement such as `ADD INDEX idx_embedding` fails, the partially configured physical snapshot table (`sqlmesh__analytics.analytics__searchable_logs__`) remains in Doris. Retrying `sqlmesh plan` while that incomplete physical table exists can skip table creation (leaving `idx_embedding` missing) or fail on already-created indexes. Before retrying `sqlmesh plan`, either: - 1. Drop the incomplete physical snapshot table (`DROP TABLE IF EXISTS sqlmesh__analytics.analytics__searchable_logs__;`) so the retry recreates the table and runs all `@runtime_stage = 'creating'` statements from scratch, or - 2. Inspect existing indexes (`SHOW INDEX FROM sqlmesh__analytics.analytics__searchable_logs__;`) and manually reconcile the missing `idx_embedding` index (`ALTER TABLE sqlmesh__analytics.analytics__searchable_logs__ ADD INDEX idx_embedding ...`) before retrying. + 1. **Only if the table is still empty (schema-creation stage) or you have a verified backup / replayable upstream source**: Drop the incomplete physical snapshot table (`DROP TABLE IF EXISTS sqlmesh__analytics.analytics__searchable_logs__;`) so the retry recreates the table and runs all `@runtime_stage = 'creating'` statements from scratch, or + 2. **If the table was created via CTAS during evaluation and already contains the initial batch from a non-replayable source**: Do **not** drop the table; instead, inspect existing indexes (`SHOW INDEX FROM sqlmesh__analytics.analytics__searchable_logs__;`) and manually reconcile the missing `idx_embedding` index (`ALTER TABLE sqlmesh__analytics.analytics__searchable_logs__ ADD INDEX idx_embedding ...`, followed by `BUILD INDEX` or `SCHEMA_CHANGE` backfill) before retrying. - **Adding indexes during forward-only migrations**: When SQLMesh creates a new physical snapshot table during a standard `sqlmesh plan`, `@runtime_stage = 'creating'` runs directly on the new physical table. However, during a **forward-only migration** (`sqlmesh plan --forward-only` or `forward_only true`) on an already-existing table, SQLMesh executes `@runtime_stage = 'creating'` against a temporary schema-diff table (`__schema_tmp`) and only applies column-level `ALTER TABLE` diffs to the existing production table—so `ADD INDEX` post-statements in `@runtime_stage = 'creating'` are not applied to the existing production table. To add an inverted or ANN index to an existing production table during a forward-only migration: diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 23e031a17c..3192fd6cec 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -4427,6 +4427,12 @@ def _create_table_from_columns( column_descriptions=column_descriptions, **kwargs, ) + thread_state = self._get_thread_state() + replace_query_target = getattr(thread_state, "replace_query_target", None) + if isinstance(replace_query_target, exp.Table) and quote_identifiers( + exp.to_table(table_name).copy() + ) == quote_identifiers(replace_query_target.copy()): + thread_state.replace_query_precreated = True def replace_query( self, @@ -4458,7 +4464,9 @@ def replace_query( ] thread_state = self._get_thread_state() prev_replace_query_target = getattr(thread_state, "replace_query_target", None) + prev_replace_query_precreated = getattr(thread_state, "replace_query_precreated", False) thread_state.replace_query_target = target_table + thread_state.replace_query_precreated = False try: super().replace_query( table_name, @@ -4472,6 +4480,7 @@ def replace_query( ) finally: thread_state.replace_query_target = prev_replace_query_target + thread_state.replace_query_precreated = prev_replace_query_precreated def _get_doris_version(self) -> t.Optional[t.Tuple[int, int]]: cached = getattr(self, "_doris_version", None) @@ -4946,8 +4955,40 @@ def _create_table_from_source_queries( target_columns_to_types = self._reorder_olap_columns(target_columns_to_types, kwargs) first_query = source_queries[0].query_factory() if source_queries else None + default_db_loaded = False + default_db: t.Optional[str] = None + default_catalog_loaded = False + default_catalog: t.Optional[str] = None + + def _is_cte_reference(node: exp.Table, quoted_table: exp.Table) -> bool: + if quoted_table.args.get("db") or quoted_table.args.get("catalog"): + return False + curr: t.Optional[exp.Expr] = node + child_cte: t.Optional[exp.CTE] = None + while curr is not None: + if isinstance(curr, exp.CTE): + child_cte = curr + with_clause = curr.args.get("with_") + if isinstance(with_clause, exp.With): + recursive = bool(with_clause.args.get("recursive")) + for cte in with_clause.expressions: + if cte is child_cte and not recursive: + break + if ( + isinstance(cte, exp.CTE) + and cte.alias + and quote_identifiers(exp.to_identifier(cte.alias)) == quoted_table.this + ): + return True + child_cte = None + curr = curr.parent + return False + def _matches_target_table(t_node: exp.Table) -> bool: + nonlocal default_db_loaded, default_db, default_catalog_loaded, default_catalog q_node = quote_identifiers(t_node.copy()) + if _is_cte_reference(t_node, q_node): + return False q_target = quote_identifiers(table.copy()) if (q_node.args.get("catalog"), q_node.args.get("db"), q_node.this) == ( q_target.args.get("catalog"), @@ -4955,30 +4996,59 @@ def _matches_target_table(t_node: exp.Table) -> bool: q_target.this, ): return True - default_cat = self.default_catalog or "internal" - norm_node = t_node.copy() - if not norm_node.catalog and norm_node.db: - norm_node.set("catalog", exp.parse_identifier(default_cat)) - norm_target = table.copy() - if not norm_target.catalog and norm_target.db: - norm_target.set("catalog", exp.parse_identifier(default_cat)) - q_norm_node = quote_identifiers(norm_node) - q_norm_target = quote_identifiers(norm_target) + if q_node.this != q_target.this or q_node.args.get("version") != q_target.args.get( + "version" + ): + return False + if bool(q_node.args.get("db")) != bool(q_target.args.get("db")): + if (not q_node.args.get("db") and q_node.args.get("catalog")) or ( + not q_target.args.get("db") and q_target.args.get("catalog") + ): + return False + if not default_db_loaded: + default_db_loaded = True + default_db = self._resolve_session_database() + if not default_db: + return False + quoted_default_db = quote_identifiers(exp.parse_identifier(default_db)) + if not q_node.args.get("db"): + q_node.set("db", quoted_default_db) + else: + q_target = q_target.copy() + q_target.set("db", quoted_default_db) + if (q_node.args.get("catalog"), q_node.args.get("db"), q_node.this) == ( + q_target.args.get("catalog"), + q_target.args.get("db"), + q_target.this, + ): + return True + if ( + q_node.args.get("db") != q_target.args.get("db") + or not q_node.args.get("db") + or (q_node.args.get("catalog") and q_target.args.get("catalog")) + ): + return False + if not default_catalog_loaded: + default_catalog_loaded = True + default_catalog = self._resolve_session_catalog() + if not default_catalog: + return False + quoted_default_cat = quote_identifiers(exp.parse_identifier(default_catalog)) + if not q_node.args.get("catalog"): + q_node.set("catalog", quoted_default_cat) + if not q_target.args.get("catalog"): + q_target = q_target.copy() + q_target.set("catalog", quoted_default_cat) return ( - q_norm_node.args.get("catalog"), - q_norm_node.args.get("db"), - q_norm_node.this, + q_node.args.get("catalog"), + q_node.args.get("db"), + q_node.this, ) == ( - q_norm_target.args.get("catalog"), - q_norm_target.args.get("db"), - q_norm_target.this, + q_target.args.get("catalog"), + q_target.args.get("db"), + q_target.this, ) - self_referencing = bool( - first_query - and any(_matches_target_table(t_node) for t_node in first_query.find_all(exp.Table)) - ) - # 1. Doris's CTAS grammar forbids `ENGINE=iceberg`, so internal/2-part Iceberg tables with known # column types must be created via schema DDL (`_create_table_from_columns`) + `INSERT INTO`. # 2. When `replace_query` is called with a self-referencing query on a non-existent table, @@ -4992,12 +5062,23 @@ def _matches_target_table(t_node: exp.Table) -> bool: "(`..
`) or specify explicit column types." ) - replace_query_target = getattr(self._get_thread_state(), "replace_query_target", None) + thread_state = self._get_thread_state() + replace_query_target = getattr(thread_state, "replace_query_target", None) from_replace_query = bool( isinstance(replace_query_target, exp.Table) and _matches_target_table(replace_query_target) ) - precreated_by_replace_query = bool(self_referencing and from_replace_query and not replace) + precreated_by_replace_query = bool( + getattr(thread_state, "replace_query_precreated", False) + and from_replace_query + and not replace + ) + self_referencing = precreated_by_replace_query or bool( + first_query + and any(_matches_target_table(t_node) for t_node in first_query.find_all(exp.Table)) + ) + if self_referencing and from_replace_query and not replace: + precreated_by_replace_query = True table_props = self._get_table_properties(kwargs) or {} raw_not_null_cols = kwargs.get("not_null_columns") or table_props.get("not_null_columns") diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index 3637174c8e..47c4348ea7 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -108,6 +108,17 @@ def set_current_catalog(self, catalog: str) -> None: """Sets the catalog name of the current connection.""" self.execute(exp.Use(this=schema_(db="information_schema", catalog=catalog))) + def get_current_database(self) -> t.Optional[str]: + try: + row = self.fetchone(exp.select(exp.column("current_schema"))) + if isinstance(row, (list, tuple)) and row: + if isinstance(row[0], str) and row[0].strip(): + return row[0].strip() + return "" + except Exception: + return "" + return None + def _resolve_session_catalog(self) -> t.Optional[str]: try: current_cat = self.get_current_catalog() diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index f651e42f7e..ce08981155 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3466,6 +3466,30 @@ def test_replace_query_cte_and_versioned_table_not_self_referencing( target_columns_to_types={"a": exp.DataType.build("INT")}, ) + # Trino resolves the active session schema via get_current_database (SELECT "current_schema") + mocker.patch.object( + adapter, + "get_current_database", + TrinoEngineAdapter.get_current_database.__get__(adapter, TrinoEngineAdapter), + ) + mocker.patch.object(adapter, "fetchone", return_value=("db",)) + adapter.cursor.execute.reset_mock() + adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test TABLESAMPLE BERNOULLI (10)", read="trino"), + target_columns_to_types={"a": exp.DataType.build("INT")}, + ) + assert to_sql_calls(adapter) == [ + 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', + 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', + 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', + 'DELETE FROM "db"."test" WHERE TRUE', + 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh" TABLESAMPLE BERNOULLI (10)', + 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', + ] + mocker.patch.object(adapter, "fetchone", return_value=None) + mocker.patch.object(adapter, "get_current_database", return_value=None) + # Database-less table reference is detected as self-referencing when active/default database matches target_table.db adapter._extra_config["schema"] = "db" adapter.cursor.execute.reset_mock() diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 85b027d015..7733689692 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6086,3 +6086,20 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: v41_adapter._to_sql(parse_one("SELECT CAST(ts AS TIMESTAMPTZ(6))", read="doris")) == "SELECT CAST(`ts` AS TIMESTAMPTZ(6))" ) + + # 10. Bare self-referencing replace_query on a non-existent table (`internal.db.test` with `SELECT a FROM test`) + # precreates the table and populates it via INSERT INTO instead of a no-op CTAS. + self_ref_adapter = make_mocked_engine_adapter(DorisEngineAdapter, database="db") + mocker.patch.object(self_ref_adapter, "table_exists", return_value=False) + self_ref_adapter.cursor.execute.reset_mock() + self_ref_adapter.replace_query( + "internal.db.test", + parse_one("SELECT 1 AS a UNION ALL SELECT a FROM test", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(self_ref_adapter) == [ + "SELECT DATABASE()", + "SELECT CURRENT_CATALOG", + "CREATE TABLE IF NOT EXISTS `internal`.`db`.`test` (`a` INT)", + "INSERT INTO `internal`.`db`.`test` (`a`) SELECT CAST(`a` AS INT) AS `a` FROM (SELECT 1 AS `a` UNION ALL SELECT `a` FROM `test`) AS `_subquery`", + ] From 880ed67be15f3f1d54e6aeae86db93176deb8e38 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:09:24 -0800 Subject: [PATCH 33/54] fix(doris): scope self-ref replace_query to Doris and address review feedback Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 10 +- sqlmesh/core/engine_adapter/base.py | 221 +-------------- sqlmesh/core/engine_adapter/doris.py | 268 +++++++++++++++++- sqlmesh/core/engine_adapter/trino.py | 20 -- sqlmesh/core/model/definition.py | 9 +- .../integration/docker/compose.doris.yaml | 3 + .../integration/test_integration_doris.py | 9 + tests/core/engine_adapter/test_base.py | 265 +---------------- tests/core/engine_adapter/test_doris.py | 61 ++++ tests/core/engine_adapter/test_starrocks.py | 2 +- 10 files changed, 357 insertions(+), 511 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 478d388c49..c474413075 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -189,7 +189,7 @@ SELECT order_id::BIGINT AS order_id, customer_id::BIGINT AS customer_id, status::VARCHAR(32) AS status, - updated_at::DATETIME AS updated_at + updated_at::DATETIME(6) AS updated_at FROM raw.orders WHERE CAST(updated_at AS DATE) BETWEEN @start_ds AND @end_ds; ``` @@ -214,14 +214,14 @@ MODEL ( ); SELECT - order_id, - status, - updated_at + order_id::BIGINT AS order_id, + status::VARCHAR(32) AS status, + updated_at::DATETIME(6) AS updated_at FROM ( SELECT order_id::BIGINT AS order_id, status::VARCHAR(32) AS status, - updated_at::DATETIME AS updated_at, + updated_at::DATETIME(6) AS updated_at, ROW_NUMBER() OVER ( PARTITION BY order_id ORDER BY updated_at DESC, changelog_id DESC diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 5ff1196755..2dd234f70a 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -295,42 +295,6 @@ def default_catalog(self) -> t.Optional[str]: ) return default_catalog - def get_current_database(self) -> t.Optional[str]: - """Returns the database/schema name of the current session, if supported.""" - return None - - def _default_schema_from_config(self) -> t.Optional[str]: - val = self._extra_config.get("schema") - if isinstance(val, str) and val.strip(): - return val.strip() - return None - - def _resolve_session_database(self) -> t.Optional[str]: - try: - current_db = self.get_current_database() - if isinstance(current_db, str): - return current_db.strip() or None - except Exception: - return None - return self._default_schema_from_config() - - def _resolve_session_catalog(self) -> t.Optional[str]: - try: - if self.default_catalog: - return self.default_catalog - except MissingDefaultCatalogError: - pass - try: - current_cat = self.get_current_catalog() - if isinstance(current_cat, str) and current_cat: - return current_cat - except Exception: - pass - return None - - def _get_table_partition_names(self, table: exp.Table) -> t.Set[str]: - return set() - @property def engine_run_mode(self) -> EngineRunMode: return EngineRunMode.SINGLE_MODE_ENGINE @@ -561,160 +525,10 @@ def replace_query( if not target_columns_to_types and table_exists: target_columns_to_types = self.columns(target_table) query = source_queries[0].query_factory() - - quoted_target = quote_identifiers(target_table.copy()) - default_catalog: t.Optional[str] = None - default_catalog_loaded = False - default_db: t.Optional[str] = None - default_db_loaded = False - - def _table_parts( - table: exp.Table, - ) -> t.Tuple[t.Optional[exp.Expr], ...]: - return ( - table.args.get("catalog"), - table.args.get("db"), - table.this, - table.args.get("version"), - ) - - def _qualified(table: exp.Table, catalog: str) -> exp.Table: - table = table.copy() - if not table.catalog and table.db: - table.set("catalog", exp.parse_identifier(catalog)) - return quote_identifiers(table) - - def _is_cte_reference(table: exp.Table, quoted_table: exp.Table) -> bool: - if quoted_table.args.get("db") or quoted_table.args.get("catalog"): - return False - curr: t.Optional[exp.Expr] = table - child_cte: t.Optional[exp.CTE] = None - while curr is not None: - if isinstance(curr, exp.CTE): - child_cte = curr - with_clause = curr.args.get("with_") - if isinstance(with_clause, exp.With): - is_recursive = bool(with_clause.args.get("recursive")) - for cte in with_clause.expressions: - if not is_recursive and cte is child_cte: - break - if ( - isinstance(cte, exp.CTE) - and cte.alias - and quote_identifiers(exp.to_identifier(cte.alias)) == quoted_table.this - ): - return True - child_cte = None - curr = curr.parent - return False - - def _is_target_table(node: exp.Expr) -> bool: - nonlocal default_catalog, default_catalog_loaded, default_db, default_db_loaded - if not isinstance(node, exp.Table): - return False - quoted_node = quote_identifiers(node.copy()) - if _is_cte_reference(node, quoted_node): - return False - target_to_check = quoted_target - if _table_parts(quoted_node) == _table_parts(target_to_check): - return True - if quoted_node.this != target_to_check.this or quoted_node.args.get( - "version" - ) != target_to_check.args.get("version"): - return False - if bool(quoted_node.args.get("db")) != bool(target_to_check.args.get("db")): - if (not quoted_node.args.get("db") and quoted_node.args.get("catalog")) or ( - not target_to_check.args.get("db") and target_to_check.args.get("catalog") - ): - return False - if not default_db_loaded: - default_db_loaded = True - default_db = self._resolve_session_database() - if not default_db: - raise SQLMeshError( - f"Cannot disambiguate unqualified schema reference '{node.sql(dialect=self.dialect)}' " - f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default database." - ) - quoted_default_db = quote_identifiers(exp.parse_identifier(default_db)) - if not quoted_node.args.get("db"): - quoted_node.set("db", quoted_default_db) - else: - target_to_check = target_to_check.copy() - target_to_check.set("db", quoted_default_db.copy()) - if _table_parts(quoted_node) == _table_parts(target_to_check): - return True - if ( - quoted_node.args.get("db") != target_to_check.args.get("db") - or not quoted_node.args.get("db") - or (quoted_node.args.get("catalog") and target_to_check.args.get("catalog")) - or self.catalog_support.is_unsupported - ): - return False - if not default_catalog_loaded: - default_catalog_loaded = True - default_catalog = self._resolve_session_catalog() - if default_catalog is None: - raise SQLMeshError( - f"Cannot disambiguate unqualified catalog reference '{node.sql(dialect=self.dialect)}' " - f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default catalog." - ) - return _table_parts(_qualified(quoted_node, default_catalog)) == _table_parts( - _qualified(target_to_check, default_catalog) - ) - - temp_table_partitions: t.Optional[t.Set[str]] = None - staged_partition_sql: t.Optional[str] = None - - def _partition_names(partition_exp: exp.Expr) -> t.Set[str]: - names: t.Set[str] = set() - for expr in partition_exp.expressions: - name = expr.name if isinstance(expr, (exp.Identifier, exp.Var, exp.Column)) else "" - if not name and isinstance(expr, exp.Partition): - for sub in expr.expressions: - if sub.name: - names.add(sub.name.lower()) - elif name: - names.add(name.lower()) - return names - - def _get_temp_table_partitions(temp_table: exp.Table) -> t.Set[str]: - nonlocal temp_table_partitions - if temp_table_partitions is not None: - return temp_table_partitions - names: t.Set[str] = set() - try: - for part_name in self._get_table_partition_names(temp_table): - if part_name: - names.add(str(part_name).lower()) - except Exception: - pass - temp_table_partitions = names - return names - - def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: - if not _is_target_table(node): - return node - replacement = replacement_table.copy() - if isinstance(node, exp.Table): - if node.args.get("alias") is not None: - replacement.set("alias", node.args["alias"].copy()) - if node.args.get("sample") is not None: - replacement.set("sample", node.args["sample"].copy()) - partition_arg = node.args.get("partition") - if partition_arg is not None: - req_names = _partition_names(partition_arg) - avail_names = _get_temp_table_partitions(replacement_table) - if req_names and req_names.issubset(avail_names): - replacement.set("partition", partition_arg.copy()) - elif staged_partition_sql != partition_arg.sql(dialect=self.dialect): - raise SQLMeshError( - f"Cannot apply partition selection '{partition_arg.sql(dialect=self.dialect)}' " - f"to temporary staging table '{replacement_table.sql(dialect=self.dialect)}' " - "because the staging table does not expose matching partitions." - ) - return replacement - - self_referencing = any(_is_target_table(table) for table in query.find_all(exp.Table)) + self_referencing = any( + quote_identifiers(table) == quote_identifiers(target_table) + for table in query.find_all(exp.Table) + ) # If a query references itself then it must have a table created regardless of approach used. if self_referencing: if not target_columns_to_types: @@ -749,31 +563,20 @@ def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.E ) if self_referencing: assert target_columns_to_types is not None - self_ref_partition_clauses: t.Dict[str, t.Optional[exp.Expr]] = {} - for sq in source_queries: - sq_ast = sq.query_factory() - for tbl in sq_ast.find_all(exp.Table): - if _is_target_table(tbl): - part_expr = tbl.args.get("partition") - part_key = ( - part_expr.sql(dialect=self.dialect) if part_expr is not None else "" - ) - self_ref_partition_clauses[part_key] = part_expr - staging_target = target_table.copy() - if len(self_ref_partition_clauses) == 1: - only_key, only_part = next(iter(self_ref_partition_clauses.items())) - if only_key and only_part is not None: - staging_target.set("partition", only_part.copy()) - staged_partition_sql = only_key with self.temp_table( - self._select_columns(target_columns_to_types).from_(staging_target), + self._select_columns(target_columns_to_types).from_(target_table), name=target_table, target_columns_to_types=target_columns_to_types, **kwargs, ) as temp_table: for source_query in source_queries: source_query.add_transform( - lambda node: _replace_target_table(node, temp_table) # type: ignore + lambda node: ( # type: ignore + temp_table # type: ignore + if isinstance(node, exp.Table) + and quote_identifiers(node) == quote_identifiers(target_table) + else node + ) ) return self._insert_overwrite_by_condition( target_table, @@ -2920,7 +2723,7 @@ def temp_table( Yields: The table expression """ - name = exp.to_table(name).copy() + name = exp.to_table(name) # ensure that we use default catalog if none is not specified if isinstance(name, exp.Table) and not name.catalog and name.db and self.default_catalog: name.set("catalog", exp.parse_identifier(self.default_catalog)) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 3192fd6cec..d166ec276e 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -46,7 +46,7 @@ from sqlmesh.core.schema_diff import TableAlterChangeColumnTypeOperation from sqlmesh.utils import columns_to_types_all_known, get_source_columns_to_types, random_id from sqlmesh.utils.connection_pool import ConnectionPool -from sqlmesh.utils.errors import SQLMeshError +from sqlmesh.utils.errors import MissingDefaultCatalogError, SQLMeshError if t.TYPE_CHECKING: from sqlmesh.core._typing import SchemaName, TableName @@ -1982,12 +1982,29 @@ def _resolve_session_database(self) -> t.Optional[str]: active_db = getattr(self, "_active_database", None) if isinstance(active_db, str) and active_db.strip(): return active_db.strip() - return super()._resolve_session_database() + try: + current_db = self.get_current_database() + if isinstance(current_db, str): + return current_db.strip() or None + except Exception: + return None + return self._default_schema_from_config() def _resolve_session_catalog(self) -> t.Optional[str]: if self._active_catalog: return self._active_catalog - return super()._resolve_session_catalog() + try: + if self.default_catalog: + return self.default_catalog + except MissingDefaultCatalogError: + pass + try: + current_cat = self.get_current_catalog() + if isinstance(current_cat, str) and current_cat: + return current_cat + except Exception: + pass + return None def _get_table_partition_names(self, table: exp.Table) -> t.Set[str]: names: t.Set[str] = set() @@ -4468,14 +4485,247 @@ def replace_query( thread_state.replace_query_target = target_table thread_state.replace_query_precreated = False try: - super().replace_query( - table_name, + target_data_object = self.get_data_object(target_table) + table_exists = target_data_object is not None + if self.drop_data_object_on_type_mismatch(target_data_object, DataObjectType.TABLE): + table_exists = False + + source_queries, target_columns_to_types = self._get_source_queries_and_columns_to_types( query_or_df, - target_columns_to_types=target_columns_to_types, - table_description=table_description, - column_descriptions=column_descriptions, + target_columns_to_types, + target_table=target_table, source_columns=source_columns, - supports_replace_table_override=supports_replace_table_override, + ) + if not target_columns_to_types and table_exists: + target_columns_to_types = self.columns(target_table) + query = source_queries[0].query_factory() + + quoted_target = quote_identifiers(target_table.copy()) + default_catalog: t.Optional[str] = None + default_catalog_loaded = False + default_db: t.Optional[str] = None + default_db_loaded = False + + def _table_parts( + table: exp.Table, + ) -> t.Tuple[t.Optional[exp.Expr], ...]: + return ( + table.args.get("catalog"), + table.args.get("db"), + table.this, + table.args.get("version"), + ) + + def _qualified(table: exp.Table, catalog: str) -> exp.Table: + table = table.copy() + if not table.catalog and table.db: + table.set("catalog", exp.parse_identifier(catalog)) + return quote_identifiers(table) + + def _is_cte_reference(table: exp.Table, quoted_table: exp.Table) -> bool: + if quoted_table.args.get("db") or quoted_table.args.get("catalog"): + return False + curr: t.Optional[exp.Expr] = table + child_cte: t.Optional[exp.CTE] = None + while curr is not None: + if isinstance(curr, exp.CTE): + child_cte = curr + with_clause = curr.args.get("with_") + if isinstance(with_clause, exp.With): + is_recursive = bool(with_clause.args.get("recursive")) + for cte in with_clause.expressions: + if not is_recursive and cte is child_cte: + break + if ( + isinstance(cte, exp.CTE) + and cte.alias + and quote_identifiers(exp.to_identifier(cte.alias)) + == quoted_table.this + ): + return True + child_cte = None + curr = curr.parent + return False + + def _is_target_table(node: exp.Expr) -> bool: + nonlocal default_catalog, default_catalog_loaded, default_db, default_db_loaded + if not isinstance(node, exp.Table): + return False + quoted_node = quote_identifiers(node.copy()) + if _is_cte_reference(node, quoted_node): + return False + target_to_check = quoted_target + if _table_parts(quoted_node) == _table_parts(target_to_check): + return True + if quoted_node.this != target_to_check.this or quoted_node.args.get( + "version" + ) != target_to_check.args.get("version"): + return False + if bool(quoted_node.args.get("db")) != bool(target_to_check.args.get("db")): + if (not quoted_node.args.get("db") and quoted_node.args.get("catalog")) or ( + not target_to_check.args.get("db") and target_to_check.args.get("catalog") + ): + return False + if not default_db_loaded: + default_db_loaded = True + default_db = self._resolve_session_database() + if not default_db: + raise SQLMeshError( + f"Cannot disambiguate unqualified schema reference '{node.sql(dialect=self.dialect)}' " + f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default database." + ) + quoted_default_db = quote_identifiers(exp.parse_identifier(default_db)) + if not quoted_node.args.get("db"): + quoted_node.set("db", quoted_default_db) + else: + target_to_check = target_to_check.copy() + target_to_check.set("db", quoted_default_db.copy()) + if _table_parts(quoted_node) == _table_parts(target_to_check): + return True + if ( + quoted_node.args.get("db") != target_to_check.args.get("db") + or not quoted_node.args.get("db") + or (quoted_node.args.get("catalog") and target_to_check.args.get("catalog")) + or self.catalog_support.is_unsupported + ): + return False + if not default_catalog_loaded: + default_catalog_loaded = True + default_catalog = self._resolve_session_catalog() + if default_catalog is None: + raise SQLMeshError( + f"Cannot disambiguate unqualified catalog reference '{node.sql(dialect=self.dialect)}' " + f"against target '{target_table.sql(dialect=self.dialect)}' without an active or default catalog." + ) + return _table_parts(_qualified(quoted_node, default_catalog)) == _table_parts( + _qualified(target_to_check, default_catalog) + ) + + temp_table_partitions: t.Optional[t.Set[str]] = None + staged_partition_sql: t.Optional[str] = None + + def _partition_names(partition_exp: exp.Expr) -> t.Set[str]: + names: t.Set[str] = set() + for expr in partition_exp.expressions: + name = ( + expr.name if isinstance(expr, (exp.Identifier, exp.Var, exp.Column)) else "" + ) + if not name and isinstance(expr, exp.Partition): + for sub in expr.expressions: + if sub.name: + names.add(sub.name.lower()) + elif name: + names.add(name.lower()) + return names + + def _get_temp_table_partitions(temp_table: exp.Table) -> t.Set[str]: + nonlocal temp_table_partitions + if temp_table_partitions is not None: + return temp_table_partitions + names: t.Set[str] = set() + try: + for part_name in self._get_table_partition_names(temp_table): + if part_name: + names.add(str(part_name).lower()) + except Exception: + pass + temp_table_partitions = names + return names + + def _replace_target_table(node: exp.Expr, replacement_table: exp.Table) -> exp.Expr: + if not _is_target_table(node): + return node + replacement = replacement_table.copy() + if isinstance(node, exp.Table): + if node.args.get("alias") is not None: + replacement.set("alias", node.args["alias"].copy()) + if node.args.get("sample") is not None: + replacement.set("sample", node.args["sample"].copy()) + partition_arg = node.args.get("partition") + if partition_arg is not None: + req_names = _partition_names(partition_arg) + avail_names = _get_temp_table_partitions(replacement_table) + if req_names and req_names.issubset(avail_names): + replacement.set("partition", partition_arg.copy()) + elif staged_partition_sql != partition_arg.sql(dialect=self.dialect): + raise SQLMeshError( + f"Cannot apply partition selection '{partition_arg.sql(dialect=self.dialect)}' " + f"to temporary staging table '{replacement_table.sql(dialect=self.dialect)}' " + "because the staging table does not expose matching partitions." + ) + return replacement + + self_referencing = any(_is_target_table(table) for table in query.find_all(exp.Table)) + if self_referencing: + if not target_columns_to_types: + raise SQLMeshError( + f"Cannot create a self-referencing table {target_table.sql(dialect=self.dialect)} without knowing the column types. " + "Try casting the columns to an expected type or defining the columns in the model metadata. " + ) + self._create_table_from_columns( + target_table, + target_columns_to_types, + exists=True, + table_description=table_description, + column_descriptions=column_descriptions, + **kwargs, + ) + + supports_replace_table = ( + self.SUPPORTS_REPLACE_TABLE + if supports_replace_table_override is None + else supports_replace_table_override + ) + if supports_replace_table or not table_exists: + return self._create_table_from_source_queries( + target_table, + source_queries, + target_columns_to_types, + replace=supports_replace_table, + table_description=table_description, + column_descriptions=column_descriptions, + **kwargs, + ) + if self_referencing: + assert target_columns_to_types is not None + self_ref_partition_clauses: t.Dict[str, t.Optional[exp.Expr]] = {} + for sq in source_queries: + sq_ast = sq.query_factory() + for tbl in sq_ast.find_all(exp.Table): + if _is_target_table(tbl): + part_expr = tbl.args.get("partition") + part_key = ( + part_expr.sql(dialect=self.dialect) if part_expr is not None else "" + ) + self_ref_partition_clauses[part_key] = part_expr + staging_target = target_table.copy() + if len(self_ref_partition_clauses) == 1: + only_key, only_part = next(iter(self_ref_partition_clauses.items())) + if only_key and only_part is not None: + staging_target.set("partition", only_part.copy()) + staged_partition_sql = only_key + with self.temp_table( + self._select_columns(target_columns_to_types).from_(staging_target), + name=target_table.copy(), + target_columns_to_types=target_columns_to_types, + **kwargs, + ) as temp_table: + + def _transform_target(node: exp.Query) -> exp.Query: + return t.cast(exp.Query, _replace_target_table(node, temp_table)) + + for source_query in source_queries: + source_query.add_transform(_transform_target) + return self._insert_overwrite_by_condition( + target_table, + source_queries, + target_columns_to_types, + **kwargs, + ) + return self._insert_overwrite_by_condition( + target_table, + source_queries, + target_columns_to_types, **kwargs, ) finally: diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index 47c4348ea7..00acddb26c 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -108,26 +108,6 @@ def set_current_catalog(self, catalog: str) -> None: """Sets the catalog name of the current connection.""" self.execute(exp.Use(this=schema_(db="information_schema", catalog=catalog))) - def get_current_database(self) -> t.Optional[str]: - try: - row = self.fetchone(exp.select(exp.column("current_schema"))) - if isinstance(row, (list, tuple)) and row: - if isinstance(row[0], str) and row[0].strip(): - return row[0].strip() - return "" - except Exception: - return "" - return None - - def _resolve_session_catalog(self) -> t.Optional[str]: - try: - current_cat = self.get_current_catalog() - if isinstance(current_cat, str) and current_cat: - return current_cat - except Exception: - return None - return None - @lru_cache() def get_catalog_type(self, catalog: t.Optional[str]) -> str: row: t.Tuple = tuple() diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 6eedc65fd1..3927de389f 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2866,15 +2866,18 @@ def _resolve_model_refs_to_physical_tables( def resolve(ref: str) -> str: table = exp.to_table(ref.strip(), dialect=dialect) physical = table_mapping.get(exp.table_name(table, identify=True)) + strip_catalog = False if not physical and default_catalog and not table.catalog: normalized_name = d.normalize_model_name( table, default_catalog=default_catalog, dialect=dialect ) physical = table_mapping.get(normalized_name) + strip_catalog = bool(physical) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). - return exp.table_name( - exp.to_table(physical, dialect=dialect) if physical else table, identify=False - ) + resolved_table = exp.to_table(physical, dialect=dialect) if physical else table + if strip_catalog: + resolved_table.set("catalog", None) + return exp.table_name(resolved_table, identify=False) return exp.Literal.string(",".join(resolve(ref) for ref in refs if ref.strip())) diff --git a/tests/core/engine_adapter/integration/docker/compose.doris.yaml b/tests/core/engine_adapter/integration/docker/compose.doris.yaml index aa1b3067ef..947ac26452 100644 --- a/tests/core/engine_adapter/integration/docker/compose.doris.yaml +++ b/tests/core/engine_adapter/integration/docker/compose.doris.yaml @@ -18,6 +18,7 @@ services: - "${DORIS_BIND_ADDRESS:-127.0.0.1}:9030:9030" volumes: - iceberg-warehouse:/tmp/doris_iceberg_warehouse + - paimon-warehouse:/tmp/doris_paimon_warehouse networks: doris_net: ipv4_address: 172.30.80.2 @@ -46,12 +47,14 @@ services: - "${DORIS_BIND_ADDRESS:-127.0.0.1}:9050:9050" volumes: - iceberg-warehouse:/tmp/doris_iceberg_warehouse + - paimon-warehouse:/tmp/doris_paimon_warehouse networks: doris_net: ipv4_address: 172.30.80.3 volumes: iceberg-warehouse: + paimon-warehouse: networks: doris_net: diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 98f42b63bd..676a6451a5 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -136,6 +136,15 @@ def test_doris_iceberg_local_catalog(ctx: TestContext): paimon_cols = adapter.columns(paimon_tbl) assert "id" in paimon_cols assert "ds" in paimon_cols + adapter.execute( + f"INSERT INTO {paimon_tbl} VALUES (1, DATE '2024-01-01')", + quote_identifiers=False, + ) + paimon_rows = adapter.fetchall( + f"SELECT id, CAST(ds AS VARCHAR(16)) FROM {paimon_tbl}", + quote_identifiers=False, + ) + assert list(paimon_rows) == [(1, "2024-01-01")] finally: try: adapter.drop_schema( diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index ce08981155..1971ba3bbc 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -13,7 +13,7 @@ from sqlmesh.core import dialect as d from sqlmesh.core.dialect import normalize_model_name from sqlmesh.core.engine_adapter import EngineAdapter, EngineAdapterWithIndexSupport -from sqlmesh.core.engine_adapter.shared import CatalogSupport, InsertOverwriteStrategy, DataObject +from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObject from sqlmesh.core.schema_diff import SchemaDiffer, TableAlterOperation, NestedSupport from sqlmesh.utils import columns_to_types_to_struct from sqlmesh.utils.date import to_ds @@ -3289,269 +3289,6 @@ def test_replace_query_self_referencing_not_exists_known( ] -def test_replace_query_self_referencing_two_part_and_aliased( - make_mocked_engine_adapter: t.Callable, mocker: MockerFixture -): - mocker.patch.object( - EngineAdapter, - "catalog_support", - new_callable=mocker.PropertyMock, - return_value=CatalogSupport.FULL_SUPPORT, - ) - adapter = make_mocked_engine_adapter(EngineAdapter, default_catalog="catalog") - adapter.SUPPORTS_REPLACE_TABLE = False - adapter.INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.DELETE_INSERT - - mocker.patch.object( - adapter, - "get_data_object", - return_value=DataObject(catalog="catalog", schema="db", name="test", type="table"), - ) - mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="abcdefgh") - - adapter.replace_query( - "catalog.db.test", - parse_one("SELECT t.a FROM db.test AS t"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "catalog"."db"."test" ("a" INT)', - 'CREATE SCHEMA IF NOT EXISTS "catalog"."db"', - 'CREATE TABLE IF NOT EXISTS "catalog"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "catalog"."db"."test") AS "_subquery"', - 'DELETE FROM "catalog"."db"."test" WHERE TRUE', - 'INSERT INTO "catalog"."db"."test" ("a") SELECT "t"."a" FROM "catalog"."db"."__temp_test_abcdefgh" AS "t"', - 'DROP TABLE IF EXISTS "catalog"."db"."__temp_test_abcdefgh"', - ] - - -def test_replace_query_cte_and_versioned_table_not_self_referencing( - make_mocked_engine_adapter: t.Callable, mocker: MockerFixture -): - adapter = make_mocked_engine_adapter(EngineAdapter) - adapter.SUPPORTS_REPLACE_TABLE = False - adapter.INSERT_OVERWRITE_STRATEGY = InsertOverwriteStrategy.DELETE_INSERT - - mocker.patch.object( - adapter, - "get_data_object", - return_value=DataObject(schema="db", name="test", type="table"), - ) - mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="abcdefgh") - - adapter.replace_query( - "test", - parse_one("WITH test AS (SELECT 2 AS a) SELECT t.a FROM test AS t"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - - assert to_sql_calls(adapter) == [ - 'DELETE FROM "test" WHERE TRUE', - 'INSERT INTO "test" ("a") WITH "test" AS (SELECT 2 AS "a") SELECT "t"."a" FROM "test" AS "t"', - ] - - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "test", - parse_one("WITH test AS (SELECT a FROM test) SELECT t.a FROM test AS t"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "test" ("a" INT)', - 'CREATE TABLE IF NOT EXISTS "__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "test") AS "_subquery"', - 'DELETE FROM "test" WHERE TRUE', - 'INSERT INTO "test" ("a") WITH "test" AS (SELECT "a" FROM "__temp_test_abcdefgh") SELECT "t"."a" FROM "test" AS "t"', - 'DROP TABLE IF EXISTS "__temp_test_abcdefgh"', - ] - - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM db.test FOR VERSION AS OF 1", read="doris"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - - assert to_sql_calls(adapter) == [ - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "db"."test" FOR VERSION AS OF 1', - ] - - mocker.patch.object( - EngineAdapter, - "catalog_support", - new_callable=mocker.PropertyMock, - return_value=CatalogSupport.FULL_SUPPORT, - ) - adapter.cursor.execute.reset_mock() - mocker.patch.object(adapter, "get_current_catalog", return_value="internal") - adapter.replace_query( - "internal.db.test", - parse_one("SELECT a FROM db.test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "internal"."db"."test" ("a" INT)', - 'CREATE SCHEMA IF NOT EXISTS "internal"."db"', - 'CREATE TABLE IF NOT EXISTS "internal"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INT) AS "a" FROM (SELECT "a" FROM "internal"."db"."test") AS "_subquery"', - 'DELETE FROM "internal"."db"."test" WHERE TRUE', - 'INSERT INTO "internal"."db"."test" ("a") SELECT "a" FROM "internal"."db"."__temp_test_abcdefgh"', - 'DROP TABLE IF EXISTS "internal"."db"."__temp_test_abcdefgh"', - ] - - adapter.cursor.execute.reset_mock() - mocker.patch.object(adapter, "get_current_catalog", return_value=None) - with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified catalog reference"): - adapter.replace_query( - "internal.db.test", - parse_one("SELECT a FROM db.test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [] - - # Trino resolves the session catalog (get_current_catalog) before _default_catalog - from sqlmesh.core.engine_adapter.trino import TrinoEngineAdapter - - adapter.dialect = "trino" - adapter._default_catalog = "dev" - mocker.patch.object(adapter, "get_current_catalog", return_value="prod") - mocker.patch.object( - adapter, - "_resolve_session_catalog", - TrinoEngineAdapter._resolve_session_catalog.__get__(adapter, TrinoEngineAdapter), - ) - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM dev.db.test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."test"', - ] - - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM prod.db.test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', - 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', - 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh"', - 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', - ] - - # When Trino get_current_catalog fails, do not fall back to _default_catalog - mocker.patch.object(adapter, "get_current_catalog", side_effect=RuntimeError("catalog error")) - adapter.cursor.execute.reset_mock() - with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified catalog reference"): - adapter.replace_query( - "prod.db.test", - parse_one("SELECT a FROM db.test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [] - mocker.patch.object(adapter, "get_current_catalog", return_value="prod") - - # Database-less table reference that matches target table name raises when no default database is known - with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - - # Trino resolves the active session schema via get_current_database (SELECT "current_schema") - mocker.patch.object( - adapter, - "get_current_database", - TrinoEngineAdapter.get_current_database.__get__(adapter, TrinoEngineAdapter), - ) - mocker.patch.object(adapter, "fetchone", return_value=("db",)) - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM test TABLESAMPLE BERNOULLI (10)", read="trino"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', - 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', - 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh" TABLESAMPLE BERNOULLI (10)', - 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', - ] - mocker.patch.object(adapter, "fetchone", return_value=None) - mocker.patch.object(adapter, "get_current_database", return_value=None) - - # Database-less table reference is detected as self-referencing when active/default database matches target_table.db - adapter._extra_config["schema"] = "db" - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM test TABLESAMPLE BERNOULLI (10)", read="trino"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'CREATE TABLE IF NOT EXISTS "db"."test" ("a" INTEGER)', - 'CREATE SCHEMA IF NOT EXISTS "dev"."db"', - 'CREATE TABLE IF NOT EXISTS "dev"."db"."__temp_test_abcdefgh" AS SELECT CAST("a" AS INTEGER) AS "a" FROM (SELECT "a" FROM "db"."test") AS "_subquery"', - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "dev"."db"."__temp_test_abcdefgh" TABLESAMPLE BERNOULLI (10)', - 'DROP TABLE IF EXISTS "dev"."db"."__temp_test_abcdefgh"', - ] - - # When an active session database (e.g., via get_current_database / SELECT DATABASE()) differs from - # configured schema/database, bare table reference is not misclassified as target_table - mocker.patch.object(adapter, "get_current_database", return_value="otherdb") - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - 'DELETE FROM "db"."test" WHERE TRUE', - 'INSERT INTO "db"."test" ("a") SELECT "a" FROM "test"', - ] - # When get_current_database fails, do not fall back to configured schema/database - mocker.patch.object(adapter, "get_current_database", side_effect=RuntimeError("db error")) - adapter.cursor.execute.reset_mock() - with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM test"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [] - mocker.patch.object(adapter, "get_current_database", return_value=None) - - # Partition selection on self-referencing table is preserved when temp table exposes matching partitions - adapter.dialect = "doris" - mocker.patch.object(adapter, "_get_table_partition_names", return_value={"p1"}) - adapter.cursor.execute.reset_mock() - adapter.replace_query( - "db.test", - parse_one("SELECT a FROM db.test PARTITION (p1)", read="doris"), - target_columns_to_types={"a": exp.DataType.build("INT")}, - ) - assert to_sql_calls(adapter) == [ - "CREATE TABLE IF NOT EXISTS `db`.`test` (`a` INT)", - "CREATE SCHEMA IF NOT EXISTS `dev`.`db`", - "CREATE TABLE IF NOT EXISTS `dev`.`db`.`__temp_test_abcdefgh` AS SELECT CAST(`a` AS INT) AS `a` FROM (SELECT `a` FROM `db`.`test` PARTITION(`p1`)) AS `_subquery`", - "DELETE FROM `db`.`test` WHERE TRUE", - "INSERT INTO `db`.`test` (`a`) SELECT `a` FROM `dev`.`db`.`__temp_test_abcdefgh` PARTITION(`p1`)", - "DROP TABLE IF EXISTS `dev`.`db`.`__temp_test_abcdefgh`", - ] - mocker.patch.object(adapter, "_get_table_partition_names", return_value=set()) - - def test_create_table_like(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture): adapter = make_mocked_engine_adapter(EngineAdapter) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 7733689692..82310fdc33 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6103,3 +6103,64 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: "CREATE TABLE IF NOT EXISTS `internal`.`db`.`test` (`a` INT)", "INSERT INTO `internal`.`db`.`test` (`a`) SELECT CAST(`a` AS INT) AS `a` FROM (SELECT 1 AS `a` UNION ALL SELECT `a` FROM `test`) AS `_subquery`", ] + + # 11. Self-referencing replace_query on an existing Doris table handles CTE shadowing, + # FOR VERSION AS OF, PARTITION(...) staging, and session database/catalog failure closed. + existing_ref_adapter = make_mocked_engine_adapter( + DorisEngineAdapter, default_catalog="internal", database="db" + ) + mocker.patch.object( + existing_ref_adapter, + "get_data_object", + return_value=DataObject(catalog="internal", schema="db", name="test", type="table"), + ) + mocker.patch("sqlmesh.core.engine_adapter.doris.random_id", return_value="abcdefgh") + existing_ref_adapter.cursor.execute.reset_mock() + existing_ref_adapter.replace_query( + "test", + parse_one("WITH test AS (SELECT 2 AS a) SELECT t.a FROM test AS t", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(existing_ref_adapter) == [ + "TRUNCATE TABLE `test`", + "INSERT INTO `test` (`a`) WITH `test` AS (SELECT 2 AS `a`) SELECT `t`.`a` FROM `test` AS `t`", + ] + + existing_ref_adapter.cursor.execute.reset_mock() + existing_ref_adapter.replace_query( + "db.test", + parse_one("SELECT a FROM db.test FOR VERSION AS OF 1", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(existing_ref_adapter) == [ + "TRUNCATE TABLE `db`.`test`", + "INSERT INTO `db`.`test` (`a`) SELECT `a` FROM `db`.`test` FOR VERSION AS OF 1", + ] + + mocker.patch.object(existing_ref_adapter, "_get_table_partition_names", return_value={"p1"}) + existing_ref_adapter.cursor.execute.reset_mock() + existing_ref_adapter.replace_query( + "db.test", + parse_one("SELECT a FROM db.test PARTITION (p1)", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(existing_ref_adapter) == [ + "CREATE TABLE IF NOT EXISTS `db`.`test` (`a` INT)", + "CREATE DATABASE IF NOT EXISTS `internal`.`db`", + "CREATE TABLE IF NOT EXISTS `internal`.`db`.`__temp_test_abcdefgh` AS SELECT CAST(`a` AS INT) AS `a` FROM (SELECT `a` FROM `db`.`test` PARTITION(`p1`)) AS `_subquery`", + "TRUNCATE TABLE `db`.`test`", + "INSERT INTO `db`.`test` (`a`) SELECT `a` FROM `internal`.`db`.`__temp_test_abcdefgh` PARTITION(`p1`)", + "DROP TABLE IF EXISTS `internal`.`db`.`__temp_test_abcdefgh`", + ] + + mocker.patch.object( + existing_ref_adapter, "get_current_database", side_effect=RuntimeError("db error") + ) + existing_ref_adapter.cursor.execute.reset_mock() + with pytest.raises(SQLMeshError, match="Cannot disambiguate unqualified schema reference"): + existing_ref_adapter.replace_query( + "db.test", + parse_one("SELECT a FROM test", read="doris"), + target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, + ) + assert to_sql_calls(existing_ref_adapter) == [] diff --git a/tests/core/engine_adapter/test_starrocks.py b/tests/core/engine_adapter/test_starrocks.py index 6464a3271d..b47dfb1e0d 100644 --- a/tests/core/engine_adapter/test_starrocks.py +++ b/tests/core/engine_adapter/test_starrocks.py @@ -2550,7 +2550,7 @@ def test_tuple_and_default_catalog_refs_are_resolved( ) catalog_base_snapshot = self._make_snapshot(catalog_base_model) catalog_physical_name = exp.to_table(catalog_base_snapshot.table_name()) - expected_catalog_physical = f"{catalog_physical_name.catalog}.{catalog_physical_name.db}.{catalog_physical_name.name}" + expected_catalog_physical = f"{catalog_physical_name.db}.{catalog_physical_name.name}" catalog_mv_model = _load_sql_model( """ From af87894a21973ce5c1d37958764499dec0a5b58b Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:59:42 -0800 Subject: [PATCH 34/54] docs(doris): clarify INCREMENTAL_BY_UNIQUE_KEY strategies in model_kinds.md Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/concepts/models/model_kinds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts/models/model_kinds.md b/docs/concepts/models/model_kinds.md index dc2ab190d2..d3a3c55df4 100644 --- a/docs/concepts/models/model_kinds.md +++ b/docs/concepts/models/model_kinds.md @@ -670,7 +670,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_UNIQUE_KEY` kind a | Redshift | MERGE ON unique key | | Postgres | MERGE ON unique key | | DuckDB | DELETE ON matched + INSERT new rows | -| Doris | DELETE ON matched + INSERT new rows | +| Doris | DELETE ON matched + INSERT new rows (or native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+; direct INSERT when a sequence column is configured) | ## FULL Models of the `FULL` kind cause the dataset associated with a model to be fully refreshed (rewritten) upon each model evaluation. From 9a47f601e985c430c3541619871e2ded80b6367a Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:32:35 -0800 Subject: [PATCH 35/54] fix(doris): address review feedback on ssl verify_mode, catalog restore, and branch+version AST Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 5 +- sqlmesh/core/config/connection.py | 64 +++++++++++++++++++--- sqlmesh/core/dialect.py | 26 ++++++--- sqlmesh/core/engine_adapter/doris.py | 21 ++++++-- sqlmesh/core/model/definition.py | 2 +- tests/core/engine_adapter/test_doris.py | 70 +++++++++++++++++++++++++ tests/core/test_connection_config.py | 21 ++++++++ tests/core/test_dialect.py | 20 +++++++ 8 files changed, 205 insertions(+), 24 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index c474413075..9a12b10fa3 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -897,7 +897,8 @@ MODEL ( storage_format = 'V3', -- Use 'size_based' on Merge-on-Write UNIQUE KEY tables ('time_series' is supported on DUPLICATE KEY append/log tables) compaction_policy = 'size_based', - -- Sequence column prevents out-of-order updates with lower sequence values from overwriting newer rows + -- Sequence column prevents out-of-order updates with lower sequence values from overwriting newer rows; + -- use DATETIME(6) (or a strictly ordered monotonic integer sequence) to preserve subsecond ordering 'function_column.sequence_col' = 'updated_at' ) ); @@ -906,7 +907,7 @@ SELECT log_id::BIGINT AS log_id, message::STRING AS message, embedding::ARRAY AS embedding, - updated_at::DATETIME AS updated_at, + updated_at::DATETIME(6) AS updated_at, created_at::DATETIME(6) AS created_at FROM raw.logs; diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index cd69e1bbe1..8dc4709064 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2792,20 +2792,61 @@ def _validate_session_variables( @property def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: + import ssl as ssl_lib + kwargs: t.Dict[str, t.Any] = { "autocommit": True, "charset": self.charset or "utf8mb4", "local_infile": False, } + raw_verify_mode = self.ssl.get("verify_mode") if isinstance(self.ssl, dict) else None + if raw_verify_mode is not None: + is_valid_raw_verify_mode = ( + isinstance(raw_verify_mode, bool) + or ( + isinstance(raw_verify_mode, int) + and raw_verify_mode + in ( + ssl_lib.CERT_NONE, + ssl_lib.CERT_OPTIONAL, + ssl_lib.CERT_REQUIRED, + ) + ) + or ( + isinstance(raw_verify_mode, str) + and raw_verify_mode.strip().lower() + in ( + "false", + "0", + "no", + "none", + "optional", + "cert_optional", + "required", + "cert_required", + "true", + "1", + "yes", + ) + ) + ) + if not is_valid_raw_verify_mode: + raise ConfigError( + f"Invalid Doris `ssl.verify_mode` value: {raw_verify_mode!r}. " + "Expected a boolean, `ssl.VerifyMode`, or one of " + "'none', 'optional', 'required', 'true', 'false'." + ) ssl_dict_requests_verification = isinstance(self.ssl, dict) and ( bool(self.ssl.get("check_hostname")) or ( - self.ssl.get("verify_mode") is not None - and self.ssl.get("verify_mode") is not False + raw_verify_mode is not None + and raw_verify_mode is not False and not ( - isinstance(self.ssl.get("verify_mode"), str) - and str(self.ssl.get("verify_mode")).strip().lower() - in ("false", "0", "no", "none") + not isinstance(raw_verify_mode, bool) and raw_verify_mode == ssl_lib.CERT_NONE + ) + and not ( + isinstance(raw_verify_mode, str) + and raw_verify_mode.strip().lower() in ("false", "0", "no", "none") ) ) ) @@ -2847,9 +2888,16 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: if self.ssl_verify_identity is not None: ssl_config["check_hostname"] = bool(self.ssl_verify_identity) verify_mode = ssl_config.get("verify_mode") - if verify_mode is False or ( - isinstance(verify_mode, str) - and verify_mode.strip().lower() in ("false", "0", "no", "none") + if isinstance(verify_mode, str): + verify_mode = verify_mode.strip() + ssl_config["verify_mode"] = verify_mode + if ( + verify_mode is False + or (not isinstance(verify_mode, bool) and verify_mode == ssl_lib.CERT_NONE) + or ( + isinstance(verify_mode, str) + and verify_mode.lower() in ("false", "0", "no", "none") + ) ): if hostname_requested: raise ConfigError( diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 4847c51382..21fe7d7b02 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -734,7 +734,13 @@ def _doris_parse_version(self: Parser) -> t.Optional[exp.Version]: ref_arg = ref_args[0] if isinstance(ref_arg, exp.Identifier) and not ref_arg.quoted: ref_arg = exp.var(ref_arg.name) - return self.expression(exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg)) + branch_ver = self.expression( + exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg) + ) + trailing_ver = self.__parse_version() # type: ignore + if trailing_ver is not None: + branch_ver.set("version", trailing_ver) + return branch_ver self._retreat(saved_idx) return None @@ -761,12 +767,12 @@ def _doris_parse_table( parse_partition=parse_partition, consume_pipe=consume_pipe, ) - if ( - isinstance(table, exp.Table) - and not table.args.get("version") - and table.meta.get("doris_version") - ): - table.set("version", table.meta.pop("doris_version")) + if isinstance(table, exp.Table) and table.meta.get("doris_version"): + branch_ver = table.meta.pop("doris_version") + existing_ver = table.args.get("version") + if isinstance(existing_ver, exp.Version) and existing_ver is not branch_ver: + branch_ver.set("version", existing_ver) + table.set("version", branch_ver) return table @@ -783,7 +789,11 @@ def _doris_version_sql(self: Generator, expression: exp.Version) -> str: and not getattr(expr_node, "quoted", False) else self.sql(expr_node) ) - return f"@{this_upper.lower()}({expr_sql})" + branch_sql = f"@{this_upper.lower()}({expr_sql})" + chained_ver = expression.args.get("version") + if isinstance(chained_ver, exp.Version): + return f"{branch_sql} {self.sql(chained_ver)}" + return branch_sql kind = expression.text("kind") this_sql = "TIME" if this_upper in ("TIMESTAMP", "TIME") else this_upper return f"FOR {this_sql} {kind} {self.sql(expression, 'expression')}" diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index d166ec276e..26c7568c33 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -288,11 +288,11 @@ def _active_catalog(self, value: t.Optional[str]) -> None: self._get_thread_state().active_catalog = value @property - def _active_catalog_conn_token(self) -> t.Optional[t.Tuple[int, int]]: + def _active_catalog_conn_token(self) -> t.Optional[t.Tuple[int, int, int]]: return getattr(self._get_thread_state(), "active_catalog_conn_token", None) @_active_catalog_conn_token.setter - def _active_catalog_conn_token(self, value: t.Optional[t.Tuple[int, int]]) -> None: + def _active_catalog_conn_token(self, value: t.Optional[t.Tuple[int, int, int]]) -> None: self._get_thread_state().active_catalog_conn_token = value @property @@ -353,8 +353,12 @@ def _pool_generation(self, value: int) -> None: except Exception: pass - def _current_conn_token(self) -> t.Tuple[int, int]: - return (self._pool_generation, id(self._connection_pool.get())) + def _current_conn_token(self) -> t.Tuple[int, int, int]: + return ( + self._pool_generation, + id(self._connection_pool.get()), + id(self._connection_pool.get_cursor()), + ) def _mark_catalog_synced(self, catalog: t.Optional[str] = None) -> None: if catalog is not None: @@ -391,7 +395,7 @@ def recycle(self) -> None: new_gen = self._pool_generation + 1 self._pool_generation = new_gen if calling_token is not None and self._active_catalog is not None: - updated_token = (new_gen, calling_token[1]) + updated_token = (new_gen, calling_token[1], calling_token[2]) self._active_catalog_conn_token = updated_token if isinstance(self._connection_pool, ConnectionPool): self._connection_pool.set_attribute("_doris_catalog_token", updated_token) @@ -723,6 +727,7 @@ def _ensure_catalog_restored(self, sql: t.Optional[str] = None) -> None: if not self._should_restore_catalog(): return try: + self._connection_pool.get_cursor() self._restore_active_catalog_on_reconnect(sql) except Exception as e: is_disconnect, is_timeout, err_name, err_code = self._extract_disconnect_info(e) @@ -774,6 +779,9 @@ def _fetch_native_df( logger.info("Connection to the database was lost. Reconnecting...") self._connection_pool.close() self._needs_catalog_restore = True + else: + if self._active_catalog: + self._needs_catalog_restore = True try: self._connection_pool.get_cursor() self._ensure_catalog_restored(sql_str) @@ -1054,6 +1062,9 @@ def _execute(self, sql: str, track_rows_processed: bool = False, **kwargs: t.Any logger.info("Connection to the database was lost. Reconnecting...") self._connection_pool.close() self._needs_catalog_restore = True + else: + if self._active_catalog: + self._needs_catalog_restore = True self._ensure_catalog_restored(sql) try: super()._execute(sql, track_rows_processed=track_rows_processed, **kwargs) diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 3927de389f..d8fcb3cf1b 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2875,7 +2875,7 @@ def resolve(ref: str) -> str: strip_catalog = bool(physical) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table - if strip_catalog: + if strip_catalog and (dialect != "doris" or resolved_table.catalog.lower() == "internal"): resolved_table.set("catalog", None) return exp.table_name(resolved_table, identify=False) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 82310fdc33..43953ee44b 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6163,4 +6163,74 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: parse_one("SELECT a FROM test", read="doris"), target_columns_to_types={"a": exp.DataType.build("INT", dialect="doris")}, ) + + # 12. With `pre_ping=True`, if `ping()` closes the cursor and `_cursor_init` switches the new cursor back + # to the configured catalog (`internal`), both `_execute` and `_fetch_native_df` restore `_active_catalog`. + pre_ping_cursors: t.List[t.Any] = [] + + def _pre_ping_conn_factory() -> t.Any: + conn = mocker.MagicMock() + + def _make_cursor() -> t.Any: + cur = mocker.MagicMock() + pre_ping_cursors.append(cur) + return cur + + conn.cursor.side_effect = _make_cursor + return conn + + def _cursor_init_internal(cur: t.Any) -> None: + cur.execute("SWITCH `internal`") + + pre_ping_adapter = DorisEngineAdapter( + _pre_ping_conn_factory, + multithreaded=False, + cursor_init=_cursor_init_internal, + pre_ping=True, + catalog="internal", + ) + mocker.patch.object( + pre_ping_adapter, + "ping", + side_effect=lambda: pre_ping_adapter._connection_pool.close_cursor(), + ) + pre_ping_adapter.set_current_catalog("iceberg_cat") + pre_ping_adapter.execute("SELECT 1") + assert [c[0][0] for c in pre_ping_cursors[-1].execute.call_args_list] == [ + "SWITCH `internal`", + "SWITCH `iceberg_cat`", + "SELECT 1", + ] + + super_fetch_pre_ping = mocker.patch( + "sqlmesh.core.engine_adapter.mixins.PandasNativeFetchDFSupportMixin._fetch_native_df", + return_value=pd.DataFrame({"id": [1]}), + ) + pre_ping_adapter._fetch_native_df("SELECT 1 AS id") + assert super_fetch_pre_ping.call_count == 1 + assert [c[0][0] for c in pre_ping_cursors[-1].execute.call_args_list] == [ + "SWITCH `internal`", + "SWITCH `iceberg_cat`", + ] + + # 13. `_resolve_model_refs_to_physical_tables` strips `internal` for Doris warehouse-local references + # but preserves external catalog qualification when `default_catalog` is an external Doris catalog. + from sqlmesh.core.model.definition import _resolve_model_refs_to_physical_tables + + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"internal"."db"."upstream_model"': "internal.sqlmesh__db.db__upstream_model__999" + }, + dialect="doris", + default_catalog="internal", + ) == exp.Literal.string("sqlmesh__db.db__upstream_model__999") + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"lake_a"."db"."upstream_model"': "lake_a.sqlmesh__db.db__upstream_model__999" + }, + dialect="doris", + default_catalog="lake_a", + ) == exp.Literal.string("lake_a.sqlmesh__db.db__upstream_model__999") assert to_sql_calls(existing_ref_adapter) == [] diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 2dea2cdaf4..d657eb2919 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2662,6 +2662,27 @@ def test_doris(make_config): "check_hostname": False, } + ssl_dict_invalid_verify_mode = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"verify_mode": "requried"}, + check_import=False, + ) + with pytest.raises(ConfigError, match="Invalid Doris `ssl.verify_mode` value"): + _ = ssl_dict_invalid_verify_mode._static_connection_kwargs + + ssl_dict_valid_verify_mode = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"verify_mode": " required "}, + check_import=False, + ) + assert ssl_dict_valid_verify_mode._static_connection_kwargs["ssl"] == { + "verify_mode": "required", + } + ssl_disabled_verify_identity = make_config( type="doris", host="doris-fe", diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index a016ab71f6..36806e5579 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1692,3 +1692,23 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): dialect="doris", ) assert mixed_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'col2', 'a')" + + branch_and_version_ast = d.parse_one( + "SELECT * FROM lance_catalog.db.tbl@branch(dev) FOR VERSION AS OF 2", + dialect="doris", + ) + assert ( + branch_and_version_ast.sql("doris") + == "SELECT * FROM lance_catalog.db.tbl @branch(dev) FOR VERSION AS OF 2" + ) + roundtrip_ast = d.parse_one(branch_and_version_ast.sql("doris"), dialect="doris") + assert roundtrip_ast == branch_and_version_ast + assert ( + roundtrip_ast.sql("doris") + == "SELECT * FROM lance_catalog.db.tbl @branch(dev) FOR VERSION AS OF 2" + ) + versions = list(roundtrip_ast.find_all(exp.Version)) + assert [(v.name.upper(), v.expression.sql("doris")) for v in versions] == [ + ("BRANCH", "dev"), + ("VERSION", "2"), + ] From 673327b0ae2adb2cea09c43b07964b838c6b53aa Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:56:52 -0800 Subject: [PATCH 36/54] fix(doris): normalize ssl.verify_mode aliases and keep changelog_id tie-breaker in docs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 12 +++++++++--- sqlmesh/core/config/connection.py | 10 ++++++++++ tests/core/test_connection_config.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 9a12b10fa3..799ef013a0 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -207,21 +207,27 @@ MODEL ( name analytics.orders_upsert, kind INCREMENTAL_BY_UNIQUE_KEY ( unique_key order_id, - when_matched WHEN MATCHED AND source.updated_at > target.updated_at THEN UPDATE SET + when_matched WHEN MATCHED AND ( + source.updated_at > target.updated_at + OR (source.updated_at = target.updated_at AND source.changelog_id > target.changelog_id) + ) THEN UPDATE SET target.status = source.status, - target.updated_at = source.updated_at + target.updated_at = source.updated_at, + target.changelog_id = source.changelog_id ) ); SELECT order_id::BIGINT AS order_id, status::VARCHAR(32) AS status, - updated_at::DATETIME(6) AS updated_at + updated_at::DATETIME(6) AS updated_at, + changelog_id::BIGINT AS changelog_id FROM ( SELECT order_id::BIGINT AS order_id, status::VARCHAR(32) AS status, updated_at::DATETIME(6) AS updated_at, + changelog_id::BIGINT AS changelog_id, ROW_NUMBER() OVER ( PARTITION BY order_id ORDER BY updated_at DESC, changelog_id DESC diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 8dc4709064..7559ae66cb 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2888,8 +2888,18 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: if self.ssl_verify_identity is not None: ssl_config["check_hostname"] = bool(self.ssl_verify_identity) verify_mode = ssl_config.get("verify_mode") + if isinstance(verify_mode, int) and not isinstance(verify_mode, bool): + verify_mode = { + int(ssl_lib.CERT_NONE): "none", + int(ssl_lib.CERT_OPTIONAL): "optional", + int(ssl_lib.CERT_REQUIRED): "required", + }[verify_mode] + ssl_config["verify_mode"] = verify_mode if isinstance(verify_mode, str): verify_mode = verify_mode.strip() + verify_mode = {"cert_required": "required", "cert_optional": "optional"}.get( + verify_mode.lower(), verify_mode + ) ssl_config["verify_mode"] = verify_mode if ( verify_mode is False diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index d657eb2919..ddbbbede2c 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2683,6 +2683,35 @@ def test_doris(make_config): "verify_mode": "required", } + import ssl as ssl_lib + + for raw_mode, expected_mode in [ + (ssl_lib.CERT_REQUIRED, "required"), + (ssl_lib.CERT_OPTIONAL, "optional"), + ("CERT_REQUIRED", "required"), + ("cert_optional", "optional"), + ]: + cfg = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"verify_mode": raw_mode}, + check_import=False, + ) + assert cfg._static_connection_kwargs["ssl"] == {"verify_mode": expected_mode} + + cfg_cert_none = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl={"verify_mode": ssl_lib.CERT_NONE}, + check_import=False, + ) + assert cfg_cert_none._static_connection_kwargs["ssl"] == { + "verify_mode": "none", + "check_hostname": False, + } + ssl_disabled_verify_identity = make_config( type="doris", host="doris-fe", From f0a037a9c266173257fdc5e52000bf8a6dd7258a Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:20:32 -0800 Subject: [PATCH 37/54] fix(doris): support USING ANALYZER and INSERT branch targets, ssl=False, and catalog ref priority Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- sqlmesh/core/config/connection.py | 2 + sqlmesh/core/dialect.py | 50 +++++++++++++++++++++---- sqlmesh/core/model/definition.py | 6 ++- tests/core/engine_adapter/test_doris.py | 10 +++-- tests/core/test_connection_config.py | 1 + tests/core/test_dialect.py | 28 ++++++++++++++ 7 files changed, 85 insertions(+), 14 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 799ef013a0..7e5042aff5 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -836,7 +836,7 @@ When SQLMesh pre-creates an empty physical snapshot table during the snapshot-cr - **`FULL`, `INCREMENTAL_BY_TIME_RANGE`, and `INCREMENTAL_BY_UNIQUE_KEY`**: Run `sqlmesh plan --restate-model ` (provided the upstream source still retains the restated interval/rows) to rewrite rows through the newly registered index. - **`INCREMENTAL_BY_PARTITION`**: Warning—`sqlmesh plan --restate-model ` drops and rebuilds the **entire** table, which loses historical partitions unless the upstream source still reproduces the full partition history; otherwise stop all writers to the table before starting the overwrite, rewrite only the unindexed partition(s) in place via `INSERT OVERWRITE TABLE PARTITION(...) SELECT ...`, and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **`INCREMENTAL_UNMANAGED`**: Restatement is disabled by default (`disable_restatement = true`), so `--restate-model` skips the model unless `disable_restatement false` is explicitly set (which truncates and reloads the full table). Choose the backfill path based on whether currently stored `UNIQUE KEY` delete/sequence tombstones (`__DORIS_DELETE_SIGN__`) must be preserved across the backfill operation: - - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure `SET show_hidden_columns = false;` is set in the session (or explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT`, because `show_hidden_columns = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM `, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. + - **Tables where `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) do not need to be retained**: To preserve live rows without truncating from source, stop all writers to the table before starting the overwrite, ensure both `SET show_hidden_columns = false;` and `SET skip_delete_sign = false;` are set in the session and explicitly filter `WHERE __DORIS_DELETE_SIGN__ = 0` in the `SELECT` (because either `show_hidden_columns = true` or `skip_delete_sign = true` disables Doris's automatic delete-sign filter and an `INSERT OVERWRITE` omitting `__DORIS_DELETE_SIGN__` would reinsert deleted rows as live rows), rewrite the existing live rows in place on the physical snapshot table after `SHOW ALTER TABLE COLUMN` reports `FINISHED`—listing columns explicitly and including `__DORIS_SEQUENCE_COL__` on `UNIQUE KEY` tables configured with `function_column.sequence_type` so hidden sequence values on live rows are preserved (`INSERT OVERWRITE TABLE (, , ..., __DORIS_SEQUENCE_COL__) SELECT , , ..., __DORIS_SEQUENCE_COL__ FROM WHERE __DORIS_DELETE_SIGN__ = 0`, or omitting `__DORIS_SEQUENCE_COL__` when `function_column.sequence_type` is not used)—and keep all writers stopped until the overwrite completes so concurrent appends are not lost. - **Tables where stored `UNIQUE KEY` sequence tombstones (`__DORIS_DELETE_SIGN__`) must not be stripped by the backfill**: Do **not** use self-`INSERT OVERWRITE` (with `show_hidden_columns = false`, `SELECT` excludes deleted rows and immediately discards their tombstones, allowing an older out-of-order write to resurrect a deleted key). Note that the procedures below only avoid stripping tombstones **during the index rebuild or replay itself**—on Doris 4.1.5 Merge-on-Write (`enable_unique_key_merge_on_write = 'true'`) tables without cluster keys (`order_by`), subsequent base compaction can still prune `__DORIS_DELETE_SIGN__` tombstones later, so ongoing protection against late out-of-order writes requires a verified Doris compaction/tombstone-retention strategy and neither procedure below is sufficient on its own without one: 1. **Build or rebuild the index via heavy `SCHEMA_CHANGE` in compute-storage decoupled mode (preserves existing segments, sequence values, and currently stored tombstones during the schema change)**: First verify `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) index-change jobs on the table (Doris rejects starting a `SCHEMA_CHANGE` job while any `IndexChangeJob` is unfinished). If the index is **not yet registered** on ``, run `SET enable_add_index_for_new_data = false;` followed by `ALTER TABLE ADD INDEX ...`, then wait until the resulting job in `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` reaches `State = 'FINISHED'` (and verify it did not end in `CANCELLED`). If the index is **already registered** (for example, added during `@runtime_stage = 'creating'` with `enable_add_index_for_new_data = true`), setting `enable_add_index_for_new_data = false` alone will not rebuild it because Doris rejects a repeated `ADD INDEX` (or skips it as a no-op with `IF NOT EXISTS`): run `SET enable_add_index_for_new_data = false;`, drop the index via `ALTER TABLE DROP INDEX `, wait until the drop's `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`), `SHOW BUILD INDEX FROM WHERE TableName = ''` has no active (`WAITING_TXN` or `RUNNING`) jobs, and `SHOW INDEX FROM ` no longer lists ``, then recreate the index in the same `enable_add_index_for_new_data = false` session via `ALTER TABLE ADD INDEX ...` and wait until the new `SHOW ALTER TABLE COLUMN FROM WHERE TableName = ''` job reaches `State = 'FINISHED'` (not `CANCELLED`). 2. **Replay into a deployable SQLMesh snapshot from an authoritative changelog/CDC source (`forward_only false` with `virtual_environment_mode: full`)** (prefer option 1 when possible, and replay only after verifying the upstream changelog/CDC source retains the required history of live rows, deleted-key tombstones, and sequence values): diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 7559ae66cb..2baf7d8fb7 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2860,6 +2860,8 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: "certificate or hostname verification (`ssl_verify_cert=True`, " "`ssl_verify_identity=True`, or `ssl` verification settings)." ) + if self.ssl is False and self.ssl_disabled is None: + kwargs["ssl_disabled"] = True if not self.ssl_disabled and self.ssl is not False: ssl_config: t.Dict[str, t.Any] = ( dict(self.ssl) if isinstance(self.ssl, dict) else {} # type: ignore[arg-type] diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 21fe7d7b02..e3922d4d1a 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -666,7 +666,14 @@ def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Opt op_name = self._curr.text.upper() self._advance() rhs = self._parse_bitwise() - this = self.expression(exp.Anonymous(this=op_name, expressions=[this, rhs])) + match_exprs: t.List[t.Optional[exp.Expr]] = [this, rhs] + if self._match_text_seq("USING", "ANALYZER"): + analyzer = self._parse_string() or self._parse_id_var(any_token=True) + if isinstance(analyzer, exp.Identifier) and not analyzer.quoted: + analyzer = exp.var(analyzer.name) + if analyzer is not None: + match_exprs.append(analyzer) + this = self.expression(exp.Anonymous(this=op_name, expressions=match_exprs)) if negated: this = self.expression(exp.Not(this=this)) elif negated: @@ -678,10 +685,15 @@ def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Opt def _doris_anonymous_sql(self: Generator, expression: exp.Anonymous) -> str: - if expression.name.upper() in _DORIS_MATCH_OPS and len(expression.expressions) == 2: + if expression.name.upper() in _DORIS_MATCH_OPS and len(expression.expressions) in (2, 3): left = self.sql(expression.expressions[0]) right = self.sql(expression.expressions[1]) - return f"{left} {expression.name.upper()} {right}" + analyzer_sql = ( + f" USING ANALYZER {self.sql(expression.expressions[2])}" + if len(expression.expressions) == 3 + else "" + ) + return f"{left} {expression.name.upper()} {right}{analyzer_sql}" return self.anonymous_sql(expression) @@ -709,6 +721,29 @@ def _doris_parse_table_parts( table.set("this", exp.Identifier(this=real_name, quoted=False)) table.set("version", ver) table.meta["doris_version"] = ver + elif isinstance(table.this, exp.Identifier): + m = re.match(r"^(.+)@(branch|tag)$", table.this.name, flags=re.IGNORECASE) + if m and self._match(TokenType.L_PAREN, advance=False): + ref_args = self._parse_wrapped_csv(lambda: self._parse_id_var(any_token=True)) + if ref_args: + real_name = m.group(1) + ref_kind = m.group(2).upper() + ref_arg = ref_args[0] + if isinstance(ref_arg, exp.Identifier) and not ref_arg.quoted: + ref_arg = exp.var(ref_arg.name) + ver = exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg) + table.set("this", exp.Identifier(this=real_name, quoted=table.this.quoted)) + table.set("version", ver) + table.meta["doris_version"] = ver + elif schema and ( + self._match_pair(TokenType.PARAMETER, TokenType.VAR, advance=False) + and self._next + and self._next.text.upper() in ("BRANCH", "TAG") + ): + parsed_ver = _doris_parse_version(self) + if parsed_ver is not None: + table.set("version", parsed_ver) + table.meta["doris_version"] = parsed_ver return table @@ -767,12 +802,13 @@ def _doris_parse_table( parse_partition=parse_partition, consume_pipe=consume_pipe, ) - if isinstance(table, exp.Table) and table.meta.get("doris_version"): - branch_ver = table.meta.pop("doris_version") - existing_ver = table.args.get("version") + target_table = table.this if isinstance(table, exp.Schema) else table + if isinstance(target_table, exp.Table) and target_table.meta.get("doris_version"): + branch_ver = target_table.meta.pop("doris_version") + existing_ver = target_table.args.get("version") if isinstance(existing_ver, exp.Version) and existing_ver is not branch_ver: branch_ver.set("version", existing_ver) - table.set("version", branch_ver) + target_table.set("version", branch_ver) return table diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index d8fcb3cf1b..84a357d1c0 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2865,14 +2865,16 @@ def _resolve_model_refs_to_physical_tables( def resolve(ref: str) -> str: table = exp.to_table(ref.strip(), dialect=dialect) - physical = table_mapping.get(exp.table_name(table, identify=True)) + physical: t.Optional[str] = None strip_catalog = False - if not physical and default_catalog and not table.catalog: + if default_catalog and not table.catalog: normalized_name = d.normalize_model_name( table, default_catalog=default_catalog, dialect=dialect ) physical = table_mapping.get(normalized_name) strip_catalog = bool(physical) + if not physical: + physical = table_mapping.get(exp.table_name(table, identify=True)) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table if strip_catalog and (dialect != "doris" or resolved_table.catalog.lower() == "internal"): diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 43953ee44b..03d039e255 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6213,14 +6213,15 @@ def _cursor_init_internal(cur: t.Any) -> None: "SWITCH `iceberg_cat`", ] - # 13. `_resolve_model_refs_to_physical_tables` strips `internal` for Doris warehouse-local references - # but preserves external catalog qualification when `default_catalog` is an external Doris catalog. + # 13. `_resolve_model_refs_to_physical_tables` prefers the `default_catalog` match over an unqualified + # match, strips `internal` for Doris warehouse-local references, and preserves external catalog qualification. from sqlmesh.core.model.definition import _resolve_model_refs_to_physical_tables assert _resolve_model_refs_to_physical_tables( exp.Literal.string("db.upstream_model"), table_mapping={ - '"internal"."db"."upstream_model"': "internal.sqlmesh__db.db__upstream_model__999" + '"db"."upstream_model"': "wrong_unqualified.db__upstream_model__111", + '"internal"."db"."upstream_model"': "internal.sqlmesh__db.db__upstream_model__999", }, dialect="doris", default_catalog="internal", @@ -6228,7 +6229,8 @@ def _cursor_init_internal(cur: t.Any) -> None: assert _resolve_model_refs_to_physical_tables( exp.Literal.string("db.upstream_model"), table_mapping={ - '"lake_a"."db"."upstream_model"': "lake_a.sqlmesh__db.db__upstream_model__999" + '"db"."upstream_model"': "wrong_unqualified.db__upstream_model__111", + '"lake_a"."db"."upstream_model"': "lake_a.sqlmesh__db.db__upstream_model__999", }, dialect="doris", default_catalog="lake_a", diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index ddbbbede2c..01cea7a5f7 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2583,6 +2583,7 @@ def test_doris(make_config): check_import=False, ) assert "ssl" not in ssl_false_config._static_connection_kwargs + assert ssl_false_config._static_connection_kwargs["ssl_disabled"] is True ssl_empty_dict_config = make_config( type="doris", diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 36806e5579..c17561eb30 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1712,3 +1712,31 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): ("BRANCH", "dev"), ("VERSION", "2"), ] + + match_analyzer_ast = d.parse_one( + "SELECT * FROM docs WHERE content MATCH 'hello' USING ANALYZER kw_analyzer", + dialect="doris", + ) + assert ( + match_analyzer_ast.sql("doris") + == "SELECT * FROM docs WHERE content MATCH 'hello' USING ANALYZER kw_analyzer" + ) + + insert_branch_ast = d.parse_one( + "INSERT INTO iceberg_tbl@branch(b1) VALUES (1, 2)", + dialect="doris", + ) + assert insert_branch_ast.sql("doris") == "INSERT INTO iceberg_tbl @branch(b1) VALUES (1, 2)" + assert d.parse_one(insert_branch_ast.sql("doris"), dialect="doris") == insert_branch_ast + + insert_branch_cols_ast = d.parse_one( + "INSERT INTO iceberg_tbl@branch(b1) (c1, c2) VALUES (1, 2)", + dialect="doris", + ) + assert ( + insert_branch_cols_ast.sql("doris") + == "INSERT INTO iceberg_tbl @branch(b1) (c1, c2) VALUES (1, 2)" + ) + assert ( + d.parse_one(insert_branch_cols_ast.sql("doris"), dialect="doris") == insert_branch_cols_ast + ) From dec8bb930d3a6cefa9efaf19ec087fde6f25cdd9 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:19:35 -0800 Subject: [PATCH 38/54] fix(doris): enforce pymysql>=1.2.0 and gate qualified excluded_trigger_tables on Doris 2.1.10+ Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 4 +- sqlmesh/core/config/connection.py | 60 ++++++++++++++++++++++++- sqlmesh/core/dialect.py | 12 +++++ sqlmesh/core/engine_adapter/doris.py | 49 ++++++++++++++------ sqlmesh/core/model/definition.py | 19 +++++++- tests/core/engine_adapter/test_doris.py | 46 ++++++++++++++++++- tests/core/test_connection_config.py | 20 +++++++++ 7 files changed, 191 insertions(+), 19 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 7e5042aff5..ed7275f163 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -782,7 +782,7 @@ GROUP BY user_id, CAST(event_time AS DATE); !!! note "Automatic Physical Table Resolution & Non-Destructive Evaluation" - Doris materialized views are only created on initial deployment or when a rebuild is required (`RECREATE_MATERIALIZED_VIEW_ON_EVALUATION = False`), preserving MTMV refresh state across routine `sqlmesh run` invocations. - When `build = 'DEFERRED'` is used on an asynchronous MTMV that also defines `audits (...)`, SQLMesh automatically submits `REFRESH MATERIALIZED VIEW AUTO` after creation. Because Doris schedules and executes initial MTMV refresh tasks asynchronously for both `BUILD IMMEDIATE` and `REFRESH MATERIALIZED VIEW`, SQLMesh does not wait for the background refresh task to finish; audits that depend on populated MTMV data can therefore run against an empty view unless a refresh-status check (via `mv_infos()` or `tasks("type" = "mv")`) confirms completion before auditing. - - When `excluded_trigger_tables` (supports a single string, comma-separated string, or tuple `('db.t1', 'db.t2')` on async MTMVs) references a SQLMesh-managed model, SQLMesh automatically resolves the model name to its underlying physical snapshot table name. + - When `excluded_trigger_tables` (supports a single string, comma-separated string, or tuple `('db.t1', 'db.t2')` on async MTMVs) references a SQLMesh-managed model, SQLMesh automatically resolves the model name to its underlying physical snapshot table name. Database- and catalog-qualified table references (`db.table` and `catalog.db.table`) in `excluded_trigger_tables` require Doris 2.1.10+ (or 3.0.6+ on the 3.0 line); earlier Doris releases only accept bare base table names in the `internal` catalog. ### Synchronous Materialized Views (Base-Table Rollup Indexes) @@ -959,7 +959,7 @@ FROM raw.logs; - **Compute-storage integrated deployments**: Run `BUILD INDEX idx_message ON sqlmesh__analytics.analytics__searchable_logs__;` and wait for `SHOW BUILD INDEX` to finish. - **Compute-storage decoupled (`storage_vault_name`) deployments**: Because Doris does not support `BUILD INDEX` in compute-storage decoupled mode, either build the index via a heavy `SCHEMA_CHANGE` job (`SET enable_add_index_for_new_data = false;` before `ALTER TABLE ... ADD INDEX`) or trigger an explicit table rebuild—by running `sqlmesh plan --restate-model analytics.searchable_logs` or bumping `stamp` in `MODEL (...)` on a non-forward-only `sqlmesh plan` (since a plain `sqlmesh plan` with no model changes and completed intervals will not re-evaluate rows), provided the upstream source still retains all data needed for the rebuild (see [Inverted & Vector Indexes, Full-Text Search & Compaction](#inverted-vector-indexes-full-text-search-compaction-doris-2150) above for model-kind-specific backfill and `INCREMENTAL_UNMANAGED` procedures). -Downstream SQLMesh models can query full-text and vector indexes using Doris's native search operators (`MATCH`, `MATCH_ANY`, `MATCH_ALL`, `MATCH_PHRASE`, `MATCH_PHRASE_PREFIX`, `MATCH_REGEXP`, `MATCH_PHRASE_EDGE`, `SEARCH(...)`) and vector distance functions (`L2_DISTANCE`, `COSINE_DISTANCE`, `INNER_PRODUCT`, `L1_DISTANCE`, and their `_APPROXIMATE` counterparts): +Downstream SQLMesh models can query full-text and vector indexes using Doris's native search operators (`MATCH`, `MATCH_ANY`, `MATCH_ALL`, `MATCH_PHRASE`, `MATCH_PHRASE_PREFIX`, `MATCH_REGEXP`, `MATCH_PHRASE_EDGE`, `SEARCH(...)`) and vector distance functions (`L2_DISTANCE`, `COSINE_DISTANCE`, `INNER_PRODUCT`, `L1_DISTANCE`). Approximate variants depend on the Doris version: `L2_DISTANCE_APPROXIMATE` and `INNER_PRODUCT_APPROXIMATE` are documented for 4.x, and `COSINE_DISTANCE_APPROXIMATE` requires a 4.2 build that includes it: ```sql SELECT diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 2baf7d8fb7..2a5a0b33cc 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -7,6 +7,7 @@ import os import pathlib import re +import types import typing as t from enum import Enum from functools import partial @@ -2770,8 +2771,64 @@ class DorisConnectionConfig(ConnectionConfig): DIALECT: t.ClassVar[t.Literal["doris"]] = "doris" DISPLAY_NAME: t.ClassVar[t.Literal["Apache Doris"]] = "Apache Doris" DISPLAY_ORDER: t.ClassVar[t.Literal[20]] = 20 + MIN_PYMYSQL_VERSION: t.ClassVar[t.Tuple[int, int, int]] = (1, 2, 0) + _base_engine_import_validator: t.ClassVar[t.Callable] = staticmethod( + _get_engine_import_validator("pymysql", "doris", decorate=False) + ) + + @classmethod + def _validate_pymysql_version(cls, pymysql_module: t.Any = None) -> None: + if pymysql_module is None: + pymysql_module = importlib.import_module("pymysql") + raw_version: t.Optional[str] = None + version_parts: t.Optional[t.Tuple[int, ...]] = None + + version_string = getattr(pymysql_module, "VERSION_STRING", None) + version_tuple = getattr(pymysql_module, "VERSION", None) + if isinstance(version_string, str) and version_string.strip(): + raw_version = version_string.strip() + elif isinstance(version_tuple, tuple): + numeric_prefix: t.List[int] = [] + for part in version_tuple: + if isinstance(part, int) and not isinstance(part, bool): + numeric_prefix.append(part) + else: + break + if numeric_prefix: + version_parts = tuple(numeric_prefix) + raw_version = ".".join(str(p) for p in numeric_prefix) + elif isinstance(pymysql_module, types.ModuleType): + from importlib.metadata import PackageNotFoundError, version as pkg_version - _engine_import_validator = _get_engine_import_validator("pymysql", "doris") + try: + raw_version = pkg_version("PyMySQL") + except PackageNotFoundError: + raw_version = None + + if version_parts is None and raw_version: + matched_nums = [int(m) for m in re.findall(r"\d+", raw_version)[:3]] + if matched_nums: + version_parts = tuple(matched_nums) + + if version_parts is not None and (*version_parts, 0, 0)[:3] < cls.MIN_PYMYSQL_VERSION: + min_ver_str = ".".join(str(p) for p in cls.MIN_PYMYSQL_VERSION) + raise ConfigError( + f"The 'doris' engine requires 'pymysql>={min_ver_str}', but found '{raw_version or version_parts}'. " + 'Please upgrade by running: `pip install --upgrade "sqlmesh[doris]"`. ' + "Older PyMySQL releases can transmit authentication packets before aborting when TLS is requested " + "and the server does not advertise TLS." + ) + + @model_validator(mode="before") + @classmethod + def _engine_import_validator(cls, data: t.Any) -> t.Any: + check_import = ( + str_to_bool(str(data.get("check_import", True))) if isinstance(data, dict) else True + ) + data = cls._base_engine_import_validator(cls, data) + if check_import: + cls._validate_pymysql_version() + return data @field_validator("session_variables", mode="after") @classmethod @@ -2963,6 +3020,7 @@ def _connection_factory(self) -> t.Callable: return connect def connect_with_required_tls(**kwargs: t.Any) -> t.Any: + self._validate_pymysql_version() conn = connect(**kwargs) if not getattr(conn, "_secure", False): try: diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index e3922d4d1a..f0264d86c3 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -977,6 +977,7 @@ def _doris_parse_alter_drop(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], DORIS_TIMESTAMPTZ_MIN_VERSION: t.Tuple[int, int, int] = (4, 1, 0) +DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION: t.Tuple[int, int, int] = (2, 1, 10) def doris_supports_timestamptz(version: t.Optional[t.Tuple[int, ...]] = None) -> bool: @@ -985,6 +986,17 @@ def doris_supports_timestamptz(version: t.Optional[t.Tuple[int, ...]] = None) -> return (*version, 0, 0)[:3] >= DORIS_TIMESTAMPTZ_MIN_VERSION +def doris_supports_qualified_excluded_trigger_tables( + version: t.Optional[t.Tuple[int, ...]] = None, +) -> bool: + if version is None: + return True + padded = (*version, 0, 0)[:3] + if padded[:2] == (3, 0): + return padded >= (3, 0, 6) + return padded >= DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION + + def _doris_datatype_sql(self: Generator, expression: exp.DataType) -> str: if expression.is_type(exp.DataType.Type.TIMESTAMPTZ) and not doris_supports_timestamptz( getattr(self.dialect, "version", None) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 26c7568c33..cf4088a5f1 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -19,6 +19,7 @@ from sqlmesh.core.dialect import ( DORIS_TIMESTAMPTZ_MIN_VERSION, + doris_supports_qualified_excluded_trigger_tables, doris_supports_timestamptz, to_schema, ) @@ -2040,6 +2041,10 @@ def _supports_timestamptz(self) -> bool: return doris_supports_timestamptz(self.TIMESTAMPTZ_MIN_VERSION) return doris_supports_timestamptz(cached_version) + def _supports_qualified_excluded_trigger_tables(self) -> bool: + cached_version = getattr(self, "_doris_version", None) + return doris_supports_qualified_excluded_trigger_tables(cached_version) + def _resolve_sync_mv_source_from_metadata( self, target_table: exp.Table, @@ -4743,7 +4748,7 @@ def _transform_target(node: exp.Query) -> exp.Query: thread_state.replace_query_target = prev_replace_query_target thread_state.replace_query_precreated = prev_replace_query_precreated - def _get_doris_version(self) -> t.Optional[t.Tuple[int, int]]: + def _get_doris_version(self) -> t.Optional[t.Tuple[int, ...]]: cached = getattr(self, "_doris_version", None) if cached is not None: return cached @@ -4760,23 +4765,28 @@ def _get_doris_version(self) -> t.Optional[t.Tuple[int, int]]: texts = [_decoded_str(v) for v in row if v is not None] for text in texts: m = re.search( - r"doris[-_\s]*(?:version\s*)?(?:doris[-_\s]*)?(\d+)\.(\d+)", + r"doris[-_\s]*(?:version\s*)?(?:doris[-_\s]*)?(\d+)\.(\d+)(?:\.(\d+))?", text, re.IGNORECASE, ) if m: - self._doris_version: t.Optional[t.Tuple[int, int]] = ( - int(m.group(1)), - int(m.group(2)), + self._doris_version: t.Optional[t.Tuple[int, ...]] = ( + (int(m.group(1)), int(m.group(2)), int(m.group(3))) + if m.group(3) is not None + else (int(m.group(1)), int(m.group(2))) ) return self._doris_version for text in texts: if text.strip() == "5.7.99" and len(texts) > 1: continue - m = re.search(r"(\d+)\.(\d+)", text) + m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?", text) if m: - ver = (int(m.group(1)), int(m.group(2))) - if ver != (5, 7): + ver: t.Tuple[int, ...] = ( + (int(m.group(1)), int(m.group(2)), int(m.group(3))) + if m.group(3) is not None + else (int(m.group(1)), int(m.group(2))) + ) + if ver[:2] != (5, 7): self._doris_version = ver return self._doris_version return None @@ -7948,11 +7958,24 @@ def _properties_to_expressions(self, properties: t.Dict[str, t.Any]) -> t.List[e raw_str = self._extract_str_value(value) val_str = raw_str if raw_str is not None else str(value) if key == "excluded_trigger_tables" and val_str: - val_str = ",".join( - re.sub(r"(?i)^`?internal`?\.", "", part.strip()) - for part in val_str.split(",") - if part.strip() - ) + supports_qualified = self._supports_qualified_excluded_trigger_tables() + cleaned_parts: t.List[str] = [] + for part in val_str.split(","): + stripped = part.strip() + if not stripped: + continue + without_internal = re.sub(r"(?i)^`?internal`?\.", "", stripped) + if not supports_qualified: + tbl = exp.to_table(stripped, dialect=self.dialect) + if tbl.catalog and tbl.catalog.lower() != "internal": + raise SQLMeshError( + f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " + f"got '{stripped}'." + ) + cleaned_parts.append(tbl.name or without_internal) + else: + cleaned_parts.append(without_internal) + val_str = ",".join(cleaned_parts) str_literal = exp.Literal.string(val_str) expressions.append(exp.Property(this=exp.Literal.string(str(key)), value=str_literal)) return expressions diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 84a357d1c0..c4bae07be6 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -735,12 +735,15 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An (render_kwargs.get("snapshots") or {}).values(), render_kwargs.get("deployability_index"), ) + doris_version = getattr(engine_adapter, "_doris_version", None) for key in keys_to_resolve: rendered[key] = _resolve_model_refs_to_physical_tables( rendered[key], table_mapping, self.dialect, default_catalog=self.default_catalog, + doris_version=doris_version, + property_name=key, ) return rendered @@ -2837,6 +2840,8 @@ def _resolve_model_refs_to_physical_tables( table_mapping: t.Dict[str, str], dialect: DialectType, default_catalog: t.Optional[str] = None, + doris_version: t.Optional[t.Tuple[int, ...]] = None, + property_name: t.Optional[str] = None, ) -> exp.Literal: """Resolve managed-model references in a property value to their physical table names. @@ -2877,7 +2882,19 @@ def resolve(ref: str) -> str: physical = table_mapping.get(exp.table_name(table, identify=True)) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table - if strip_catalog and (dialect != "doris" or resolved_table.catalog.lower() == "internal"): + if ( + dialect == "doris" + and property_name in (None, "excluded_trigger_tables") + and not d.doris_supports_qualified_excluded_trigger_tables(doris_version) + ): + if resolved_table.catalog and resolved_table.catalog.lower() != "internal": + raise SQLMeshError( + f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " + f"got '{exp.table_name(resolved_table, identify=False)}'." + ) + resolved_table.set("catalog", None) + resolved_table.set("db", None) + elif strip_catalog and (dialect != "doris" or resolved_table.catalog.lower() == "internal"): resolved_table.set("catalog", None) return exp.table_name(resolved_table, identify=False) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 03d039e255..4137dfac0b 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -3300,7 +3300,10 @@ def test_union_time_transform_delete_agg_ctas_mv_df_order_and_limit_fold( ] # 11. `DorisConnectionConfig` with `ssl=True` populates a non-empty `ssl` dict so PyMySQL enables TLS. - mocker.patch("sqlmesh.core.config.connection.importlib.import_module") + mocker.patch( + "sqlmesh.core.config.connection.importlib.import_module", + return_value=mocker.MagicMock(VERSION_STRING="1.2.0"), + ) ssl_conn_cfg = DorisConnectionConfig(host="fe.internal", user="root", ssl=True) assert ssl_conn_cfg._static_connection_kwargs["ssl"] == {"check_hostname": False} assert bool(ssl_conn_cfg._static_connection_kwargs["ssl"]) is True @@ -3423,7 +3426,10 @@ def test_doris_connection_config_tls_matrix(mocker: MockerFixture): from sqlmesh.core.config.connection import DorisConnectionConfig from sqlmesh.utils.errors import ConfigError - mocker.patch("sqlmesh.core.config.connection.importlib.import_module") + mocker.patch( + "sqlmesh.core.config.connection.importlib.import_module", + return_value=mocker.MagicMock(VERSION_STRING="1.2.0"), + ) # 1. `ssl_verify_identity=True` requires `ssl_ca` (or `ssl.ca` / `ssl.capath`) because PyMySQL # disables hostname verification when no CA is configured; with `ssl_ca`, it populates `check_hostname=True`. @@ -4297,6 +4303,7 @@ def test_doris_sync_mv_source_tables_persistence_and_isolation( # 4. `DorisConnectionConfig._connection_factory` enforces TLS when SSL options are configured pymysql = pytest.importorskip("pymysql") + mocker.patch.object(pymysql, "VERSION_STRING", "1.2.0", create=True) tls_cfg = DorisConnectionConfig( host="fe.doris.cluster", @@ -6235,4 +6242,39 @@ def _cursor_init_internal(cur: t.Any) -> None: dialect="doris", default_catalog="lake_a", ) == exp.Literal.string("lake_a.sqlmesh__db.db__upstream_model__999") + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"internal"."db"."upstream_model"': "internal.sqlmesh__db.db__upstream_model__999", + }, + dialect="doris", + default_catalog="internal", + doris_version=(2, 1, 9), + property_name="excluded_trigger_tables", + ) == exp.Literal.string("db__upstream_model__999") + with pytest.raises(SQLMeshError, match="Doris 2.1.10\\+"): + _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"lake_a"."db"."upstream_model"': "lake_a.sqlmesh__db.db__upstream_model__999", + }, + dialect="doris", + default_catalog="lake_a", + doris_version=(2, 1, 9), + property_name="excluded_trigger_tables", + ) + v219_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + v219_adapter._doris_version = (2, 1, 9) + assert v219_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("internal.db.t1,db.t2")} + ) == [ + exp.Property( + this=exp.Literal.string("excluded_trigger_tables"), + value=exp.Literal.string("t1,t2"), + ) + ] + with pytest.raises(SQLMeshError, match="Doris 2.1.10\\+"): + v219_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("lake_a.db.t1")} + ) assert to_sql_calls(existing_ref_adapter) == [] diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 01cea7a5f7..7b55647e77 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2791,3 +2791,23 @@ def test_doris(make_config): session_variables={"timeout": float("nan")}, check_import=False, ) + + import pymysql + + with patch.object(pymysql, "VERSION_STRING", "1.1.1", create=True): + with pytest.raises(ConfigError, match="requires 'pymysql>=1.2.0', but found '1.1.1'"): + make_config( + type="doris", + host="doris-fe", + user="admin", + check_import=True, + ) + + with patch.object(pymysql, "VERSION_STRING", "1.2.0", create=True): + valid_ver_cfg = make_config( + type="doris", + host="doris-fe", + user="admin", + check_import=True, + ) + assert isinstance(valid_ver_cfg, DorisConnectionConfig) From ecef824e615659f6fe23c44b366644c32ad67b7c Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:49:10 -0800 Subject: [PATCH 39/54] fix(doris): resolve CodeRabbit review findings on port default, MAP arrays, and MTMV property resolution Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 15 ++++++++++----- sqlmesh/core/config/connection.py | 6 ++++++ sqlmesh/core/dialect.py | 21 ++++++++++++++++++++- sqlmesh/core/engine_adapter/doris.py | 5 +++++ sqlmesh/core/model/definition.py | 19 +++++++++++++++---- tests/core/engine_adapter/test_doris.py | 24 ++++++++++++++++++++++++ tests/core/test_connection_config.py | 3 +++ tests/core/test_dialect.py | 11 +++++++++++ 8 files changed, 94 insertions(+), 10 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index ed7275f163..6f609f780f 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -1012,17 +1012,22 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; | --- | :---: | :---: | :---: | :---: | | Source / Upstream `SELECT` | Supported | Supported (incl. `FOR TIME/VERSION AS OF`, `@branch`, `@tag`) | Supported (Doris 2.1+) | Supported (Read-Only via corresponding External Catalog; Lance also supports `s3()` / `local()` TVFs) | | `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `TRUNCATE TABLE` + `INSERT`; non-atomic, so a failed `INSERT` leaves the table empty) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | -| `INCREMENTAL_BY_TIME_RANGE` | Supported (`DELETE WHERE time_col >= low AND time_col <= high` + `INSERT`) | Supported (`DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No row-level `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA row-level `DELETE`) | -| `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`, `DELETE ... USING` + `INSERT` or native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | -| `INCREMENTAL_BY_PARTITION` | Supported (Column & time-truncation range `DELETE` + `INSERT`) | Supported (Partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | +| `INCREMENTAL_BY_TIME_RANGE` | Supported (`DELETE WHERE time_col >= low AND time_col <= high` + `INSERT`; non-atomic, so concurrent readers may observe a gap and a failed `INSERT` leaves the interval missing) | Supported (`DELETE` + `INSERT`; non-atomic; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No row-level `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA row-level `DELETE`) | +| `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`; non-atomic `DELETE ... USING` + `INSERT`, or single-statement native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Non-atomic batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | +| `INCREMENTAL_BY_PARTITION` | Supported (Non-atomic column & time-truncation range `DELETE` + `INSERT`) | Supported (Non-atomic partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | | `SCD_TYPE_2_BY_TIME` / `SCD_TYPE_2_BY_COLUMN` | Supported | Supported (Requires Doris 4.1.0+ & Iceberg v2 for row-level `DELETE`) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | | `VIEW` | Supported (`CREATE OR REPLACE VIEW`; can query external catalog tables) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | Not supported as a `VIEW` target catalog (define the `VIEW` model in the `internal` catalog and reference external tables in its query) | | `VIEW (materialized true)` | Supported (Asynchronous MTMVs; synchronous MVs are base-table rollup indexes created separately on the base table) | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | Not supported by Doris inside external catalogs | | `SEED` / Python DataFrame Models | Supported (Automatic `VARCHAR(255)` OLAP key coercion) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Paimon: Read-Only in Doris 2.1–4.1.5; Lance/Hudi/Delta/LakeSoul: Write Guarded | | `CUSTOM ('doris_ddl_only')` | Supported (DDL + `ALTER TABLE` schema evolution only) | Supported | Supported (Requires Doris 2.1.6+ for writes) | Not supported (Read-Only) | -!!! note "Recovering from a Failed In-Place OLAP `FULL` Refresh" - Because Doris does not execute `TRUNCATE TABLE` and `INSERT INTO` inside a multi-statement transaction, an in-place OLAP `FULL` refresh that fails during `INSERT INTO` leaves the target table empty. To recover after fixing the failure, re-run the model (for example via `sqlmesh plan --restate-model ` or by re-applying the failed `sqlmesh plan`), since each `FULL` run rebuilds the table from its query. +!!! warning "Non-Atomic Replacements and Recovery for `FULL` and Incremental Strategies" + Doris does not execute `TRUNCATE TABLE` + `INSERT INTO` or `DELETE` + `INSERT INTO` inside a multi-statement transaction (`SUPPORTS_TRANSACTIONS = False`). As a result, concurrent readers can observe a temporary gap between the deletion and insertion steps, and if `INSERT INTO` fails after `TRUNCATE` or `DELETE` has already committed, the deleted rows are not rolled back automatically. Recover according to the model's strategy after resolving the underlying error: + + - **`FULL`**: A failed `INSERT` after `TRUNCATE TABLE` (or Iceberg full-table `DELETE`) leaves the target table empty. Re-run `sqlmesh plan --restate-model ` (or re-apply the failed `sqlmesh plan`) to rebuild the entire table from its query. + - **`INCREMENTAL_BY_TIME_RANGE`**: `DorisEngineAdapter` deletes rows where `time_column` falls within the batch `[start, end]` interval before inserting the replacement rows. If the `INSERT` fails, that `[start, end]` interval remains missing; inspect the failed batch window in the SQLMesh logs and run `sqlmesh plan --restate-model --start --end ` (or `sqlmesh run` if the interval was not marked completed in state) to re-populate the deleted time range. + - **`INCREMENTAL_BY_UNIQUE_KEY`**: When `when_matched` and `merge_filter` are omitted, `DorisEngineAdapter` stages incoming rows, executes `DELETE ... USING` (or batched key `DELETE` on Iceberg) for matching `unique_key` values, and then runs `INSERT INTO`. If the `INSERT` fails after the `DELETE`, those keys are missing from the target table. If the source query still contains the full history for those keys within the failed interval, restate that interval (`sqlmesh plan --restate-model --start --end `); if prior versions of those keys only existed in earlier intervals or an ephemeral source, restore from a table backup/snapshot or run a full restatement (`sqlmesh plan --restate-model `) against a complete source. (On Doris 4.1+ internal OLAP `UNIQUE KEY` tables with `when_matched` or `merge_filter`, updates use a single-statement native `MERGE INTO` instead of `DELETE` + `INSERT`.) + - **`INCREMENTAL_BY_PARTITION`**: `DorisEngineAdapter` determines the distinct partition keys (or `DATE_TRUNC` time buckets) present in the incoming source query, deletes all existing target rows matching those partitions, and then inserts the incoming rows. If the `INSERT` fails after the partition `DELETE`, all rows for those partition key values remain missing; re-run or restate the model (`sqlmesh plan --restate-model `) so the source query re-emits the complete data for the affected partitions. --- diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 2a5a0b33cc..b1f045d4c9 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2830,6 +2830,11 @@ def _engine_import_validator(cls, data: t.Any) -> t.Any: cls._validate_pymysql_version() return data + @field_validator("port", mode="before") + @classmethod + def _default_port_when_none(cls, v: t.Any) -> t.Any: + return 9030 if v is None else v + @field_validator("session_variables", mode="after") @classmethod def _validate_session_variables( @@ -2855,6 +2860,7 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: "autocommit": True, "charset": self.charset or "utf8mb4", "local_infile": False, + "port": self.port if self.port is not None else 9030, } raw_verify_mode = self.ssl.get("verify_mode") if isinstance(self.ssl, dict) else None if raw_verify_mode is not None: diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index f0264d86c3..d95f9b41e2 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -978,6 +978,7 @@ def _doris_parse_alter_drop(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], DORIS_TIMESTAMPTZ_MIN_VERSION: t.Tuple[int, int, int] = (4, 1, 0) DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION: t.Tuple[int, int, int] = (2, 1, 10) +DORIS_MAP_FROM_ARRAYS_MIN_VERSION: t.Tuple[int, int, int] = (4, 2, 0) def doris_supports_timestamptz(version: t.Optional[t.Tuple[int, ...]] = None) -> bool: @@ -990,13 +991,19 @@ def doris_supports_qualified_excluded_trigger_tables( version: t.Optional[t.Tuple[int, ...]] = None, ) -> bool: if version is None: - return True + return False padded = (*version, 0, 0)[:3] if padded[:2] == (3, 0): return padded >= (3, 0, 6) return padded >= DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION +def doris_supports_map_from_arrays(version: t.Optional[t.Tuple[int, ...]] = None) -> bool: + if not version or version[0] == sys.maxsize: + return False + return (*version, 0, 0)[:3] >= DORIS_MAP_FROM_ARRAYS_MIN_VERSION + + def _doris_datatype_sql(self: Generator, expression: exp.DataType) -> str: if expression.is_type(exp.DataType.Type.TIMESTAMPTZ) and not doris_supports_timestamptz( getattr(self.dialect, "version", None) @@ -1036,6 +1043,18 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: if isinstance(keys, exp.Array) and isinstance(values, exp.Array): return var_map_sql(self, expression) if keys is not None or values is not None: + if ( + isinstance(keys, exp.Expr) + and isinstance(values, exp.Expr) + and keys.is_type(exp.DataType.Type.ARRAY) + and values.is_type(exp.DataType.Type.ARRAY) + ): + if not doris_supports_map_from_arrays(getattr(self.dialect, "version", None)): + raise SQLMeshError( + "Doris does not support constructing a MAP from array columns prior to Doris 4.2 " + "(MAP_FROM_ARRAYS requires Doris 4.2+)." + ) + return self.func("MAP_FROM_ARRAYS", keys, values) return self.func("MAP", keys, values) return self.func("MAP", *expression.expressions) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index cf4088a5f1..a5b9499a51 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -18,6 +18,7 @@ from sqlglot.optimizer.qualify_columns import quote_identifiers from sqlmesh.core.dialect import ( + DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION, DORIS_TIMESTAMPTZ_MIN_VERSION, doris_supports_qualified_excluded_trigger_tables, doris_supports_timestamptz, @@ -2043,6 +2044,10 @@ def _supports_timestamptz(self) -> bool: def _supports_qualified_excluded_trigger_tables(self) -> bool: cached_version = getattr(self, "_doris_version", None) + if cached_version is None: + return doris_supports_qualified_excluded_trigger_tables( + DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION + ) return doris_supports_qualified_excluded_trigger_tables(cached_version) def _resolve_sync_mv_source_from_metadata( diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index c4bae07be6..7dd757ce77 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -735,7 +735,13 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An (render_kwargs.get("snapshots") or {}).values(), render_kwargs.get("deployability_index"), ) - doris_version = getattr(engine_adapter, "_doris_version", None) + discover_doris_version = getattr(engine_adapter, "_get_doris_version", None) + doris_version = ( + discover_doris_version() + if discover_doris_version + else getattr(engine_adapter, "_doris_version", None) + ) + engine_dialect = getattr(engine_adapter, "dialect", None) for key in keys_to_resolve: rendered[key] = _resolve_model_refs_to_physical_tables( rendered[key], @@ -744,6 +750,7 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An default_catalog=self.default_catalog, doris_version=doris_version, property_name=key, + engine_dialect=engine_dialect, ) return rendered @@ -2842,6 +2849,7 @@ def _resolve_model_refs_to_physical_tables( default_catalog: t.Optional[str] = None, doris_version: t.Optional[t.Tuple[int, ...]] = None, property_name: t.Optional[str] = None, + engine_dialect: t.Optional[DialectType] = None, ) -> exp.Literal: """Resolve managed-model references in a property value to their physical table names. @@ -2882,9 +2890,10 @@ def resolve(ref: str) -> str: physical = table_mapping.get(exp.table_name(table, identify=True)) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table + effective_engine_dialect = engine_dialect if engine_dialect is not None else dialect if ( - dialect == "doris" - and property_name in (None, "excluded_trigger_tables") + effective_engine_dialect == "doris" + and property_name == "excluded_trigger_tables" and not d.doris_supports_qualified_excluded_trigger_tables(doris_version) ): if resolved_table.catalog and resolved_table.catalog.lower() != "internal": @@ -2894,7 +2903,9 @@ def resolve(ref: str) -> str: ) resolved_table.set("catalog", None) resolved_table.set("db", None) - elif strip_catalog and (dialect != "doris" or resolved_table.catalog.lower() == "internal"): + elif strip_catalog and ( + effective_engine_dialect != "doris" or resolved_table.catalog.lower() == "internal" + ): resolved_table.set("catalog", None) return exp.table_name(resolved_table, identify=False) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 4137dfac0b..5d980eac0f 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -1821,6 +1821,7 @@ def test_resolve_table_refs_in_physical_properties_for_mv_and_sync_mv( ) ), ) + adapter._doris_version = (2, 1, 10) rendered_async_props = async_mv_model.render_physical_properties( snapshots=snapshots, engine_adapter=adapter ) @@ -2726,6 +2727,7 @@ def test_table_type_keys_random_distribution_mv_exists_and_tuple_ref_resolution( default_catalog="internal", ), ) + adapter._doris_version = (2, 1, 10) rendered_tuple_props = tuple_mv_model.render_physical_properties( snapshots=snapshots, engine_adapter=adapter ) @@ -6242,6 +6244,18 @@ def _cursor_init_internal(cur: t.Any) -> None: dialect="doris", default_catalog="lake_a", ) == exp.Literal.string("lake_a.sqlmesh__db.db__upstream_model__999") + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"db"."upstream_model"': "wrong_unqualified.db__upstream_model__111", + '"lake_a"."db"."upstream_model"': "lake_a.sqlmesh__db.db__upstream_model__999", + }, + dialect="mysql", + default_catalog="lake_a", + doris_version=(2, 1, 10), + property_name="excluded_trigger_tables", + engine_dialect="doris", + ) == exp.Literal.string("lake_a.sqlmesh__db.db__upstream_model__999") assert _resolve_model_refs_to_physical_tables( exp.Literal.string("db.upstream_model"), table_mapping={ @@ -6252,6 +6266,16 @@ def _cursor_init_internal(cur: t.Any) -> None: doris_version=(2, 1, 9), property_name="excluded_trigger_tables", ) == exp.Literal.string("db__upstream_model__999") + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db.upstream_model"), + table_mapping={ + '"internal"."db"."upstream_model"': "internal.sqlmesh__db.db__upstream_model__999", + }, + dialect="doris", + default_catalog="internal", + doris_version=None, + property_name="excluded_trigger_tables", + ) == exp.Literal.string("db__upstream_model__999") with pytest.raises(SQLMeshError, match="Doris 2.1.10\\+"): _resolve_model_refs_to_physical_tables( exp.Literal.string("db.upstream_model"), diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 7b55647e77..d841bdcf0d 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2808,6 +2808,9 @@ def test_doris(make_config): type="doris", host="doris-fe", user="admin", + port=None, check_import=True, ) assert isinstance(valid_ver_cfg, DorisConnectionConfig) + assert valid_ver_cfg.port == 9030 + assert valid_ver_cfg._connection_factory_with_kwargs.keywords["port"] == 9030 diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index c17561eb30..03fc883fa1 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1740,3 +1740,14 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): assert ( d.parse_one(insert_branch_cols_ast.sql("doris"), dialect="doris") == insert_branch_cols_ast ) + + k_arr = exp.column("k_arr") + k_arr.type = exp.DataType.build("ARRAY", dialect="doris") + v_arr = exp.column("v_arr") + v_arr.type = exp.DataType.build("ARRAY", dialect="doris") + map_from_arr_expr = exp.Map(keys=k_arr, values=v_arr) + with pytest.raises(d.SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"): + map_from_arr_expr.sql("doris") + with pytest.raises(d.SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"): + map_from_arr_expr.sql("doris, version=4.1.5") + assert map_from_arr_expr.sql("doris, version=4.2.0") == "MAP_FROM_ARRAYS(k_arr, v_arr)" From 8deab7ccf86537da585b5c2a8b35929a396f30ff Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:04:38 -0800 Subject: [PATCH 40/54] docs(doris): clarify native MERGE target requirements in model_kinds.md Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/concepts/models/model_kinds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts/models/model_kinds.md b/docs/concepts/models/model_kinds.md index d3a3c55df4..8fad86a7d7 100644 --- a/docs/concepts/models/model_kinds.md +++ b/docs/concepts/models/model_kinds.md @@ -670,7 +670,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_UNIQUE_KEY` kind a | Redshift | MERGE ON unique key | | Postgres | MERGE ON unique key | | DuckDB | DELETE ON matched + INSERT new rows | -| Doris | DELETE ON matched + INSERT new rows (or native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+; direct INSERT when a sequence column is configured) | +| Doris | DELETE ON matched + INSERT new rows (or native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+ internal OLAP `UNIQUE KEY` tables — external catalogs and `DUPLICATE KEY` / `AGGREGATE KEY` tables are not supported for native MERGE; direct INSERT when a sequence column is configured) | ## FULL Models of the `FULL` kind cause the dataset associated with a model to be fully refreshed (rewritten) upon each model evaluation. From 21eb5aa1ffcd74862984fc5c5e78b50d107873d7 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:29:12 -0800 Subject: [PATCH 41/54] docs,test(doris): link non-atomic incremental recovery in model_kinds.md and tighten Doris tests Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/concepts/models/model_kinds.md | 8 ++++---- .../engine_adapter/integration/test_integration_doris.py | 8 ++++++++ tests/core/test_connection_config.py | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/concepts/models/model_kinds.md b/docs/concepts/models/model_kinds.md index 8fad86a7d7..832715ba3d 100644 --- a/docs/concepts/models/model_kinds.md +++ b/docs/concepts/models/model_kinds.md @@ -444,7 +444,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_TIME_RANGE` kind a | Redshift | DELETE by time range, then INSERT | | Postgres | DELETE by time range, then INSERT | | DuckDB | DELETE by time range, then INSERT | -| Doris | DELETE by time range, then INSERT | +| Doris | DELETE by time range, then INSERT (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave the deleted interval missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)) | ## INCREMENTAL_BY_UNIQUE_KEY @@ -670,7 +670,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_UNIQUE_KEY` kind a | Redshift | MERGE ON unique key | | Postgres | MERGE ON unique key | | DuckDB | DELETE ON matched + INSERT new rows | -| Doris | DELETE ON matched + INSERT new rows (or native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+ internal OLAP `UNIQUE KEY` tables — external catalogs and `DUPLICATE KEY` / `AGGREGATE KEY` tables are not supported for native MERGE; direct INSERT when a sequence column is configured) | +| Doris | DELETE ON matched + INSERT new rows (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave deleted keys missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix); or single-statement native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+ internal OLAP `UNIQUE KEY` tables — external catalogs and `DUPLICATE KEY` / `AGGREGATE KEY` tables are not supported for native MERGE; direct INSERT when a sequence column is configured) | ## FULL Models of the `FULL` kind cause the dataset associated with a model to be fully refreshed (rewritten) upon each model evaluation. @@ -763,7 +763,7 @@ Depending on the target engine, models of the `FULL` kind are materialized using | Redshift | DROP TABLE, CREATE TABLE, INSERT | | Postgres | DROP TABLE, CREATE TABLE, INSERT | | DuckDB | CREATE OR REPLACE TABLE | -| Doris | CREATE TABLE ... AS SELECT (new OLAP), DELETE + INSERT (existing OLAP & Iceberg), INSERT OVERWRITE (unpartitioned Hive) | +| Doris | CREATE TABLE ... AS SELECT (new OLAP), TRUNCATE / DELETE + INSERT (existing OLAP & Doris 4.1+ Iceberg v2 — non-atomic: concurrent readers may observe a gap and failed inserts leave the table empty; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)), INSERT OVERWRITE (unpartitioned Hive on Doris 2.1.6+) | ## VIEW The model kinds described so far cause the output of a model query to be materialized and stored in a physical table. @@ -1633,7 +1633,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_PARTITION` kind ar | Redshift | DELETE by partitioning key, then INSERT | | Postgres | DELETE by partitioning key, then INSERT | | DuckDB | DELETE by partitioning key, then INSERT | -| Doris | DELETE by partitioning key, then INSERT | +| Doris | DELETE by partitioning key, then INSERT (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave deleted partitions missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)) | ## INCREMENTAL_UNMANAGED diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 676a6451a5..30cb5d0448 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -196,6 +196,12 @@ def test_materialized_view_sync(ctx: TestContext): }, ) + def rollup_present() -> bool: + rows = ctx.engine_adapter.fetchall(f"DESC {ctx.engine_adapter._to_sql(source_table)} ALL") + return any(bool(row and str(row[0]).strip().lower() == view.name.lower()) for row in rows) + + wait_until(rollup_present, attempts=30, wait=1) + def drop_view_success() -> bool: try: ctx.engine_adapter.drop_view( @@ -212,6 +218,7 @@ def drop_view_success() -> bool: return False wait_until(drop_view_success, attempts=30, wait=1) + wait_until(lambda: not rollup_present(), attempts=30, wait=1) def test_materialized_view_async_mtmv(ctx: TestContext): @@ -739,6 +746,7 @@ def test_doris_4_1_native_merge_segment_v3_and_ann_index(ctx: TestContext): f"INSERT INTO {ctx.engine_adapter._to_sql(merge_tbl)} (`id`, `name`, `val`) VALUES (1, 'orig_a', 10), (2, 'orig_b', 20)" ) src_query = parse_one( + "SELECT 0 AS id, 'filtered_zero' AS name, 99 AS val UNION ALL " "SELECT 1 AS id, 'new_a' AS name, 15 AS val UNION ALL " "SELECT 2 AS id, 'new_b' AS name, 5 AS val UNION ALL " "SELECT 3 AS id, 'new_c' AS name, 30 AS val", diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index d841bdcf0d..be96e5e026 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2792,7 +2792,7 @@ def test_doris(make_config): check_import=False, ) - import pymysql + pymysql = pytest.importorskip("pymysql") with patch.object(pymysql, "VERSION_STRING", "1.1.1", create=True): with pytest.raises(ConfigError, match="requires 'pymysql>=1.2.0', but found '1.1.1'"): From 27aa1feee2e1dc8a9cbf8d760727cbefd049fa2c Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:28:37 -0800 Subject: [PATCH 42/54] fix(doris): support ADD INDEX IF NOT EXISTS, invalidate dispatch cache, and fix merge_filter assertion Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- docs/integrations/engines/doris.md | 2 +- sqlmesh/core/config/connection.py | 11 +++++++ sqlmesh/core/dialect.py | 32 ++++++++++++++++++- .../integration/test_integration_doris.py | 3 +- tests/core/test_dialect.py | 16 ++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 6f609f780f..7607d5e428 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -53,7 +53,7 @@ pip install "sqlmesh[doris,postgres]" | `password` | The password to use for authentication with Doris FE (Default: `None`) | string | N | | `port` | The query port number of the Doris FE server (Default: `9030`) | int | N | | `database` | The optional default database (schema) name | string | N | -| `catalog` | The optional default catalog name (e.g., `internal` or an external `iceberg` catalog) | string | N | +| `catalog` | The optional default catalog name (e.g., `internal` or an external `iceberg` catalog). Omit in Doris-only projects for 2-part (`db.table`) names; set explicitly (e.g., `catalog: internal`) in multi-gateway projects that pair Doris with a catalog-qualified gateway (such as DuckDB, Snowflake, or Databricks) so model identifier depths match across gateways | string | N | | `charset` | The character set used for the connection (Default: `utf8mb4`) | string | N | | `collation` | The collation used for the connection | string | N | | `connect_timeout` | Connection timeout in seconds (Default: `10`) | int | N | diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index b1f045d4c9..6d4908c25f 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2725,6 +2725,9 @@ class DorisConnectionConfig(ConnectionConfig): port: The query port number of the Doris FE node. Default is 9030. database: The optional database name. catalog: The optional default catalog name (e.g., 'internal' or an external Iceberg catalog). + Defaults to None so Doris-only projects use 2-part (db.table) identifiers; in mixed-gateway + projects combining Doris with catalog-qualified engines (e.g., DuckDB, Snowflake, Databricks), + set catalog (e.g., 'internal') so model identifier depths match across gateways. charset: The optional character set (defaults to 'utf8mb4'). collation: The optional collation. connect_timeout: Timeout in seconds for establishing a connection. Default is 10. @@ -3068,6 +3071,14 @@ def init(cursor: t.Any) -> None: return init def get_catalog(self) -> t.Optional[str]: + """Return the configured default catalog. + + Defaults to ``None`` so single-gateway Doris projects use 2-part (``db.table``) names + without requiring 3-part catalog qualification in every SQL query. In mixed-gateway + projects that combine Doris with a catalog-qualified gateway (such as DuckDB, Snowflake, + or Databricks), configure ``catalog`` (for example, ``catalog: internal``) on the Doris + connection so model identifier depths match across gateways. + """ return self.catalog diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index d95f9b41e2..1ae99e1b74 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -14,6 +14,7 @@ from sqlglot.dialects.dialect import DialectType, rename_func, var_map_sql from sqlglot.dialects import Doris, DuckDB, Snowflake, TSQL import sqlglot.dialects.athena as athena +from sqlglot.generator import _DISPATCH_CACHE import sqlglot.generators.athena as athena_generators from sqlglot.parser import build_var_map from sqlglot.parsers.athena import AthenaTrinoParser @@ -838,7 +839,14 @@ def _doris_version_sql(self: Generator, expression: exp.Version) -> str: def _doris_parse_index_constraint( self: Parser, kind: t.Optional[str] = None ) -> exp.IndexColumnConstraint: - constraint = self.__parse_index_constraint(kind=kind) # type: ignore + if kind: + self._match_texts(("INDEX", "KEY")) + exists = self._parse_exists(not_=True) + constraint = self.__parse_index_constraint(kind=None) # type: ignore + if kind: + constraint.set("kind", kind) + if exists: + constraint.set("exists", True) options = list(constraint.args.get("options") or []) updated = False while True: @@ -866,6 +874,25 @@ def _doris_parse_index_constraint( _doris_parse_index_constraint.__name__ = "_parse_index_constraint" +def _doris_indexcolumnconstraint_sql(self: Generator, expression: exp.IndexColumnConstraint) -> str: + if not expression.args.get("exists"): + return self._indexcolumnconstraint_sql(expression) # type: ignore + kind = self.sql(expression, "kind") + kind = f"{kind} INDEX" if kind else "INDEX" + this = self.sql(expression, "this") + this = f" {this}" if this else "" + index_type = self.sql(expression, "index_type") + index_type = f" USING {index_type}" if index_type else "" + expressions = self.expressions(expression, flat=True) + expressions = f" ({expressions})" if expressions else "" + options = self.expressions(expression, key="options", sep=" ") + options = f" {options}" if options else "" + return f"{kind} IF NOT EXISTS{this}{index_type}{expressions}{options}" + + +_doris_indexcolumnconstraint_sql.__name__ = "indexcolumnconstraint_sql" + + def _doris_indexconstraintoption_sql(self: Generator, expression: exp.IndexConstraintOption) -> str: props = expression.args.get("parser") if isinstance(props, exp.Properties): @@ -1941,6 +1968,7 @@ def tokenize(self, sql: str) -> t.List[Token]: exp.Version: _doris_version_sql, } ) + _DISPATCH_CACHE.pop(Doris.Generator, None) tokenizers = {Tokenizer} parsers = {Parser} @@ -2041,9 +2069,11 @@ def tokenize(self, sql: str) -> t.List[Token]: _override(Doris.Generator, altercolumn_sql) _override(Doris.Generator, columndef_sql) _override(Doris.Generator, _doris_datatype_sql) + _override(Doris.Generator, _doris_indexcolumnconstraint_sql) _override(Doris.Generator, _doris_indexconstraintoption_sql) _override(Doris.Generator, _doris_indexparameters_sql) _override(Doris.Generator, _doris_alter_sql) + _DISPATCH_CACHE.pop(Doris.Generator, None) # DuckDB's prefix absolute power operator `@` clashes with the macro syntax DuckDB.Parser.NO_PAREN_FUNCTION_PARSERS.pop("@", None) diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 30cb5d0448..3e17cd8925 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -766,8 +766,9 @@ def test_doris_4_1_native_merge_segment_v3_and_ann_index(ctx: TestContext): merged_rows = ctx.engine_adapter.fetchall( exp.select("id", "name", "val").from_(merge_tbl).order_by("id") ) - # Row 1 updated val (15 > 10) while keeping name='orig_a'; Row 2 unchanged (5 < 20); Row 3 inserted + # Row 0 unmatched due to merge_filter and inserted by WHEN NOT MATCHED; Row 1 updated val (15 > 10) while keeping name='orig_a'; Row 2 unchanged (5 < 20); Row 3 inserted assert list(merged_rows) == [ + (0, "filtered_zero", 99), (1, "orig_a", 15), (2, "orig_b", 20), (3, "new_c", 30), diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 03fc883fa1..98c6a80c16 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1751,3 +1751,19 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): with pytest.raises(d.SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"): map_from_arr_expr.sql("doris, version=4.1.5") assert map_from_arr_expr.sql("doris, version=4.2.0") == "MAP_FROM_ARRAYS(k_arr, v_arr)" + + alter_idx_if_not_exists_sql = "ALTER TABLE t ADD INDEX IF NOT EXISTS idx_body (body) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'fts'" + alter_idx_if_not_exists_ast = d.parse_one(alter_idx_if_not_exists_sql, dialect="doris") + assert isinstance(alter_idx_if_not_exists_ast, exp.Alter) + assert isinstance(alter_idx_if_not_exists_ast.args["actions"][0], exp.IndexColumnConstraint) + assert alter_idx_if_not_exists_ast.args["actions"][0].args.get("exists") is True + assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql + + # Verify extend_sqlglot() invalidates _DISPATCH_CACHE for Doris.Generator + from sqlglot.dialects import Doris + from sqlglot.generator import _DISPATCH_CACHE + + _DISPATCH_CACHE[Doris.Generator] = {} + d.extend_sqlglot() + assert Doris.Generator not in _DISPATCH_CACHE + assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql From e4d841c6e4eb17e9cd3c50d34e85975517009852 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:45:59 -0800 Subject: [PATCH 43/54] fix(doris): probe server version for TIMESTAMPTZ DDL, reject NO_BACKSLASH_ESCAPES, and bound FE config wait Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- .github/scripts/wait-for-db.sh | 18 ++++++- docs/concepts/models/model_kinds.md | 2 +- docs/integrations/engines/doris.md | 4 +- sqlmesh/core/config/connection.py | 9 ++++ sqlmesh/core/engine_adapter/doris.py | 17 ++++++- tests/core/engine_adapter/test_doris.py | 15 +++++- tests/core/test_connection_config.py | 10 ++++ tests/core/test_dialect.py | 62 +++++++++++++++++++++++++ 8 files changed, 130 insertions(+), 7 deletions(-) diff --git a/.github/scripts/wait-for-db.sh b/.github/scripts/wait-for-db.sh index 118ae5b053..bd38cf5a54 100755 --- a/.github/scripts/wait-for-db.sh +++ b/.github/scripts/wait-for-db.sh @@ -201,7 +201,23 @@ doris_ready() { done # set default replication num to 1 (there is only one be in the docker compose file) - docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" + cfg_out="" + while true; do + remaining=$((deadline - SECONDS)) + if [ "$remaining" -le 0 ]; then + echo "ERROR: Timed out after ${timeout_secs}s configuring Doris frontend: $cfg_out" >&2 + exit 1 + fi + if cfg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" 2>&1); then + break + fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: Timed out after ${timeout_secs}s configuring Doris frontend: $cfg_out" >&2 + exit 1 + fi + echo "WARN: Failed to configure Doris frontend (will retry): $cfg_out" >&2 + sleep 2 + done } echo "Waiting for $ENGINE to be ready..." diff --git a/docs/concepts/models/model_kinds.md b/docs/concepts/models/model_kinds.md index 832715ba3d..5acac37aee 100644 --- a/docs/concepts/models/model_kinds.md +++ b/docs/concepts/models/model_kinds.md @@ -763,7 +763,7 @@ Depending on the target engine, models of the `FULL` kind are materialized using | Redshift | DROP TABLE, CREATE TABLE, INSERT | | Postgres | DROP TABLE, CREATE TABLE, INSERT | | DuckDB | CREATE OR REPLACE TABLE | -| Doris | CREATE TABLE ... AS SELECT (new OLAP), TRUNCATE / DELETE + INSERT (existing OLAP & Doris 4.1+ Iceberg v2 — non-atomic: concurrent readers may observe a gap and failed inserts leave the table empty; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)), INSERT OVERWRITE (unpartitioned Hive on Doris 2.1.6+) | +| Doris | CREATE TABLE ... AS SELECT (new OLAP), TRUNCATE / DELETE + INSERT (existing OLAP & Doris 4.1+ Iceberg v2 — non-atomic: concurrent readers may observe a gap and failed inserts leave the table empty or partially populated; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)), INSERT OVERWRITE (unpartitioned Hive on Doris 2.1.6+) | ## VIEW The model kinds described so far cause the output of a model query to be materialized and stored in a physical table. diff --git a/docs/integrations/engines/doris.md b/docs/integrations/engines/doris.md index 7607d5e428..ada9e07b27 100644 --- a/docs/integrations/engines/doris.md +++ b/docs/integrations/engines/doris.md @@ -1011,7 +1011,7 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; | SQLMesh Model Kind | Doris Internal OLAP Catalog (`internal` on Local Disk or S3/GCS Storage Vault) | External Iceberg Catalog (`table_format iceberg` on S3/GCS/HDFS) | External Hive Catalog (`table_format hive` on S3/GCS/HDFS) | External Read-Only Lakehouse Catalogs (`paimon`, `lance`, `hudi`, `deltalake`, `lakesoul`) | | --- | :---: | :---: | :---: | :---: | | Source / Upstream `SELECT` | Supported | Supported (incl. `FOR TIME/VERSION AS OF`, `@branch`, `@tag`) | Supported (Doris 2.1+) | Supported (Read-Only via corresponding External Catalog; Lance also supports `s3()` / `local()` TVFs) | -| `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `TRUNCATE TABLE` + `INSERT`; non-atomic, so a failed `INSERT` leaves the table empty) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | +| `FULL` | Supported (`CREATE TABLE ... AS SELECT` / `TRUNCATE TABLE` + `INSERT`; non-atomic, so a failed `INSERT` leaves the table empty or partially populated) | Supported (`CREATE TABLE ... ENGINE=iceberg` / `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2 for existing-table `DELETE`) | Supported on unpartitioned Hive tables (`CREATE TABLE ... ENGINE=hive` / `INSERT OVERWRITE TABLE`, requires Doris 2.1.6+ for writes; partitioned Hive `FULL` refreshes are not supported until full-table replacement is available) | Paimon: Read-Only in Doris 2.1–4.1.5 (write support tracked in `apache/doris#65086`); Lance/Hudi/Delta/LakeSoul: Read-Only (Write Guarded) | | `INCREMENTAL_BY_TIME_RANGE` | Supported (`DELETE WHERE time_col >= low AND time_col <= high` + `INSERT`; non-atomic, so concurrent readers may observe a gap and a failed `INSERT` leaves the interval missing) | Supported (`DELETE` + `INSERT`; non-atomic; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No row-level `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA row-level `DELETE`) | | `INCREMENTAL_BY_UNIQUE_KEY` | Supported (Auto-promotes `unique_key` to `UNIQUE KEY`; non-atomic `DELETE ... USING` + `INSERT`, or single-statement native `MERGE INTO` on Doris 4.1+ `UNIQUE KEY` tables when `when_matched` / `merge_filter` is specified) | Supported (Non-atomic batched key `DELETE` + `INSERT` when `when_matched` and `merge_filter` are omitted; native `MERGE INTO` is not supported on Iceberg targets; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE` / `MERGE INTO`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE` / `MERGE INTO`) | | `INCREMENTAL_BY_PARTITION` | Supported (Non-atomic column & time-truncation range `DELETE` + `INSERT`) | Supported (Non-atomic partition `DELETE` + `INSERT`; requires Doris 4.1.0+ & Iceberg v2) | Not supported by Hive in Doris (No `DELETE`) | Not supported in Doris 2.1–4.1.5 (No GA `DELETE`) | @@ -1024,7 +1024,7 @@ WHERE order_date BETWEEN @start_ds AND @end_ds; !!! warning "Non-Atomic Replacements and Recovery for `FULL` and Incremental Strategies" Doris does not execute `TRUNCATE TABLE` + `INSERT INTO` or `DELETE` + `INSERT INTO` inside a multi-statement transaction (`SUPPORTS_TRANSACTIONS = False`). As a result, concurrent readers can observe a temporary gap between the deletion and insertion steps, and if `INSERT INTO` fails after `TRUNCATE` or `DELETE` has already committed, the deleted rows are not rolled back automatically. Recover according to the model's strategy after resolving the underlying error: - - **`FULL`**: A failed `INSERT` after `TRUNCATE TABLE` (or Iceberg full-table `DELETE`) leaves the target table empty. Re-run `sqlmesh plan --restate-model ` (or re-apply the failed `sqlmesh plan`) to rebuild the entire table from its query. + - **`FULL`**: A failed `INSERT` after `TRUNCATE TABLE` (or Iceberg full-table `DELETE`) leaves the target table empty or partially populated. Re-run `sqlmesh plan --restate-model ` (or re-apply the failed `sqlmesh plan`) to rebuild the entire table from its query. - **`INCREMENTAL_BY_TIME_RANGE`**: `DorisEngineAdapter` deletes rows where `time_column` falls within the batch `[start, end]` interval before inserting the replacement rows. If the `INSERT` fails, that `[start, end]` interval remains missing; inspect the failed batch window in the SQLMesh logs and run `sqlmesh plan --restate-model --start --end ` (or `sqlmesh run` if the interval was not marked completed in state) to re-populate the deleted time range. - **`INCREMENTAL_BY_UNIQUE_KEY`**: When `when_matched` and `merge_filter` are omitted, `DorisEngineAdapter` stages incoming rows, executes `DELETE ... USING` (or batched key `DELETE` on Iceberg) for matching `unique_key` values, and then runs `INSERT INTO`. If the `INSERT` fails after the `DELETE`, those keys are missing from the target table. If the source query still contains the full history for those keys within the failed interval, restate that interval (`sqlmesh plan --restate-model --start --end `); if prior versions of those keys only existed in earlier intervals or an ephemeral source, restore from a table backup/snapshot or run a full restatement (`sqlmesh plan --restate-model `) against a complete source. (On Doris 4.1+ internal OLAP `UNIQUE KEY` tables with `when_matched` or `merge_filter`, updates use a single-statement native `MERGE INTO` instead of `DELETE` + `INSERT`.) - **`INCREMENTAL_BY_PARTITION`**: `DorisEngineAdapter` determines the distinct partition keys (or `DATE_TRUNC` time buckets) present in the incoming source query, deletes all existing target rows matching those partitions, and then inserts the incoming rows. If the `INSERT` fails after the partition `DELETE`, all rows for those partition key values remain missing; re-run or restate the model (`sqlmesh plan --restate-model `) so the source query re-emits the complete data for the affected partitions. diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 6d4908c25f..594ee73c4f 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2853,6 +2853,15 @@ def _validate_session_variables( isinstance(val, float) and (math.isnan(val) or math.isinf(val)) ): raise ConfigError(f"Invalid Doris session variable value for '{k}': {val!r}") + if ( + k.lower() == "sql_mode" + and isinstance(val, str) + and "no_backslash_escapes" in val.lower() + ): + raise ConfigError( + "Doris session variable 'sql_mode' cannot enable 'NO_BACKSLASH_ESCAPES' " + "because SQLGlot string literal escaping assumes standard backslash escaping." + ) return v @property diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index a5b9499a51..cdfdeac3b7 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -2038,6 +2038,8 @@ def _get_table_partition_names(self, table: exp.Table) -> t.Set[str]: def _supports_timestamptz(self) -> bool: cached_version = getattr(self, "_doris_version", None) + if cached_version is None: + cached_version = self._get_doris_version() if cached_version is None: return doris_supports_timestamptz(self.TIMESTAMPTZ_MIN_VERSION) return doris_supports_timestamptz(cached_version) @@ -5894,7 +5896,7 @@ def _collect_auto_branches(node: exp.Expr) -> None: new_select_exprs.append(proj) branch.set("expressions", new_select_exprs) - return super()._build_create_table_exp( + create_exp = super()._build_create_table_exp( table_name_or_schema, expression=expression, exists=exists, @@ -5904,6 +5906,19 @@ def _collect_auto_branches(node: exp.Expr) -> None: table_kind=table_kind, **kwargs, ) + if any(dtype.is_type("timestamptz") for dtype in create_exp.find_all(exp.DataType)): + if not self._supports_timestamptz(): + create_exp = create_exp.copy() + for dtype in reversed(list(create_exp.find_all(exp.DataType))): + if dtype.is_type("timestamptz"): + dtype.replace( + exp.DataType.build( + "DATETIME", + expressions=list(dtype.expressions), + dialect=self.dialect, + ) + ) + return create_exp @staticmethod def _is_integer_indicator_projection(proj: exp.Expr) -> bool: diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 5d980eac0f..c54c67b635 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6084,13 +6084,24 @@ def _exec_side_effect(sql_val: str, *a: t.Any, **kw: t.Any) -> None: assert exp.DataType.Type.TIMESTAMPTZ in v41_kwargs["parameterized_type_defaults"] v21_adapter = make_mocked_engine_adapter(DorisEngineAdapter) - v21_adapter._doris_version = (2, 1) + v21_adapter.cursor.fetchone.return_value = ("Doris version doris-2.1.7-rc03", "5.7.99") + v21_adapter.create_table( + "db.ts_tbl", + { + "id": exp.DataType.build("INT", dialect="doris"), + "ts": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + }, + ) + assert to_sql_calls(v21_adapter) == [ + "SELECT @@version_comment, VERSION()", + "CREATE TABLE IF NOT EXISTS `db`.`ts_tbl` (`id` INT, `ts` DATETIME(6))", + ] assert ( v21_adapter._to_sql(parse_one("SELECT CAST(ts AS TIMESTAMPTZ(6))", read="doris")) == "SELECT CAST(`ts` AS DATETIME(6))" ) v41_adapter = make_mocked_engine_adapter(DorisEngineAdapter) - v41_adapter._doris_version = (4, 1) + v41_adapter.cursor.fetchone.return_value = ("Doris version doris-4.1.0-rc01", "5.7.99") assert ( v41_adapter._to_sql(parse_one("SELECT CAST(ts AS TIMESTAMPTZ(6))", read="doris")) == "SELECT CAST(`ts` AS TIMESTAMPTZ(6))" diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index be96e5e026..11626f9b81 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2792,6 +2792,16 @@ def test_doris(make_config): check_import=False, ) + for forbidden_mode in ("NO_BACKSLASH_ESCAPES", "STRICT_TRANS_TABLES, no_backslash_escapes"): + with pytest.raises(ConfigError, match="NO_BACKSLASH_ESCAPES"): + make_config( + type="doris", + host="doris-fe", + user="admin", + session_variables={"SQL_MODE": forbidden_mode}, + check_import=False, + ) + pymysql = pytest.importorskip("pymysql") with patch.object(pymysql, "VERSION_STRING", "1.1.1", create=True): diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 98c6a80c16..719f5fe439 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1672,6 +1672,68 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): == "DATETIME(6)" ) + from unittest.mock import MagicMock + from sqlmesh.core.engine_adapter.doris import DorisEngineAdapter + + def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapter: + conn = MagicMock() + cur = MagicMock() + cur.fetchone.return_value = version_row + conn.cursor.return_value = cur + return DorisEngineAdapter(lambda: conn) + + fresh_v21_adapter = _make_fresh_doris_adapter(("Doris version doris-2.1.7-rc03", "5.7.99")) + v21_create_exp = fresh_v21_adapter._build_create_table_exp( + exp.Schema( + this=exp.to_table("db.events", dialect="doris"), + expressions=[ + exp.ColumnDef( + this=exp.to_identifier("id"), + kind=exp.DataType.build("INT", dialect="doris"), + ), + exp.ColumnDef( + this=exp.to_identifier("ts"), + kind=exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + ), + ], + ), + None, + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "ts": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + }, + ) + assert ( + fresh_v21_adapter._to_sql(v21_create_exp) + == "CREATE TABLE IF NOT EXISTS `db`.`events` (`id` INT, `ts` DATETIME(6))" + ) + + fresh_v41_adapter = _make_fresh_doris_adapter(("Doris version doris-4.1.0-rc01", "5.7.99")) + v41_create_exp = fresh_v41_adapter._build_create_table_exp( + exp.Schema( + this=exp.to_table("db.events", dialect="doris"), + expressions=[ + exp.ColumnDef( + this=exp.to_identifier("id"), + kind=exp.DataType.build("INT", dialect="doris"), + ), + exp.ColumnDef( + this=exp.to_identifier("ts"), + kind=exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + ), + ], + ), + None, + target_columns_to_types={ + "id": exp.DataType.build("INT", dialect="doris"), + "ts": exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris"), + }, + ) + assert ( + fresh_v41_adapter._to_sql(v41_create_exp) + == "CREATE TABLE IF NOT EXISTS `db`.`events` (`id` INT, `ts` TIMESTAMPTZ(6))" + ) + negated_match_ast = d.parse_one( "SELECT * FROM docs WHERE body NOT MATCH_ANY 'error warning'", dialect="doris", From 800046a90ec7dbefcaf0aa1282444bd6e29bb149 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:38:41 -0800 Subject: [PATCH 44/54] fix(doris): address CodeRabbit review feedback on sql_mode, ADD INDEX, and StarRocks refs Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/config/connection.py | 33 +++++++++++++++------ sqlmesh/core/dialect.py | 17 +++++++++-- sqlmesh/core/model/definition.py | 6 +++- tests/core/engine_adapter/test_starrocks.py | 25 ++++++++++++++++ tests/core/test_connection_config.py | 14 +++++++++ tests/core/test_dialect.py | 15 ++++++++++ 6 files changed, 97 insertions(+), 13 deletions(-) diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 594ee73c4f..8cda4e6a52 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2853,15 +2853,30 @@ def _validate_session_variables( isinstance(val, float) and (math.isnan(val) or math.isinf(val)) ): raise ConfigError(f"Invalid Doris session variable value for '{k}': {val!r}") - if ( - k.lower() == "sql_mode" - and isinstance(val, str) - and "no_backslash_escapes" in val.lower() - ): - raise ConfigError( - "Doris session variable 'sql_mode' cannot enable 'NO_BACKSLASH_ESCAPES' " - "because SQLGlot string literal escaping assumes standard backslash escaping." - ) + if k.lower() == "sql_mode": + if not isinstance(val, str): + raise ConfigError( + f"Doris session variable 'sql_mode' must be a string, got {val!r}." + ) + lowered_sql_mode = val.lower() + sql_modes = { + mode.strip().upper() for mode in re.split(r"[\s,]+", val) if mode.strip() + } + if "no_backslash_escapes" in lowered_sql_mode: + raise ConfigError( + "Doris session variable 'sql_mode' cannot enable 'NO_BACKSLASH_ESCAPES' " + "because SQLGlot string literal escaping assumes standard backslash escaping." + ) + if "pipes_as_concat" in lowered_sql_mode or "ANSI" in sql_modes: + raise ConfigError( + "Doris session variable 'sql_mode' cannot enable 'PIPES_AS_CONCAT' (or 'ANSI') " + "because the Doris SQLGlot dialect parses '||' as logical OR." + ) + if "ansi_quotes" in lowered_sql_mode: + raise ConfigError( + "Doris session variable 'sql_mode' cannot enable 'ANSI_QUOTES' " + "because the Doris SQLGlot dialect parses double-quoted tokens as string literals." + ) return v @property diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 1ae99e1b74..e5b1201bb1 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -993,7 +993,17 @@ def _doris_parse_alter_raw_action(self: Parser) -> exp.Var: def _doris_parse_alter_add(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], None]: if self._match(TokenType.INDEX): - return _doris_parse_index_constraint(self) + actions: t.List[exp.Expr] = [_doris_parse_index_constraint(self)] + while True: + comma_index = self._index + if not self._match(TokenType.COMMA): + break + self._match_text_seq("ADD") + if not self._match(TokenType.INDEX): + self._retreat(comma_index) + break + actions.append(_doris_parse_index_constraint(self)) + return actions return self._parse_alter_table_add() @@ -1056,8 +1066,9 @@ def _doris_alter_sql(self: Generator, expression: exp.Alter) -> str: else "" ) return f"BUILD INDEX {idx_sql} ON {tbl_sql}{parts_sql}" - if actions and isinstance(actions[0], exp.IndexColumnConstraint): - return f"ALTER TABLE {self.sql(expression, 'this')} ADD {self.expressions(expression, key='actions', flat=True)}" + if actions and all(isinstance(action, exp.IndexColumnConstraint) for action in actions): + add_actions = ", ".join(f"ADD {self.sql(action)}" for action in actions) + return f"ALTER TABLE {self.sql(expression, 'this')} {add_actions}" return self._alter_sql(expression) # type: ignore diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 7dd757ce77..fb87e0f859 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2887,10 +2887,14 @@ def resolve(ref: str) -> str: physical = table_mapping.get(normalized_name) strip_catalog = bool(physical) if not physical: - physical = table_mapping.get(exp.table_name(table, identify=True)) + physical = table_mapping.get(exp.table_name(table, identify=True)) or table_mapping.get( + d.normalize_model_name(table, default_catalog=default_catalog, dialect=dialect) + ) # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table effective_engine_dialect = engine_dialect if engine_dialect is not None else dialect + if physical and effective_engine_dialect != "doris" and resolved_table.catalog: + strip_catalog = True if ( effective_engine_dialect == "doris" and property_name == "excluded_trigger_tables" diff --git a/tests/core/engine_adapter/test_starrocks.py b/tests/core/engine_adapter/test_starrocks.py index b47dfb1e0d..95ee671680 100644 --- a/tests/core/engine_adapter/test_starrocks.py +++ b/tests/core/engine_adapter/test_starrocks.py @@ -2577,6 +2577,31 @@ def test_tuple_and_default_catalog_refs_are_resolved( f"{expected_catalog_physical},external_db.raw_table" ) + # Also verify explicit 3-part reference resolution strips the catalog to a 2-part StarRocks name + explicit_catalog_mv_model = _load_sql_model( + """ + MODEL ( + name starrocks.test_mv_explicit_catalog, + kind VIEW (materialized true), + dialect starrocks, + columns (a INT), + physical_properties ( + refresh_scheme = ASYNC, + excluded_trigger_tables = (default_catalog.starrocks.managed_model, external_db.raw_table) + ) + ); + SELECT 1 AS a; + """, + default_catalog="default_catalog", + ) + rendered_explicit_catalog = explicit_catalog_mv_model.render_physical_properties( + snapshots={catalog_base_snapshot.name: catalog_base_snapshot}, + engine_adapter=adapter, + ) + assert rendered_explicit_catalog["excluded_trigger_tables"] == exp.Literal.string( + f"{expected_catalog_physical},external_db.raw_table" + ) + class TestClusteringKeyOrdering: """StarRocks uses ClusteredByMixin but must KEEP the historical alter order. diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 11626f9b81..48b9ed4bc6 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2802,6 +2802,20 @@ def test_doris(make_config): check_import=False, ) + for forbidden_concat_mode in ( + "PIPES_AS_CONCAT", + "STRICT_TRANS_TABLES, pipes_as_concat", + "ANSI", + ): + with pytest.raises(ConfigError, match="PIPES_AS_CONCAT"): + make_config( + type="doris", + host="doris-fe", + user="admin", + session_variables={"sql_mode": forbidden_concat_mode}, + check_import=False, + ) + pymysql = pytest.importorskip("pymysql") with patch.object(pymysql, "VERSION_STRING", "1.1.1", create=True): diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 719f5fe439..61c6f44961 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1655,6 +1655,21 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): assert isinstance(drop_branch_ast, exp.Alter) assert drop_branch_ast.sql("doris") == "ALTER TABLE @this_model DROP BRANCH IF EXISTS b1" + multi_add_index_ast = d.parse_one( + "ALTER TABLE db.model ADD INDEX i1(a) USING INVERTED, ADD INDEX i2(b) USING INVERTED", + dialect="doris", + ) + assert isinstance(multi_add_index_ast, exp.Alter) + assert len(multi_add_index_ast.args["actions"]) == 2 + assert all( + isinstance(action, exp.IndexColumnConstraint) + for action in multi_add_index_ast.args["actions"] + ) + assert ( + multi_add_index_ast.sql("doris") + == "ALTER TABLE db.model ADD INDEX i1 (a) USING INVERTED, ADD INDEX i2 (b) USING INVERTED" + ) + assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" assert ( From 9e4d9033abc1a525c88943a6c8698e92d2bc53a6 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:58:55 -0800 Subject: [PATCH 45/54] fix(doris): address CodeRabbit feedback on project_init state_connection and quoted MATCH aliases Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/cli/project_init.py | 17 ++++++++++------- sqlmesh/core/dialect.py | 6 +++++- tests/core/engine_adapter/test_doris.py | 12 ++++++++++++ tests/core/test_dialect.py | 6 ++++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/sqlmesh/cli/project_init.py b/sqlmesh/cli/project_init.py index b149b44bb9..8638aba5a4 100644 --- a/sqlmesh/cli/project_init.py +++ b/sqlmesh/cli/project_init.py @@ -88,13 +88,16 @@ def _gen_config( " # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection\n" f" # {doc_link.format(engine_link=engine_link)}\n{connection_settings}" ) - if engine_type == "doris": - connection_settings += ( - " # Doris cannot be used as a state_connection; use DuckDB for local dev or Postgres/MySQL in production:\n" - " state_connection:\n" - " type: duckdb\n" - " database: state.db\n" - ) + + if engine_type == "doris" and template != ProjectTemplate.DBT: + if not connection_settings.endswith("\n"): + connection_settings += "\n" + connection_settings += ( + " # Doris cannot be used as a state_connection; use DuckDB for local dev or Postgres/MySQL in production:\n" + " state_connection:\n" + " type: duckdb\n" + " database: state.db\n" + ) default_configs = { ProjectTemplate.DEFAULT: f"""# --- Gateway Connection --- diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index e5b1201bb1..3387146de7 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -663,7 +663,11 @@ def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Opt if this is not None: idx = self._index negated = self._match(TokenType.NOT) - if self._curr and self._curr.text.upper() in _DORIS_MATCH_OPS: + if ( + self._curr + and self._curr.token_type not in (TokenType.IDENTIFIER, TokenType.STRING) + and self._curr.text.upper() in _DORIS_MATCH_OPS + ): op_name = self._curr.text.upper() self._advance() rhs = self._parse_bitwise() diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index c54c67b635..6ddba8d803 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6313,3 +6313,15 @@ def _cursor_init_internal(cur: t.Any) -> None: {"excluded_trigger_tables": exp.Literal.string("lake_a.db.t1")} ) assert to_sql_calls(existing_ref_adapter) == [] + + # 14. `_gen_config` appends the Doris `state_connection` even when custom settings are supplied (e.g. DLT template). + from sqlmesh.cli.project_init import InitCliMode, ProjectTemplate, _gen_config + + dlt_doris_config = _gen_config( + engine_type="doris", + settings=" type: doris\n host: 127.0.0.1\n user: root", + start="2024-01-01", + template=ProjectTemplate.DLT, + cli_mode=InitCliMode.DEFAULT, + ) + assert "state_connection:\n type: duckdb\n database: state.db" in dlt_doris_config diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 61c6f44961..757988d4bf 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1670,6 +1670,12 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): == "ALTER TABLE db.model ADD INDEX i1 (a) USING INVERTED, ADD INDEX i2 (b) USING INVERTED" ) + quoted_alias_ast = d.parse_one( + "SELECT body `match_any` FROM docs", + dialect="doris", + ) + assert quoted_alias_ast.sql("doris") == "SELECT body AS `match_any` FROM docs" + assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" assert ( From 4202ff529b851c36bd3a14a7b229a74a4004c734 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:26:27 -0800 Subject: [PATCH 46/54] fix(doris): address CodeRabbit review feedback on TLS, catalog handshake, and MAP_FROM_ARRAYS Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/config/connection.py | 10 ++++++-- sqlmesh/core/dialect.py | 20 ++++++--------- .../integration/test_integration_doris.py | 12 +++++++++ tests/core/engine_adapter/test_doris.py | 9 +++++-- tests/core/test_connection_config.py | 25 ++++++++++++++++++- tests/core/test_dialect.py | 5 ++++ 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 8cda4e6a52..e85044dba0 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2940,6 +2940,12 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]: ) ) ) + if (self.ssl_disabled is True and (self.ssl is True or isinstance(self.ssl, dict))) or ( + self.ssl is False and self.ssl_disabled is False + ): + raise ConfigError( + "Doris `ssl_disabled` and `ssl` have conflicting values (`ssl_disabled=True` with `ssl=True`/dict or `ssl=False` with `ssl_disabled=False`)." + ) if (self.ssl_disabled or self.ssl is False) and ( self.ssl_verify_cert is True or self.ssl_verify_identity is True @@ -3022,7 +3028,7 @@ def _extra_engine_config(self) -> t.Dict[str, t.Any]: @property def _connection_kwargs_keys(self) -> t.Set[str]: - is_external_catalog = bool(self.catalog and self.catalog.lower() != "internal") + has_catalog = bool(self.catalog) return { k for k in ( @@ -3038,7 +3044,7 @@ def _connection_kwargs_keys(self) -> t.Set[str]: "write_timeout", "ssl_disabled", ) - if getattr(self, k) is not None and not (k == "database" and is_external_catalog) + if getattr(self, k) is not None and not (k == "database" and has_catalog) } @property diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 3387146de7..0dc4a28777 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -1085,19 +1085,13 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: if isinstance(keys, exp.Array) and isinstance(values, exp.Array): return var_map_sql(self, expression) if keys is not None or values is not None: - if ( - isinstance(keys, exp.Expr) - and isinstance(values, exp.Expr) - and keys.is_type(exp.DataType.Type.ARRAY) - and values.is_type(exp.DataType.Type.ARRAY) - ): - if not doris_supports_map_from_arrays(getattr(self.dialect, "version", None)): - raise SQLMeshError( - "Doris does not support constructing a MAP from array columns prior to Doris 4.2 " - "(MAP_FROM_ARRAYS requires Doris 4.2+)." - ) - return self.func("MAP_FROM_ARRAYS", keys, values) - return self.func("MAP", keys, values) + # exp.Map with non-literal-array keys/values always means array-to-map construction. + if not doris_supports_map_from_arrays(getattr(self.dialect, "version", None)): + raise SQLMeshError( + "Doris does not support constructing a MAP from array columns prior to Doris 4.2 " + "(MAP_FROM_ARRAYS requires Doris 4.2+)." + ) + return self.func("MAP_FROM_ARRAYS", keys, values) return self.func("MAP", *expression.expressions) diff --git a/tests/core/engine_adapter/integration/test_integration_doris.py b/tests/core/engine_adapter/integration/test_integration_doris.py index 3e17cd8925..6454edaff4 100644 --- a/tests/core/engine_adapter/integration/test_integration_doris.py +++ b/tests/core/engine_adapter/integration/test_integration_doris.py @@ -95,6 +95,18 @@ def test_doris_iceberg_local_catalog(ctx: TestContext): ) adapter.execute(create_branch_expr, quote_identifiers=False) + insert_main_expr = parse_one( + f"INSERT INTO {table_fqn} VALUES (1, DATE '2024-01-01', 'us-east')", + read="doris", + ) + adapter.execute(insert_main_expr, quote_identifiers=False) + + main_read_expr = parse_one( + f"SELECT id FROM {table_fqn}", + read="doris", + ) + assert list(adapter.fetchall(main_read_expr, quote_identifiers=False)) == [(1,)] + branch_read_expr = parse_one( f"SELECT id FROM {table_fqn}@branch(audit_branch)", read="doris", diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 6ddba8d803..0b14088efd 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -3455,12 +3455,17 @@ def test_doris_connection_config_tls_matrix(mocker: MockerFixture): cfg_cert = DorisConnectionConfig(host="fe.internal", user="root", ssl_verify_cert=True) assert cfg_cert._static_connection_kwargs["ssl"] == {"verify_mode": True} - # 3. `ssl_disabled=True` overrides `ssl=True` and omits the `ssl` dictionary. + # 3. `ssl_disabled=True` rejects conflicting `ssl=True`, and omits `ssl` when `ssl_ca` is set without verification. cfg_disabled = DorisConnectionConfig( - host="fe.internal", user="root", ssl=True, ssl_disabled=True + host="fe.internal", user="root", ssl_ca="/path/to/ca.pem", ssl_disabled=True ) assert "ssl" not in cfg_disabled._static_connection_kwargs + with pytest.raises(ConfigError, match="conflicting values"): + _ = DorisConnectionConfig( + host="fe.internal", user="root", ssl=True, ssl_disabled=True + )._static_connection_kwargs + def test_doris_snapshot_evaluator_end_to_end_model_kinds( make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter], diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 48b9ed4bc6..003b20c2bd 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2816,6 +2816,29 @@ def test_doris(make_config): check_import=False, ) + ssl_true_and_disabled = make_config( + type="doris", + host="doris-fe", + user="admin", + ssl=True, + ssl_disabled=True, + check_import=False, + ) + with pytest.raises(ConfigError, match="conflicting values"): + _ = ssl_true_and_disabled._static_connection_kwargs + + internal_catalog_cfg = make_config( + type="doris", + host="doris-fe", + user="admin", + catalog="internal", + database="demo", + check_import=False, + ) + assert "database" not in internal_catalog_cfg._connection_kwargs_keys + + +def test_doris_pymysql_version(make_config): pymysql = pytest.importorskip("pymysql") with patch.object(pymysql, "VERSION_STRING", "1.1.1", create=True): @@ -2837,4 +2860,4 @@ def test_doris(make_config): ) assert isinstance(valid_ver_cfg, DorisConnectionConfig) assert valid_ver_cfg.port == 9030 - assert valid_ver_cfg._connection_factory_with_kwargs.keywords["port"] == 9030 + assert getattr(valid_ver_cfg._connection_factory_with_kwargs, "keywords")["port"] == 9030 diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 757988d4bf..95895ac945 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1835,6 +1835,11 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte map_from_arr_expr.sql("doris, version=4.1.5") assert map_from_arr_expr.sql("doris, version=4.2.0") == "MAP_FROM_ARRAYS(k_arr, v_arr)" + unannotated_map_from_arr = exp.Map(keys=exp.column("k_arr"), values=exp.column("v_arr")) + with pytest.raises(d.SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"): + unannotated_map_from_arr.sql("doris") + assert unannotated_map_from_arr.sql("doris, version=4.2.0") == "MAP_FROM_ARRAYS(k_arr, v_arr)" + alter_idx_if_not_exists_sql = "ALTER TABLE t ADD INDEX IF NOT EXISTS idx_body (body) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'fts'" alter_idx_if_not_exists_ast = d.parse_one(alter_idx_if_not_exists_sql, dialect="doris") assert isinstance(alter_idx_if_not_exists_ast, exp.Alter) From d730dfd713f9d0e103dbb95a838a86f6db3aaf19 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:52:19 -0800 Subject: [PATCH 47/54] fix(doris): preserve quoted branch/tag names and mock pymysql in test_doris Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/dialect.py | 22 ++++++++++++++-------- tests/core/test_connection_config.py | 3 ++- tests/core/test_dialect.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 0dc4a28777..b62368b5ba 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -721,8 +721,12 @@ def _doris_parse_table_parts( ref_kind = m.group(2).upper() ref_arg = table.this.expressions[0] if isinstance(ref_arg, exp.Column) and not ref_arg.table: - ref_arg = exp.var(ref_arg.name) - ver = exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg) + ref_arg = ( + ref_arg.this + if isinstance(ref_arg.this, exp.Identifier) and ref_arg.this.quoted + else exp.var(ref_arg.name) + ) + ver = self.expression(exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg)) table.set("this", exp.Identifier(this=real_name, quoted=False)) table.set("version", ver) table.meta["doris_version"] = ver @@ -736,13 +740,14 @@ def _doris_parse_table_parts( ref_arg = ref_args[0] if isinstance(ref_arg, exp.Identifier) and not ref_arg.quoted: ref_arg = exp.var(ref_arg.name) - ver = exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg) + ver = self.expression(exp.Version(this=ref_kind, kind="AS OF", expression=ref_arg)) table.set("this", exp.Identifier(this=real_name, quoted=table.this.quoted)) table.set("version", ver) table.meta["doris_version"] = ver - elif schema and ( - self._match_pair(TokenType.PARAMETER, TokenType.VAR, advance=False) - and self._next + elif ( + self._curr is not None + and self._curr.token_type == TokenType.PARAMETER + and self._next is not None and self._next.text.upper() in ("BRANCH", "TAG") ): parsed_ver = _doris_parse_version(self) @@ -760,8 +765,9 @@ def _doris_parse_version(self: Parser) -> t.Optional[exp.Version]: if ver is not None: return ver if ( - self._match_pair(TokenType.PARAMETER, TokenType.VAR, advance=False) - and self._next + self._curr is not None + and self._curr.token_type == TokenType.PARAMETER + and self._next is not None and self._next.text.upper() in ("BRANCH", "TAG") ): saved_idx = self._index diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 003b20c2bd..d757b4fa81 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2545,7 +2545,8 @@ def test_doris(make_config): assert advanced_config.get_catalog() == "iceberg_cat" assert advanced_config.concurrent_tasks == 10 assert "database" not in advanced_config._connection_kwargs_keys - assert "database" not in advanced_config._connection_factory_with_kwargs.keywords + with patch.dict("sys.modules", {"pymysql": MagicMock()}): + assert "database" not in advanced_config._connection_factory_with_kwargs.keywords assert advanced_config._static_connection_kwargs["ssl"] == { "ca": "/path/to/ca.pem", "verify_mode": True, diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 95895ac945..815f5cadfd 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1824,6 +1824,18 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte d.parse_one(insert_branch_cols_ast.sql("doris"), dialect="doris") == insert_branch_cols_ast ) + for quoted_ref_sql, expected_ref_sql in ( + ("SELECT * FROM tbl@branch(`release-2026`)", "SELECT * FROM tbl @branch(`release-2026`)"), + ("SELECT * FROM tbl@tag(`v1.0.0`)", "SELECT * FROM tbl @tag(`v1.0.0`)"), + ( + "INSERT INTO tbl@branch(`release-2026`) VALUES (1)", + "INSERT INTO tbl @branch(`release-2026`) VALUES (1)", + ), + ): + quoted_ref_ast = d.parse_one(quoted_ref_sql, dialect="doris") + assert quoted_ref_ast.sql("doris") == expected_ref_sql + assert d.parse_one(quoted_ref_ast.sql("doris"), dialect="doris") == quoted_ref_ast + k_arr = exp.column("k_arr") k_arr.type = exp.DataType.build("ARRAY", dialect="doris") v_arr = exp.column("v_arr") From f5dd5363f2d1b90ab8dedbaa7316bd08ae1d5f03 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:10:39 -0800 Subject: [PATCH 48/54] fix(doris): support DROP PARTITION DDL, adapter versioned MAP_FROM_ARRAYS, and disambiguate excluded_trigger_tables Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/dialect.py | 58 ++++++++++++++++++++++++- sqlmesh/core/engine_adapter/doris.py | 54 ++++++++++++++++++++++- sqlmesh/core/model/definition.py | 48 ++++++++++++++++++-- tests/core/engine_adapter/test_doris.py | 41 +++++++++++++++++ tests/core/test_dialect.py | 15 +++++++ 5 files changed, 209 insertions(+), 7 deletions(-) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index b62368b5ba..079ee27e62 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -1017,9 +1017,48 @@ def _doris_parse_alter_add(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], return self._parse_alter_table_add() +def _doris_parse_drop_partition_action(self: Parser) -> t.Optional[exp.DropPartition]: + start_index = self._index + temporary = bool(self._match(TokenType.TEMPORARY) or self._match_text_seq("TEMPORARY")) + if not (self._match(TokenType.PARTITION) or self._match_text_seq("PARTITION")): + self._retreat(start_index) + return None + exists = self._parse_exists() + if self._match(TokenType.L_PAREN, advance=False): + part_expr: t.Optional[exp.Expr] = self.expression( + exp.Partition(expressions=self._parse_wrapped_csv(self._parse_assignment)) + ) + else: + part_expr = self._parse_id_var(any_token=False) + if part_expr is None: + self._retreat(start_index) + return None + force = bool(self._match_text_seq("FORCE")) + drop_part = self.expression(exp.DropPartition(expressions=[part_expr], exists=exists)) + if temporary: + drop_part.set("temporary", True) + if force: + drop_part.set("force", True) + return drop_part + + def _doris_parse_alter_drop(self: Parser) -> t.Union[exp.Expr, t.List[exp.Expr], None]: if self._match_texts(("BRANCH", "TAG"), advance=False): return _doris_parse_alter_raw_action(self) + drop_partition = _doris_parse_drop_partition_action(self) + if drop_partition is not None: + actions: t.List[exp.Expr] = [drop_partition] + while True: + comma_index = self._index + if not self._match(TokenType.COMMA): + break + self._match(TokenType.DROP) + next_drop = _doris_parse_drop_partition_action(self) + if next_drop is None: + self._retreat(comma_index) + break + actions.append(next_drop) + return actions return self._parse_alter_table_drop() @@ -1085,6 +1124,19 @@ def _doris_alter_sql(self: Generator, expression: exp.Alter) -> str: _doris_alter_sql.__name__ = "alter_sql" +def _doris_droppartition_sql(self: Generator, expression: exp.DropPartition) -> str: + if any(isinstance(e, exp.Partition) for e in expression.expressions): + return self._droppartition_sql(expression) # type: ignore + temporary = " TEMPORARY" if expression.args.get("temporary") else "" + exists = " IF EXISTS" if expression.args.get("exists") else "" + force = " FORCE" if expression.args.get("force") else "" + partitions = self.expressions(expression, flat=True) + return f"DROP{temporary} PARTITION{exists} {partitions}{force}" + + +_doris_droppartition_sql.__name__ = "droppartition_sql" + + def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: keys = expression.args.get("keys") values = expression.args.get("values") @@ -1092,7 +1144,10 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: return var_map_sql(self, expression) if keys is not None or values is not None: # exp.Map with non-literal-array keys/values always means array-to-map construction. - if not doris_supports_map_from_arrays(getattr(self.dialect, "version", None)): + doris_version = expression.meta.get("doris_version") or getattr( + self.dialect, "version", None + ) + if not doris_supports_map_from_arrays(doris_version): raise SQLMeshError( "Doris does not support constructing a MAP from array columns prior to Doris 4.2 " "(MAP_FROM_ARRAYS requires Doris 4.2+)." @@ -2088,6 +2143,7 @@ def tokenize(self, sql: str) -> t.List[Token]: _override(Doris.Generator, _doris_indexconstraintoption_sql) _override(Doris.Generator, _doris_indexparameters_sql) _override(Doris.Generator, _doris_alter_sql) + _override(Doris.Generator, _doris_droppartition_sql) _DISPATCH_CACHE.pop(Doris.Generator, None) # DuckDB's prefix absolute power operator `@` clashes with the macro syntax diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index cdfdeac3b7..fefc314d39 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -1783,6 +1783,7 @@ def _create_materialized_view( else None ), table_kind="MATERIALIZED_VIEW", + query_expression=query, ) if is_sync_mv: @@ -7388,7 +7389,12 @@ def _build_table_properties_exp( # Default replication_num = 1 for single-BE dev/CI when not explicitly configured on OLAP tables # Note: Only emit explicit table_properties_copy entries so unit test expectations remain clean. if table_properties_copy: - properties.extend(self._properties_to_expressions(table_properties_copy)) + properties.extend( + self._properties_to_expressions( + table_properties_copy, + query_expression=kwargs.get("query_expression"), + ) + ) if properties: return exp.Properties(expressions=properties) @@ -7675,6 +7681,18 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) -> expression = struct_replacement else: dtype.replace(struct_replacement) + if any( + not ( + isinstance(m.args.get("keys"), exp.Array) + and isinstance(m.args.get("values"), exp.Array) + ) + and (m.args.get("keys") is not None or m.args.get("values") is not None) + for m in expression.find_all(exp.Map) + ): + self._get_doris_version() + doris_version = getattr(self, "_doris_version", None) + if doris_version and "dialect" not in kwargs: + kwargs["dialect"] = f"{self.dialect}, version={'.'.join(str(p) for p in doris_version)}" return super()._to_sql(expression, quote=quote, **kwargs) def _build_distributed_by_exp( @@ -7945,7 +7963,11 @@ def _get_temp_table( return table_copy - def _properties_to_expressions(self, properties: t.Dict[str, t.Any]) -> t.List[exp.Expr]: + def _properties_to_expressions( + self, + properties: t.Dict[str, t.Any], + query_expression: t.Optional[exp.Expr] = None, + ) -> t.List[exp.Expr]: """Convert a dictionary of properties to a list of Doris PROPERTIES ('k'='v') expressions.""" expressions: t.List[exp.Expr] = [] skip_keys = { @@ -7979,6 +8001,29 @@ def _properties_to_expressions(self, properties: t.Dict[str, t.Any]) -> t.List[e val_str = raw_str if raw_str is not None else str(value) if key == "excluded_trigger_tables" and val_str: supports_qualified = self._supports_qualified_excluded_trigger_tables() + dbs_by_table_name: t.Dict[str, t.Set[str]] = {} + if not supports_qualified: + if isinstance(query_expression, exp.Expr): + cte_names = { + cte.alias.lower() + for cte in query_expression.find_all(exp.CTE) + if cte.alias + } + for q_tbl in query_expression.find_all(exp.Table): + if q_tbl.name and ( + q_tbl.db or q_tbl.catalog or q_tbl.name.lower() not in cte_names + ): + dbs_by_table_name.setdefault(q_tbl.name.lower(), set()).add( + (q_tbl.db or "").lower() + ) + for raw_part in val_str.split(","): + raw_stripped = raw_part.strip() + if raw_stripped: + p_tbl = exp.to_table(raw_stripped, dialect=self.dialect) + if p_tbl.name and p_tbl.db: + dbs_by_table_name.setdefault(p_tbl.name.lower(), set()).add( + p_tbl.db.lower() + ) cleaned_parts: t.List[str] = [] for part in val_str.split(","): stripped = part.strip() @@ -7992,6 +8037,11 @@ def _properties_to_expressions(self, properties: t.Dict[str, t.Any]) -> t.List[e f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " f"got '{stripped}'." ) + if tbl.db and len(dbs_by_table_name.get(tbl.name.lower(), set())) > 1: + raise SQLMeshError( + f"Doris 'excluded_trigger_tables' prior to 2.1.10 (and 3.0.6) matches tables by bare name only " + f"and cannot disambiguate '{exp.table_name(tbl, identify=False)}' when the view references multiple tables named '{tbl.name}'." + ) cleaned_parts.append(tbl.name or without_internal) else: cleaned_parts.append(without_internal) diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index fb87e0f859..0601ffe256 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -742,6 +742,18 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An else getattr(engine_adapter, "_doris_version", None) ) engine_dialect = getattr(engine_adapter, "dialect", None) + referenced_tables: t.List[str | exp.Table] = list(self.depends_on or ()) + model_query = getattr(self, "query", None) + if isinstance(model_query, exp.Expr): + cte_names = { + cte.alias.lower() for cte in model_query.find_all(exp.CTE) if cte.alias + } + referenced_tables.extend( + t_node + for t_node in model_query.find_all(exp.Table) + if t_node.name + and (t_node.db or t_node.catalog or t_node.name.lower() not in cte_names) + ) for key in keys_to_resolve: rendered[key] = _resolve_model_refs_to_physical_tables( rendered[key], @@ -751,6 +763,7 @@ def render_physical_properties(self, **render_kwargs: t.Any) -> t.Dict[str, t.An doris_version=doris_version, property_name=key, engine_dialect=engine_dialect, + referenced_tables=referenced_tables, ) return rendered @@ -2850,6 +2863,7 @@ def _resolve_model_refs_to_physical_tables( doris_version: t.Optional[t.Tuple[int, ...]] = None, property_name: t.Optional[str] = None, engine_dialect: t.Optional[DialectType] = None, + referenced_tables: t.Optional[t.Iterable[str | exp.Table]] = None, ) -> exp.Literal: """Resolve managed-model references in a property value to their physical table names. @@ -2876,8 +2890,10 @@ def _resolve_model_refs_to_physical_tables( else: refs = [value.sql(dialect=dialect)] - def resolve(ref: str) -> str: - table = exp.to_table(ref.strip(), dialect=dialect) + def _to_physical_table(ref: str | exp.Table) -> t.Tuple[exp.Table, t.Optional[str], bool]: + table = ( + ref.copy() if isinstance(ref, exp.Table) else exp.to_table(ref.strip(), dialect=dialect) + ) physical: t.Optional[str] = None strip_catalog = False if default_catalog and not table.catalog: @@ -2890,9 +2906,25 @@ def resolve(ref: str) -> str: physical = table_mapping.get(exp.table_name(table, identify=True)) or table_mapping.get( d.normalize_model_name(table, default_catalog=default_catalog, dialect=dialect) ) - # Managed model -> physical table; otherwise keep the reference (just unquoted/normalized). resolved_table = exp.to_table(physical, dialect=dialect) if physical else table - effective_engine_dialect = engine_dialect if engine_dialect is not None else dialect + return resolved_table, physical, strip_catalog + + effective_engine_dialect = engine_dialect if engine_dialect is not None else dialect + dbs_by_table_name: t.Dict[str, t.Set[str]] = {} + if ( + effective_engine_dialect == "doris" + and property_name == "excluded_trigger_tables" + and not d.doris_supports_qualified_excluded_trigger_tables(doris_version) + ): + for candidate in [*(referenced_tables or ()), *(r for r in refs if r.strip())]: + cand_tbl, _, _ = _to_physical_table(candidate) + if cand_tbl.name and (cand_tbl.db or isinstance(candidate, exp.Table)): + dbs_by_table_name.setdefault(cand_tbl.name.lower(), set()).add( + (cand_tbl.db or "").lower() + ) + + def resolve(ref: str) -> str: + resolved_table, physical, strip_catalog = _to_physical_table(ref) if physical and effective_engine_dialect != "doris" and resolved_table.catalog: strip_catalog = True if ( @@ -2905,6 +2937,14 @@ def resolve(ref: str) -> str: f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " f"got '{exp.table_name(resolved_table, identify=False)}'." ) + if ( + resolved_table.db + and len(dbs_by_table_name.get(resolved_table.name.lower(), set())) > 1 + ): + raise SQLMeshError( + f"Doris 'excluded_trigger_tables' prior to 2.1.10 (and 3.0.6) matches tables by bare name only " + f"and cannot disambiguate '{exp.table_name(resolved_table, identify=False)}' when the view references multiple tables named '{resolved_table.name}'." + ) resolved_table.set("catalog", None) resolved_table.set("db", None) elif strip_catalog and ( diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index 0b14088efd..fa0321c3e5 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6317,6 +6317,47 @@ def _cursor_init_internal(cur: t.Any) -> None: v219_adapter._properties_to_expressions( {"excluded_trigger_tables": exp.Literal.string("lake_a.db.t1")} ) + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + _resolve_model_refs_to_physical_tables( + exp.Literal.string("db_a.orders"), + table_mapping={}, + dialect="doris", + doris_version=(2, 1, 9), + property_name="excluded_trigger_tables", + referenced_tables=[ + exp.to_table("db_a.orders", dialect="doris"), + exp.to_table("db_b.orders", dialect="doris"), + ], + ) + assert _resolve_model_refs_to_physical_tables( + exp.Literal.string("db_a.orders"), + table_mapping={}, + dialect="doris", + doris_version=(2, 1, 10), + property_name="excluded_trigger_tables", + referenced_tables=[ + exp.to_table("db_a.orders", dialect="doris"), + exp.to_table("db_b.orders", dialect="doris"), + ], + ) == exp.Literal.string("db_a.orders") + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + v219_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("db_a.orders")}, + query_expression=parse_one( + "SELECT * FROM db_a.orders JOIN db_b.orders ON db_a.orders.id = db_b.orders.id", + dialect="doris", + ), + ) + v420_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + v420_adapter.cursor.fetchone.return_value = ("Doris version doris-4.2.0-rc01", "5.7.99") + assert ( + v420_adapter._to_sql(exp.Map(keys=exp.column("k_arr"), values=exp.column("v_arr"))) + == "MAP_FROM_ARRAYS(`k_arr`, `v_arr`)" + ) + v415_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + v415_adapter.cursor.fetchone.return_value = ("Doris version doris-4.1.5-rc02", "5.7.99") + with pytest.raises(SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"): + v415_adapter._to_sql(exp.Map(keys=exp.column("k_arr"), values=exp.column("v_arr"))) assert to_sql_calls(existing_ref_adapter) == [] # 14. `_gen_config` appends the Doris `state_connection` even when custom settings are supplied (e.g. DLT template). diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 815f5cadfd..5f87fcff86 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1867,3 +1867,18 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte d.extend_sqlglot() assert Doris.Generator not in _DISPATCH_CACHE assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql + + for drop_part_sql in ( + "ALTER TABLE db.t DROP PARTITION p1", + "ALTER TABLE db.t DROP PARTITION IF EXISTS p1", + "ALTER TABLE db.t DROP PARTITION IF EXISTS p1 FORCE", + "ALTER TABLE db.t DROP TEMPORARY PARTITION IF EXISTS p1 FORCE", + "ALTER TABLE db.t DROP PARTITION p1, DROP PARTITION p2 FORCE", + ): + drop_part_ast = d.parse_one(drop_part_sql, dialect="doris") + assert isinstance(drop_part_ast, exp.Alter) + assert isinstance(drop_part_ast.args["actions"][0], exp.DropPartition) + assert drop_part_ast.copy().sql("doris") == drop_part_sql + + drop_col_sql = "ALTER TABLE db.t DROP COLUMN c1" + assert d.parse_one(drop_col_sql, dialect="doris").sql("doris") == drop_col_sql From e5cbf6cd8e74b9badcdcc9c66c64b22aeae7ffd1 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:04:03 -0800 Subject: [PATCH 49/54] fix(doris): preserve MATCH precedence after simplify_parens and validate MAP literal array lengths Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/dialect.py | 33 ++++++++++++++++++++++++++++++--- tests/core/test_dialect.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 079ee27e62..ba0f831143 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -691,14 +691,36 @@ def _doris_parse_range(self: Parser, this: t.Optional[exp.Expr] = None) -> t.Opt def _doris_anonymous_sql(self: Generator, expression: exp.Anonymous) -> str: if expression.name.upper() in _DORIS_MATCH_OPS and len(expression.expressions) in (2, 3): - left = self.sql(expression.expressions[0]) - right = self.sql(expression.expressions[1]) + left_expr = expression.expressions[0] + right_expr = expression.expressions[1] + left = self.sql(left_expr) + if isinstance(left_expr, (exp.Connector, exp.Not, exp.Predicate)) or ( + isinstance(left_expr, exp.Anonymous) and left_expr.name.upper() in _DORIS_MATCH_OPS + ): + left = f"({left})" + right = self.sql(right_expr) + if isinstance(right_expr, (exp.Connector, exp.Not, exp.Predicate)) or ( + isinstance(right_expr, exp.Anonymous) and right_expr.name.upper() in _DORIS_MATCH_OPS + ): + right = f"({right})" analyzer_sql = ( f" USING ANALYZER {self.sql(expression.expressions[2])}" if len(expression.expressions) == 3 else "" ) - return f"{left} {expression.name.upper()} {right}{analyzer_sql}" + sql = f"{left} {expression.name.upper()} {right}{analyzer_sql}" + if ( + isinstance( + expression.parent, + (exp.Binary, exp.Unary, exp.Predicate, exp.Between, exp.In), + ) + and not isinstance(expression.parent, (exp.Connector, exp.Not, exp.Paren)) + ) or ( + isinstance(expression.parent, exp.Anonymous) + and expression.parent.name.upper() in _DORIS_MATCH_OPS + ): + return f"({sql})" + return sql return self.anonymous_sql(expression) @@ -1141,6 +1163,11 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: keys = expression.args.get("keys") values = expression.args.get("values") if isinstance(keys, exp.Array) and isinstance(values, exp.Array): + if len(keys.expressions) != len(values.expressions): + raise SQLMeshError( + "Doris MAP requires key and value arrays to have the same number of elements " + f"(got {len(keys.expressions)} keys and {len(values.expressions)} values)." + ) return var_map_sql(self, expression) if keys is not None or values is not None: # exp.Map with non-literal-array keys/values always means array-to-map construction. diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 5f87fcff86..a6daf0fd20 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1882,3 +1882,35 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte drop_col_sql = "ALTER TABLE db.t DROP COLUMN c1" assert d.parse_one(drop_col_sql, dialect="doris").sql("doris") == drop_col_sql + + with pytest.raises( + d.SQLMeshError, + match="Doris MAP requires key and value arrays to have the same number of elements", + ): + exp.Map( + keys=exp.Array(expressions=[exp.Literal.number(1), exp.Literal.number(2)]), + values=exp.Array(expressions=[exp.Literal.number(10)]), + ).sql("doris") + + from sqlmesh.core.model import load_sql_based_model + + match_arith_model = load_sql_based_model( + d.parse( + """ + MODEL ( + name db.match_scores, + kind FULL, + dialect doris + ); + SELECT (body MATCH_ANY 'x') + 1 AS score FROM db.docs + """, + default_dialect="doris", + ), + mapping_schema={"db": {"docs": {"body": "STRING"}}}, + ) + rendered_match_query = match_arith_model.render_query(optimize=True) + assert rendered_match_query is not None + assert ( + rendered_match_query.sql("doris") + == "SELECT (`docs`.`body` MATCH_ANY 'x') + 1 AS `score` FROM `db`.`docs` AS `docs`" + ) From f7f68b33b51b966986e4594510f27a8411c66602 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:47:56 -0800 Subject: [PATCH 50/54] fix(doris): unify unknown version excluded_trigger_tables handling and evict failed cursor_init connections Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/engine_adapter/doris.py | 5 +---- sqlmesh/utils/connection_pool.py | 20 ++++++++++++++++++++ tests/core/engine_adapter/test_doris.py | 15 +++++++++++++++ tests/core/test_connection_config.py | 22 +++++++++++++++++----- 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index fefc314d39..502904bba0 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -18,7 +18,6 @@ from sqlglot.optimizer.qualify_columns import quote_identifiers from sqlmesh.core.dialect import ( - DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION, DORIS_TIMESTAMPTZ_MIN_VERSION, doris_supports_qualified_excluded_trigger_tables, doris_supports_timestamptz, @@ -2048,9 +2047,7 @@ def _supports_timestamptz(self) -> bool: def _supports_qualified_excluded_trigger_tables(self) -> bool: cached_version = getattr(self, "_doris_version", None) if cached_version is None: - return doris_supports_qualified_excluded_trigger_tables( - DORIS_QUALIFIED_EXCLUDED_TRIGGER_TABLES_MIN_VERSION - ) + cached_version = self._get_doris_version() return doris_supports_qualified_excluded_trigger_tables(cached_version) def _resolve_sync_mv_source_from_metadata( diff --git a/sqlmesh/utils/connection_pool.py b/sqlmesh/utils/connection_pool.py index 2a3d5a40ef..12701b8600 100644 --- a/sqlmesh/utils/connection_pool.py +++ b/sqlmesh/utils/connection_pool.py @@ -146,6 +146,7 @@ def get_cursor(self) -> t.Any: self._cursor_init(cursor) except Exception: _try_close(cursor, "cursor") + self._reset_connection_on_cursor_init_failure(thread_id) raise self._thread_cursors[thread_id] = cursor return self._thread_cursors[thread_id] @@ -195,6 +196,9 @@ def _discard_transaction(self, thread_id: t.Hashable) -> None: with self._thread_transactions_lock: self._thread_transactions.discard(thread_id) + def _reset_connection_on_cursor_init_failure(self, thread_id: t.Hashable) -> None: + self._discard_transaction(thread_id) + class ThreadLocalConnectionPool(_ThreadLocalBase): def __init__( @@ -206,6 +210,12 @@ def __init__( self._thread_connections: t.Dict[t.Hashable, t.Any] = {} self._thread_connections_lock = Lock() + def _reset_connection_on_cursor_init_failure(self, thread_id: t.Hashable) -> None: + with self._thread_connections_lock: + if thread_id in self._thread_connections: + _try_close(self._thread_connections.pop(thread_id), "connection") + self._discard_transaction(thread_id) + def get(self) -> t.Any: thread_id = get_ident() with self._thread_connections_lock: @@ -246,6 +256,13 @@ def __init__( self._connection: t.Optional[t.Any] = None self._connection_lock = Lock() + def _reset_connection_on_cursor_init_failure(self, thread_id: t.Hashable) -> None: + with self._connection_lock: + if not self._thread_cursors and self._connection is not None: + _try_close(self._connection, "connection") + self._connection = None + self._discard_transaction(thread_id) + def get(self) -> t.Any: with self._connection_lock: if self._connection is None: @@ -297,6 +314,9 @@ def get_cursor(self) -> t.Any: self._cursor_init(cursor) except Exception: _try_close(cursor, "cursor") + _try_close(self._connection, "connection") + self._connection = None + self._is_transaction_active = False raise self._cursor = cursor return self._cursor diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index fa0321c3e5..f4ebb0c2b0 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -2426,6 +2426,7 @@ def test_unannotated_view_column_comments_and_starrocks_property_parity( # 2. Parenthesized string forms for unique_key and order_by, structured distributed_by with `columns=...`, # and tuple generic properties (e.g. excluded_trigger_tables = ('db.t1', 'db.t2')). + adapter._doris_version = (2, 1, 10) adapter.create_table( "db.paren_keys_tbl", target_columns_to_types={ @@ -6317,6 +6318,20 @@ def _cursor_init_internal(cur: t.Any) -> None: v219_adapter._properties_to_expressions( {"excluded_trigger_tables": exp.Literal.string("lake_a.db.t1")} ) + unknown_ver_adapter = make_mocked_engine_adapter(DorisEngineAdapter) + mocker.patch.object(unknown_ver_adapter, "_get_doris_version", return_value=None) + assert unknown_ver_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("internal.db.t1,db.t2")} + ) == [ + exp.Property( + this=exp.Literal.string("excluded_trigger_tables"), + value=exp.Literal.string("t1,t2"), + ) + ] + with pytest.raises(SQLMeshError, match="Doris 2.1.10\\+"): + unknown_ver_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("lake_a.db.t1")} + ) with pytest.raises(SQLMeshError, match="cannot disambiguate"): _resolve_model_refs_to_physical_tables( exp.Literal.string("db_a.orders"), diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index d757b4fa81..bf3dd76b4a 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2748,9 +2748,17 @@ def test_doris(make_config): with pytest.raises(ConfigError, match="requires `ssl_ca`"): _ = ssl_dict_check_hostname_no_ca._static_connection_kwargs - from sqlmesh.utils.connection_pool import SingletonConnectionPool, ThreadLocalConnectionPool + from sqlmesh.utils.connection_pool import ( + SingletonConnectionPool, + ThreadLocalConnectionPool, + ThreadLocalSharedConnectionPool, + ) - for pool_cls in (SingletonConnectionPool, ThreadLocalConnectionPool): + for pool_cls in ( + SingletonConnectionPool, + ThreadLocalConnectionPool, + ThreadLocalSharedConnectionPool, + ): failing_cursor = MagicMock() failing_cursor.execute.side_effect = [ None, @@ -2758,13 +2766,17 @@ def test_doris(make_config): RuntimeError("SWITCH failed"), ] healthy_cursor = MagicMock() - mock_conn = MagicMock() - mock_conn.cursor.side_effect = [failing_cursor, healthy_cursor] - pool = pool_cls(lambda: mock_conn, cursor_init=advanced_config._cursor_init) + failing_conn = MagicMock() + failing_conn.cursor.return_value = failing_cursor + healthy_conn = MagicMock() + healthy_conn.cursor.return_value = healthy_cursor + conns = iter([failing_conn, healthy_conn]) + pool = pool_cls(lambda: next(conns), cursor_init=advanced_config._cursor_init) with pytest.raises(RuntimeError, match="SWITCH failed"): pool.get_cursor() failing_cursor.close.assert_called() + failing_conn.close.assert_called_once() recovered_cursor = pool.get_cursor() assert recovered_cursor is healthy_cursor From cb211e1302f43bb67691c2dc43fc06394cd45050 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:07:30 -0800 Subject: [PATCH 51/54] fix(doris): map TIMESTAMP keyword to DATETIME and preserve db case in excluded_trigger_tables Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/dialect.py | 1 + sqlmesh/core/engine_adapter/doris.py | 4 ++-- sqlmesh/core/model/definition.py | 4 +--- tests/core/engine_adapter/test_doris.py | 20 ++++++++++++++++++++ tests/core/test_dialect.py | 9 +++++++++ 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index ba0f831143..ee9ce21304 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -1987,6 +1987,7 @@ class _DorisTokenizer(Doris.tokenizer_class): # type: ignore "JSONB": TokenType.JSONB, "LARGEINT": TokenType.INT128, "QUANTILE_STATE": TokenType.USERDEFINED, + "TIMESTAMP": TokenType.DATETIME, "TIMESTAMPTZ": ( TokenType.TIMESTAMPTZ if doris_supports_timestamptz(TIMESTAMPTZ_MIN_VERSION) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index 502904bba0..a71675883d 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -8011,7 +8011,7 @@ def _properties_to_expressions( q_tbl.db or q_tbl.catalog or q_tbl.name.lower() not in cte_names ): dbs_by_table_name.setdefault(q_tbl.name.lower(), set()).add( - (q_tbl.db or "").lower() + q_tbl.db or "" ) for raw_part in val_str.split(","): raw_stripped = raw_part.strip() @@ -8019,7 +8019,7 @@ def _properties_to_expressions( p_tbl = exp.to_table(raw_stripped, dialect=self.dialect) if p_tbl.name and p_tbl.db: dbs_by_table_name.setdefault(p_tbl.name.lower(), set()).add( - p_tbl.db.lower() + p_tbl.db ) cleaned_parts: t.List[str] = [] for part in val_str.split(","): diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 0601ffe256..a6fe4a9d4d 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2919,9 +2919,7 @@ def _to_physical_table(ref: str | exp.Table) -> t.Tuple[exp.Table, t.Optional[st for candidate in [*(referenced_tables or ()), *(r for r in refs if r.strip())]: cand_tbl, _, _ = _to_physical_table(candidate) if cand_tbl.name and (cand_tbl.db or isinstance(candidate, exp.Table)): - dbs_by_table_name.setdefault(cand_tbl.name.lower(), set()).add( - (cand_tbl.db or "").lower() - ) + dbs_by_table_name.setdefault(cand_tbl.name.lower(), set()).add(cand_tbl.db or "") def resolve(ref: str) -> str: resolved_table, physical, strip_catalog = _to_physical_table(ref) diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index f4ebb0c2b0..ff552a5e26 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6363,6 +6363,26 @@ def _cursor_init_internal(cur: t.Any) -> None: dialect="doris", ), ) + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + _resolve_model_refs_to_physical_tables( + exp.Literal.string("Sales.orders"), + table_mapping={}, + dialect="doris", + doris_version=(2, 1, 9), + property_name="excluded_trigger_tables", + referenced_tables=[ + exp.to_table("Sales.orders", dialect="doris"), + exp.to_table("sales.orders", dialect="doris"), + ], + ) + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + v219_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("Sales.orders")}, + query_expression=parse_one( + "SELECT * FROM Sales.orders JOIN sales.orders ON Sales.orders.id = sales.orders.id", + dialect="doris", + ), + ) v420_adapter = make_mocked_engine_adapter(DorisEngineAdapter) v420_adapter.cursor.fetchone.return_value = ("Doris version doris-4.2.0-rc01", "5.7.99") assert ( diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index a6daf0fd20..d6c6143e55 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1914,3 +1914,12 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte rendered_match_query.sql("doris") == "SELECT (`docs`.`body` MATCH_ANY 'x') + 1 AS `score` FROM `db`.`docs` AS `docs`" ) + + assert ( + d.parse_one("SELECT TIMESTAMP '2020-01-01 00:00:00'", dialect="doris").sql("doris") + == "SELECT CAST('2020-01-01 00:00:00' AS DATETIME)" + ) + assert ( + d.parse_one("SELECT TIMESTAMPTZ '2020-01-01 00:00:00+08:00'", dialect="doris").sql("doris") + == "SELECT CAST('2020-01-01 00:00:00+08:00' AS TIMESTAMPTZ)" + ) From 57c0b3a838db4639fe2fb6b8a657b25400af5349 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:33:04 -0800 Subject: [PATCH 52/54] test(doris): split Doris dialect tests by feature and isolate dispatch cache test Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- tests/core/test_dialect.py | 206 ++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 106 deletions(-) diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index d6c6143e55..928586d4eb 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1607,7 +1607,7 @@ def test_extend_sqlglot_is_idempotent(): assert Generator.UNWRAPPED_INTERVAL_VALUES == before -def test_doris_userdefined_isolation_and_modify_column_not_null(): +def test_doris_userdefined_isolation(): # Non-Doris dialects must keep standard INFORMATION_SCHEMA `USER-DEFINED` behavior (`kind=None`) for other_dialect in ("postgres", "mysql", "snowflake"): other_udt = exp.DataType.build("USER-DEFINED", dialect=other_dialect) @@ -1619,6 +1619,8 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): doris_udt = exp.DataType.build(doris_udt_sql, dialect="doris", udt=True) assert doris_udt.sql("doris") == doris_udt_sql + +def test_doris_alter_table_parsing(): # Doris `ALTER TABLE ... MODIFY COLUMN ... NOT NULL` parses as structured `exp.Alter` # (enabling `@this_model` macro resolution) and preserves `NOT NULL` on generation. alter_ast = d.parse_one( @@ -1670,11 +1672,32 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): == "ALTER TABLE db.model ADD INDEX i1 (a) USING INVERTED, ADD INDEX i2 (b) USING INVERTED" ) - quoted_alias_ast = d.parse_one( - "SELECT body `match_any` FROM docs", - dialect="doris", - ) - assert quoted_alias_ast.sql("doris") == "SELECT body AS `match_any` FROM docs" + alter_idx_if_not_exists_sql = "ALTER TABLE t ADD INDEX IF NOT EXISTS idx_body (body) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'fts'" + alter_idx_if_not_exists_ast = d.parse_one(alter_idx_if_not_exists_sql, dialect="doris") + assert isinstance(alter_idx_if_not_exists_ast, exp.Alter) + assert isinstance(alter_idx_if_not_exists_ast.args["actions"][0], exp.IndexColumnConstraint) + assert alter_idx_if_not_exists_ast.args["actions"][0].args.get("exists") is True + assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql + + for drop_part_sql in ( + "ALTER TABLE db.t DROP PARTITION p1", + "ALTER TABLE db.t DROP PARTITION IF EXISTS p1", + "ALTER TABLE db.t DROP PARTITION IF EXISTS p1 FORCE", + "ALTER TABLE db.t DROP TEMPORARY PARTITION IF EXISTS p1 FORCE", + "ALTER TABLE db.t DROP PARTITION p1, DROP PARTITION p2 FORCE", + ): + drop_part_item = d.parse_one(drop_part_sql, dialect="doris") + assert isinstance(drop_part_item, exp.Alter) + assert isinstance(drop_part_item.args["actions"][0], exp.DropPartition) + assert drop_part_item.copy().sql("doris") == drop_part_sql + + drop_col_sql = "ALTER TABLE db.t DROP COLUMN c1" + assert d.parse_one(drop_col_sql, dialect="doris").sql("doris") == drop_col_sql + + +def test_doris_timestamptz_and_timestamp_literals(): + from unittest.mock import MagicMock + from sqlmesh.core.engine_adapter.doris import DorisEngineAdapter assert exp.DataType.build("TIMESTAMPTZ", dialect="doris").sql("doris") == "TIMESTAMPTZ" assert exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris") == "TIMESTAMPTZ(6)" @@ -1692,9 +1715,14 @@ def test_doris_userdefined_isolation_and_modify_column_not_null(): exp.DataType.build("TIMESTAMPTZ(6)", dialect="doris").sql("doris, version=2.1") == "DATETIME(6)" ) - - from unittest.mock import MagicMock - from sqlmesh.core.engine_adapter.doris import DorisEngineAdapter + assert ( + d.parse_one("SELECT TIMESTAMP '2020-01-01 00:00:00'", dialect="doris").sql("doris") + == "SELECT CAST('2020-01-01 00:00:00' AS DATETIME)" + ) + assert ( + d.parse_one("SELECT TIMESTAMPTZ '2020-01-01 00:00:00+08:00'", dialect="doris").sql("doris") + == "SELECT CAST('2020-01-01 00:00:00+08:00' AS TIMESTAMPTZ)" + ) def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapter: conn = MagicMock() @@ -1755,46 +1783,24 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte == "CREATE TABLE IF NOT EXISTS `db`.`events` (`id` INT, `ts` TIMESTAMPTZ(6))" ) - negated_match_ast = d.parse_one( - "SELECT * FROM docs WHERE body NOT MATCH_ANY 'error warning'", - dialect="doris", - ) - assert ( - negated_match_ast.sql("doris") - == "SELECT * FROM docs WHERE NOT body MATCH_ANY 'error warning'" - ) - aliased_struct_ast = d.parse_one( - "SELECT STRUCT(1 AS id, 'a' AS name)", - dialect="doris", - ) - assert aliased_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'name', 'a')" +def test_doris_match_struct_and_map_expressions(): + from sqlmesh.core.model import load_sql_based_model - mixed_struct_ast = d.parse_one( - "SELECT STRUCT(1 AS id, 'a')", + quoted_alias_ast = d.parse_one( + "SELECT body `match_any` FROM docs", dialect="doris", ) - assert mixed_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'col2', 'a')" + assert quoted_alias_ast.sql("doris") == "SELECT body AS `match_any` FROM docs" - branch_and_version_ast = d.parse_one( - "SELECT * FROM lance_catalog.db.tbl@branch(dev) FOR VERSION AS OF 2", + negated_match_ast = d.parse_one( + "SELECT * FROM docs WHERE body NOT MATCH_ANY 'error warning'", dialect="doris", ) assert ( - branch_and_version_ast.sql("doris") - == "SELECT * FROM lance_catalog.db.tbl @branch(dev) FOR VERSION AS OF 2" - ) - roundtrip_ast = d.parse_one(branch_and_version_ast.sql("doris"), dialect="doris") - assert roundtrip_ast == branch_and_version_ast - assert ( - roundtrip_ast.sql("doris") - == "SELECT * FROM lance_catalog.db.tbl @branch(dev) FOR VERSION AS OF 2" + negated_match_ast.sql("doris") + == "SELECT * FROM docs WHERE NOT body MATCH_ANY 'error warning'" ) - versions = list(roundtrip_ast.find_all(exp.Version)) - assert [(v.name.upper(), v.expression.sql("doris")) for v in versions] == [ - ("BRANCH", "dev"), - ("VERSION", "2"), - ] match_analyzer_ast = d.parse_one( "SELECT * FROM docs WHERE content MATCH 'hello' USING ANALYZER kw_analyzer", @@ -1805,36 +1811,17 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte == "SELECT * FROM docs WHERE content MATCH 'hello' USING ANALYZER kw_analyzer" ) - insert_branch_ast = d.parse_one( - "INSERT INTO iceberg_tbl@branch(b1) VALUES (1, 2)", + aliased_struct_ast = d.parse_one( + "SELECT STRUCT(1 AS id, 'a' AS name)", dialect="doris", ) - assert insert_branch_ast.sql("doris") == "INSERT INTO iceberg_tbl @branch(b1) VALUES (1, 2)" - assert d.parse_one(insert_branch_ast.sql("doris"), dialect="doris") == insert_branch_ast + assert aliased_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'name', 'a')" - insert_branch_cols_ast = d.parse_one( - "INSERT INTO iceberg_tbl@branch(b1) (c1, c2) VALUES (1, 2)", + mixed_struct_ast = d.parse_one( + "SELECT STRUCT(1 AS id, 'a')", dialect="doris", ) - assert ( - insert_branch_cols_ast.sql("doris") - == "INSERT INTO iceberg_tbl @branch(b1) (c1, c2) VALUES (1, 2)" - ) - assert ( - d.parse_one(insert_branch_cols_ast.sql("doris"), dialect="doris") == insert_branch_cols_ast - ) - - for quoted_ref_sql, expected_ref_sql in ( - ("SELECT * FROM tbl@branch(`release-2026`)", "SELECT * FROM tbl @branch(`release-2026`)"), - ("SELECT * FROM tbl@tag(`v1.0.0`)", "SELECT * FROM tbl @tag(`v1.0.0`)"), - ( - "INSERT INTO tbl@branch(`release-2026`) VALUES (1)", - "INSERT INTO tbl @branch(`release-2026`) VALUES (1)", - ), - ): - quoted_ref_ast = d.parse_one(quoted_ref_sql, dialect="doris") - assert quoted_ref_ast.sql("doris") == expected_ref_sql - assert d.parse_one(quoted_ref_ast.sql("doris"), dialect="doris") == quoted_ref_ast + assert mixed_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'col2', 'a')" k_arr = exp.column("k_arr") k_arr.type = exp.DataType.build("ARRAY", dialect="doris") @@ -1852,37 +1839,6 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte unannotated_map_from_arr.sql("doris") assert unannotated_map_from_arr.sql("doris, version=4.2.0") == "MAP_FROM_ARRAYS(k_arr, v_arr)" - alter_idx_if_not_exists_sql = "ALTER TABLE t ADD INDEX IF NOT EXISTS idx_body (body) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'fts'" - alter_idx_if_not_exists_ast = d.parse_one(alter_idx_if_not_exists_sql, dialect="doris") - assert isinstance(alter_idx_if_not_exists_ast, exp.Alter) - assert isinstance(alter_idx_if_not_exists_ast.args["actions"][0], exp.IndexColumnConstraint) - assert alter_idx_if_not_exists_ast.args["actions"][0].args.get("exists") is True - assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql - - # Verify extend_sqlglot() invalidates _DISPATCH_CACHE for Doris.Generator - from sqlglot.dialects import Doris - from sqlglot.generator import _DISPATCH_CACHE - - _DISPATCH_CACHE[Doris.Generator] = {} - d.extend_sqlglot() - assert Doris.Generator not in _DISPATCH_CACHE - assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql - - for drop_part_sql in ( - "ALTER TABLE db.t DROP PARTITION p1", - "ALTER TABLE db.t DROP PARTITION IF EXISTS p1", - "ALTER TABLE db.t DROP PARTITION IF EXISTS p1 FORCE", - "ALTER TABLE db.t DROP TEMPORARY PARTITION IF EXISTS p1 FORCE", - "ALTER TABLE db.t DROP PARTITION p1, DROP PARTITION p2 FORCE", - ): - drop_part_ast = d.parse_one(drop_part_sql, dialect="doris") - assert isinstance(drop_part_ast, exp.Alter) - assert isinstance(drop_part_ast.args["actions"][0], exp.DropPartition) - assert drop_part_ast.copy().sql("doris") == drop_part_sql - - drop_col_sql = "ALTER TABLE db.t DROP COLUMN c1" - assert d.parse_one(drop_col_sql, dialect="doris").sql("doris") == drop_col_sql - with pytest.raises( d.SQLMeshError, match="Doris MAP requires key and value arrays to have the same number of elements", @@ -1892,8 +1848,6 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte values=exp.Array(expressions=[exp.Literal.number(10)]), ).sql("doris") - from sqlmesh.core.model import load_sql_based_model - match_arith_model = load_sql_based_model( d.parse( """ @@ -1915,11 +1869,51 @@ def _make_fresh_doris_adapter(version_row: tuple[str, str]) -> DorisEngineAdapte == "SELECT (`docs`.`body` MATCH_ANY 'x') + 1 AS `score` FROM `db`.`docs` AS `docs`" ) - assert ( - d.parse_one("SELECT TIMESTAMP '2020-01-01 00:00:00'", dialect="doris").sql("doris") - == "SELECT CAST('2020-01-01 00:00:00' AS DATETIME)" - ) - assert ( - d.parse_one("SELECT TIMESTAMPTZ '2020-01-01 00:00:00+08:00'", dialect="doris").sql("doris") - == "SELECT CAST('2020-01-01 00:00:00+08:00' AS TIMESTAMPTZ)" - ) + +@pytest.mark.parametrize( + "raw_sql,expected_sql", + [ + ( + "SELECT * FROM lance_catalog.db.tbl@branch(dev) FOR VERSION AS OF 2", + "SELECT * FROM lance_catalog.db.tbl @branch(dev) FOR VERSION AS OF 2", + ), + ( + "INSERT INTO iceberg_tbl@branch(b1) VALUES (1, 2)", + "INSERT INTO iceberg_tbl @branch(b1) VALUES (1, 2)", + ), + ( + "INSERT INTO iceberg_tbl@branch(b1) (c1, c2) VALUES (1, 2)", + "INSERT INTO iceberg_tbl @branch(b1) (c1, c2) VALUES (1, 2)", + ), + ( + "SELECT * FROM tbl@branch(`release-2026`)", + "SELECT * FROM tbl @branch(`release-2026`)", + ), + ( + "SELECT * FROM tbl@tag(`v1.0.0`)", + "SELECT * FROM tbl @tag(`v1.0.0`)", + ), + ( + "INSERT INTO tbl@branch(`release-2026`) VALUES (1)", + "INSERT INTO tbl @branch(`release-2026`) VALUES (1)", + ), + ], +) +def test_doris_branch_and_tag_references(raw_sql: str, expected_sql: str): + parsed_ast = d.parse_one(raw_sql, dialect="doris") + assert parsed_ast.sql("doris") == expected_sql + roundtrip_ast = d.parse_one(parsed_ast.sql("doris"), dialect="doris") + assert roundtrip_ast == parsed_ast + assert roundtrip_ast.sql("doris") == expected_sql + + +def test_doris_extend_sqlglot_dispatch_cache_invalidation(): + from sqlglot.dialects import Doris + from sqlglot.generator import _DISPATCH_CACHE + + alter_idx_if_not_exists_sql = "ALTER TABLE t ADD INDEX IF NOT EXISTS idx_body (body) USING INVERTED PROPERTIES ('parser'='english') COMMENT 'fts'" + alter_idx_if_not_exists_ast = d.parse_one(alter_idx_if_not_exists_sql, dialect="doris") + _DISPATCH_CACHE[Doris.Generator] = {} + d.extend_sqlglot() + assert Doris.Generator not in _DISPATCH_CACHE + assert alter_idx_if_not_exists_ast.sql("doris") == alter_idx_if_not_exists_sql From 727b24a75ee14f2dbb7d02768df23bfe97381408 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:57:10 -0800 Subject: [PATCH 53/54] fix(doris): preserve equality expressions in native STRUCT calls Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/dialect.py | 21 +++++++++++++++++++++ tests/core/test_dialect.py | 12 ++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index ee9ce21304..1ba829b7d9 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -1183,6 +1183,23 @@ def _doris_map_sql(self: Generator, expression: exp.Map | exp.VarMap) -> str: return self.func("MAP", *expression.expressions) +def _doris_parse_struct(self: Parser) -> exp.Struct: + args: t.List[exp.Expr] = [] + for arg in self._parse_function_args(alias=True): + if isinstance(arg, exp.Alias): + args.append( + self.expression( + exp.PropertyEQ( + this=exp.to_identifier(arg.alias), + expression=arg.this, + ) + ) + ) + else: + args.append(arg) + return self.expression(exp.Struct(expressions=args)) + + def _doris_struct_sql(self: Generator, expression: exp.Struct) -> str: if any(isinstance(e, (exp.PropertyEQ, exp.Alias)) for e in expression.expressions): args: t.List[str] = [] @@ -2014,6 +2031,10 @@ def tokenize(self, sql: str) -> t.List[Token]: TokenType.ANTI, TokenType.SEMI, } + Doris.Parser.FUNCTION_PARSERS = { + **Doris.Parser.FUNCTION_PARSERS, + "STRUCT": _doris_parse_struct, + } Doris.Parser.FUNCTIONS.update( { "INNER_PRODUCT": exp.DotProduct.from_arg_list, diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 928586d4eb..01b0e5aa1b 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -1823,6 +1823,18 @@ def test_doris_match_struct_and_map_expressions(): ) assert mixed_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('id', 1, 'col2', 'a')" + eq_struct_ast = d.parse_one( + "SELECT STRUCT(a = b, c = 1)", + dialect="doris", + ) + assert eq_struct_ast.sql("doris") == "SELECT STRUCT(a = b, c = 1)" + + aliased_eq_struct_ast = d.parse_one( + "SELECT STRUCT(a = b AS is_eq, c)", + dialect="doris", + ) + assert aliased_eq_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('is_eq', a = b, 'col2', c)" + k_arr = exp.column("k_arr") k_arr.type = exp.DataType.build("ARRAY", dialect="doris") v_arr = exp.column("v_arr") From 8b36b0e5959483656ef31a291d23af3d12331141 Mon Sep 17 00:00:00 2001 From: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:19:20 -0800 Subject: [PATCH 54/54] fix(doris): include catalog in excluded_trigger_tables ambiguity check Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com> --- sqlmesh/core/engine_adapter/doris.py | 15 ++++++++++----- sqlmesh/core/model/definition.py | 19 ++++++++++++------- tests/core/engine_adapter/test_doris.py | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/sqlmesh/core/engine_adapter/doris.py b/sqlmesh/core/engine_adapter/doris.py index a71675883d..439dc7bbf7 100644 --- a/sqlmesh/core/engine_adapter/doris.py +++ b/sqlmesh/core/engine_adapter/doris.py @@ -7998,8 +7998,9 @@ def _properties_to_expressions( val_str = raw_str if raw_str is not None else str(value) if key == "excluded_trigger_tables" and val_str: supports_qualified = self._supports_qualified_excluded_trigger_tables() - dbs_by_table_name: t.Dict[str, t.Set[str]] = {} + dbs_by_table_name: t.Dict[str, t.Set[t.Tuple[str, str]]] = {} if not supports_qualified: + default_cat_lower = str(self._extra_config.get("catalog") or "internal").lower() if isinstance(query_expression, exp.Expr): cte_names = { cte.alias.lower() @@ -8010,16 +8011,18 @@ def _properties_to_expressions( if q_tbl.name and ( q_tbl.db or q_tbl.catalog or q_tbl.name.lower() not in cte_names ): + q_cat = (q_tbl.catalog or default_cat_lower).lower() dbs_by_table_name.setdefault(q_tbl.name.lower(), set()).add( - q_tbl.db or "" + (q_cat, q_tbl.db or "") ) for raw_part in val_str.split(","): raw_stripped = raw_part.strip() if raw_stripped: p_tbl = exp.to_table(raw_stripped, dialect=self.dialect) - if p_tbl.name and p_tbl.db: + if p_tbl.name and (p_tbl.db or p_tbl.catalog): + p_cat = (p_tbl.catalog or default_cat_lower).lower() dbs_by_table_name.setdefault(p_tbl.name.lower(), set()).add( - p_tbl.db + (p_cat, p_tbl.db or "") ) cleaned_parts: t.List[str] = [] for part in val_str.split(","): @@ -8034,7 +8037,9 @@ def _properties_to_expressions( f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " f"got '{stripped}'." ) - if tbl.db and len(dbs_by_table_name.get(tbl.name.lower(), set())) > 1: + if (tbl.db or tbl.catalog) and len( + dbs_by_table_name.get(tbl.name.lower(), set()) + ) > 1: raise SQLMeshError( f"Doris 'excluded_trigger_tables' prior to 2.1.10 (and 3.0.6) matches tables by bare name only " f"and cannot disambiguate '{exp.table_name(tbl, identify=False)}' when the view references multiple tables named '{tbl.name}'." diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index a6fe4a9d4d..c681b2962b 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2910,16 +2910,22 @@ def _to_physical_table(ref: str | exp.Table) -> t.Tuple[exp.Table, t.Optional[st return resolved_table, physical, strip_catalog effective_engine_dialect = engine_dialect if engine_dialect is not None else dialect - dbs_by_table_name: t.Dict[str, t.Set[str]] = {} + dbs_by_table_name: t.Dict[str, t.Set[t.Tuple[str, str]]] = {} if ( effective_engine_dialect == "doris" and property_name == "excluded_trigger_tables" and not d.doris_supports_qualified_excluded_trigger_tables(doris_version) ): + default_cat_lower = (default_catalog or "internal").lower() for candidate in [*(referenced_tables or ()), *(r for r in refs if r.strip())]: cand_tbl, _, _ = _to_physical_table(candidate) - if cand_tbl.name and (cand_tbl.db or isinstance(candidate, exp.Table)): - dbs_by_table_name.setdefault(cand_tbl.name.lower(), set()).add(cand_tbl.db or "") + if cand_tbl.name and ( + cand_tbl.db or cand_tbl.catalog or isinstance(candidate, exp.Table) + ): + cand_cat = (cand_tbl.catalog or default_cat_lower).lower() + dbs_by_table_name.setdefault(cand_tbl.name.lower(), set()).add( + (cand_cat, cand_tbl.db or "") + ) def resolve(ref: str) -> str: resolved_table, physical, strip_catalog = _to_physical_table(ref) @@ -2935,10 +2941,9 @@ def resolve(ref: str) -> str: f"Doris 'excluded_trigger_tables' only supports non-internal catalog references in Doris 2.1.10+ (and 3.0.6+); " f"got '{exp.table_name(resolved_table, identify=False)}'." ) - if ( - resolved_table.db - and len(dbs_by_table_name.get(resolved_table.name.lower(), set())) > 1 - ): + if (resolved_table.db or resolved_table.catalog) and len( + dbs_by_table_name.get(resolved_table.name.lower(), set()) + ) > 1: raise SQLMeshError( f"Doris 'excluded_trigger_tables' prior to 2.1.10 (and 3.0.6) matches tables by bare name only " f"and cannot disambiguate '{exp.table_name(resolved_table, identify=False)}' when the view references multiple tables named '{resolved_table.name}'." diff --git a/tests/core/engine_adapter/test_doris.py b/tests/core/engine_adapter/test_doris.py index ff552a5e26..bd11183955 100644 --- a/tests/core/engine_adapter/test_doris.py +++ b/tests/core/engine_adapter/test_doris.py @@ -6383,6 +6383,26 @@ def _cursor_init_internal(cur: t.Any) -> None: dialect="doris", ), ) + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + _resolve_model_refs_to_physical_tables( + exp.Literal.string("internal.sales.orders"), + table_mapping={}, + dialect="doris", + doris_version=(2, 1, 9), + property_name="excluded_trigger_tables", + referenced_tables=[ + exp.to_table("internal.sales.orders", dialect="doris"), + exp.to_table("hive.sales.orders", dialect="doris"), + ], + ) + with pytest.raises(SQLMeshError, match="cannot disambiguate"): + v219_adapter._properties_to_expressions( + {"excluded_trigger_tables": exp.Literal.string("internal.sales.orders")}, + query_expression=parse_one( + "SELECT * FROM internal.sales.orders JOIN hive.sales.orders ON internal.sales.orders.id = hive.sales.orders.id", + dialect="doris", + ), + ) v420_adapter = make_mocked_engine_adapter(DorisEngineAdapter) v420_adapter.cursor.fetchone.return_value = ("Doris version doris-4.2.0-rc01", "5.7.99") assert (