From 27479f90d8b707aa9e7bfdab4e5e140e1f65c563 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 18:55:45 +0000 Subject: [PATCH 1/6] perf: import pandas on first row conversion instead of in connect() client.py and result_set.py imported pandas at module load, so the first connect() in a process spent ~165 ms importing it under the import lock while every other connecting thread waited. pandas is only used when an Arrow table is converted to rows, so import it there. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- CHANGELOG.md | 1 + src/databricks/sql/client.py | 1 - src/databricks/sql/result_set.py | 5 ++- tests/unit/test_lazy_pandas_import.py | 59 +++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_lazy_pandas_import.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bae4c54e..9516075e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) - Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most. +- Speed up the first `connect()` in a process by importing pandas only when rows are first converted through it. # 4.9.0 (2026-11-04) - Upgrade Databricks SQL Kernel to 1.4.0, which keeps Lakehouse Real-Time session affinity and flushes shared telemetry only after the final session closes. diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 613b62386..eb7516b1e 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -12,7 +12,6 @@ BinaryIO, TYPE_CHECKING, ) -import pandas try: import pyarrow diff --git a/src/databricks/sql/result_set.py b/src/databricks/sql/result_set.py index f24a07505..ca039e115 100644 --- a/src/databricks/sql/result_set.py +++ b/src/databricks/sql/result_set.py @@ -4,7 +4,6 @@ from typing import List, Optional, TYPE_CHECKING, Tuple import logging -import pandas try: import pyarrow @@ -103,6 +102,10 @@ def _convert_arrow_table(self, table): ResultRow(*[v.as_py() for v in r]) for r in zip(*table.itercolumns()) ] + # Imported on first use: pandas takes ~165 ms to import, and connect() + # used to pay that under the import lock for every thread connecting. + import pandas + # Need to use nullable types, as otherwise type can change when there are missing values. # See https://arrow.apache.org/docs/python/pandas.html#nullable-types # NOTE: This api is epxerimental https://pandas.pydata.org/pandas-docs/stable/user_guide/integer_na.html diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py new file mode 100644 index 000000000..60685f59f --- /dev/null +++ b/tests/unit/test_lazy_pandas_import.py @@ -0,0 +1,59 @@ +"""pandas must load on the first row conversion, not when connect() is imported. + +Each check runs in a fresh interpreter because ``sys.modules`` is process-global. +""" + +import subprocess +import sys +import textwrap + +import pytest + +pytest.importorskip("pyarrow") + +CONNECT_PATH_MODULES = [ + "databricks.sql.client", + "databricks.sql.result_set", + "databricks.sql.backend.kernel.client", + "databricks.sql.backend.kernel.result_set", + "databricks.sql.backend.sea.backend", +] + + +def _run(script: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], + capture_output=True, + text=True, + ) + + +@pytest.mark.parametrize("module_name", CONNECT_PATH_MODULES) +def test_connect_path_does_not_import_pandas(module_name): + proc = _run(f""" + import sys + import {module_name} # noqa: F401 + sys.exit(1 if "pandas" in sys.modules else 0) + """) + if proc.returncode not in (0, 1): + pytest.skip(f"{module_name} not importable here:\n{proc.stderr}") + assert proc.returncode == 0, f"importing {module_name} imported pandas" + + +def test_row_conversion_imports_pandas_on_first_use(): + proc = _run(""" + import sys + from unittest.mock import MagicMock + import pyarrow + from databricks.sql.result_set import ResultSet + + assert "pandas" not in sys.modules + rs = MagicMock() + rs.connection.disable_pandas = False + rs.description = [("a",), ("b",)] + table = pyarrow.table({"a": [1, None], "b": ["x", "y"]}) + rows = ResultSet._convert_arrow_table(rs, table) + assert "pandas" in sys.modules + assert [tuple(r) for r in rows] == [(1, "x"), (None, "y")], rows + """) + assert proc.returncode == 0, proc.stderr From 4041f33dd06c6c3d4e30fca6a4faa4d254197a92 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 19:11:55 +0000 Subject: [PATCH 2/6] test: tell an import failure apart from pandas being loaded Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- tests/unit/test_lazy_pandas_import.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py index 60685f59f..9fbaed2ce 100644 --- a/tests/unit/test_lazy_pandas_import.py +++ b/tests/unit/test_lazy_pandas_import.py @@ -28,16 +28,28 @@ def _run(script: str) -> subprocess.CompletedProcess: ) +# Sentinel exit codes, distinct from the 1 of an uncaught exception. +_NO_PANDAS, _PANDAS_LOADED, _IMPORT_FAILED = 10, 11, 12 + + @pytest.mark.parametrize("module_name", CONNECT_PATH_MODULES) def test_connect_path_does_not_import_pandas(module_name): proc = _run(f""" import sys - import {module_name} # noqa: F401 - sys.exit(1 if "pandas" in sys.modules else 0) + try: + import {module_name} # noqa: F401 + except BaseException: + import traceback + traceback.print_exc() + sys.exit({_IMPORT_FAILED}) + sys.exit({_PANDAS_LOADED} if "pandas" in sys.modules else {_NO_PANDAS}) """) - if proc.returncode not in (0, 1): + if proc.returncode == _IMPORT_FAILED: pytest.skip(f"{module_name} not importable here:\n{proc.stderr}") - assert proc.returncode == 0, f"importing {module_name} imported pandas" + assert proc.returncode != _PANDAS_LOADED, f"importing {module_name} imported pandas" + assert ( + proc.returncode == _NO_PANDAS + ), f"unexpected exit {proc.returncode}:\n{proc.stderr}" def test_row_conversion_imports_pandas_on_first_use(): From 9b9111581e0bf235f9cc4678e1c7757f4c6f7633 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 20:32:48 +0000 Subject: [PATCH 3/6] test: skip the pandas import guard only when an optional extra is missing Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- tests/unit/test_lazy_pandas_import.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py index 9fbaed2ce..301345a9b 100644 --- a/tests/unit/test_lazy_pandas_import.py +++ b/tests/unit/test_lazy_pandas_import.py @@ -29,7 +29,9 @@ def _run(script: str) -> subprocess.CompletedProcess: # Sentinel exit codes, distinct from the 1 of an uncaught exception. -_NO_PANDAS, _PANDAS_LOADED, _IMPORT_FAILED = 10, 11, 12 +_NO_PANDAS, _PANDAS_LOADED, _MISSING_OPTIONAL_DEP = 10, 11, 12 +# Only a missing optional extra may skip; any other import error fails. +_OPTIONAL_DEPS = ("databricks_sql_kernel", "pyarrow") @pytest.mark.parametrize("module_name", CONNECT_PATH_MODULES) @@ -38,14 +40,15 @@ def test_connect_path_does_not_import_pandas(module_name): import sys try: import {module_name} # noqa: F401 - except BaseException: - import traceback - traceback.print_exc() - sys.exit({_IMPORT_FAILED}) + except ModuleNotFoundError as e: + if e.name not in {_OPTIONAL_DEPS!r}: + raise + print(e.name) + sys.exit({_MISSING_OPTIONAL_DEP}) sys.exit({_PANDAS_LOADED} if "pandas" in sys.modules else {_NO_PANDAS}) """) - if proc.returncode == _IMPORT_FAILED: - pytest.skip(f"{module_name} not importable here:\n{proc.stderr}") + if proc.returncode == _MISSING_OPTIONAL_DEP: + pytest.skip(f"{module_name} needs {proc.stdout.strip()}, not installed") assert proc.returncode != _PANDAS_LOADED, f"importing {module_name} imported pandas" assert ( proc.returncode == _NO_PANDAS From 1c07208c3d3010c28be7c187b650dd0f79f36a2a Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 20:53:58 +0000 Subject: [PATCH 4/6] test: skip the pandas import guard when the kernel wheel is missing The kernel backend re-raises a missing databricks_sql_kernel as an ImportError, so look for the missing extra along the exception chain. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- tests/unit/test_lazy_pandas_import.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py index 301345a9b..1bb0fcfec 100644 --- a/tests/unit/test_lazy_pandas_import.py +++ b/tests/unit/test_lazy_pandas_import.py @@ -40,10 +40,18 @@ def test_connect_path_does_not_import_pandas(module_name): import sys try: import {module_name} # noqa: F401 - except ModuleNotFoundError as e: - if e.name not in {_OPTIONAL_DEPS!r}: + except ImportError as e: + # The kernel backend re-raises a missing wheel as a friendlier + # ImportError, so look for the missing extra along the chain. + cause = e + while cause is not None and not ( + isinstance(cause, ModuleNotFoundError) + and cause.name in {_OPTIONAL_DEPS!r} + ): + cause = cause.__cause__ or cause.__context__ + if cause is None: raise - print(e.name) + print(cause.name) sys.exit({_MISSING_OPTIONAL_DEP}) sys.exit({_PANDAS_LOADED} if "pandas" in sys.modules else {_NO_PANDAS}) """) From 916638bb76e17fa400e70f51c0d194947925989f Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 21:09:41 +0000 Subject: [PATCH 5/6] test: pin that _disable_pandas never imports pandas; bound subprocess time Build the test table from IPC bytes, since constructing it from Python lists makes pyarrow import pandas itself. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- tests/unit/test_lazy_pandas_import.py | 56 ++++++++++++++++++++------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py index 1bb0fcfec..fc239abaf 100644 --- a/tests/unit/test_lazy_pandas_import.py +++ b/tests/unit/test_lazy_pandas_import.py @@ -25,9 +25,44 @@ def _run(script: str) -> subprocess.CompletedProcess: [sys.executable, "-c", textwrap.dedent(script)], capture_output=True, text=True, + timeout=120, ) +def _table_ipc_hex() -> str: + # Children read the table from IPC bytes: building it from Python lists + # makes pyarrow import pandas itself, which would hide what the + # conversion imports. + import io + + import pyarrow + import pyarrow.ipc + + table = pyarrow.table({"a": [1, None], "b": ["x", "y"]}) + sink = io.BytesIO() + with pyarrow.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + return sink.getvalue().hex() + + +def _convert_rows_script(disable_pandas: bool) -> str: + return f""" + import sys + from unittest.mock import MagicMock + import pyarrow.ipc + from databricks.sql.result_set import ResultSet + + table = pyarrow.ipc.open_stream(bytes.fromhex("{_table_ipc_hex()}")).read_all() + assert "pandas" not in sys.modules + rs = MagicMock() + rs.connection.disable_pandas = {disable_pandas} + rs.description = [("a",), ("b",)] + rows = ResultSet._convert_arrow_table(rs, table) + assert [tuple(r) for r in rows] == [(1, "x"), (None, "y")], rows + print("pandas" in sys.modules) + """ + + # Sentinel exit codes, distinct from the 1 of an uncaught exception. _NO_PANDAS, _PANDAS_LOADED, _MISSING_OPTIONAL_DEP = 10, 11, 12 # Only a missing optional extra may skip; any other import error fails. @@ -64,19 +99,12 @@ def test_connect_path_does_not_import_pandas(module_name): def test_row_conversion_imports_pandas_on_first_use(): - proc = _run(""" - import sys - from unittest.mock import MagicMock - import pyarrow - from databricks.sql.result_set import ResultSet + proc = _run(_convert_rows_script(disable_pandas=False)) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "True", "the pandas path should import pandas" - assert "pandas" not in sys.modules - rs = MagicMock() - rs.connection.disable_pandas = False - rs.description = [("a",), ("b",)] - table = pyarrow.table({"a": [1, None], "b": ["x", "y"]}) - rows = ResultSet._convert_arrow_table(rs, table) - assert "pandas" in sys.modules - assert [tuple(r) for r in rows] == [(1, "x"), (None, "y")], rows - """) + +def test_disable_pandas_never_imports_pandas(): + proc = _run(_convert_rows_script(disable_pandas=True)) assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "False", "_disable_pandas=True imported pandas" From c1937110f5370b2a1d22dbceed0293773b304665 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 21:28:39 +0000 Subject: [PATCH 6/6] test: guard the default Thrift connect path against importing pandas Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- tests/unit/test_lazy_pandas_import.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_lazy_pandas_import.py b/tests/unit/test_lazy_pandas_import.py index fc239abaf..235b43faf 100644 --- a/tests/unit/test_lazy_pandas_import.py +++ b/tests/unit/test_lazy_pandas_import.py @@ -12,8 +12,11 @@ pytest.importorskip("pyarrow") CONNECT_PATH_MODULES = [ + "databricks.sql", "databricks.sql.client", "databricks.sql.result_set", + "databricks.sql.utils", + "databricks.sql.backend.thrift_backend", "databricks.sql.backend.kernel.client", "databricks.sql.backend.kernel.result_set", "databricks.sql.backend.sea.backend",