fix: keep NaN distinct from NULL in float columns when pandas is enabled

CHANGELOG.md

@@ -3,6 +3,7 @@
 # Unreleased
 - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
 - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
+- Fix: with pandas enabled (the default), a floating-point NaN in a FLOAT or DOUBLE column is now returned as `float('nan')` instead of `None`, so it can be told apart from SQL NULL. This matches what `_disable_pandas=True` already returned.
 - Speed up Thrift response handling: read replies in 1 MiB chunks and decode them with Thrift's C-accelerated protocol, and stop stringifying every request/response for the default no-op debug logger. Wide result schemas, metadata calls with many rows, and inline Arrow results benefit most.
 
 # 4.9.0 (2026-11-04)

src/databricks/sql/result_set.py

@@ -8,6 +8,7 @@
 
 try:
     import pyarrow
+    import pyarrow.compute
 except ImportError:
     pyarrow = None
 
@@ -130,6 +131,19 @@ def _convert_arrow_table(self, table):
         )
 
         res = df.to_numpy(na_value=None, dtype="object")
+
+        # The nullable float dtypes turn IEEE NaN into NA, which would make it
+        # indistinguishable from SQL NULL, so restore NaN values from Arrow.
+        for i, field in enumerate(table_renamed.schema):
+            if not pyarrow.types.is_floating(field.type):
+                continue
+            nan_mask = pyarrow.compute.is_nan(table_renamed.column(i))
+            if not pyarrow.compute.any(nan_mask).as_py():
+                continue
+            if nan_mask.null_count:
+                nan_mask = nan_mask.fill_null(False)
+            res[nan_mask.to_numpy(), i] = float("nan")
+
         return [ResultRow(*v) for v in res]
 
     @property

tests/unit/test_pandas_compatibility.py

@@ -7,6 +7,7 @@
 """
 
 import datetime
+import math
 import unittest
 from decimal import Decimal
 from unittest.mock import Mock
@@ -122,6 +123,50 @@ def test_float_types(self):
         self.assertIsNone(rows[1].float32_col)
         self.assertAlmostEqual(rows[2].float64_col, 4.5)
 
+    def _nan_and_null_table(self):
+        values = [[float("nan"), None], [float("inf"), float("-inf"), 1.5]]
+        table = pa.table(
+            {
+                "id": pa.chunked_array([[1, 2], [3, 4, 5]], type=pa.int64()),
+                "float32_col": pa.chunked_array(values, type=pa.float32()),
+                "float64_col": pa.chunked_array(values, type=pa.float64()),
+            }
+        )
+        description = [
+            ("id", "bigint", None, None, None, None, None),
+            ("float32_col", "float", None, None, None, None, None),
+            ("float64_col", "double", None, None, None, None, None),
+        ]
+        return table, description
+
+    def test_float_nan_is_not_converted_to_null(self):
+        table, description = self._nan_and_null_table()
+
+        rows = _make_result_set(description)._convert_arrow_table(table)
+
+        self.assertEqual([row.id for row in rows], [1, 2, 3, 4, 5])
+        for column in ("float32_col", "float64_col"):
+            values = [getattr(row, column) for row in rows]
+            self.assertIsInstance(values[0], float)
+            self.assertTrue(math.isnan(values[0]))
+            self.assertEqual(values[1:], [None, float("inf"), float("-inf"), 1.5])
+
+    def test_float_nan_and_null_match_disable_pandas_path(self):
+        table, description = self._nan_and_null_table()
+
+        def normalized(rows):
+            return [
+                ["NaN" if isinstance(v, float) and math.isnan(v) else v for v in row]
+                for row in rows
+            ]
+
+        with_pandas = _make_result_set(description)._convert_arrow_table(table)
+        without_pandas = _make_result_set(
+            description, disable_pandas=True
+        )._convert_arrow_table(table)
+
+        self.assertEqual(normalized(with_pandas), normalized(without_pandas))
+
     def test_boolean_type(self):
         table = pa.table(
             {