Add Parquet-to-NumPy C entry point with column projection

.github/workflows/ci.yml

@@ -25,7 +25,30 @@
           name: dist-sdist
           path: dist/*
 
+  parquet:
+    name: Native Parquet / ubuntu-24.04 / Python 3.14
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/setup-python@master
+        with:
+          python-version: 3.14
+      - run: echo '::add-matcher::.github/problem-matchers/gcc.json'
+      - name: Install Arrow/Parquet C++ libraries
+        run: |
+          wget -q https://packages.apache.org/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-noble.deb
+          sudo apt-get install -y ./apache-arrow-apt-source-latest-noble.deb
+          sudo apt-get update
+          sudo apt-get install -y libarrow-dev libparquet-dev
+          pkg-config --modversion arrow parquet
+      - run: python -m pip install -r requirements-dev-3_14.txt pyarrow
+      - run: python -m pip install -v .
+        env:
+          ARRAYKIT_WITH_PARQUET: '1'
+      - run: python -c "import arraykit._parquet"
+      - run: python -m pytest test/test_parquet_to_arrays.py test/test_parquet_build.py
+
   matrix_config:
     name: Matrix Runner Config
     runs-on: ubuntu-22.04
     outputs:

MANIFEST.in

@@ -1,3 +1,4 @@
 include src/__init__.pyi src/py.typed
 include src/*.h
+include src/*.cpp
 include VERSION

README.rst

@@ -32,6 +32,70 @@ ArrayKit requires the following:
 - numpy>=1.24.3
 
 
+Reading Parquet
+---------------
+
+``parquet_to_arrays(file, *, columns_select=None)`` returns a list of immutable,
+one-dimensional NumPy arrays, one per top-level Parquet column. It uses the native
+Arrow and Parquet C++ libraries, not Python PyArrow or ``arrow_python``.
+
+Parquet support is opt-in at build time. Install Arrow/Parquet >=24 development
+libraries and a C++20 compiler, then build ArrayKit from source::
+
+    ARRAYKIT_WITH_PARQUET=1 pip install --no-binary=arraykit arraykit
+
+The build finds libraries using ``pkg-config``. Alternatively set ``ARROW_HOME``
+to a native installation prefix containing ``include`` and ``lib``, or set both
+``ARROW_INCLUDE_DIR`` and ``ARROW_LIBRARY_DIR``. These settings are build-time
+only; the resulting extension must be able to load the same native libraries at
+runtime. Explicit library directories are embedded as runtime search paths on
+Unix. On Windows, make the Arrow DLL directory available to the loader (for
+example with ``os.add_dll_directory()`` before calling the reader). Keep native
+library installations ABI-compatible with the version used to build the extension.
+Locally built wheels with this backend are not portable unless their native
+dependencies are also supplied.
+
+Normal builds still require only NumPy. Calling ``parquet_to_arrays`` without
+the native backend raises ``ImportError`` with build instructions. PyArrow is
+used only to generate fixtures and reference results in optional tests, never
+for loading Parquet in ArrayKit.
+
+``file`` can be a string, bytes, or path-like filesystem path, or a seekable
+binary file such as an open file or ``io.BytesIO``. Caller-owned files are not
+closed; their position can change. Paths identify individual files, not datasets
+or directories.
+
+``columns_select`` can be a list of unique column names. Only those columns are
+decoded, in selection order. ``None`` loads all columns in file order; ``[]``
+returns an empty list. Missing or ambiguous names and repeated selections raise
+``ValueError``. Names select whole top-level columns (including nested columns),
+not dotted paths to nested fields.
+
+The native extension reads paths through Arrow's random-access file API.
+Open Python files use binary ``read`` (returning any bytes-like object),
+``seek``, and ``tell`` callbacks; the Parquet decoder and array conversion
+remain native. It releases the GIL during
+reading and decoding. Arrow handles compression and page encodings; ArrayKit
+copies decoded columns directly into NumPy-owned arrays through the C API,
+without a Python Arrow table, pandas, or Python loops over columns or values.
+Performance relative to Python PyArrow depends on the file, column types, and
+selection.
+
+Dtypes and null handling preserve numeric types where possible; nullable integers
+become floating-point arrays with ``NaN``, nullable booleans and strings use object
+arrays with ``None``, timestamps and dates use ``datetime64`` with ``NaT``, and
+durations use ``timedelta64`` with ``NaT``. Timestamp time zones are dropped;
+values are UTC. Large nullable integers can lose precision when converted to
+floating point. Dictionary columns are decoded to values, decimal columns use
+``decimal.Decimal`` objects, time-of-day columns use ``datetime.time`` objects
+(nanosecond values that are not whole microseconds raise ``ValueError``), and
+nested struct/list columns use object arrays. Temporal values nested in structs
+use NumPy ``datetime64`` or ``timedelta64`` scalars. Other Arrow types may not
+have a supported NumPy conversion. Unsupported conversions raise ``NotImplementedError``;
+malformed files raise ``ValueError`` or ``OSError``. Exceptions from Python file
+methods are preserved.
+
+
 What is New in ArrayKit
 -------------------------
 

performance/parquet/fixtures.py

@@ -0,0 +1,125 @@
+import atexit
+import io
+import os
+import shutil
+import tempfile
+import typing as tp
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+COLUMNS = 10
+NULL_INTERVAL = 10  # every n-th value is null in nullable fixtures
+
+DIRECTORY = tempfile.mkdtemp(prefix='arraykit-parquet-')
+atexit.register(shutil.rmtree, DIRECTORY, ignore_errors=True)
+
+
+class PayLoad:
+    def __init__(self, label: str, size: int, table: pa.Table):
+        self.table = table
+        self.size = size
+        self.path = os.path.join(DIRECTORY, f'{label}-{size}.parquet')
+        pq.write_table(table, self.path)
+        with open(self.path, 'rb') as f:
+            self.data = f.read()
+        self.columns = table.column_names
+        self.columns_select = self.columns[::2]
+
+    def bytes_io(self) -> io.BytesIO:
+        return io.BytesIO(self.data)
+
+
+# -------------------------------------------------------------------------------
+def get_table(get_array: tp.Callable[[int, int], pa.Array], size: int) -> pa.Table:
+    return pa.table({f'c{i}': get_array(size, i) for i in range(COLUMNS)})
+
+
+def get_null_mask(size: int) -> np.ndarray:
+    return (np.arange(size) % NULL_INTERVAL) == 0
+
+
+class FixtureFactory:
+    NAME = ''
+    SORT = 0
+    CACHE = {}  # can be shared for all classes
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        raise NotImplementedError()
+
+    @classmethod
+    def get_label_payload(cls, size: int) -> tp.Tuple[str, PayLoad]:
+        key = (cls, size)
+        if key not in cls.CACHE:
+            table = get_table(cls.get_array, size)
+            cls.CACHE[key] = PayLoad(cls.NAME, size, table)
+        return cls.NAME, cls.CACHE[key]
+
+
+class FFInt64(FixtureFactory):
+    NAME = 'int64'
+    SORT = 0
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        return pa.array(np.arange(column, column + size, dtype=np.int64))
+
+
+class FFInt64Null(FixtureFactory):
+    NAME = 'int64 (nulls)'
+    SORT = 1
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        values = np.arange(column, column + size, dtype=np.int64)
+        return pa.array(values, mask=get_null_mask(size))
+
+
+class FFFloat64(FixtureFactory):
+    NAME = 'float64'
+    SORT = 2
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        return pa.array(np.arange(column, column + size, dtype=np.float64) * 0.5)
+
+
+class FFBool(FixtureFactory):
+    NAME = 'bool'
+    SORT = 3
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        return pa.array((np.arange(size) + column) % 3 == 0)
+
+
+class FFDTns(FixtureFactory):
+    NAME = 'dt64[ns]'
+    SORT = 4
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        values = np.arange(size, dtype=np.int64) * 1_000_000_007 + column
+        return pa.array(values.astype('datetime64[ns]'))
+
+
+class FFString(FixtureFactory):
+    NAME = 'string'
+    SORT = 5
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        return pa.array([hex(e) * 2 for e in range(column, column + size)])
+
+
+class FFStringDict(FixtureFactory):
+    NAME = 'string (dict)'
+    SORT = 6
+
+    @staticmethod
+    def get_array(size: int, column: int) -> pa.Array:
+        labels = np.array([f'label-{i}' for i in range(100)], dtype=object)
+        return pa.array(labels[(np.arange(size) + column) % 100]).dictionary_encode()