FEAT-#4144: Implement dataframe exchange protocol for pandas storage format

.github/workflows/ci.yml

@@ -466,6 +466,7 @@ jobs:
       - run: python -m pytest modin/pandas/test/test_io.py --verbose
       - run: python -m pytest modin/experimental/pandas/test/test_io_exp.py
       - run: pip install "dfsql>=0.4.2" "pyparsing<=2.4.7" && pytest modin/experimental/sql/test/test_sql.py
+      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
       - uses: codecov/codecov-action@v2
 
   test-experimental:

.github/workflows/push.yml

@@ -168,6 +168,7 @@ jobs:
       - run: python -m pytest -n 2 modin/pandas/test/test_general.py
       - run: python -m pytest modin/pandas/test/test_io.py
       - run: python -m pytest modin/experimental/pandas/test/test_io_exp.py
+      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
       - uses: codecov/codecov-action@v2
 
   test-windows:

modin/conftest.py

@@ -234,7 +234,7 @@ def from_arrow(cls, at, data_cls):
     def free(self):
         pass
 
-    def to_dataframe(self, nan_as_null: bool = False, allow_copy: bool = True) -> dict:
+    def to_dataframe(self, nan_as_null: bool = False, allow_copy: bool = True):
         raise NotImplementedError(
             "The selected execution does not implement the DataFrame exchange protocol."
         )

modin/core/dataframe/base/exchange/dataframe_protocol/__init__.py

@@ -16,7 +16,3 @@
 
 See more in https://data-apis.org/dataframe-protocol/latest/index.html.
 """
-
-from .dataframe import ProtocolDataframe
-
-__all__ = ["ProtocolDataframe"]

modin/core/dataframe/base/exchange/dataframe_protocol/dataframe.py

@@ -330,6 +330,10 @@ def get_chunks(self, n_chunks: Optional[int] = None) -> Iterable["ProtocolColumn
         ------
         DataFrame
             A ``DataFrame`` object(s).
+
+        Raises
+        ------
+        ``RuntimeError`` if ``n_chunks`` is not a multiple of ``self.num_chunks()``.
         """
         pass
 
@@ -539,5 +543,9 @@ def get_chunks(
         ------
         ProtocolDataframe
             A ``ProtocolDataframe`` object(s).
+
+        Raises
+        ------
+        ``RuntimeError`` if ``n_chunks`` is not a multiple of ``self.num_chunks()``.
         """
         pass

modin/core/dataframe/pandas/dataframe/dataframe.py

@@ -2826,3 +2826,36 @@ def finalize(self):
         that were used to build it.
         """
         self._partition_mgr_cls.finalize(self._partitions)
+
+    def __dataframe__(self, nan_as_null: bool = False, allow_copy: bool = True):
+        """
+        Get a Modin DataFrame that implements the dataframe exchange protocol.
+
+        See more about the protocol in https://data-apis.org/dataframe-protocol/latest/index.html.
+
+        Parameters
+        ----------
+        nan_as_null : bool, default: False
+            A keyword intended for the consumer to tell the producer
+            to overwrite null values in the data with ``NaN`` (or ``NaT``).
+            This currently has no effect; once support for nullable extension
+            dtypes is added, this value should be propagated to columns.
+        allow_copy : bool, default: True
+            A keyword that defines whether or not the library is allowed
+            to make a copy of the data. For example, copying data would be necessary
+            if a library supports strided buffers, given that this protocol
+            specifies contiguous buffers. Currently, if the flag is set to ``False``
+            and a copy is needed, a ``RuntimeError`` will be raised.
+
+        Returns
+        -------
+        ProtocolDataframe
+            A dataframe object following the dataframe protocol specification.
+        """
+        from modin.core.dataframe.pandas.exchange.dataframe_protocol.dataframe import (
+            PandasProtocolDataframe,
+        )
+
+        return PandasProtocolDataframe(
+            self, nan_as_null=nan_as_null, allow_copy=allow_copy
+        )

modin/core/dataframe/pandas/exchange/__init__.py

@@ -0,0 +1,14 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""Base Modin Dataframe functionality related to data exchange protocols and optimized for pandas storage format."""

modin/core/dataframe/pandas/exchange/dataframe_protocol/__init__.py

@@ -0,0 +1,18 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""
+Base Modin Dataframe functionality related to the dataframe exchange protocol and optimized for pandas storage format.
+
+See more in https://data-apis.org/dataframe-protocol/latest/index.html.
+"""

modin/core/dataframe/pandas/exchange/dataframe_protocol/buffer.py

@@ -0,0 +1,116 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""
+Dataframe exchange protocol implementation.
+
+See more in https://data-apis.org/dataframe-protocol/latest/index.html.
+
+Notes
+-----
+- Interpreting a raw pointer (as in ``Buffer.ptr``) is annoying and unsafe to
+  do in pure Python. It's more general but definitely less friendly than having
+  ``to_arrow`` and ``to_numpy`` methods. So for the buffers which lack
+  ``__dlpack__`` (e.g., because the column dtype isn't supported by DLPack),
+  this is worth looking at again.
+"""
+
+import enum
+import numpy as np
+from typing import Tuple
+
+from modin.core.dataframe.base.exchange.dataframe_protocol.dataframe import (
+    ProtocolBuffer,
+)
+from modin.utils import _inherit_docstrings
+
+
+@_inherit_docstrings(ProtocolBuffer)
+class PandasProtocolBuffer(ProtocolBuffer):
+    """
+    Data in the buffer is guaranteed to be contiguous in memory.
+
+    Note that there is no dtype attribute present, a buffer can be thought of
+    as simply a block of memory. However, if the column that the buffer is
+    attached to has a dtype that's supported by DLPack and ``__dlpack__`` is
+    implemented, then that dtype information will be contained in the return
+    value from ``__dlpack__``.
+
+    This distinction is useful to support both (a) data exchange via DLPack on a
+    buffer and (b) dtypes like variable-length strings which do not have a
+    fixed number of bytes per element.
+
+    Parameters
+    ----------
+    x : np.ndarray
+        Data to be held by ``Buffer``.
+    allow_copy : bool, default: True
+        A keyword that defines whether or not the library is allowed
+        to make a copy of the data. For example, copying data would be necessary
+        if a library supports strided buffers, given that this protocol
+        specifies contiguous buffers. Currently, if the flag is set to ``False``
+        and a copy is needed, a ``RuntimeError`` will be raised.
+    """
+
+    def __init__(self, x: np.ndarray, allow_copy: bool = True) -> None:
+        if not x.strides == (x.dtype.itemsize,):
+            # The protocol does not support strided buffers, so a copy is
+            # necessary. If that's not allowed, we need to raise an exception.
+            if allow_copy:
+                x = x.copy()
+            else:
+                raise RuntimeError(
+                    "Exports cannot be zero-copy in the case "
+                    + "of a non-contiguous buffer"
+                )
+
+        # Store the numpy array in which the data resides as a private
+        # attribute, so we can use it to retrieve the public attributes
+        self._x = x
+
+    @property
+    def bufsize(self) -> int:
+        return self._x.size * self._x.dtype.itemsize
+
+    @property
+    def ptr(self) -> int:
+        return self._x.__array_interface__["data"][0]
+
+    def __dlpack__(self):
+        raise NotImplementedError("__dlpack__")
+
+    def __dlpack_device__(self) -> Tuple[enum.IntEnum, int]:
+        class Device(enum.IntEnum):
+            CPU = 1
+
+        return (Device.CPU, None)
+
+    def __repr__(self) -> str:
+        """
+        Return a string representation for a particular ``PandasProtocolBuffer``.
+
+        Returns
+        -------
+        str
+        """
+        return (
+            "Buffer("
+            + str(
+                {
+                    "bufsize": self.bufsize,
+                    "ptr": self.ptr,
+                    "device": self.__dlpack_device__()[0].name,
+                }
+            )
+            + ")"
+        )