Remove exposure on column construction and unwrap buffers on pylibcudf conversion (#20980)

This PR removes the ability to construct columns and buffers that are already exposed, which is not actually ever possible in the current cudf data model. This change allows us to simplify the various column constructors and standardize the validation process.

Relatedly, this PR ensures that conversion from cudf ColumnBase to pylibcudf Column unwraps Buffers so that you do not expose the pylibcudf representation to cudf's Buffer semantics. That change should allow us to fully decouple the internal representation of pylibcudf Columns inside cudf from how they are exposed to public APIs, which also ensures that we do not break CoW and spilling functionality by making too many Buffer copies that we shouldn't.

Authors:
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Matthew Roeschke (https://github.com/mroeschke)

URL: #20980

Author: vyasr

python/cudf/cudf/core/buffer/buffer.py

@@ -91,19 +91,18 @@ def __init__(
         ptr: int,
         size: int,
         owner: object,
-        exposed: bool,
     ):
         if size < 0:
             raise ValueError("size cannot be negative")
 
         self._ptr = ptr
         self._size = size
         self._owner = owner
-        self._exposed = exposed
+        self._exposed = False
         self._slices = weakref.WeakSet()
 
     @classmethod
-    def from_device_memory(cls, data: Any, exposed: bool) -> Self:
+    def from_device_memory(cls, data: Any) -> Self:
         """Create from an object providing a `__cuda_array_interface__`.
 
         No data is being copied.
@@ -112,10 +111,6 @@ def from_device_memory(cls, data: Any, exposed: bool) -> Self:
         ----------
         data : device-buffer-like
             An object implementing the CUDA Array Interface.
-        exposed : bool
-            Mark the buffer as permanently exposed. This is used by
-            copy-on-write to determine when a deep copy is required
-            and by SpillableBuffer to mark the buffer unspillable.
 
         Returns
         -------
@@ -135,7 +130,7 @@ def from_device_memory(cls, data: Any, exposed: bool) -> Self:
             size = data.size
         else:
             ptr, size = get_ptr_and_size(data.__cuda_array_interface__)
-        return cls(ptr=ptr, size=size, owner=data, exposed=exposed)
+        return cls(ptr=ptr, size=size, owner=data)
 
     @classmethod
     def from_host_memory(cls, data: Any) -> Self:
@@ -166,7 +161,7 @@ def from_host_memory(cls, data: Any) -> Self:
         # Copy to device memory
         buf = rmm.DeviceBuffer(ptr=ptr, size=size)
         # Create from device memory
-        return cls.from_device_memory(buf, exposed=False)
+        return cls.from_device_memory(buf)
 
     @property
     def size(self) -> int:
@@ -398,7 +393,6 @@ def copy(self, deep: bool = True) -> Self:
                 ptr=self._owner.ptr + self._offset,
                 size=self.size,
             ),
-            exposed=False,
         )
         return self.__class__(owner=owner, offset=0, size=owner.size)
 
@@ -477,7 +471,7 @@ def deserialize(cls, header: dict, frames: list) -> Self:
             header["owner-type-serialized-name"]
         ]
         if hasattr(frame, "__cuda_array_interface__"):
-            owner = owner_type.from_device_memory(frame, exposed=False)
+            owner = owner_type.from_device_memory(frame)
         else:
             owner = owner_type.from_host_memory(frame)
         return cls(

python/cudf/cudf/core/buffer/spillable_buffer.py

@@ -217,7 +217,7 @@ def _finalize_init(self, ptr_desc: dict[str, Any]) -> None:
         self._manager.add(self)
 
     @classmethod
-    def from_device_memory(cls, data: Any, exposed: bool) -> Self:
+    def from_device_memory(cls, data: Any) -> Self:
         """Create a spillabe buffer from device memory.
 
         No data is being copied.
@@ -226,15 +226,13 @@ def from_device_memory(cls, data: Any, exposed: bool) -> Self:
         ----------
         data : device-buffer-like
             An object implementing the CUDA Array Interface.
-        exposed : bool
-            Mark the buffer as permanently exposed (unspillable).
 
         Returns
         -------
         SpillableBufferOwner
             Buffer representing the same device memory as `data`
         """
-        ret = super().from_device_memory(data, exposed=exposed)
+        ret = super().from_device_memory(data)
         ret._finalize_init(ptr_desc={"type": "gpu"})
         return ret
 
@@ -268,7 +266,7 @@ def from_host_memory(cls, data: Any) -> Self:
         data = data.cast("B")  # Make sure itemsize==1
 
         # Create an already spilled buffer
-        ret = cls(ptr=0, size=data.nbytes, owner=None, exposed=False)
+        ret = cls(ptr=0, size=data.nbytes, owner=None)
         ret._finalize_init(ptr_desc={"type": "cpu", "memoryview": data})
         return ret
 
@@ -575,7 +573,6 @@ def serialize(self) -> tuple[dict, list]:
                     Buffer(
                         owner=BufferOwner.from_device_memory(
                             SpillableBufferCAIWrapper(self),
-                            exposed=False,
                         )
                     )
                 ]

python/cudf/cudf/core/buffer/utils.py

@@ -43,8 +43,6 @@ def get_buffer_owner(data: Any) -> BufferOwner | None:
 
 def as_buffer(
     data: Any,
-    *,
-    exposed: bool = False,
 ) -> Buffer:
     """Factory function to wrap `data` in a Buffer object.
 
@@ -68,10 +66,6 @@ def as_buffer(
     ----------
     data : buffer-like or array-like
         A buffer-like or array-like object.
-    exposed : bool, optional
-        Mark the buffer as permanently exposed. This is used by
-        copy-on-write to determine when a deep copy is required and
-        by SpillableBuffer to mark the buffer unspillable.
 
     Return
     ------
@@ -94,16 +88,12 @@ def as_buffer(
 
     # Handle host memory,
     if not hasattr(data, "__cuda_array_interface__"):
-        if exposed:
-            raise ValueError("cannot created exposed host memory")
         return buffer_class(owner=owner_class.from_host_memory(data))
 
     # Check if `data` is owned by a known class
     owner = get_buffer_owner(data)
     if owner is None:  # `data` is new device memory
-        return buffer_class(
-            owner=owner_class.from_device_memory(data, exposed=exposed)
-        )
+        return buffer_class(owner=owner_class.from_device_memory(data))
 
     # At this point, we know that `data` is owned by a known class, which
     # should be the same class as specified by the current config (see above)

python/cudf/cudf/core/column/categorical.py

@@ -103,14 +103,12 @@ def base_data(self) -> None:
         return None
 
     def _get_children_from_pylibcudf_column(
-        self, plc_column: plc.Column, dtype: DtypeObj, exposed: bool
+        self, plc_column: plc.Column, dtype: DtypeObj
     ) -> tuple[ColumnBase]:
         """
         This column considers the plc_column (i.e. codes) as children
         """
-        return (
-            type(self).from_pylibcudf(plc_column, data_ptr_exposed=exposed),
-        )
+        return (type(self).from_pylibcudf(plc_column),)
 
     def __contains__(self, item: ScalarLike) -> bool:
         try:
@@ -137,8 +135,8 @@ def codes(self) -> NumericalColumn:
     def ordered(self) -> bool | None:
         return self.dtype.ordered
 
-    def to_pylibcudf(self, mode: Literal["read", "write"]) -> plc.Column:
-        return self.children[0].to_pylibcudf(mode)
+    def to_pylibcudf(self) -> plc.Column:
+        return self.children[0].to_pylibcudf()
 
     def __setitem__(self, key: Any, value: Any) -> None:
         if is_scalar(value) and _is_null_host_scalar(value):
@@ -680,7 +678,6 @@ def _with_type_metadata(self: Self, dtype: DtypeObj) -> Self:
             return type(self)(
                 plc_column=self.plc_column,
                 dtype=dtype,
-                exposed=False,
             )
 
         return self