Add dynamic_accessibility_property to all memory resources

Author: pciolkosz

libcudacxx/include/cuda/__memory_resource/any_resource.h

@@ -88,7 +88,8 @@ template <class _Property>
 using __iproperty = typename __with_property<_Property>::template __iproperty<>;
 
 template <class... _Properties>
-using __iproperty_set = ::cuda::__iset<__iproperty<_Properties>...>;
+using __iproperty_set =
+  ::cuda::__iset<__iproperty<_Properties>..., __iproperty<::cuda::mr::dynamic_accessibility_property>>;
 
 // Wrap the calls of the allocate and deallocate member functions
 // because of NVBUG#4967486
@@ -359,6 +360,15 @@ struct _CCCL_DECLSPEC_EMPTY_BASES resource_ref
   }
 };
 
+template <class... _Properties>
+inline constexpr bool __disable_default_dynamic_accessibility_property<any_resource<_Properties...>> = true;
+template <class... _Properties>
+inline constexpr bool __disable_default_dynamic_accessibility_property<resource_ref<_Properties...>> = true;
+template <class... _Properties>
+inline constexpr bool __disable_default_dynamic_accessibility_property<any_synchronous_resource<_Properties...>> = true;
+template <class... _Properties>
+inline constexpr bool __disable_default_dynamic_accessibility_property<synchronous_resource_ref<_Properties...>> = true;
+
 _CCCL_TEMPLATE(class... _Properties, class _Resource)
 _CCCL_REQUIRES(mr::synchronous_resource_with<_Resource, _Properties...>)
 synchronous_resource_ref<_Properties...> __as_resource_ref(_Resource& __mr) noexcept

libcudacxx/include/cuda/__memory_resource/get_property.h

@@ -23,6 +23,7 @@
 
 #if _CCCL_HAS_CTK()
 
+#  include <cuda/__memory_resource/properties.h>
 #  include <cuda/std/__concepts/same_as.h>
 #  include <cuda/std/__type_traits/remove_const_ref.h>
 #  include <cuda/std/__type_traits/void_t.h>
@@ -107,9 +108,19 @@ _CCCL_CONCEPT_FRAGMENT(
   __has_upstream_resource_,
   requires(const _Resource& __res)(
     requires(::cuda::std::same_as<::cuda::std::__remove_const_ref_t<decltype(__res.upstream_resource())>, _Upstream>)));
+
+template <class _Resource, class _Upstream>
+_CCCL_CONCEPT_FRAGMENT(__has_get_resource_,
+                       requires(const _Resource& __res)(requires(
+                         ::cuda::std::same_as<::cuda::std::__remove_const_ref_t<decltype(__res.get())>, _Upstream>)));
+template <class _Resource, class _Upstream>
+_CCCL_CONCEPT __has_get_resource = _CCCL_FRAGMENT(__has_get_resource_, _Resource, _Upstream);
+
 #  endif // ^^^ _CCCL_DOXYGEN_INVOKED ^^^
 template <class _Resource, class _Upstream>
-_CCCL_CONCEPT __has_upstream_resource = _CCCL_FRAGMENT(__has_upstream_resource_, _Resource, _Upstream);
+_CCCL_CONCEPT __has_forwarded_resource =
+  _CCCL_FRAGMENT(__has_upstream_resource_, _Resource, _Upstream)
+  || _CCCL_FRAGMENT(__has_get_resource_, _Resource, _Upstream);
 
 _CCCL_BEGIN_NAMESPACE_CPO(__forward_property)
 template <class _Derived, class _Upstream>
@@ -120,14 +131,35 @@ struct __fn
   _CCCL_REQUIRES((!property_with_value<_Property>) _CCCL_AND has_property<_Upstream, _Property>)
   _CCCL_API friend constexpr void get_property(const _Derived&, _Property) noexcept {}
 
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_API friend constexpr auto
+  get_property(const _Derived& __res, ::cuda::mr::dynamic_accessibility_property __prop) noexcept
+  {
+    if constexpr (_CCCL_FRAGMENT(__has_upstream_resource_, _Derived, _Upstream))
+    {
+      return get_property(__res.upstream_resource(), __prop);
+    }
+    else
+    {
+      return get_property(__res.get(), __prop);
+    }
+  }
+
   // The indirection is needed, otherwise the compiler might believe that _Derived is an incomplete type
   _CCCL_EXEC_CHECK_DISABLE
   _CCCL_TEMPLATE(class _Property, class _Derived2 = _Derived)
   _CCCL_REQUIRES(property_with_value<_Property> _CCCL_AND has_property<_Upstream, _Property> _CCCL_AND
-                   __has_upstream_resource<_Derived2, _Upstream>)
+                   __has_forwarded_resource<_Derived2, _Upstream>)
   _CCCL_API friend constexpr __property_value_t<_Property> get_property(const _Derived& __res, _Property __prop)
   {
-    return get_property(__res.upstream_resource(), __prop);
+    if constexpr (_CCCL_FRAGMENT(__has_upstream_resource_, _Derived, _Upstream))
+    {
+      return get_property(__res.upstream_resource(), __prop);
+    }
+    else
+    {
+      return get_property(__res.get(), __prop);
+    }
   }
 };
 _CCCL_END_NAMESPACE_CPO
@@ -145,15 +177,36 @@ _CCCL_END_NAMESPACE_CPO
 //!
 //! .. note::
 //!
-//!    In order to forward stateful properties, a type needs do implement an `upstream_resource()` method that returns
-//!    an instance of the upstream.
+//!    In order to forward stateful properties, a type needs to implement either:
+//!    - an `upstream_resource()` method returning the upstream resource, or
+//!    - a `get()` method returning the upstream resource.
 //!
 //! @endrst
 template <class _Derived, class _Upstream>
 using forward_property = __forward_property::__fn<_Derived, _Upstream>;
 
 _CCCL_END_NAMESPACE_CUDA
 
+_CCCL_BEGIN_NAMESPACE_CUDA_MR
+
+template <class _Tp>
+inline constexpr bool __disable_default_dynamic_accessibility_property = false;
+
+//! Default implementation: infer from has_property<Resource, host_accessible> and
+//! has_property<Resource, device_accessible>. Resources can override by providing
+//! their own get_property(..., dynamic_accessibility_property).
+//! Excluded for type-erased wrappers (any_resource, resource_ref, etc.) so that
+//! get_property dispatches via their interface to the stored concrete resource.
+_CCCL_TEMPLATE(class _Resource)
+_CCCL_REQUIRES((!__disable_default_dynamic_accessibility_property<_Resource>) )
+_CCCL_API constexpr __memory_accessability
+get_property([[maybe_unused]] const _Resource& __res, dynamic_accessibility_property) noexcept
+{
+  return __memory_accessability_from_static_properties<::cuda::has_property<_Resource, host_accessible>,
+                                                       ::cuda::has_property<_Resource, device_accessible>>();
+}
+_CCCL_END_NAMESPACE_CUDA_MR
+
 #  include <cuda/std/__cccl/epilogue.h>
 
 #endif // _CCCL_HAS_CTK()

libcudacxx/include/cuda/__memory_resource/properties.h

@@ -110,19 +110,35 @@ struct __copy_default_queries<_Resource, false>
 
 enum class __memory_accessability
 {
+  __unknown,
   __host,
   __device,
   __host_device,
 };
 
+//! @brief The dynamic_accessibility_property reports the resource's memory accessibility at runtime.
+//! Compared to the static properties, it can be used to query the memory accessibility of a resource that is not known
+//! at compile time.
+struct dynamic_accessibility_property
+{
+  using value_type = __memory_accessability;
+};
+
+template <bool _HostAccessible, bool _DeviceAccessible>
+_CCCL_API constexpr __memory_accessability __memory_accessability_from_static_properties() noexcept
+{
+  return _HostAccessible && _DeviceAccessible ? __memory_accessability::__host_device
+       : _DeviceAccessible                    ? __memory_accessability::__device
+       : _HostAccessible                      ? __memory_accessability::__host
+                                              : __memory_accessability::__unknown;
+}
+
 template <class... _Properties>
 struct __memory_accessability_from_properties
 {
   static constexpr __memory_accessability value =
-    ::cuda::mr::__is_host_device_accessible<_Properties...> ? __memory_accessability::__host_device
-    : ::cuda::mr::__is_device_accessible<_Properties...>
-      ? __memory_accessability::__device
-      : __memory_accessability::__host;
+    __memory_accessability_from_static_properties<::cuda::mr::__is_host_accessible<_Properties...>,
+                                                  ::cuda::mr::__is_device_accessible<_Properties...>>();
 };
 
 _CCCL_END_NAMESPACE_CUDA_MR

libcudacxx/include/cuda/__memory_resource/shared_resource.h

@@ -50,7 +50,9 @@ _CCCL_BEGIN_NAMESPACE_CUDA_MR
 //! @tparam _Resource The resource type to hold.
 //! @endrst
 template <class _Resource>
-struct shared_resource : ::cuda::mr::__copy_default_queries<_Resource>
+struct shared_resource
+    : ::cuda::mr::__copy_default_queries<_Resource>
+    , ::cuda::forward_property<shared_resource<_Resource>, _Resource>
 {
   static_assert(::cuda::mr::synchronous_resource<_Resource>, "");
 
@@ -255,19 +257,6 @@ struct shared_resource : ::cuda::mr::__copy_default_queries<_Resource>
     return !(__lhs == __rhs);
   }
 
-  //! @brief Forwards the stateless properties
-  _CCCL_TEMPLATE(class _Property)
-  _CCCL_REQUIRES((!property_with_value<_Property>) _CCCL_AND(has_property<_Resource, _Property>))
-  friend void get_property(const shared_resource&, _Property) noexcept {}
-
-  //! @brief Forwards the stateful properties
-  _CCCL_TEMPLATE(class _Property)
-  _CCCL_REQUIRES(property_with_value<_Property> _CCCL_AND(has_property<_Resource, _Property>))
-  [[nodiscard]] friend __property_value_t<_Property> get_property(const shared_resource& __self, _Property) noexcept
-  {
-    return get_property(__self.__control_block->__resource, _Property{});
-  }
-
 private:
   // Use a custom shared_ptr implementation because (a) we don't need to support weak_ptr so we only
   // need one pointer, not two, and (b) this implementation can work on device also.

libcudacxx/test/libcudacxx/cuda/memory_resource/any_resource/any_resource.cu

@@ -228,6 +228,34 @@ struct host_device_resource
 static_assert(cuda::has_property<host_device_resource, cuda::mr::device_accessible>);
 static_assert(cuda::has_property<host_device_resource, cuda::mr::host_accessible>);
 
+struct explicit_dynamic_resource
+{
+  void* allocate(cuda::stream_ref, size_t, size_t)
+  {
+    return nullptr;
+  }
+  void deallocate(cuda::stream_ref, void*, size_t, size_t) noexcept {}
+  void* allocate_sync(size_t, size_t)
+  {
+    return nullptr;
+  }
+  void deallocate_sync(void*, size_t, size_t) noexcept {}
+  friend bool operator==(const explicit_dynamic_resource&, const explicit_dynamic_resource&) noexcept
+  {
+    return true;
+  }
+  friend bool operator!=(const explicit_dynamic_resource&, const explicit_dynamic_resource&) noexcept
+  {
+    return false;
+  }
+  friend constexpr void get_property(const explicit_dynamic_resource&, cuda::mr::host_accessible) noexcept {}
+  friend constexpr cuda::mr::__memory_accessability
+  get_property(const explicit_dynamic_resource&, cuda::mr::dynamic_accessibility_property) noexcept
+  {
+    return cuda::mr::__memory_accessability::__device;
+  }
+};
+
 void requires_host(cuda::mr::resource_ref<cuda::mr::host_accessible>) {}
 void requires_device(cuda::mr::resource_ref<cuda::mr::device_accessible>) {}
 
@@ -265,6 +293,22 @@ TEST_CASE("resource_ref regression test for cccl#6839", "[container][resource]")
   CHECK(checks_device_runtime_resource_ref(ref)); // Test that we are device accessible
 }
 
+TEST_CASE("dynamic_accessibility_property in type-erased wrappers", "[container][resource]")
+{
+  cuda::mr::any_resource<cuda::mr::host_accessible> host_device_any{host_device_resource{}};
+  CHECK(get_property(host_device_any, cuda::mr::dynamic_accessibility_property{})
+        == cuda::mr::__memory_accessability::__host_device);
+
+  cuda::mr::any_resource<cuda::mr::host_accessible> explicit_any{explicit_dynamic_resource{}};
+  CHECK(get_property(explicit_any, cuda::mr::dynamic_accessibility_property{})
+        == cuda::mr::__memory_accessability::__device);
+
+  host_device_resource host_device_mr{};
+  cuda::mr::resource_ref<cuda::mr::host_accessible> host_device_ref{host_device_mr};
+  CHECK(get_property(host_device_ref, cuda::mr::dynamic_accessibility_property{})
+        == cuda::mr::__memory_accessability::__host_device);
+}
+
 TEMPLATE_TEST_CASE_METHOD(test_fixture, "Empty property set", "[container][resource]", big_resource, small_resource)
 {
   using TestResource = TestType;

libcudacxx/test/libcudacxx/cuda/memory_resource/properties/properties.pass.cpp

@@ -36,6 +36,46 @@ static_assert(!cuda::mr::__is_host_device_accessible<cuda::mr::host_accessible>,
 static_assert(!cuda::mr::__is_host_device_accessible<cuda::mr::device_accessible>, "");
 static_assert(cuda::mr::__is_host_device_accessible<cuda::mr::host_accessible, cuda::mr::device_accessible>, "");
 
+struct host_only_resource
+{
+  friend constexpr void get_property(const host_only_resource&, cuda::mr::host_accessible) noexcept {}
+};
+
+struct device_only_resource
+{
+  friend constexpr void get_property(const device_only_resource&, cuda::mr::device_accessible) noexcept {}
+};
+
+struct host_device_resource
+{
+  friend constexpr void get_property(const host_device_resource&, cuda::mr::host_accessible) noexcept {}
+  friend constexpr void get_property(const host_device_resource&, cuda::mr::device_accessible) noexcept {}
+};
+
+struct explicit_dynamic_resource
+{
+  friend constexpr void get_property(const explicit_dynamic_resource&, cuda::mr::host_accessible) noexcept {}
+  friend constexpr cuda::mr::__memory_accessability
+  get_property(const explicit_dynamic_resource&, cuda::mr::dynamic_accessibility_property) noexcept
+  {
+    return cuda::mr::__memory_accessability::__device;
+  }
+};
+
+static_assert(cuda::has_property<host_only_resource, cuda::mr::dynamic_accessibility_property>);
+static_assert(cuda::has_property<device_only_resource, cuda::mr::dynamic_accessibility_property>);
+static_assert(cuda::has_property<host_device_resource, cuda::mr::dynamic_accessibility_property>);
+static_assert(cuda::has_property<explicit_dynamic_resource, cuda::mr::dynamic_accessibility_property>);
+
+static_assert(get_property(host_only_resource{}, cuda::mr::dynamic_accessibility_property{})
+              == cuda::mr::__memory_accessability::__host);
+static_assert(get_property(device_only_resource{}, cuda::mr::dynamic_accessibility_property{})
+              == cuda::mr::__memory_accessability::__device);
+static_assert(get_property(host_device_resource{}, cuda::mr::dynamic_accessibility_property{})
+              == cuda::mr::__memory_accessability::__host_device);
+static_assert(get_property(explicit_dynamic_resource{}, cuda::mr::dynamic_accessibility_property{})
+              == cuda::mr::__memory_accessability::__device);
+
 int main(int, char**)
 {
   return 0;

libcudacxx/test/libcudacxx/cuda/memory_resource/shared_resource.cu

@@ -163,6 +163,12 @@ TEMPLATE_TEST_CASE_METHOD(test_fixture, "shared_resource", "[container][resource
     CHECK(this->counts == expected);
   }
 
+  SECTION("dynamic_accessibility_property forwards through shared_resource")
+  {
+    cuda::mr::shared_resource mr{cuda::std::in_place_type<TestResource>, 42, this};
+    CHECK(get_property(mr, cuda::mr::dynamic_accessibility_property{}) == cuda::mr::__memory_accessability::__host);
+  }
+
   // Reset the counters:
   this->counts = Counts();
 

libcudacxx/test/libcudacxx/cuda/memory_resource/synchronous_adapter.cu

@@ -28,7 +28,23 @@ constexpr bool passed_property =
 template <class Resource>
 constexpr bool same_properties =
   passed_property<Resource, cuda::mr::device_accessible> && passed_property<Resource, cuda::mr::host_accessible>
-  && passed_property<Resource, extra_property> && passed_property<Resource, get_data>;
+  && passed_property<Resource, extra_property> && passed_property<Resource, get_data>
+  && passed_property<Resource, cuda::mr::dynamic_accessibility_property>;
+
+struct explicit_dynamic_resource
+{
+  void* allocate_sync(size_t, size_t)
+  {
+    return nullptr;
+  }
+  void deallocate_sync(void*, size_t, size_t) noexcept {}
+  friend constexpr void get_property(const explicit_dynamic_resource&, cuda::mr::host_accessible) noexcept {}
+  friend constexpr cuda::mr::__memory_accessability
+  get_property(const explicit_dynamic_resource&, cuda::mr::dynamic_accessibility_property) noexcept
+  {
+    return cuda::mr::__memory_accessability::__device;
+  }
+};
 
 C2H_CCCLRT_TEST("synchronous_resource_adapter", "[memory_resource]")
 {
@@ -70,4 +86,12 @@ C2H_CCCLRT_TEST("synchronous_resource_adapter", "[memory_resource]")
     STATIC_CHECK(same_default_queries<cuda::mr::legacy_pinned_memory_resource>);
     STATIC_CHECK(same_default_queries<small_resource>);
   }
+
+  SECTION("explicit dynamic_accessibility_property overrides template")
+  {
+    cuda::mr::synchronous_resource_adapter<explicit_dynamic_resource> adapter{explicit_dynamic_resource{}};
+    STATIC_CHECK(cuda::has_property<decltype(adapter), cuda::mr::dynamic_accessibility_property>);
+    CHECK(get_property(adapter, cuda::mr::dynamic_accessibility_property{})
+          == cuda::mr::__memory_accessability::__device);
+  }
 }