Add experimental implementation of CHAI using the strategy pattern

host-configs/lc/blueos_3_ppc64le_ib_p9/nvcc_clang.cmake

@@ -30,3 +30,5 @@ set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "")
 set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")
 set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=--gcc-toolchain=${GCC_HOME}" CACHE STRING "")
+
+set(UMPIRE_FMT_TARGET "fmt::fmt" CACHE STRING "")

host-configs/lc/toss_4_x86_64_ib_cray/amdclang.cmake

@@ -6,8 +6,8 @@
 ##############################################################################
 
 # Set up software versions
-set(ROCM_VERSION "6.2.0" CACHE PATH "")
-set(GCC_VERSION "12.2.1" CACHE PATH "")
+set(ROCM_VERSION "6.4.1" CACHE PATH "")
+set(GCC_VERSION "13.3.1" CACHE PATH "")
 
 # Set up compilers
 set(COMPILER_BASE "/usr/tce/packages/rocmcc/rocmcc-${ROCM_VERSION}-magic" CACHE PATH "")

src/chai/CMakeLists.txt

@@ -20,7 +20,13 @@ set (chai_headers
   ManagedArray.inl
   managed_ptr.hpp
   PointerRecord.hpp
-  Types.hpp)
+  Types.hpp
+  expt/Array.hpp
+  expt/ExecutionContext.hpp
+  expt/Manager.hpp
+  expt/HostManager.hpp
+  expt/PageableManager.hpp
+  expt/PinnedManager.hpp)
 
 if(CHAI_DISABLE_RM)
   set(chai_headers
@@ -29,7 +35,19 @@ if(CHAI_DISABLE_RM)
 endif ()
 
 set (chai_sources
-  ArrayManager.cpp)
+  ArrayManager.cpp
+  expt/Manager.cpp
+  expt/HostManager.cpp)
+
+if(CHAI_ENABLE_CUDA OR CHAI_ENABLE_HIP OR CHAI_ENABLE_GPU_SIMULATION_MODE)
+  set(chai_headers
+      ${chai_headers}
+      expt/CopyHidingManager.hpp)
+
+  set(chai_sources
+      ${chai_sources}
+      expt/CopyHidingManager.cpp)
+endif()
 
 set (chai_depends
   umpire)

src/chai/expt/Array.hpp

@@ -0,0 +1,122 @@
+#ifndef CHAI_ARRAY_HPP
+#define CHAI_ARRAY_HPP
+
+#include "chai/Manager.hpp"
+
+namespace chai {
+namespace expt {
+  /*!
+   * \class Array
+   *
+   * \brief An array class that manages coherency across the CPU and GPU.
+   *        How the coherence is obtained is controlled by the array manager.
+   *
+   * \tparam T The type of element in the array.
+   */
+  template <typename T>
+  class Array {
+    public:
+      /*!
+       * \brief Constructs an empty array without an array manager.
+       */
+      Array() = default;
+
+      /*!
+       * \brief Constructs an array from a manager.
+       *
+       * \param manager The array manager controls the coherence of the array.
+       *
+       * \note The array takes ownership of the manager.
+       */
+      Array(Manager* manager) :
+        m_manager{manager}
+      {
+      }
+
+      /*!
+       * \brief Constructs a shallow copy of an array from another and makes
+       *        the data coherent in the current execution space.
+       *
+       * \param other The other array.
+       *
+       * \note This is a shallow copy.
+       */
+      CHAI_HOST_DEVICE Array(const Array& other) :
+        m_size{other.m_size},
+        m_data{other.m_data},
+        m_manager{other.m_manager}
+      {
+#if !defined(CHAI_DEVICE_COMPILE)
+        if (m_manager) {
+          m_manager->update(m_data, !std::is_const<T>::value);
+        }
+      }
+
+      /*!
+       * \brief Frees the resources associated with this array.
+       *
+       * \note Once free has been called, it is invalid to use any other copies
+       *       of this array (since copies are shallow).
+       */
+      void free() {
+        m_size = 0;
+        m_data = nullptr;
+        delete m_manager;
+        m_manager = nullptr;
+      }
+
+      /*!
+       * \brief Get the number of elements in the array.
+       *
+       * \pre The copy constructor has been called with the execution space
+       *      set to CPU or GPU (e.g. by the RAJA plugin).
+       */
+      CHAI_HOST_DEVICE size_t size() const {
+        return m_size;
+      }
+
+      /*!
+       * \brief Get the ith element in the array.
+       *
+       * \param i The index of the element to retrieve.
+       *
+       * \pre The copy constructor has been called with the execution space
+       *      set to CPU or GPU (e.g. by the RAJA plugin).
+       */
+      CHAI_HOST_DEVICE T& operator[](size_t i) const {
+        return m_data[i];
+      }
+
+    private:
+      /*!
+       * The number of elements in the array.
+       */
+      size_t m_size = 0;
+
+      /*!
+       * The array that is coherent in the current execution space.
+       */
+      T* m_data = nullptr;
+
+      /*!
+       * The array manager controls the coherence of the array.
+       */
+      Manager* m_manager = nullptr;
+  };  // class Array
+
+  /*!
+   * \brief Constructs an array by creating a new manager object.
+   *
+   * \tparam Manager The type of array manager.
+   * \tparam Args The type of the arguments used to construct the array manager.
+   *
+   * \param args The arguments to construct an array manager.
+   */
+  template <typename Manager, typename... Args>
+  Array<T> makeArray(Args&&... args) {
+    return Array<T>(new Manager(std::forward<Args>(args)...));
+  }
+}  // namespace expt
+}  // namespace chai
+
+#endif  // CHAI_ARRAY_HPP

src/chai/expt/CopyHidingManager.cpp

@@ -0,0 +1,75 @@
+#include "chai/expt/CopyHidingManager.hpp"
+#include "umpire/ResourceManager.hpp"
+
+namespace chai {
+namespace expt {
+  CopyHidingManager::CopyHidingManager(int hostAllocatorID,
+                                       int deviceAllocatorID,
+                                       std::size_t size) :
+    Manager{},
+    m_host_allocator_id{hostAllocatorID},
+    m_device_allocator_id{deviceAllocatorID},
+    m_size{size}
+  {
+  }
+
+  CopyHidingManager::~CopyHidingManager() {
+    umpire::ResourceManager::getInstance().getAllocator(m_device_allocator_id).deallocate(m_device_data);
+    umpire::ResourceManager::getInstance().getAllocator(m_host_allocator_id).deallocate(m_host_data);
+  }
+
+  size_t CopyHidingManager::size() const {
+    return m_size;
+  }
+
+  void* CopyHidingManager::data(ExecutionContext context, bool touch) {
+    if (context == ExecutionContext::HOST) {
+      if (!m_host_data) {
+        m_host_data = umpire::ResourceManager::getInstance().getAllocator(m_host_allocator_id).allocate(m_size);
+      }
+
+      if (m_touch == ExecutionContext::DEVICE) {
+        umpire::ResourceManager::getInstance().copy(m_host_data, m_device_data, m_size);
+        m_touch = ExecutionContext::NONE;
+      }
+
+      if (touch) {
+        m_touch = ExecutionContext::HOST;
+      }
+
+      return m_host_data;
+    }
+    else if (context == ExecutionContext::DEVICE) {
+      if (!m_device_data) {
+        m_device_data = umpire::ResourceManager::getInstance().getAllocator(m_device_allocator_id).allocate(m_size);
+      }
+
+      if (m_touch == ExecutionContext::HOST) {
+        umpire::ResourceManager::getInstance().copy(m_device_data, m_host_data, m_size);
+        m_touch = ExecutionContext::NONE;
+      }
+
+      if (touch) {
+        m_touch = ExecutionContext::DEVICE;
+      }
+
+      return m_device_data;
+    }
+    else {
+      return nullptr;
+    }
+  }
+
+  int CopyHidingManager::getHostAllocatorID() const {
+    return m_host_allocator_id;
+  }
+
+  int CopyHidingManager::getDeviceAllocatorID() const {
+    return m_device_allocator_id;
+  }
+
+  ExecutionContext CopyHidingManager::getTouch() const {
+    return m_touch;
+  }
+}  // namespace expt
+}  // namespace chai

src/chai/expt/CopyHidingManager.hpp

@@ -0,0 +1,75 @@
+#ifndef CHAI_COPY_HIDING_MANAGER_HPP
+#define CHAI_COPY_HIDING_MANAGER_HPP
+
+#include "chai/expt/Manager.hpp"
+
+namespace chai {
+namespace expt {
+  /*!
+   * \class CopyHidingManager
+   *
+   * \brief Controls the coherence of an array on the host and device.
+   */
+  class CopyHidingManager : public Manager {
+    public:
+      /*!
+       * \brief Constructs a host array manager.
+       */
+      CopyHidingManager(int hostAllocatorID,
+                        int deviceAllocatorID,
+                        std::size_t size);
+
+      /*!
+       * \brief Copy constructor is deleted.
+       */
+      CopyHidingManager(const CopyHidingManager&) = delete;
+
+      /*!
+       * \brief Copy assignment operator is deleted.
+       */
+      CopyHidingManager& operator=(const CopyHidingManager&) = delete;
+
+      /*!
+       * \brief Virtual destructor.
+       */
+      virtual ~CopyHidingManager();
+
+      /*!
+       * \brief Get the number of elements.
+       */
+      virtual std::size_t size() const override;
+
+      /*!
+       * \brief Updates the data to be coherent in the current execution space.
+       *
+       * \param data [out] A coherent array in the current execution space.
+       */
+      virtual void* data(ExecutionContext context, bool touch) override;
+
+      /*!
+       * \brief Get the host allocator ID.
+       */
+      int getHostAllocatorID() const;
+
+      /*!
+       * \brief Get the device allocator ID.
+       */
+      int getDeviceAllocatorID() const;
+
+      /*!
+       * \brief Get the last touch.
+       */
+      ExecutionContext getTouch() const;
+
+    private:
+      int m_host_allocator_id{-1};
+      int m_device_allocator_id{-1};
+      std::size_t m_size{0};
+      void* m_host_data{nullptr};
+      void* m_device_data{nullptr};
+      ExecutionContext m_touch{ExecutionContext::NONE};
+  };  // class CopyHidingManager
+}  // namespace expt
+}  // namespace chai
+
+#endif  // CHAI_COPY_HIDING_MANAGER_HPP

src/chai/expt/ExecutionContext.hpp

@@ -0,0 +1,24 @@
+#ifndef CHAI_EXECUTION_CONTEXT_HPP
+#define CHAI_EXECUTION_CONTEXT_HPP
+
+#include "chai/config.hpp"
+
+namespace chai {
+namespace expt {
+  /*!
+   * \brief Enum listing possible execution contexts.
+   */
+  enum class ExecutionContext {
+    /*! Default, no execution space. */
+    NONE = 0,
+    /*! Executing in CPU space */
+    HOST,
+#if defined(CHAI_ENABLE_CUDA) || defined(CHAI_ENABLE_HIP) || defined(CHAI_ENABLE_GPU_SIMULATION_MODE)
+    /*! Executing in GPU space */
+    DEVICE
+#endif
+  };  // enum class ExecutionContext
+}  // namespace expt
+}  // namespace chai
+
+#endif  // CHAI_EXECUTION_CONTEXT_HPP

src/chai/expt/HostManager.cpp

@@ -0,0 +1,35 @@
+#include "chai/expt/HostManager.hpp"
+#include "umpire/ResourceManager.hpp"
+
+namespace chai {
+namespace expt {
+  HostManager::HostManager(int allocatorID, std::size_t size) :
+    Manager{},
+    m_allocator_id{allocatorID},
+    m_size{size}
+  {
+    m_data = umpire::ResourceManager::getInstance().getAllocator(m_allocator_id).allocate(size);
+  }
+
+  HostManager::~HostManager() {
+    umpire::ResourceManager::getInstance().getAllocator(m_allocator_id).deallocate(m_data);
+  }
+
+  std::size_t HostManager::size() const {
+    return m_size;
+  }
+
+  void* HostManager::data(ExecutionContext context, bool /* touch */) {
+    if (context == ExecutionContext::HOST) {
+      return m_data;
+    }
+    else {
+      return nullptr;
+    }
+  }
+
+  int HostManager::getAllocatorID() const {
+    return m_allocator_id;
+  }
+}  // namespace expt
+}  // namespace chai