Perform a memory copy for simulation buffer located in incorrect global memory address range

Author: sophimao

include/acl_mem.h

@@ -32,6 +32,10 @@ int acl_submit_mem_transfer_device_op(cl_event event);
 
 int acl_submit_migrate_mem_device_op(cl_event event);
 
+int acl_realloc_buffer_for_simulator(cl_mem mem,
+                                     const unsigned int physical_device_id,
+                                     const unsigned int mem_id);
+
 // Actually execute the memory transfer device operation.
 // In the normal case source and destination are different, in which case
 // the HAL is called and the transfer is non-blocking.

src/acl_kernel.cpp

@@ -2917,6 +2917,14 @@ static cl_int l_copy_and_adjust_arguments_for_device(
                                                   [needed_mem_id]);
 #endif
 
+        int env_override = 0;
+        (void)acl_get_offline_device_user_setting(&env_override);
+        if (env_override == ACL_CONTEXT_MPSIM) {
+          if (!acl_realloc_buffer_for_simulator(mem_obj, needed_physical_id,
+                                                needed_mem_id)) {
+            return CL_MEM_OBJECT_ALLOCATION_FAILURE;
+          }
+        }
         // copy the address of the reserved allocation into the invocation
         // image:
         const void *mem_addr =

src/acl_mem.cpp

@@ -4224,8 +4224,6 @@ static void l_get_working_range(const acl_block_allocation_t *block_allocation,
   acl_assert_locked();
 
   if (block_allocation->region == &(acl_platform.global_mem)) {
-    int env_override = 0;
-    (void)acl_get_offline_device_user_setting(&env_override);
     const auto *global_mem_defs = &(acl_platform.device[physical_device_id]
                                         .def.autodiscovery_def.global_mem_defs);
 
@@ -6583,6 +6581,84 @@ void acl_copy_device_buffers_from_host_after_programming(
   }
 }
 
+int acl_realloc_buffer_for_simulator(cl_mem mem,
+                                     const unsigned int physical_device_id,
+                                     const unsigned int mem_id) {
+  // Only reallocate and migrate if mem resides in global memory
+  if (mem->block_allocation->region == &(acl_platform.global_mem)) {
+    int mem_on_host;
+    void *old_mem_address;
+    void *new_mem_address;
+
+    acl_addr_range_t global_mem_range =
+        acl_platform.device[physical_device_id]
+            .def.autodiscovery_def.global_mem_defs[mem_id]
+            .get_usable_range();
+
+    // Save old address
+    old_mem_address = l_get_address_of_writable_copy(mem, physical_device_id,
+                                                     &mem_on_host, CL_FALSE);
+
+    // The mem migration is only needed if the buffer is binded to the device
+    // before global memory range is confirmed (i.e., before reprogram), and
+    // assumed address range before reprogram is different from actual
+    // Therefore, check if:
+    //   1. allocation is deferred (if so auto migration will happen)
+    //   2. buffer is on host
+    //   3. buffer appears to be "at the destination"
+    //   4. block allocation is outside the global memory range
+    if (!mem->allocation_deferred &&
+        !(mem->mem_cpy_host_ptr_pending || mem_on_host) &&
+        (mem->block_allocation ==
+         mem->reserved_allocations[physical_device_id][mem_id]) &&
+        (mem->block_allocation->range.begin >= global_mem_range.next ||
+         mem->block_allocation->range.next < global_mem_range.begin)) {
+
+      // mem_id should align if block allocation is the same as reserved
+      // allocation
+      assert(mem->mem_id == mem_id);
+
+      // Okay to set this to NULL, memory tracked in mem->block_allocation
+      mem->reserved_allocations[physical_device_id][mem_id] = NULL;
+      // We will reallocate block, so remove it from linked list first
+      acl_block_allocation_t **block_ptr =
+          &(mem->block_allocation->region->first_block);
+      acl_block_allocation_t *block = *block_ptr;
+      assert(block != NULL); // Should be at least one block
+      while (*block_ptr) {
+        if (block == mem->block_allocation) {
+          *block_ptr = block->next_block_in_region;
+          break;
+        }
+        // Advance to the next block in the region
+        block_ptr = &(block->next_block_in_region);
+        block = *block_ptr;
+      }
+      if (!acl_do_physical_buffer_allocation(physical_device_id, mem)) {
+        return 0;
+      }
+
+      new_mem_address =
+          mem->reserved_allocations[physical_device_id][mem_id]->range.begin;
+      const acl_hal_t *const hal = acl_get_hal();
+
+#ifdef MEM_DEBUG_MSG
+      printf("reallocating mem obj for simulation after getting global mem "
+             "info, device %u ([0]%zx -> [0]%zx) ",
+             physical_device_id,
+             (size_t)(ACL_STRIP_PHYSICAL_ID(old_mem_address)),
+             (size_t)(ACL_STRIP_PHYSICAL_ID(new_mem_address)));
+#endif
+
+      // do blocking copy, this is for simulation only so performance is
+      // probably not a huge concern
+      hal->copy_globalmem_to_globalmem(0, old_mem_address, new_mem_address,
+                                       mem->size);
+    }
+  }
+  return 1;
+}
+
 static void acl_print_all_mem_in_region(acl_mem_region_t *region);
 void acl_print_all_mem(void) {
   acl_assert_locked();