Add device global in device autodiscovery definition

sherry-yuan · sherry-yuan · commit f89b75951824 · 2022-02-14T09:30:42.000-05:00
----------------------------------------------------------

The autodiscovery specifies how many device global there are in the device

Specify for each device global how many attribute it has (currently 3)

The 3 attribute are: device global name, address, size

The device global name are used in runtime to get address of device global when given the name in clEnqueueReadGlobalVariableINTEL or clEnqueueWriteGlobalVariableINTEL
diff --git a/include/acl.h b/include/acl.h
@@ -7,6 +7,7 @@
 #include <array>
 #include <assert.h>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include <CL/cl_ext.h>
@@ -230,8 +231,6 @@ typedef struct {
       fast_launch_depth; /* How many kernels can be buffered on the device, 0
                             means no buffering just one can execute*/
   unsigned int is_sycl_compile; /* [1] SYCL compile; [0] OpenCL compile*/
-  unsigned int device_global_address; /* Address of kernel's device global*/
-  unsigned int device_global_size; /* Size of address space of device global used by this kernel*/
 } acl_accel_def_t;
 
 /* An ACL system definition.
@@ -480,6 +479,12 @@ typedef class acl_device_program_info_t *acl_device_program_info;
  */
 #define ACL_MEM_CAPABILITY_P2P (1 << 3)
 
+typedef struct acl_device_global_mem_def_t {
+  std::string name;
+  unsigned int address;
+  unsigned int size;
+} acl_device_global_mem_def_t;
+
 // Part of acl_device_def_t where members are populated from the information
 // in the autodiscovery string. This will get updated every time the device
 // is programmed with a new device binary as the new binary would contain a
@@ -498,6 +503,11 @@ typedef struct acl_device_def_autodiscovery_t {
   std::array<acl_system_global_mem_def_t, ACL_MAX_GLOBAL_MEM> global_mem_defs;
 
   std::vector<acl_hostpipe_info_t> acl_hostpipe_info;
+
+  // device global definition
+  unsigned int num_device_global;
+  std::unordered_map<std::string, acl_device_global_mem_def_t>
+      device_global_mem_defs;
 } acl_device_def_autodiscovery_t;
 
 typedef struct acl_device_def_t {
diff --git a/src/acl_auto_configure.cpp b/src/acl_auto_configure.cpp
@@ -96,6 +96,7 @@ static bool read_uint_counters(const std::string &str,
     UNREFERENCED_PARAMETER(e);
     return false;
   }
+
   return true;
 }
 
@@ -493,6 +494,61 @@ bool acl_load_device_def_from_str(const std::string &config_str,
                                 counters);
   }
 
+  // Read device global information
+  unsigned int num_device_global = 0;
+  if (result && counters.back() > 0) {
+    result =
+        read_uint_counters(config_str, curr_pos, num_device_global, counters);
+    devdef.num_device_global = num_device_global;
+
+    for (auto i = 0U; result && (i < num_device_global);
+         i++) { // device_global_memories
+      // read total number of fields in global_memories
+      int total_fields_device_global = 0;
+      if (counters.back() > 0) {
+        result = read_int_counters(config_str, curr_pos,
+                                   total_fields_device_global, counters);
+      }
+
+      counters.emplace_back(total_fields_device_global);
+
+      // read device global name
+      std::string device_global_name;
+      if (result && counters.back() > 0) {
+        result = read_string_counters(config_str, curr_pos, device_global_name,
+                                      counters);
+      }
+
+      // read device global address
+      unsigned int dev_global_addr = 0; // Default
+      if (result && counters.back() > 0) {
+        result =
+            read_uint_counters(config_str, curr_pos, dev_global_addr, counters);
+      }
+      // read device global address size
+      unsigned int dev_global_size = 0; // Default
+      if (result && counters.back() > 0) {
+        result =
+            read_uint_counters(config_str, curr_pos, dev_global_size, counters);
+      }
+
+      acl_device_global_mem_def_t dev_global_def = {
+          device_global_name, dev_global_addr, dev_global_size};
+      devdef.device_global_mem_defs[device_global_name] = dev_global_def;
+
+      // forward compatibility: bypassing remaining fields at the end of global
+      // memory
+      while (result && counters.size() > 0 &&
+             counters.back() > 0) { // total_fields_device_global>0
+        std::string tmp;
+        result =
+            result && read_string_counters(config_str, curr_pos, tmp, counters);
+        check_section_counters(counters);
+      }
+      counters.pop_back(); // removing total_fields_device_global
+    }                      // device_global_memories
+  }
+
   // forward compatibility: bypassing remaining fields at the end of device
   // description section
   while (result && counters.size() > 0 &&
@@ -873,30 +929,6 @@ bool acl_load_device_def_from_str(const std::string &config_str,
                                     devdef.accel[i].is_sycl_compile, counters);
       }
 
-      devdef.accel[i].device_global_address =
-          0; // Initializing for backward compatability
-      std::cerr << result << std::endl;
-      std::cerr << (counters.back() > 0) << std::endl;
-      if (result && counters.back() > 0) {
-        std::cerr << "read dev global address" << std::endl;
-        result = read_uint_counters(config_str, curr_pos,
-                                    devdef.accel[i].device_global_address, counters);
-      }else {
-        std::cerr << "read dev global address fail" << std::endl;
-      }
-        
-
-      devdef.accel[i].device_global_size =
-          0; // Initializing for backward compatability
-      if (result && counters.back() > 0) {
-        std::cerr << "read dev global size" << std::endl;
-        result = read_uint_counters(config_str, curr_pos,
-                                    devdef.accel[i].device_global_size, counters);
-      }else {
-        std::cerr << "read dev global size fail" << std::endl;
-
-      }
-
       // forward compatibility: bypassing remaining fields at the end of kernel
       // description section
       while (result && counters.size() > 0 &&
diff --git a/src/acl_mem.cpp b/src/acl_mem.cpp
@@ -421,10 +421,12 @@ CL_API_ENTRY cl_int clEnqueueReadGlobalVariableINTEL(
   }
 
   // dev_addr_t dev_global_address =
-  uintptr_t dev_global_address = kernel->accel_def->device_global_address;
-  assert(kernel->accel_def->device_global_address == 4096); // TODO: remove when merging
-  // uintptr_t dev_global_address = 0x4000000;
-  // TODO: add checks for whether the copy will be out of bound for device global
+  // uintptr_t dev_global_address =
+  // kernel->dev_bin->get_devdef().autodiscovery_def.device_global_mem_defs[name];
+  // uintptr_t dev_global_address = kernel->accel_def->device_global_address;
+  uintptr_t dev_global_address = 0x4000000;
+  // TODO: add checks for whether the copy will be out of bound for device
+  // global
   void *dev_global_ptr =
       (void *)(dev_global_address + offset * 8); // 1 unit of offset is 8 bits
   status = set_kernel_arg_mem_pointer_without_checks(kernel, 0, dev_global_ptr);
diff --git a/test/acl_auto_configure_test.cpp b/test/acl_auto_configure_test.cpp
@@ -36,6 +36,7 @@ TEST(auto_configure, simple) {
 #define VERSIONIDSTRINGIFY(x) #x
 #define VERSIONIDTOSTR(x) VERSIONIDSTRINGIFY(x)
 #define DEVICE_FIELDS " 23"
+#define DEVICE_FIELDS_DEV_GLOBAL " 32"
 #define DEVICE_FIELDS_OLD " 18"
 #define BOARDNAME "de4_gen2x4_swdimm"
 #define BOARDNAME2 "pcie385_a7"
@@ -96,31 +97,32 @@ TEST(auto_configure, simple) {
 #define IS_SYCL_COMPILE " 1"
 #define IS_NOT_SYCL_COMPILE " 0"
 
-// device global information
-#define KERNEL_DEVICE_GLOBAL_ADDRESS " 4096"
-#define KERNEL_DEVICE_GLOBAL_SIZE " 2048"
+// device global
+#define NUM_DEV_GLOBAL " 2"
+#define NUM_DEV_GLOBAL_FIELD " 3" // containing dev_globa_name, address, size
+#define DEV_GLOBAL_1                                                           \
+  " kernel15_dev_global 4096 2048" // in format of dev_globa_name, address, size
+#define DEV_GLOBAL_2 " kernel15_dev_global2 2048 1024"
 
   int parsed;
   std::string err_str;
   std::string autodiscovery = std::string(
-              VERSIONIDTOSTR(ACL_AUTO_CONFIGURE_VERSIONID)
-                  DEVICE_FIELDS RANDOM_HASH
-              " " BOARDNAME IS_NOT_BIG_ENDIAN MEM HOSTPIPE KERNEL_ARG_INFO_NONE
-              " 1 84 foo" KERNEL_CRA KERNEL_FAST_LAUNCH_DEPTH KERNEL_PERF_MON // 84 = number of kernel field
-                  KERNEL_WORKGROUP_VARIANT KERNEL_WORKITEM_VARIANT
-                      KERNEL_NUM_VECTOR_LANES1 KERNEL_PROFILE_SCANCHAIN_LENGTH
-                          ARGS_LOCAL_GLOBAL_LONG_PROF KERNEL_PRINTF_FORMATSTRINGS
-                              LD_1024 KERNEL_REQD_WORK_GROUP_SIZE_NONE
-                                  KERNEL_MAX_WORK_GROUP_SIZE_NONE
-                                      KERNEL_MAX_GLOBAL_WORK_DIM_NONE
-                                          KERNEL_USES_GLOBAL_WORK_OFFSET_ENABLED
-                                              IS_SYCL_COMPILE KERNEL_DEVICE_GLOBAL_ADDRESS KERNEL_DEVICE_GLOBAL_SIZE);
-  std::cerr << autodiscovery << std::endl;
-  ACL_LOCKED(
-      parsed = acl_load_device_def_from_str(
-          autodiscovery,
-          m_device_def.autodiscovery_def, err_str));
-  std::cerr << err_str << std::endl;
+      VERSIONIDTOSTR(ACL_AUTO_CONFIGURE_VERSIONID)
+          DEVICE_FIELDS_DEV_GLOBAL RANDOM_HASH
+      " " BOARDNAME IS_NOT_BIG_ENDIAN MEM HOSTPIPE KERNEL_ARG_INFO_NONE
+          NUM_DEV_GLOBAL NUM_DEV_GLOBAL_FIELD DEV_GLOBAL_1 NUM_DEV_GLOBAL_FIELD
+              DEV_GLOBAL_2
+      " 1 82 foo" KERNEL_CRA KERNEL_FAST_LAUNCH_DEPTH KERNEL_PERF_MON
+          KERNEL_WORKGROUP_VARIANT KERNEL_WORKITEM_VARIANT
+              KERNEL_NUM_VECTOR_LANES1 KERNEL_PROFILE_SCANCHAIN_LENGTH
+                  ARGS_LOCAL_GLOBAL_LONG_PROF KERNEL_PRINTF_FORMATSTRINGS
+                      LD_1024 KERNEL_REQD_WORK_GROUP_SIZE_NONE
+                          KERNEL_MAX_WORK_GROUP_SIZE_NONE
+                              KERNEL_MAX_GLOBAL_WORK_DIM_NONE
+                                  KERNEL_USES_GLOBAL_WORK_OFFSET_ENABLED
+                                      IS_SYCL_COMPILE);
+  ACL_LOCKED(parsed = acl_load_device_def_from_str(
+                 autodiscovery, m_device_def.autodiscovery_def, err_str));
   CHECK_EQUAL(1, parsed);
 
   CHECK_EQUAL(1, m_device_def.autodiscovery_def.num_global_mem_systems);
@@ -267,9 +269,26 @@ TEST(auto_configure, simple) {
   CHECK_EQUAL(0,
               (int)m_device_def.autodiscovery_def.accel[0].max_work_group_size);
   CHECK_EQUAL(1, (int)m_device_def.autodiscovery_def.accel[0].is_sycl_compile);
-  CHECK_EQUAL(4096, (int)m_device_def.autodiscovery_def.accel[0].device_global_address);
-  CHECK_EQUAL(2048, (int)m_device_def.autodiscovery_def.accel[0].device_global_size);
 
+  CHECK_EQUAL(2, (int)m_device_def.autodiscovery_def.num_device_global);
+  CHECK(m_device_def.autodiscovery_def.device_global_mem_defs.find(
+            "kernel15_dev_global") !=
+        m_device_def.autodiscovery_def.device_global_mem_defs.end());
+  CHECK(m_device_def.autodiscovery_def.device_global_mem_defs.find(
+            "kernel15_dev_global2") !=
+        m_device_def.autodiscovery_def.device_global_mem_defs.end());
+  CHECK_EQUAL(4096, m_device_def.autodiscovery_def
+                        .device_global_mem_defs["kernel15_dev_global"]
+                        .address);
+  CHECK_EQUAL(2048, m_device_def.autodiscovery_def
+                        .device_global_mem_defs["kernel15_dev_global"]
+                        .size);
+  CHECK_EQUAL(2048, m_device_def.autodiscovery_def
+                        .device_global_mem_defs["kernel15_dev_global2"]
+                        .address);
+  CHECK_EQUAL(1024, m_device_def.autodiscovery_def
+                        .device_global_mem_defs["kernel15_dev_global2"]
+                        .size);
 
   // Check a second parsing.
   // It should allocate a new string for the name.
@@ -470,8 +489,8 @@ TEST(auto_configure, many_ok_forward_compatibility) {
       ACL_AUTO_CONFIGURE_VERSIONID) " 28 "
                                     "sample40byterandomhash000000000000000000 "
                                     "a10gx 0 1 15 DDR 2 1 6 0 2147483648 100 "
-                                    "100 100 100 200 200 200 200 0 0 0 0 400 "
-                                    "400 400 400 400 47 "
+                                    "100 100 100 200 200 200 200 0 0 0 0 2 "
+                                    "1 1 1 400 47 "
                                     "40 external_sort_stage_0 0 128 1 0 0 1 0 "
                                     "1 0 1 10 0 0 4 1 0 0 500 500 500 500 0 0 "
                                     "0 0 1 1 1 3 1 1 1 3 1 800 800 800 800 800 "
@@ -1185,7 +1204,7 @@ TEST(auto_configure, hostpipe) {
                                     "200 "
                                     "2 9 host_to_dev 1 0 32 32768 300 300 300 "
                                     "300 dev_to_host 0 1 32 32768 300 300 300 "
-                                    "300 400 400 400 400 400 0 "
+                                    "300 400 1 3 400 400 0 "
                                     "1 29 foo 0 128 1 0 0 1 0 1 0 0 0 0 0 0 1 "
                                     "1 1 3 1 1 1 3 1 800 800 800 800 800 900 "
                                     "900"
diff --git a/test/acl_globals_test.cpp b/test/acl_globals_test.cpp
@@ -198,25 +198,24 @@ static std::vector<acl_accel_def_t> acltest_complex_system_device0_accel = {
      {},
      {32768, 0, 0},
      1},
-    {14,                                                // id
-     ACL_RANGE_FROM_ARRAY(acltest_devicelocal[11]),     // mem
-     acltest_kernels[14],                               // iface
-     acltest_laspace_info, // local_aspaces
-     {0, 0, 0}, // compile_work_group_size
-     0, // is_workgroup_invariant
-     0, // is_workitem_invariant
-     1, // num_vector_lanes
-     0, // profiling_words_to_readback
-     32768, // max_work_group_size
-     3, // max_global_work_dim
-     {}, // printf_format_info
-     {32768, 0, 0}, // max_work_group_size_arr
-     1, // uses_global_work_offset
-     0, // fast_launch_depth
-     1, // is_sycl_compile
-     4096, // device_global_address
-     2048, // device_global_size
-     },
+    {
+        14,                                            // id
+        ACL_RANGE_FROM_ARRAY(acltest_devicelocal[11]), // mem
+        acltest_kernels[14],                           // iface
+        acltest_laspace_info,                          // local_aspaces
+        {0, 0, 0},     // compile_work_group_size
+        0,             // is_workgroup_invariant
+        0,             // is_workitem_invariant
+        1,             // num_vector_lanes
+        0,             // profiling_words_to_readback
+        32768,         // max_work_group_size
+        3,             // max_global_work_dim
+        {},            // printf_format_info
+        {32768, 0, 0}, // max_work_group_size_arr
+        1,             // uses_global_work_offset
+        0,             // fast_launch_depth
+        1,             // is_sycl_compile
+    },
     {1,
      ACL_RANGE_FROM_ARRAY(acltest_devicelocal[1]),
      acltest_kernels[1],
