[clang] [offload] [XteamReduction] Do not use blocksize any more to choose the DeviceRTL entrypoint.

Till now, for XteamReduction, the compiler would use the blocksize to
choose the runtime entrypoint to call. For example, for blocksize of
256, an entrypoint with the prefix 4x64 would be chosen for wavefront
size of 64. 4x64 indicates a maximum of 4 wavefronts, each of size 64.
Unfortunately, this approach leads to a large number of entrypoints and
maintaining them is hard.

This patch reduces the number of entrypoints manyfold by keeping support
for only 1 prefix per wavefront size. Specifically, only 16x64 and 32x32
prefixes will be supported, the rest are removed. This implies a maximum
of 16 wavefronts for wavefront size 64 and a maximum of 32 wavefronts
for wavefront size 32. Hence, these entrypoints are now independent of
the blocksize known at compile-time. Note that users can still change
the blocksize by using OpenMP clauses, compile-time options, or runtime
environment variables --- none of that support is changing.

The only downside of the new approach is that LDS will now be allocated
by DeviceRTL based on the maximum number of wavefronts, either 16 or 32.
When the default XteamReduction blocksize of 1024 is used, the LDS
allocation is still optimal. But if the blocksize is reduced by one of
the ways mentioned above, there will be LDS over-allocation. However,
given that the absolute size of this LDS usage is small, the over-allocation
should not be a problem.

The total number of entrypoints reduced from 264 to 48.

Change-Id: I06b0232a3103180c0102191a47739ffd042f4508

Author: dhruvachak

clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp

@@ -3904,325 +3904,63 @@ llvm::Value *CGOpenMPRuntimeGPU::getXteamRedSum(
   if (SumType->isIntegerTy()) {
     if (SumType->getPrimitiveSizeInBits() == 32) {
       if (WarpSize == 32) {
-        switch (BlockSize) {
-        default:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_1x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_1x32),
-              Args);
-        case 64:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_2x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_2x32),
-              Args);
-        case 128:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_4x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_4x32),
-              Args);
-        case 256:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_8x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_8x32),
-              Args);
-        case 512:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_16x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_16x32),
-              Args);
-        case 1024:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_32x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_32x32),
-              Args);
-        }
-      } else {
-        switch (BlockSize) {
-        default:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_1x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_1x64),
-              Args);
-        case 128:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_2x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_2x64),
-              Args);
-        case 256:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_4x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_4x64),
-              Args);
-        case 512:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_8x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_8x64),
-              Args);
-        case 1024:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ui_16x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ui_16x64),
-              Args);
-        }
-      }
-    }
-    if (SumType->getPrimitiveSizeInBits() == 64) {
-      if (WarpSize == 32) {
-        switch (BlockSize) {
-        default:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_1x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_1x32),
-              Args);
-        case 64:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_2x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_2x32),
-              Args);
-        case 128:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_4x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_4x32),
-              Args);
-        case 256:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_8x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_8x32),
-              Args);
-        case 512:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_16x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_16x32),
-              Args);
-        case 1024:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_32x32_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_32x32),
-              Args);
-        }
-      } else {
-        switch (BlockSize) {
-        default:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_1x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_1x64),
-              Args);
-        case 128:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_2x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_2x64),
-              Args);
-        case 256:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_4x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_4x64),
-              Args);
-        case 512:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_8x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_8x64),
-              Args);
-        case 1024:
-          return CGF.EmitRuntimeCall(
-              OMPBuilder.getOrCreateRuntimeFunction(
-                  CGM.getModule(), IsFast
-                                       ? OMPRTL___kmpc_xteamr_ul_16x64_fast_sum
-                                       : OMPRTL___kmpc_xteamr_ul_16x64),
-              Args);
-        }
-      }
-    }
-  }
-  if (SumType->isFloatTy()) {
-    if (WarpSize == 32) {
-      switch (BlockSize) {
-      default:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_1x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_1x32),
-            Args);
-      case 64:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_2x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_2x32),
-            Args);
-      case 128:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_4x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_4x32),
-            Args);
-      case 256:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_8x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_8x32),
-            Args);
-      case 512:
         return CGF.EmitRuntimeCall(
             OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_16x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_16x32),
+                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_ui_32x32_fast_sum
+                                        : OMPRTL___kmpc_xteamr_ui_32x32),
             Args);
-      case 1024:
+      } else {
         return CGF.EmitRuntimeCall(
             OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_32x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_32x32),
+                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_ui_16x64_fast_sum
+                                        : OMPRTL___kmpc_xteamr_ui_16x64),
             Args);
       }
-    } else {
-      switch (BlockSize) {
-      default:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_1x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_1x64),
-            Args);
-      case 128:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_2x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_2x64),
-            Args);
-      case 256:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_4x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_4x64),
-            Args);
-      case 512:
+    }
+    if (SumType->getPrimitiveSizeInBits() == 64) {
+      if (WarpSize == 32) {
         return CGF.EmitRuntimeCall(
             OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_8x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_8x64),
+                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_ul_32x32_fast_sum
+                                        : OMPRTL___kmpc_xteamr_ul_32x32),
             Args);
-      case 1024:
+      } else {
         return CGF.EmitRuntimeCall(
             OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_16x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_f_16x64),
+                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_ul_16x64_fast_sum
+                                        : OMPRTL___kmpc_xteamr_ul_16x64),
             Args);
       }
     }
   }
+  if (SumType->isFloatTy()) {
+    if (WarpSize == 32) {
+      return CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(
+              CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_32x32_fast_sum
+                                      : OMPRTL___kmpc_xteamr_f_32x32),
+          Args);
+    } else {
+      return CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(
+              CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_f_16x64_fast_sum
+                                      : OMPRTL___kmpc_xteamr_f_16x64),
+          Args);
+    }
+  }
   if (SumType->isDoubleTy()) {
     if (WarpSize == 32) {
-      switch (BlockSize) {
-      default:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_1x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_1x32),
-            Args);
-      case 64:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_2x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_2x32),
-            Args);
-      case 128:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_4x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_4x32),
-            Args);
-      case 256:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_8x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_8x32),
-            Args);
-      case 512:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_16x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_16x32),
-            Args);
-      case 1024:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_32x32_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_32x32),
-            Args);
-      }
+      return CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(
+              CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_32x32_fast_sum
+                                      : OMPRTL___kmpc_xteamr_d_32x32),
+          Args);
     } else {
-      switch (BlockSize) {
-      default:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_1x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_1x64),
-            Args);
-      case 128:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_2x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_2x64),
-            Args);
-      case 256:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_4x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_4x64),
-            Args);
-      case 512:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_8x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_8x64),
-            Args);
-      case 1024:
-        return CGF.EmitRuntimeCall(
-            OMPBuilder.getOrCreateRuntimeFunction(
-                CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_16x64_fast_sum
-                                        : OMPRTL___kmpc_xteamr_d_16x64),
-            Args);
-      }
+      return CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(
+              CGM.getModule(), IsFast ? OMPRTL___kmpc_xteamr_d_16x64_fast_sum
+                                      : OMPRTL___kmpc_xteamr_d_16x64),
+          Args);
     }
   }
   llvm_unreachable("No support for other types currently.");

clang/test/OpenMP/fast_red_codegen.cpp

@@ -1128,7 +1128,7 @@ int main()
 // CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[DOTADDR_ASCAST]], align 8
 // CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[DOTADDR1_ASCAST]], align 8
 // CHECK-NEXT:    [[TMP31:%.*]] = load double, ptr addrspace(5) [[TMP5]], align 8
-// CHECK-NEXT:    call void @__kmpc_xteamr_d_8x64_fast_sum(double [[TMP31]], ptr [[TMP2]], ptr [[TMP29]], ptr [[TMP30]], ptr @__kmpc_rfun_sum_d, ptr @__kmpc_rfun_sum_lds_d, double 0.000000e+00, i64 [[TMP17]], i32 [[TMP16]])
+// CHECK-NEXT:    call void @__kmpc_xteamr_d_16x64_fast_sum(double [[TMP31]], ptr [[TMP2]], ptr [[TMP29]], ptr [[TMP30]], ptr @__kmpc_rfun_sum_d, ptr @__kmpc_rfun_sum_lds_d, double 0.000000e+00, i64 [[TMP17]], i32 [[TMP16]])
 // CHECK-NEXT:    ret void
 //
 //

clang/test/OpenMP/xteam_red_codegen.cpp

@@ -1128,7 +1128,7 @@ int main()
 // CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[DOTADDR_ASCAST]], align 8
 // CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[DOTADDR1_ASCAST]], align 8
 // CHECK-NEXT:    [[TMP31:%.*]] = load double, ptr addrspace(5) [[TMP5]], align 8
-// CHECK-NEXT:    call void @__kmpc_xteamr_d_8x64(double [[TMP31]], ptr [[TMP2]], ptr [[TMP29]], ptr [[TMP30]], ptr @__kmpc_rfun_sum_d, ptr @__kmpc_rfun_sum_lds_d, double 0.000000e+00, i64 [[TMP17]], i32 [[TMP16]])
+// CHECK-NEXT:    call void @__kmpc_xteamr_d_16x64(double [[TMP31]], ptr [[TMP2]], ptr [[TMP29]], ptr [[TMP30]], ptr @__kmpc_rfun_sum_d, ptr @__kmpc_rfun_sum_lds_d, double 0.000000e+00, i64 [[TMP17]], i32 [[TMP16]])
 // CHECK-NEXT:    ret void
 //
 //