merge amd-stg-open into amd-mainline-open

  Bulk Promotion from July 20th, 2023

  Merge commit '821c4a5d622158015e6ff4847ab917f348228beb' into HEAD

Change-Id: If49761004bf0a9856eb9eada0a7d3f2025c5dd62

Author: David Salinas

amd/comgr/CMakeLists.txt

@@ -1,8 +1,30 @@
 cmake_minimum_required(VERSION 3.13.4)
 
-project(amd_comgr VERSION "2.5.0" LANGUAGES C CXX)
+project(amd_comgr VERSION "2.6.0" LANGUAGES C CXX)
 set(amd_comgr_NAME "${PROJECT_NAME}")
 
+# Get git branch and commit hash to add to log for easier debugging.
+# Modeled after https://github.com/pmirshad/cmake-with-git-metadata.
+# Current working branch
+if(EXISTS "${CMAKE_SOURCE_DIR}/../../.git")
+execute_process(
+  COMMAND git rev-parse --abbrev-ref HEAD
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE AMD_COMGR_GIT_BRANCH
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+# Shortened commit hash
+execute_process(
+  COMMAND git log -1 --format=%h
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE AMD_COMGR_GIT_COMMIT
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+else(EXISTS "${CMAKE_SOURCE_DIR}/../../.git")
+  set(AMD_COMGR_GIT_BRANCH "not-available")
+  set(AMD_COMGR_GIT_COMMIT "not-available")
+endif(EXISTS "${CMAKE_SOURCE_DIR}/../../.git")
+
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 # Disable file reorg backward compatibility for ASAN build
@@ -111,7 +133,6 @@ message("--CLANG_INCLUDE_DIRS: ${CLANG_INCLUDE_DIRS}")
 message("----LLD_INCLUDE_DIRS: ${LLD_INCLUDE_DIRS}")
 message("---AMDDeviceLibs_DIR: ${AMDDeviceLibs_DIR}")
 message("------------ROCM_DIR: ${ROCM_DIR}")
-message("")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
@@ -139,13 +160,19 @@ set(AMD_COMGR_PRIVATE_COMPILE_DEFINITIONS ${LLVM_DEFINITIONS})
 set(AMD_COMGR_PUBLIC_LINKER_OPTIONS)
 set(AMD_COMGR_PRIVATE_LINKER_OPTIONS)
 
+list(APPEND AMD_COMGR_PRIVATE_COMPILE_OPTIONS "-DAMD_COMGR_GIT_COMMIT=${AMD_COMGR_GIT_COMMIT}")
+list(APPEND AMD_COMGR_PRIVATE_COMPILE_OPTIONS "-DAMD_COMGR_GIT_BRANCH=${AMD_COMGR_GIT_BRANCH}")
+message("----COMGR_GIT_COMMIT: ${AMD_COMGR_GIT_COMMIT}")
+message("----COMGR_GIT_BRANCH: ${AMD_COMGR_GIT_BRANCH}")
+message("")
+
 if (UNIX)
   list(APPEND AMD_COMGR_PRIVATE_COMPILE_OPTIONS
     -fno-rtti -Wall -Wno-attributes -fms-extensions -fvisibility=hidden)
   # TODO: Confirm this is actually needed due to LLVM/Clang code
   list(APPEND AMD_COMGR_PRIVATE_COMPILE_OPTIONS -fno-strict-aliasing)
   list(APPEND AMD_COMGR_PRIVATE_COMPILE_DEFINITIONS
-    _GNU_SOURCE __STDC_LIMIT_MACROS __STDC_CONSTANT_MACROS)
+    _GNU_SOURCE __STDC_LIMIT_MACROS __STDC_CONSTANT_MACROS AMD_COMGR_BUILD)
   list(APPEND AMD_COMGR_PUBLIC_LINKER_OPTIONS -pthread)
   if (NOT APPLE AND COMGR_BUILD_SHARED_LIBS)
     configure_file(

amd/comgr/cmake/opencl_pch.cmake

@@ -38,7 +38,7 @@ function(generate_pch version)
 
   add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/opencl${version}-c.pch
     COMMAND $<TARGET_FILE:clang> -cc1 -x cl-header -triple amdgcn-amd-amdhsa
-      -Werror -O3 -Dcl_khr_fp64 -Dcl_khr_fp16 -DNDEBUG -cl-std=CL${version}
+    -Werror -Wno-builtin-macro-redefined -O3 -Dcl_khr_fp64 -Dcl_khr_fp16 -DNDEBUG -cl-std=CL${version}
       -emit-pch -o ${CMAKE_CURRENT_BINARY_DIR}/opencl${version}-c.pch
       < ${OPENCL_C_H}
     DEPENDS clang ${OPENCL_C_H} ${clang_resource_headers_gen}

amd/comgr/docs/ReleaseNotes.md

@@ -19,6 +19,8 @@ code bases.
 legal comgr-objdump option. However registering this as an LLVM option by Comgr
 prevents other LLVM tools or instances from registering a -h option in the same
 process, which is an issue because -h is a common short form for -help.
+-  Updated default code object version used when linking code object specific
+device library from v4 to v5
 
 New Features
 ------------
@@ -49,6 +51,14 @@ prevented correct execution of
 COMPILE\_SOURCE\_WITH\_DEVICE\_LIBS\_TO\_BC action.
 - Fixed a multi-threading bug where programs would hang when calling Comgr APIs
 like amd\_comgr\_iterate\_symbols() from multiple threads
+- Fixed an issue where providing DataObjects with an empty name to the bitcode
+linking action caused errors when AMD\_COMGR\_SAVE\_TEMPS was enabled, or when
+linking bitcode bundles.
+- Updated to use lld::lldMain() introduced in D110949 instead of the older
+lld::elf::link in Comgr's linkWithLLD()
+- Added -x assembler option to assembly compilation. Before, if an assembly file
+did not end with a .s file extension, it was not handled properly by the Comgr
+ASSEMBLE\_SOURCE\_TO\_RELOCATABLE action.
 
 
 New APIs
@@ -58,6 +68,15 @@ New APIs
     - Support bitcode and executable name lowering. The first call populates a
     list of mangled names for a given data object, while the second fetches a
     name from a given object and index.
+- amd\_comgr\_populate\_name\_expression\_map() (v2.6)
+- amd\_comgr\_map\_name\_expression\_to\_symbol\_name() (v2.6)
+    - Support bitcode and code object name expression mapping. The first call
+    populates a map of name expressions for a given comgr data object, using
+    LLVM APIs to traverse the bitcode or code object. The second call returns
+    a value (mangled symbol name) from the map for a given key (unmangled
+    name expression). These calls assume that names of interest have been
+    enclosed the HIP runtime using a stub attribute containg the following
+    string in the name: "__amdgcn_name_expr".
 
 Deprecated APIs
 ---------------
@@ -67,6 +86,10 @@ Removed APIs
 
 New Comgr Actions and Data Types
 --------------------------------
+- (Action) AMD\_COMGR\_ACTION\_COMPILE\_SOURCE\_TO\_RELOCATABLE
+  - This action performs compile-to-bitcode, linking device libraries, and
+codegen-to-relocatable in a single step. By doing so, clients are able to defer more
+of the flag handling to toolchain. Currently only supports HIP.
 - (Data Type) AMD\_COMGR\_DATA\_KIND\_BC\_BUNDLE
 - (Data Type) AMD\_COMGR\_DATA\_KIND\_AR\_BUNDLE
   - These data kinds can now be passed to an AMD\_COMGR\_ACTION\_LINK\_BC\_TO\_BC
@@ -95,6 +118,12 @@ metadata querys for code object v2 objects.
 deprecation of code object v3 in LLVM. However, we still test loading and
 metadata querys for code object v3 objects.
 - Revamp symbolizer test to fail on errors, among other improvments
+- Improve linking and unbundling log to correctly store temporary files in /tmp,
+and to output clang-offload-bundler command to allow users to re-create Comgr
+unbundling.
+- Add git branch and commit hash for Comgr, and commit hash for LLVM to log
+output for Comgr actions. This can help us debug issues more quickly in cases
+where reporters provide Comgr logs.
 
 New Targets
 -----------

amd/comgr/include/amd_comgr.h.in

@@ -51,6 +51,13 @@
 // Add deprecation support for Comgr on Linux
 // This can be removed in favor of generic [[deprecated]] in C23, which should
 // also allow us to more easily include support on Windows
+
+#ifndef AMD_COMGR_DEPRECATED
+#ifdef AMD_COMGR_BUILD
+#define AMD_COMGR_DEPRECATED(msg) // empty
+#endif
+#endif
+
 #ifndef AMD_COMGR_DEPRECATED
 #if defined __GNUC__ && (__GNUC__ > 5 || defined __clang__)
 #define AMD_COMGR_DEPRECATED(msg) __attribute__((deprecated(msg)))
@@ -203,6 +210,12 @@ extern "C" {
  */
 #define AMD_COMGR_VERSION_2_5
 
+/**
+ * The function was introduced or changed in version 2.6 of the interface
+ * and has the symbol version string of ``"@amd_comgr_NAME@_2.6"``.
+ */
+#define AMD_COMGR_VERSION_2_6
+
 /** @} */
 
 /**
@@ -1572,8 +1585,9 @@ typedef enum amd_comgr_action_kind_s {
    * Return @p AMD_COMGR_STATUS_ERROR if the link or unbundling fails.
    *
    * Return @p AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT
-   * if isa name is not set in @p info and does not match the isa name
-   * of all bc data objects in @p input.
+   * if IsaName is not set in @p info and does not match the isa name
+   * of all bc data objects in @p input, or if the Name field is not set for
+   * any DataObject in the input set.
    */
   AMD_COMGR_ACTION_LINK_BC_TO_BC = 0x4,
   /**
@@ -1731,10 +1745,27 @@ typedef enum amd_comgr_action_kind_s {
    * if isa name or language is not set in @p info.
    */
   AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC = 0xF,
+  /**
+   * Compile a single source data object in @p input in order. For each
+   * successful compilation add a relocatable data object to @p result.
+   * Resolve any include source names using the names of include data objects
+   * in @p input. Resolve any include relative path names using the
+   * working directory path in @p info. Produce relocatable for hip name in @p
+   * info. Compile the source for the language in @p info. Link against
+   * the device-specific and language-specific bitcode device libraries
+   * required for compilation. Currently only supports HIP language.
+   *
+   * Return @p AMD_COMGR_STATUS_ERROR if any compilation
+   * fails.
+   *
+   * Return @p AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT
+   * if isa name or language is not set in @p info.
+   */
+  AMD_COMGR_ACTION_COMPILE_SOURCE_TO_RELOCATABLE = 0x10,
   /**
    * Marker for last valid action kind.
    */
-  AMD_COMGR_ACTION_LAST = AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC
+  AMD_COMGR_ACTION_LAST = AMD_COMGR_ACTION_COMPILE_SOURCE_TO_RELOCATABLE
 } amd_comgr_action_kind_t;
 
 /**
@@ -2386,6 +2417,76 @@ amd_comgr_get_mangled_name(
     size_t *size,
     char *mangled_name) AMD_COMGR_VERSION_2_5;
 
+/**
+ * @brief Populate a name expression map from a given code object.
+ *
+ * Used to map stub names *__amdgcn_name_expr_* in bitcodes and code
+ * objects generated by hip runtime to an associated (unmangled) name
+ * expression and (mangled) symbol name.
+ *
+ * @param[in] data A data object of kind @p
+ * AMD_COMGR_DATA_KIND_EXECUTABLE or @p AMD_COMGR_DATA_KIND_BC
+ *
+ * @param[out] count The number of name expressions mapped. This value
+ * can be used as an upper bound to the Index provided to the corresponding
+ * amd_comgr_map_name_expression_to_symbol_name() call.
+ *
+ * @retval ::AMD_COMGR_STATUS_SUCCESS The function executed successfully.
+ *
+ * @retval ::AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT @p data is
+ * an invalid data object or not of kind @p AMD_COMGR_DATA_KIND_EXECUTABLE or
+ * @p AMD_COMGR_DATA_KIND_BC.
+ *
+ * @retval ::AMD_COMGR_STATUS_ERROR LLVM API failure, which should be
+ * accompanied by an LLVM error message to stderr
+ *
+ */
+amd_comgr_status_t AMD_COMGR_API
+amd_comgr_populate_name_expression_map(
+    amd_comgr_data_t data,
+    size_t *count) AMD_COMGR_VERSION_2_6;
+
+/**
+ * @brief Fetch a related symbol name for a given name expression;
+ * or that name's length.
+ *
+ * The @p data must have had its name expression map populated with @p
+ * amd_comgr_populate_name_expression_map.
+ *
+ * @param[in] data A data object of kind @p
+ * AMD_COMGR_DATA_KIND_EXECUTABLE or @p AMD_COMGR_DATA_KIND_BC used to
+ * identify which map of name expressions to retrieve from.
+ *
+ * @param[in, out] size For out, the size of @p symbol_name. For in,
+ * if @symbol_name is NULL, set to the size of the Nth option string including
+ * the terminating null character.
+ *
+ * @param[in] name_expression A character array of a name expression. This name
+ * is used as the key to the name expression map in order to locate the desired
+ * @symbol_name.
+ *
+ * @param[out] symbol_name If not NULL, then the first @p size characters of
+ * the symbol name string mapped from @name_expression are copied into @p
+ * symbol_name. If NULL, no symbol name string is copied, and only @p size is
+ * updated (useful in order to find the size of the buffer required to copy the
+ * symbol_name string).
+ *
+ * @retval ::AMD_COMGR_STATUS_SUCCESS The function executed successfully.
+ *
+ * @retval ::AMD_COMGR_STATUS_ERROR @p data object is not valid (NULL or not of
+ * type bitcode or code object)
+ *
+ * @retval ::AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT @p name_expression is not
+ * present in the name expression map.
+ *
+ */
+amd_comgr_status_t AMD_COMGR_API
+amd_comgr_map_name_expression_to_symbol_name(
+    amd_comgr_data_t data,
+    size_t *size,
+    char *name_expression,
+    char *symbol_name) AMD_COMGR_VERSION_2_6;
+
 /**
  * @brief A data structure for Code object information.
  */