Move API notes out of the arch-specific subdirectories in lib/swift/ (#21690)

They're all the same anyway, and no longer even need to be compiled,
just copied in as text.

And drastically simplify how we "generate" them. Instead of attaching
their build jobs to the appropriate overlays, if present, "just" have
one job to copy them all and attach it to the Darwin overlay. That's
what we do for the overlay shim headers, and it's good enough.
(Eventually we want to get out of the business of shipping them
altogether.)

This does have the same flaw as the shim headers: if you /just/ change
API notes, the corresponding overlay does not get rebuilt. You have to
touch that too. But in practice that'll happen most of the time
anyway.

Part of rdar://problem/43545560

Author: jrose-apple

CMakeLists.txt

@@ -991,23 +991,14 @@ if(SWIFT_NEED_EXPLICIT_LIBDISPATCH)
                         ${SWIFT_PATH_TO_LIBDISPATCH_SOURCE})
 endif()
 
-#
-# Set up global CMake variables for API notes.
-#
-set(SWIFT_API_NOTES_PATH "${SWIFT_SOURCE_DIR}/apinotes")
-include("${SWIFT_API_NOTES_PATH}/CMakeLists.txt")
-if(NOT DEFINED SWIFT_API_NOTES_INPUTS)
-  message(FATAL_ERROR "API notes are not available in ${SWIFT_API_NOTES_PATH}")
-endif()
-
 # Add all of the subdirectories, where we actually do work.
 
 ###############
 # PLEASE READ #
 ###############
 #
 # We have to include stdlib/ before tools/.
-# Do not move add_subdirectory(stdlib) after add_subdirectory(tools)!                                             
+# Do not move add_subdirectory(stdlib) after add_subdirectory(tools)!
 #
 # We must include stdlib/ before tools/ because stdlib/CMakeLists.txt
 # declares the swift-stdlib-* set of targets. These targets will then
@@ -1023,6 +1014,14 @@ endif()
 # https://bugs.swift.org/browse/SR-5975
 add_subdirectory(stdlib)
 
+if(SWIFT_BUILD_SDK_OVERLAY)
+  list_intersect("${SWIFT_APPLE_PLATFORMS}" "${SWIFT_SDKS}"
+                 building_darwin_sdks)
+  if(building_darwin_sdks)
+    add_subdirectory(apinotes)
+  endif()
+endif()
+
 add_subdirectory(include)
 
 if(SWIFT_INCLUDE_TOOLS)

apinotes/CMakeLists.txt

@@ -1,24 +1,33 @@
-set(SWIFT_API_NOTES_INPUTS
-  Accelerate
-  Dispatch
-  ScriptingBridge
-  os
+set(sources
+  Accelerate.apinotes
+  Dispatch.apinotes
+  ScriptingBridge.apinotes
+  os.apinotes
 )
 
-if(NOT DEFINED SWIFT_API_NOTES_PATH)
-  message(FATAL_ERROR "Define SWIFT_API_NOTES_PATH before including this file")
-endif()
+set(output_dir "${SWIFTLIB_DIR}/apinotes")
 
-foreach(module ${SWIFT_API_NOTES_INPUTS})
-  if(NOT EXISTS "${SWIFT_API_NOTES_PATH}/${module}.apinotes")
-    message(SEND_ERROR "Missing apinotes for ${module}")
-  endif()
+set(inputs)
+set(outputs)
+foreach(input ${sources})
+  list(APPEND inputs "${CMAKE_CURRENT_SOURCE_DIR}/${input}")
+  list(APPEND outputs "${output_dir}/${input}")
 endforeach()
 
-file(GLOB SWIFT_API_NOTES_INPUT_FILES "${SWIFT_API_NOTES_PATH}/*.apinotes")
-foreach(file ${SWIFT_API_NOTES_INPUT_FILES})
-  get_filename_component(name "${file}" NAME_WE)
-  if(NOT "${name}" IN_LIST SWIFT_API_NOTES_INPUTS)
-    message(SEND_ERROR "Found apinotes for ${name}; please add to CMakeLists.txt")
-  endif()
-endforeach()
+add_custom_command(
+    OUTPUT "${output_dir}"
+    COMMAND ${CMAKE_COMMAND} "-E" "make_directory" "${output_dir}")
+add_custom_command(
+    OUTPUT ${outputs}
+    DEPENDS ${inputs} "${output_dir}"
+    COMMAND
+      "${CMAKE_COMMAND}" "-E" "copy_if_different" ${inputs} "${output_dir}/")
+
+add_custom_target("copy_apinotes"
+    DEPENDS "${outputs}" "${output_dir}"
+    COMMENT "Copying API notes to ${output_dir}"
+    SOURCES "${sources}")
+
+swift_install_in_component(sdk-overlay
+    FILES ${sources}
+    DESTINATION "lib/swift/apinotes")

cmake/modules/AddSwift.cmake

@@ -606,7 +606,6 @@ endfunction()
 #     [C_COMPILE_FLAGS flag1...]
 #     [SWIFT_COMPILE_FLAGS flag1...]
 #     [LINK_FLAGS flag1...]
-#     [API_NOTES_NON_OVERLAY]
 #     [FILE_DEPENDS target1 ...]
 #     [DONT_EMBED_BITCODE]
 #     [IS_STDLIB]
@@ -661,9 +660,6 @@ endfunction()
 # LINK_FLAGS
 #   Extra linker flags.
 #
-# API_NOTES_NON_OVERLAY
-#   Generate API notes for non-overlayed modules with this target.
-#
 # FILE_DEPENDS
 #   Additional files this library depends on.
 #
@@ -686,7 +682,6 @@ endfunction()
 #   Sources to add into this library
 function(_add_swift_library_single target name)
   set(SWIFTLIB_SINGLE_options
-        API_NOTES_NON_OVERLAY
         DONT_EMBED_BITCODE
         FORCE_BUILD_OPTIMIZED
         IS_SDK_OVERLAY
@@ -805,22 +800,6 @@ function(_add_swift_library_single target name)
       SWIFTLIB_SINGLE_SOURCES
       "${SWIFTLIB_SINGLE_ARCHITECTURE}")
 
-  # Figure out whether and which API notes to create.
-  set(SWIFTLIB_SINGLE_API_NOTES)
-  if(SWIFTLIB_SINGLE_API_NOTES_NON_OVERLAY)
-    # Adopt all of the non-overlay API notes.
-    foreach(framework_name ${SWIFT_API_NOTES_INPUTS})
-      if (${framework_name} STREQUAL "WatchKit" AND
-          ${SWIFTLIB_SINGLE_SDK} STREQUAL "OSX")
-        # HACK: don't build WatchKit API notes for OS X.
-      else()
-        # Always build the "non-overlay" apinotes to keep them in sync
-        # rdar://40496966
-        list(APPEND SWIFTLIB_SINGLE_API_NOTES "${framework_name}")
-      endif()
-    endforeach()
-  endif()
-
   # Remove the "swift" prefix from the name to determine the module name.
   if(SWIFTLIB_IS_STDLIB_CORE)
     set(module_name "Swift")
@@ -870,7 +849,6 @@ function(_add_swift_library_single target name)
         ${SWIFTLIB_SINGLE_INTERFACE_LINK_LIBRARIES}
       SDK ${SWIFTLIB_SINGLE_SDK}
       ARCHITECTURE ${SWIFTLIB_SINGLE_ARCHITECTURE}
-      API_NOTES ${SWIFTLIB_SINGLE_API_NOTES}
       MODULE_NAME ${module_name}
       COMPILE_FLAGS ${SWIFTLIB_SINGLE_SWIFT_COMPILE_FLAGS}
       ${SWIFTLIB_SINGLE_IS_STDLIB_keyword}
@@ -1471,7 +1449,6 @@ endfunction()
 #     [SWIFT_COMPILE_FLAGS flag1...]
 #     [LINK_FLAGS flag1...]
 #     [DONT_EMBED_BITCODE]
-#     [API_NOTES_NON_OVERLAY]
 #     [INSTALL]
 #     [IS_STDLIB]
 #     [IS_STDLIB_CORE]
@@ -1550,9 +1527,6 @@ endfunction()
 # LINK_FLAGS
 #   Extra linker flags.
 #
-# API_NOTES_NON_OVERLAY
-#   Generate API notes for non-overlayed modules with this target.
-#
 # DONT_EMBED_BITCODE
 #   Don't embed LLVM bitcode in this target, even if it is enabled globally.
 #
@@ -1589,7 +1563,6 @@ endfunction()
 #   Sources to add into this library.
 function(add_swift_target_library name)
   set(SWIFTLIB_options
-        API_NOTES_NON_OVERLAY
         DONT_EMBED_BITCODE
         FORCE_BUILD_OPTIMIZED
         HAS_SWIFT_CONTENT
@@ -1904,7 +1877,6 @@ function(add_swift_target_library name)
         INCORPORATE_OBJECT_LIBRARIES ${SWIFTLIB_INCORPORATE_OBJECT_LIBRARIES}
         INCORPORATE_OBJECT_LIBRARIES_SHARED_ONLY ${SWIFTLIB_INCORPORATE_OBJECT_LIBRARIES_SHARED_ONLY}
         ${SWIFTLIB_DONT_EMBED_BITCODE_keyword}
-        ${SWIFTLIB_API_NOTES_NON_OVERLAY_keyword}
         ${SWIFTLIB_IS_STDLIB_keyword}
         ${SWIFTLIB_IS_STDLIB_CORE_keyword}
         ${SWIFTLIB_IS_SDK_OVERLAY_keyword}

cmake/modules/SwiftSource.cmake

@@ -17,7 +17,7 @@ function(handle_swift_sources
   cmake_parse_arguments(SWIFTSOURCES
       "IS_MAIN;IS_STDLIB;IS_STDLIB_CORE;IS_SDK_OVERLAY;EMBED_BITCODE"
       "SDK;ARCHITECTURE;INSTALL_IN_COMPONENT"
-      "DEPENDS;API_NOTES;COMPILE_FLAGS;MODULE_NAME"
+      "DEPENDS;COMPILE_FLAGS;MODULE_NAME"
       ${ARGN})
   translate_flag(${SWIFTSOURCES_IS_MAIN} "IS_MAIN" IS_MAIN_arg)
   translate_flag(${SWIFTSOURCES_IS_STDLIB} "IS_STDLIB" IS_STDLIB_arg)
@@ -90,7 +90,6 @@ function(handle_swift_sources
         FLAGS ${swift_compile_flags}
         SDK ${SWIFTSOURCES_SDK}
         ARCHITECTURE ${SWIFTSOURCES_ARCHITECTURE}
-        API_NOTES ${SWIFTSOURCES_API_NOTES}
         MODULE_NAME ${SWIFTSOURCES_MODULE_NAME}
         ${IS_MAIN_arg}
         ${IS_STDLIB_arg}
@@ -159,7 +158,7 @@ function(_compile_swift_files
   cmake_parse_arguments(SWIFTFILE
     "IS_MAIN;IS_STDLIB;IS_STDLIB_CORE;IS_SDK_OVERLAY;EMBED_BITCODE"
     "OUTPUT;MODULE_NAME;INSTALL_IN_COMPONENT"
-    "SOURCES;FLAGS;DEPENDS;SDK;ARCHITECTURE;API_NOTES;OPT_FLAGS;MODULE_DIR"
+    "SOURCES;FLAGS;DEPENDS;SDK;ARCHITECTURE;OPT_FLAGS;MODULE_DIR"
     ${ARGN})
 
   # Check arguments.
@@ -360,28 +359,6 @@ function(_compile_swift_files
     set(swift_compiler_tool_dep "swift")
   endif()
 
-  # Generate API notes if requested.
-  set(command_create_apinotes)
-  set(depends_create_apinotes)
-  set(apinote_files)
-
-  foreach(apinote_module ${SWIFTFILE_API_NOTES})
-    set(apinote_file "${module_dir}/${apinote_module}.apinotes")
-    set(apinote_input_file
-      "${SWIFT_API_NOTES_PATH}/${apinote_module}.apinotes")
-
-    list(APPEND command_create_apinotes
-      COMMAND
-      "${CMAKE_COMMAND}" "-E" "copy_if_different"
-      "${apinote_input_file}" "${apinote_file}")
-    list(APPEND depends_create_apinotes "${apinote_input_file}")
-
-    list(APPEND apinote_files "${apinote_file}")
-    swift_install_in_component("${SWIFTFILE_INSTALL_IN_COMPONENT}"
-      FILES ${apinote_file}
-      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/${library_subdir}")
-  endforeach()
-
   # If there are more than one output files, we assume that they are specified
   # otherwise e.g. with an output file map.
   set(output_option)
@@ -405,7 +382,6 @@ function(_compile_swift_files
   endif()
 
   set(standard_outputs ${SWIFTFILE_OUTPUT})
-  set(apinotes_outputs ${apinote_files})
   set(sib_outputs "${sib_file}")
   set(sibopt_outputs "${sibopt_file}")
   set(sibgen_outputs "${sibgen_file}")
@@ -425,8 +401,6 @@ function(_compile_swift_files
     # always gets updated.
     set(command_touch_standard_outputs
       COMMAND "${CMAKE_COMMAND}" -E touch ${standard_outputs})
-    set(command_touch_apinotes_outputs
-      COMMAND "${CMAKE_COMMAND}" -E touch ${apinotes_outputs})
     set(command_touch_module_outputs
       COMMAND "${CMAKE_COMMAND}" -E touch ${module_outputs})
     set(command_touch_sib_outputs
@@ -445,22 +419,6 @@ function(_compile_swift_files
       OUTPUT ${obj_dirs}
       COMMENT "Generating obj dirs for ${first_output}")
 
-  # Generate the api notes if we need them.
-  if (apinotes_outputs)
-    add_custom_command_target(
-        api_notes_dependency_target
-        # Create API notes before compiling, because this will affect the APIs
-        # the overlay sees.
-        ${command_create_apinotes}
-        ${command_touch_apinotes_outputs}
-        COMMAND ""
-        OUTPUT ${apinotes_outputs}
-        DEPENDS
-          ${depends_create_apinotes}
-          ${obj_dirs_dependency_target}
-        COMMENT "Copying API notes for ${first_output}")
-  endif()
-
   # Then we can compile both the object files and the swiftmodule files
   # in parallel in this target for the object file, and ...
 
@@ -484,7 +442,7 @@ function(_compile_swift_files
       DEPENDS
         ${swift_compiler_tool_dep}
         ${file_path} ${source_files} ${SWIFTFILE_DEPENDS}
-        ${swift_ide_test_dependency} ${api_notes_dependency_target}
+        ${swift_ide_test_dependency}
         ${obj_dirs_dependency_target}
       COMMENT "Compiling ${first_output}")
   set("${dependency_target_out_var_name}" "${dependency_target}" PARENT_SCOPE)
@@ -518,7 +476,7 @@ function(_compile_swift_files
         DEPENDS
           ${swift_compiler_tool_dep}
           ${source_files} ${SWIFTFILE_DEPENDS}
-          ${swift_ide_test_dependency} ${api_notes_dependency_target}
+          ${swift_ide_test_dependency}
           ${obj_dirs_dependency_target}
         COMMENT "Generating ${module_file}")
     set("${dependency_module_target_out_var_name}" "${module_dependency_target}" PARENT_SCOPE)

lib/ClangImporter/ClangImporter.cpp

@@ -612,10 +612,12 @@ getNormalInvocationArguments(std::vector<std::string> &invocationArgStrs,
 
   // Enable API notes alongside headers/in frameworks.
   invocationArgStrs.push_back("-fapinotes-modules");
-  invocationArgStrs.push_back("-iapinotes-modules");
-  invocationArgStrs.push_back(searchPathOpts.RuntimeLibraryImportPath);
   invocationArgStrs.push_back("-fapinotes-swift-version=" +
-      languageVersion.asAPINotesVersionString());
+                              languageVersion.asAPINotesVersionString());
+  invocationArgStrs.push_back("-iapinotes-modules");
+  invocationArgStrs.push_back((llvm::Twine(searchPathOpts.RuntimeResourcePath) +
+                               llvm::sys::path::get_separator() +
+                               "apinotes").str());
 }
 
 static void

stdlib/public/Platform/CMakeLists.txt

@@ -14,7 +14,10 @@ add_swift_target_library(swiftDarwin ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_SDK_
     SWIFT_COMPILE_FLAGS -Xfrontend -disable-objc-attr-requires-foundation-module "${SWIFT_RUNTIME_SWIFT_COMPILE_FLAGS}"
     LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
     TARGET_SDKS ALL_APPLE_PLATFORMS
-    API_NOTES_NON_OVERLAY)
+
+    # This is overly conservative, but we have so few API notes files that
+    # haven't migrated to the Swift repo that it's probably fine in practice.
+    DEPENDS copy_apinotes)
 
 add_swift_target_library(swiftGlibc ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_SDK_OVERLAY
     Glibc.swift.gyb

stdlib/public/SwiftShims/CMakeLists.txt

@@ -64,7 +64,8 @@ foreach(input ${sources})
       COMMAND
         "${CMAKE_COMMAND}" "-E" "copy_if_different"
         "${CMAKE_CURRENT_SOURCE_DIR}/${input}"
-        "${output_dir}/${input}")
+        "${output_dir}/${input}"
+      COMMENT "Copying ${input} to ${output_dir}")
   list(APPEND outputs "${output_dir}/${input}")
 endforeach()
 # Put the output dir itself last so that it isn't considered the primary output.