Merge remote-tracking branch 'origin/master' into master-rebranch

Author: swift-ci

utils/api_checker/sdk-module-lists/fixed-modules-appletvos.txt renamed to utils/api_checker/sdk-module-lists/fixed-clang-modules-appletvos.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-modules-common.txt renamed to utils/api_checker/sdk-module-lists/fixed-clang-modules-common.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-modules-watchos.txt renamed to utils/api_checker/sdk-module-lists/fixed-clang-modules-iosmac.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-modules-iphoneos.txt renamed to utils/api_checker/sdk-module-lists/fixed-clang-modules-iphoneos.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-modules-macosx.txt renamed to utils/api_checker/sdk-module-lists/fixed-clang-modules-macosx.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-clang-modules-watchos.txt

@@ -0,0 +1 @@
+CreateML

utils/api_checker/sdk-module-lists/fixed-swift-modules-appletvos.txt

@@ -0,0 +1 @@
+// Empty

utils/api_checker/sdk-module-lists/fixed-swift-modules-common.txt

@@ -44,24 +44,38 @@ def check_output(cmd, verbose=False):
 
 
 def get_sdk_path(platform):
+    if platform.startswith('iosmac'):
+        platform = 'macosx'
     return check_output(['xcrun', '-sdk', platform, '-show-sdk-path'])
 
 
+def write_fixed_module(file, platform, infix, verbose):
+    common_modules_path = os.path.join(INFER_IMPORT_DIR, 'fixed-' + infix +
+                                       '-modules-common.txt')
+    platform_modules_path = os.path.join(INFER_IMPORT_DIR, 'fixed-' + infix +
+                                         '-modules-' + platform + '.txt')
+    with open(common_modules_path, 'r') as extra:
+        if verbose:
+            print('Including modules in: ' + common_modules_path)
+        file.write(extra.read())
+    with open(platform_modules_path, 'r') as extra:
+        if verbose:
+            print('Including modules in: ' + platform_modules_path)
+        file.write(extra.read())
+
+
 def prepare_module_list(platform, file, verbose, module_filter_flags,
-                        include_fixed_modules):
+                        include_fixed_clang_modules):
     cmd = [INFER_IMPORT_PATH, '-s', get_sdk_path(platform)]
     cmd.extend(module_filter_flags)
     if verbose:
         cmd.extend(['--v'])
     check_call(cmd, output=file)
-    # The fixed modules are all objc frameworks.
-    if not include_fixed_modules:
-        return
-    with open(INFER_IMPORT_DIR + '/fixed-modules-common.txt', 'r') as extra:
-        file.write(extra.read())
-    with open(INFER_IMPORT_DIR + '/fixed-modules-' + platform + '.txt',
-              'r') as extra:
-        file.write(extra.read())
+    # Always include fixed swift modules
+    write_fixed_module(file, platform, 'swift', verbose)
+    # Check if we need fixed clang modules
+    if include_fixed_clang_modules:
+        write_fixed_module(file, platform, 'clang', verbose)
 
 
 def get_api_digester_path(tool_path):
@@ -82,50 +96,69 @@ def __init__(self, tool_path, platform):
             'macosx': 'x86_64-apple-macosx10.15',
             'appletvos': 'arm64-apple-tvos13.0',
             'watchos': 'armv7k-apple-watchos6.0',
+            'iosmac': 'x86_64-apple-ios13.0-macabi',
         }
         self.tool_path = get_api_digester_path(tool_path)
         self.platform = platform
         self.target = target_map[platform]
         self.sdk = get_sdk_path(platform)
+        self.inputs = []
+        if self.platform == 'macosx':
+            # We need this input search path for CreateML
+            self.inputs.extend([self.sdk + '/usr/lib/swift/'])
         self.frameworks = [
             self.sdk + '/System/Library/Frameworks/',
             os.path.realpath(self.sdk + '/../../Library/Frameworks/')]
+        if self.platform.startswith('iosmac'):
+            # Catalyst modules need this extra framework dir
+            iOSSupport = self.sdk + \
+                '/System/iOSSupport/System/Library/Frameworks'
+            self.frameworks.extend([iOSSupport])
 
     def run(self, output, module, swift_ver, opts, verbose,
-            module_filter_flags, include_fixed_modules, separate_by_module):
+            module_filter_flags, include_fixed_clang_modules,
+            separate_by_module):
         cmd = [self.tool_path, '-sdk', self.sdk, '-target',
                self.target, '-dump-sdk', '-module-cache-path',
                '/tmp/ModuleCache', '-swift-version',
                swift_ver, '-abort-on-module-fail']
+        _environ = dict(os.environ)
+        _environ['SWIFT_FORCE_MODULE_LOADING'] = 'prefer-interface'
         for path in self.frameworks:
             cmd.extend(['-iframework', path])
+        for path in self.inputs:
+            cmd.extend(['-I', path])
         cmd.extend(['-' + o for o in opts])
         if verbose:
             cmd.extend(['-v'])
         if module:
             cmd.extend(['-module', module])
             cmd.extend(['-o', output])
-            check_call(cmd, verbose=verbose)
+            check_call(cmd, env=_environ, verbose=verbose)
         else:
             with tempfile.NamedTemporaryFile() as tmp:
                 prepare_module_list(self.platform, tmp, verbose,
-                                    module_filter_flags, include_fixed_modules)
+                                    module_filter_flags,
+                                    include_fixed_clang_modules)
                 if separate_by_module:
                     tmp.seek(0)
                     create_directory(output)
                     for module in [name.strip() for name in tmp.readlines()]:
+                        # Skip comments
+                        if module.startswith('//'):
+                            continue
                         dir_path = os.path.realpath(output + '/' + module)
                         file_path = os.path.realpath(dir_path + '/' +
                                                      self.platform + '.json')
                         create_directory(dir_path)
                         current_cmd = list(cmd)
                         current_cmd.extend(['-module', module])
                         current_cmd.extend(['-o', file_path])
-                        check_call(current_cmd, verbose=verbose)
+                        check_call(current_cmd, env=_environ, verbose=verbose)
                 else:
                     cmd.extend(['-o', output])
                     cmd.extend(['-module-list-file', tmp.name])
-                    check_call(cmd, verbose=verbose)
+                    check_call(cmd, env=_environ, verbose=verbose)
 
 
 class DiagnoseConfig:
@@ -214,21 +247,21 @@ def main():
             fatal_error("Need to specify --output")
         if args.module_filter == '':
             module_filter_flags = []
-            include_fixed_modules = True
+            include_fixed_clang_modules = True
         elif args.module_filter == 'swift-frameworks-only':
             module_filter_flags = ['--swift-frameworks-only']
-            include_fixed_modules = False
+            include_fixed_clang_modules = False
         elif args.module_filter == 'swift-overlay-only':
             module_filter_flags = ['--swift-overlay-only']
-            include_fixed_modules = False
+            include_fixed_clang_modules = False
         else:
             fatal_error("cannot recognize --module-filter")
         runner = DumpConfig(tool_path=args.tool_path, platform=args.target)
         runner.run(output=args.output, module=args.module,
                    swift_ver=args.swift_version, opts=args.opts,
                    verbose=args.v,
                    module_filter_flags=module_filter_flags,
-                   include_fixed_modules=include_fixed_modules,
+                   include_fixed_clang_modules=include_fixed_clang_modules,
                    separate_by_module=args.separate_by_module)
     elif args.action == 'diagnose':
         if not args.dump_before: