[3.8] bpo-41100: Support macOS 11 Big Sur and Apple Silicon Macs (#25806)

* bpo-41100: Support macOS 11 and Apple Silicon on Python 3.8

This is a partial backport of bpo-41100 changes `e8b1c038b14b5fc8120aab62c9bf5fb840274cb6` and `96d906b144e6e6aa96c5ffebecbcc5d38034bbda` for Python 3.8. We introduce the ability to build Python from source for `arm64` on macOS, but we do not make a promise of support. This allows us to omit support for Universal2 binaries as well as weak-linking of symbols from the macOS SDK based on the deployment target, which are larger changes much more difficult to merge.

This also includes a backport of subsequent bpo-42688 change `7e729978fa08a360cbf936dc215ba7dd25a06a08` to fix build errors with external `libffi`.

* bpo-41116: Ensure system supplied libraries are found on macOS 11 (GH-23301) (GH-23455)

On macOS system provided libraries are in a shared library cache
and not at their usual location. This PR teaches distutils to search
in the SDK, even if there was no "-sysroot" argument in
the compiler flags.
(cherry picked from commit 404a719)

* bpo-42504: fix for MACOSX_DEPLOYMENT_TARGET=11 (GH-23556)

macOS releases numbering has changed as of macOS 11 Big Sur.  Previously, major releases were of the form 10.x, 10.x+1, 10.x+2, etc; as of Big Sur, they are now x, x+1, etc, so, for example, 10.15, 10.15.1, ..., 10.15.7, 11, 11.0.1, 11.1, ..., 12, 12.1, etc. Allow Python to build with single-digit deployment target values. Patch provided by FX Coudert.
(cherry picked from commit 5291639)

* bpo-42504: Ensure that get_config_var('MACOSX_DEPLOYMENT_TARGET') is a string (GH-24341) (GH-24410)

* bpo-42504: Ensure that get_config_var('MACOSX_DEPLOYMENT_TARGET') is a string
(cherry picked from commit 49926cf)

Co-authored-by: Ronald Oussoren <[email protected]>
Co-authored-by: FX Coudert <[email protected]>
Co-authored-by: Max Bélanger <[email protected]>

Author: 4 people

Doc/whatsnew/3.8.rst

@@ -2273,3 +2273,25 @@ IPv4 address sent from the remote server when setting up a passive data
 channel.  We reuse the ftp server IP address instead.  For unusual code
 requiring the old behavior, set a ``trust_server_pasv_ipv4_address``
 attribute on your FTP instance to ``True``.  (See :issue:`43285`)
+
+Notable changes in Python 3.8.10
+================================
+
+macOS 11.0 (Big Sur) and Apple Silicon Mac support
+--------------------------------------------------
+
+As of 3.8.10, Python now supports building and running on macOS 11
+(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture).
+A new universal build variant, ``universal2``, is now available to natively
+support both ``ARM64`` and ``Intel 64`` in one set of executables.
+Note that support for "weaklinking", building binaries targeted for newer
+versions of macOS that will also run correctly on older versions by
+testing at runtime for missing features, is not included in this backport
+from Python 3.9; to support a range of macOS versions, continue to target
+for and build on the oldest version in the range.
+
+(Originally contributed by Ronald Oussoren and Lawrence D'Anna in :issue:`41100`,
+with fixes by FX Coudert and Eli Rykoff, and backported to 3.8 by Maxime Bélanger
+and Ned Deily)
+
+

Lib/_osx_support.py

@@ -52,7 +52,7 @@ def _find_executable(executable, path=None):
         return executable
 
 
-def _read_output(commandstring):
+def _read_output(commandstring, capture_stderr=False):
     """Output from successful command execution or None"""
     # Similar to os.popen(commandstring, "r").read(),
     # but without actually using os.popen because that
@@ -67,7 +67,10 @@ def _read_output(commandstring):
             os.getpid(),), "w+b")
 
     with contextlib.closing(fp) as fp:
-        cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        if capture_stderr:
+            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
+        else:
+            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
         return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
 
 
@@ -110,6 +113,26 @@ def _get_system_version():
 
     return _SYSTEM_VERSION
 
+_SYSTEM_VERSION_TUPLE = None
+def _get_system_version_tuple():
+    """
+    Return the macOS system version as a tuple
+
+    The return value is safe to use to compare
+    two version numbers.
+    """
+    global _SYSTEM_VERSION_TUPLE
+    if _SYSTEM_VERSION_TUPLE is None:
+        osx_version = _get_system_version()
+        if osx_version:
+            try:
+                _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
+            except ValueError:
+                _SYSTEM_VERSION_TUPLE = ()
+
+    return _SYSTEM_VERSION_TUPLE
+
+
 def _remove_original_values(_config_vars):
     """Remove original unmodified values for testing"""
     # This is needed for higher-level cross-platform tests of get_platform.
@@ -125,21 +148,52 @@ def _save_modified_value(_config_vars, cv, newvalue):
         _config_vars[_INITPRE + cv] = oldvalue
     _config_vars[cv] = newvalue
 
+
+_cache_default_sysroot = None
+def _default_sysroot(cc):
+    """ Returns the root of the default SDK for this system, or '/' """
+    global _cache_default_sysroot
+
+    if _cache_default_sysroot is not None:
+        return _cache_default_sysroot
+
+    contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
+    in_incdirs = False
+    for line in contents.splitlines():
+        if line.startswith("#include <...>"):
+            in_incdirs = True
+        elif line.startswith("End of search list"):
+            in_incdirs = False
+        elif in_incdirs:
+            line = line.strip()
+            if line == '/usr/include':
+                _cache_default_sysroot = '/'
+            elif line.endswith(".sdk/usr/include"):
+                _cache_default_sysroot = line[:-12]
+    if _cache_default_sysroot is None:
+        _cache_default_sysroot = '/'
+
+    return _cache_default_sysroot
+
 def _supports_universal_builds():
     """Returns True if universal builds are supported on this system"""
     # As an approximation, we assume that if we are running on 10.4 or above,
     # then we are running with an Xcode environment that supports universal
     # builds, in particular -isysroot and -arch arguments to the compiler. This
     # is in support of allowing 10.4 universal builds to run on 10.3.x systems.
 
-    osx_version = _get_system_version()
-    if osx_version:
-        try:
-            osx_version = tuple(int(i) for i in osx_version.split('.'))
-        except ValueError:
-            osx_version = ''
+    osx_version = _get_system_version_tuple()
     return bool(osx_version >= (10, 4)) if osx_version else False
 
+def _supports_arm64_builds():
+    """Returns True if arm64 builds are supported on this system"""
+    # There are two sets of systems supporting macOS/arm64 builds:
+    # 1. macOS 11 and later, unconditionally
+    # 2. macOS 10.15 with Xcode 12.2 or later
+    # For now the second category is ignored.
+    osx_version = _get_system_version_tuple()
+    return osx_version >= (11, 0) if osx_version else False
+
 
 def _find_appropriate_compiler(_config_vars):
     """Find appropriate C compiler for extension module builds"""
@@ -331,6 +385,12 @@ def compiler_fixup(compiler_so, cc_args):
             except ValueError:
                 break
 
+    elif not _supports_arm64_builds():
+        # Look for "-arch arm64" and drop that
+        for idx in reversed(range(len(compiler_so))):
+            if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
+                del compiler_so[idx:idx+2]
+
     if 'ARCHFLAGS' in os.environ and not stripArch:
         # User specified different -arch flags in the environ,
         # see also distutils.sysconfig
@@ -481,6 +541,8 @@ def get_platform_osx(_config_vars, osname, release, machine):
 
             if len(archs) == 1:
                 machine = archs[0]
+            elif archs == ('arm64', 'x86_64'):
+                machine = 'universal2'
             elif archs == ('i386', 'ppc'):
                 machine = 'fat'
             elif archs == ('i386', 'x86_64'):

Lib/ctypes/macholib/dyld.py

@@ -6,6 +6,11 @@
 from ctypes.macholib.framework import framework_info
 from ctypes.macholib.dylib import dylib_info
 from itertools import *
+try:
+    from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+    def _dyld_shared_cache_contains_path(*args):
+        raise NotImplementedError
 
 __all__ = [
     'dyld_find', 'framework_find',
@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
                 dyld_executable_path_search(name, executable_path),
                 dyld_default_search(name, env),
             ), env):
+
         if os.path.isfile(path):
             return path
+        try:
+            if _dyld_shared_cache_contains_path(path):
+                return path
+        except NotImplementedError:
+            pass
+
     raise ValueError("dylib %s could not be found" % (name,))
 
 def framework_find(fn, executable_path=None, env=None):

Lib/ctypes/test/test_macholib.py

@@ -45,19 +45,22 @@ def find_lib(name):
 class MachOTest(unittest.TestCase):
     @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
     def test_find(self):
-
-        self.assertEqual(find_lib('pthread'),
-                             '/usr/lib/libSystem.B.dylib')
+        # On Mac OS 11, system dylibs are only present in the shared cache,
+        # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+        # be resolved by dyld_find
+        self.assertIn(find_lib('pthread'),
+                              ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
 
         result = find_lib('z')
         # Issue #21093: dyld default search path includes $HOME/lib and
         # /usr/local/lib before /usr/lib, which caused test failures if
         # a local copy of libz exists in one of them. Now ignore the head
         # of the path.
-        self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
+        self.assertRegex(result, r".*/lib/libz.*\.dylib")
 
-        self.assertEqual(find_lib('IOKit'),
-                             '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
+        self.assertIn(find_lib('IOKit'),
+                              ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+                              '/System/Library/Frameworks/IOKit.framework/IOKit'))
 
 if __name__ == "__main__":
     unittest.main()

Lib/distutils/tests/test_build_ext.py

@@ -492,12 +492,16 @@ def _try_compile_deployment_target(self, operator, target):
         # format the target value as defined in the Apple
         # Availability Macros.  We can't use the macro names since
         # at least one value we test with will not exist yet.
-        if target[1] < 10:
+        if target[:2] < (10, 10):
             # for 10.1 through 10.9.x -> "10n0"
             target = '%02d%01d0' % target
         else:
             # for 10.10 and beyond -> "10nn00"
-            target = '%02d%02d00' % target
+            if len(target) >= 2:
+                target = '%02d%02d00' % target
+            else:
+                # 11 and later can have no minor version (11 instead of 11.0)
+                target = '%02d0000' % target
         deptarget_ext = Extension(
             'deptarget',
             [deptarget_c],

Lib/distutils/unixccompiler.py

@@ -290,7 +290,7 @@ def find_library_file(self, dirs, lib, debug=0):
             cflags = sysconfig.get_config_var('CFLAGS')
             m = re.search(r'-isysroot\s*(\S+)', cflags)
             if m is None:
-                sysroot = '/'
+                sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC'))
             else:
                 sysroot = m.group(1)
 

Lib/sysconfig.py

@@ -18,6 +18,11 @@
     'parse_config_h',
 ]
 
+# Keys for get_config_var() that are never converted to Python integers.
+_ALWAYS_STR = {
+    'MACOSX_DEPLOYMENT_TARGET',
+}
+
 _INSTALL_SCHEMES = {
     'posix_prefix': {
         'stdlib': '{installed_base}/lib/python{py_version_short}',
@@ -242,6 +247,9 @@ def _parse_makefile(filename, vars=None):
                 notdone[n] = v
             else:
                 try:
+                    if n in _ALWAYS_STR:
+                        raise ValueError
+
                     v = int(v)
                 except ValueError:
                     # insert literal `$'
@@ -300,6 +308,8 @@ def _parse_makefile(filename, vars=None):
                         notdone[name] = value
                     else:
                         try:
+                            if name in _ALWAYS_STR:
+                                raise ValueError
                             value = int(value)
                         except ValueError:
                             done[name] = value.strip()
@@ -461,6 +471,8 @@ def parse_config_h(fp, vars=None):
         if m:
             n, v = m.group(1, 2)
             try:
+                if n in _ALWAYS_STR:
+                    raise ValueError
                 v = int(v)
             except ValueError:
                 pass

Lib/test/test_bytes.py

@@ -963,6 +963,7 @@ def test_from_format(self):
             c_char_p)
 
         PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
+        PyBytes_FromFormat.argtypes = (c_char_p,)
         PyBytes_FromFormat.restype = py_object
 
         # basic tests

Lib/test/test_platform.py

@@ -209,15 +209,18 @@ def test_mac_ver(self):
             # On Snow Leopard, sw_vers reports 10.6.0 as 10.6
             if len_diff > 0:
                 expect_list.extend(['0'] * len_diff)
-            self.assertEqual(result_list, expect_list)
+            # For compatibility with older binaries, macOS 11.x may report
+            # itself as '10.16' rather than '11.x.y'.
+            if result_list != ['10', '16']:
+                self.assertEqual(result_list, expect_list)
 
             # res[1] claims to contain
             # (version, dev_stage, non_release_version)
             # That information is no longer available
             self.assertEqual(res[1], ('', '', ''))
 
             if sys.byteorder == 'little':
-                self.assertIn(res[2], ('i386', 'x86_64'))
+                self.assertIn(res[2], ('i386', 'x86_64', 'arm64'))
             else:
                 self.assertEqual(res[2], 'PowerPC')
 

Lib/test/test_unicode.py

@@ -2454,11 +2454,13 @@ class CAPITest(unittest.TestCase):
     def test_from_format(self):
         support.import_module('ctypes')
         from ctypes import (
+            c_char_p,
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,
             c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
         name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.argtypes = (c_char_p,)
         _PyUnicode_FromFormat.restype = py_object
 
         def PyUnicode_FromFormat(format, *args):