[recipe-stl] Rework of libtorrent and boost (#1971)

opacam · inclement · commit 9e93257f576d · 2019-08-29T21:52:47.000+01:00
which: - fix build for android's NDK r19+ - allow us to build `boost+libtorrent` for any arch - Update boost to version `1.69.0` - update libtorrent to version `1.2.1` The build method needs to be changed because one of the scripts used in `boost+libtorrent` build (make-standalone-toolchain.sh), has been removed from android's NDK. [This is because](https://developer.android.com/ndk/guides/standalone_toolchain): `As of r19, the NDK's default toolchains are standalone toolchains, which renders this process unnecessary.` Note: since 3887d2b, `python-for-android` uses android's NDK r19 as the minimum supported and also changed the build method used by p4a as [described in here](https://developer.android.com/ndk/guides/other_build_systems)
diff --git a/pythonforandroid/recipes/boost/__init__.py b/pythonforandroid/recipes/boost/__init__.py
@@ -1,18 +1,22 @@
-from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory
+from pythonforandroid.util import current_directory, build_platform
+from pythonforandroid.recipe import Recipe
+from pythonforandroid.logger import shprint
 from os.path import join, exists
 from os import environ
+import shutil
 import sh
 
 """
-This recipe creates a custom toolchain and bootstraps Boost from source to build Boost.Build
+This recipe bootstraps Boost from source to build Boost.Build
 including python bindings
 """
 
 
 class BoostRecipe(Recipe):
     # Todo: make recipe compatible with all p4a architectures
     '''
-    .. note:: This recipe can be built only against API 21+ and arch armeabi-v7a
+    .. note:: This recipe can be built only against API 21+ and an android
+              ndk >= r19
 
     .. versionchanged:: 0.6.0
          Rewrote recipe to support clang's build. The following changes has
@@ -24,22 +28,33 @@ class BoostRecipe(Recipe):
             - Default compiler for ndk's toolchain set to clang
             - Python version will be detected via user-config.jam
             - Changed stl's lib from ``gnustl_shared`` to ``c++_shared``
+
+    .. versionchanged:: 2019.08.09.1.dev0
+
+            - Bumped version number to 1.68.0
+            - Adapted to work with ndk-r19+
     '''
-    version = '1.68.0'
-    url = 'http://downloads.sourceforge.net/project/boost/' \
-          'boost/{version}/boost_{version_underscore}.tar.bz2'
+    version = '1.69.0'
+    url = (
+        'http://downloads.sourceforge.net/project/boost/'
+        'boost/{version}/boost_{version_underscore}.tar.bz2'
+    )
     depends = [('python2', 'python3')]
-    patches = ['disable-so-version.patch',
-               'use-android-libs.patch',
-               'fix-android-issues.patch']
+    patches = [
+        'disable-so-version.patch',
+        'use-android-libs.patch',
+        'fix-android-issues.patch',
+    ]
+    need_stl_shared = True
 
     @property
     def versioned_url(self):
         if self.url is None:
             return None
         return self.url.format(
             version=self.version,
-            version_underscore=self.version.replace('.', '_'))
+            version_underscore=self.version.replace('.', '_'),
+        )
 
     def should_build(self, arch):
         return not exists(join(self.get_build_dir(arch.arch), 'b2'))
@@ -48,56 +63,50 @@ def prebuild_arch(self, arch):
         super(BoostRecipe, self).prebuild_arch(arch)
         env = self.get_recipe_env(arch)
         with current_directory(self.get_build_dir(arch.arch)):
-            if not exists(env['CROSSHOME']):
-                # Make custom toolchain
-                bash = sh.Command('bash')
-                shprint(bash, join(self.ctx.ndk_dir, 'build/tools/make-standalone-toolchain.sh'),
-                        '--arch=' + env['ARCH'],
-                        '--platform=android-' + str(self.ctx.android_api),
-                        '--toolchain=' + env['CROSSHOST'] + '-' + self.ctx.toolchain_version + ':-llvm',
-                        '--use-llvm',
-                        '--stl=libc++',
-                        '--install-dir=' + env['CROSSHOME']
-                        )
             # Set custom configuration
-            shutil.copyfile(join(self.get_recipe_dir(), 'user-config.jam'),
-                            join(env['BOOST_BUILD_PATH'], 'user-config.jam'))
+            shutil.copyfile(
+                join(self.get_recipe_dir(), 'user-config.jam'),
+                join(env['BOOST_BUILD_PATH'], 'user-config.jam'),
+            )
 
     def build_arch(self, arch):
         super(BoostRecipe, self).build_arch(arch)
         env = self.get_recipe_env(arch)
         env['PYTHON_HOST'] = self.ctx.hostpython
         with current_directory(self.get_build_dir(arch.arch)):
-            # Compile Boost.Build engine with this custom toolchain
-            bash = sh.Command('bash')
-            shprint(bash, 'bootstrap.sh')  # Do not pass env
-        # Install app stl
-        shutil.copyfile(
-            join(self.ctx.ndk_dir, 'sources/cxx-stl/llvm-libc++/libs/'
-                                   'armeabi-v7a/libc++_shared.so'),
-            join(self.ctx.get_libs_dir(arch.arch), 'libc++_shared.so'))
-
-    def select_build_arch(self, arch):
-        return arch.arch.replace('eabi-v7a', '').replace('eabi', '')
+            if not exists('b2'):
+                # Compile Boost.Build engine with this custom toolchain
+                bash = sh.Command('bash')
+                shprint(bash, 'bootstrap.sh')  # Do not pass env
 
     def get_recipe_env(self, arch):
         # We don't use the normal env because we
         # are building with a standalone toolchain
         env = environ.copy()
 
-        env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch)  # find user-config.jam
-        env['BOOST_ROOT'] = env['BOOST_BUILD_PATH']  # find boost source
+        # find user-config.jam
+        env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch)
+        # find boost source
+        env['BOOST_ROOT'] = env['BOOST_BUILD_PATH']
 
         env['PYTHON_ROOT'] = self.ctx.python_recipe.link_root(arch.arch)
         env['PYTHON_INCLUDE'] = self.ctx.python_recipe.include_root(arch.arch)
         env['PYTHON_MAJOR_MINOR'] = self.ctx.python_recipe.version[:3]
-        env['PYTHON_LINK_VERSION'] = self.ctx.python_recipe.major_minor_version_string
+        env[
+            'PYTHON_LINK_VERSION'
+        ] = self.ctx.python_recipe.major_minor_version_string
         if 'python3' in self.ctx.python_recipe.name:
             env['PYTHON_LINK_VERSION'] += 'm'
 
-        env['ARCH'] = self.select_build_arch(arch)
-        env['CROSSHOST'] = env['ARCH'] + '-linux-androideabi'
-        env['CROSSHOME'] = join(env['BOOST_ROOT'], 'standalone-' + env['ARCH'] + '-toolchain')
+        env['ARCH'] = arch.arch.replace('-', '')
+        env['TARGET_TRIPLET'] = arch.target
+        env['CROSSHOST'] = arch.command_prefix
+        env['CROSSHOME'] = join(
+            self.ctx.ndk_dir,
+            'toolchains/llvm/prebuilt/{build_platform}'.format(
+                build_platform=build_platform
+            ),
+        )
         return env
 
 
diff --git a/pythonforandroid/recipes/boost/fix-android-issues.patch b/pythonforandroid/recipes/boost/fix-android-issues.patch
@@ -1,10 +1,26 @@
-diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/user.hpp
---- boost_1_68_0.orig/boost/config/user.hpp	2018-08-01 22:50:46.000000000 +0200
-+++ boost_1_68_0/boost/config/user.hpp	2018-08-27 15:43:38.000000000 +0200
+diff -u -r boost_1_69_0.orig/boost/asio/detail/config.hpp boost_1_69_0/boost/asio/detail/config.hpp
+--- boost_1_69_0.orig/boost/asio/detail/config.hpp	2018-12-05 20:58:15.000000000 +0100
++++ boost_1_69_0/boost/asio/detail/config.hpp	2018-12-13 14:52:06.000000000 +0100
+@@ -815,7 +815,11 @@
+ #    if (_LIBCPP_VERSION < 7000)
+ #     if (__cplusplus >= 201402)
+ #      if __has_include(<experimental/string_view>)
+-#       define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
++#     if __clang_major__ >= 7
++#      undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW
++#     else
++#      define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
++#     endif // __clang_major__ >= 7
+ #      endif // __has_include(<experimental/string_view>)
+ #     endif // (__cplusplus >= 201402)
+ #    endif // (_LIBCPP_VERSION < 7000)
+diff -u -r boost_1_69_0.orig/boost/config/user.hpp boost_1_69_0/boost/config/user.hpp
+--- boost_1_69_0.orig/boost/config/user.hpp	2018-12-05 20:58:16.000000000 +0100
++++ boost_1_69_0/boost/config/user.hpp	2018-12-13 14:35:29.000000000 +0100
 @@ -13,6 +13,12 @@
  //  configuration policy:
  //
- 
+
 +// Android defines
 +// There is problem with std::atomic on android (and some other platforms).
 +// See this link for more info:
@@ -13,41 +29,25 @@ diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/use
 +
  // define this to locate a compiler config file:
  // #define BOOST_COMPILER_CONFIG <myheader>
- 
-diff -u -r boost_1_68_0.orig/boost/asio/detail/config.hpp boost_1_68_0/boost/asio/detail/config.hpp
---- boost_1_68_0.orig/boost/asio/detail/config.hpp  2018-08-01 22:50:46.000000000 +0200
-+++ boost_1_68_0/boost/asio/detail/config.hpp   2018-09-19 12:39:56.000000000 +0200
-@@ -804,7 +804,11 @@
- #  if defined(__clang__)
- #   if (__cplusplus >= 201402)
- #    if __has_include(<experimental/string_view>)
--#     define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
-+#     if __clang_major__ >= 7
-+#      undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW
-+#     else
-+#      define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
-+#     endif // __clang_major__ >= 7
- #    endif // __has_include(<experimental/string_view>)
- #   endif // (__cplusplus >= 201402)
- #  endif // defined(__clang__)
-diff -u -r boost_1_68_0.orig/boost/system/error_code.hpp boost_1_68_0/boost/system/error_code.hpp
---- boost_1_68_0.orig/boost/system/error_code.hpp	2018-08-01 22:50:53.000000000 +0200
-+++ boost_1_68_0/boost/system/error_code.hpp	2018-08-27 15:44:29.000000000 +0200
-@@ -17,6 +17,7 @@
- #include <boost/assert.hpp>
- #include <boost/noncopyable.hpp>
- #include <boost/utility/enable_if.hpp>
+
+diff -u -r boost_1_69_0.orig/boost/system/error_code.hpp boost_1_69_0/boost/system/error_code.hpp
+--- boost_1_69_0.orig/boost/system/error_code.hpp	2018-12-05 20:58:23.000000000 +0100
++++ boost_1_69_0/boost/system/error_code.hpp	2018-12-13 14:53:33.000000000 +0100
+@@ -14,6 +14,7 @@
+ #include <boost/system/detail/config.hpp>
+ #include <boost/cstdint.hpp>
+ #include <boost/config.hpp>
 +#include <stdio.h>
  #include <ostream>
  #include <string>
- #include <stdexcept>
-diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/libs/filesystem/src/operations.cpp
---- boost_1_68_0.orig/libs/filesystem/src/operations.cpp	2018-08-01 22:50:47.000000000 +0200
-+++ boost_1_68_0/libs/filesystem/src/operations.cpp	2018-08-27 15:47:15.000000000 +0200
+ #include <functional>
+diff -u -r boost_1_69_0.orig/libs/filesystem/src/operations.cpp boost_1_69_0/libs/filesystem/src/operations.cpp
+--- boost_1_69_0.orig/libs/filesystem/src/operations.cpp	2018-12-05 20:58:17.000000000 +0100
++++ boost_1_69_0/libs/filesystem/src/operations.cpp	2018-12-13 14:55:41.000000000 +0100
 @@ -232,6 +232,21 @@
- 
+
  # if defined(BOOST_POSIX_API)
- 
+
 +#  if defined(__ANDROID__)
 +#  define truncate libboost_truncate_wrapper
 +// truncate() is present in Android libc only starting from ABI 21, so here's a simple wrapper
@@ -64,5 +64,23 @@ diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/lib
 +#  endif
 +
  typedef int err_t;
- 
+
  //  POSIX uses a 0 return to indicate success
+diff -u -r boost_1_69_0.orig/tools/build/src/tools/common.jam boost_1_69_0/tools/build/src/tools/common.jam
+--- boost_1_69_0.orig/tools/build/src/tools/common.jam	2019-01-25 23:18:34.544755629 +0200
++++ boost_1_69_0/tools/build/src/tools/common.jam	2019-01-25 23:20:42.309047754 +0200
+@@ -976,10 +976,10 @@
+     }
+
+     # Ditto, from Clang 4
+-    if $(tag) in clang clangw && [ numbers.less 3 $(version[1]) ]
+-    {
+-        version = $(version[1]) ;
+-    }
++    #if $(tag) in clang clangw && [ numbers.less 3 $(version[1]) ]
++    #{
++    #    version = $(version[1]) ;
++    #}
+
+     # On intel, version is not added, because it does not matter and it is the
+     # version of vc used as backend that matters. Ideally, we should encode the
diff --git a/pythonforandroid/recipes/boost/user-config.jam b/pythonforandroid/recipes/boost/user-config.jam
@@ -1,49 +1,30 @@
 import os ;
 
 local ARCH = [ os.environ ARCH ] ;
+local TARGET_TRIPLET = [ os.environ TARGET_TRIPLET ] ;
 local CROSSHOME = [ os.environ CROSSHOME ] ;
 local PYTHON_HOST = [ os.environ PYTHON_HOST ] ;
 local PYTHON_ROOT = [ os.environ PYTHON_ROOT ] ;
 local PYTHON_INCLUDE = [ os.environ PYTHON_INCLUDE ] ;
 local PYTHON_LINK_VERSION = [ os.environ PYTHON_LINK_VERSION ] ;
 local PYTHON_MAJOR_MINOR = [ os.environ PYTHON_MAJOR_MINOR ] ;
 
-using clang : $(ARCH) : $(CROSSHOME)/bin/arm-linux-androideabi-clang++ :
-<archiver>$(CROSSHOME)/bin/arm-linux-androideabi-ar
-<root>$(CROSSHOME)/sysroot
-<architecture>$(ARCH)
-<compileflags>-fexceptions
-<compileflags>-frtti
-<compileflags>-fpic
+using clang : $(ARCH) : $(CROSSHOME)/bin/$(TARGET_TRIPLET)-clang++ :
+<archiver>$(CROSSHOME)/bin/llvm-ar
+<compileflags>-fPIC
 <compileflags>-ffunction-sections
+<compileflags>-fdata-sections
 <compileflags>-funwind-tables
-<compileflags>-march=armv7-a
-<compileflags>-msoft-float
-<compileflags>-mfpu=neon
-<compileflags>-mthumb
-<linkflags>-march=armv7-a
-<linkflags>-Wl,--fix-cortex-a8
-<compileflags>-Os
-<compileflags>-fomit-frame-pointer
-<compileflag>-fno-strict-aliasing
-<compileflags>-DANDROID
-<compileflags>-D__ANDROID__
-<compileflags>-DANDROID_TOOLCHAIN=clang
-<compileflags>-DANDROID_ABI=armv7-a
-<compileflags>-DANDROID_STL=c++_shared
-<compileflags>-DBOOST_ALL_NO_LIB
-#<compileflags>-DNDEBUG
-<compileflags>-O2
+<compileflags>-fstack-protector-strong
+<compileflags>-no-canonical-prefixes
+<compileflags>-Wformat
+<compileflags>-Werror=format-security
+<compileflags>-frtti
+<compileflags>-fexceptions
+<compileflags>-DNDEBUG
 <compileflags>-g
-<compileflags>-fvisibility=hidden
-<compileflags>-fvisibility-inlines-hidden
-<compileflags>-fdata-sections
-<cxxflags>-D__arm__
-<cxxflags>-D_REENTRANT
-<cxxflags>-D_GLIBCXX__PTHREADS
-<compileflags>-Wno-long-long
-<compileflags>-Wno-missing-field-initializers
-<compileflags>-Wno-unused-variable
+<compileflags>-Oz
+<compileflags>-mthumb
 <linkflags>-Wl,-z,relro
 <linkflags>-Wl,-z,now
 <linkflags>-lc++_shared
diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py
@@ -5,8 +5,8 @@
 import sh
 
 # This recipe builds libtorrent with Python bindings
-# It depends on Boost.Build and the source of several Boost libraries present in BOOST_ROOT,
-# which is all provided by the boost recipe
+# It depends on Boost.Build and the source of several Boost libraries present
+# in BOOST_ROOT, which is all provided by the boost recipe
 
 
 def get_lib_from(search_directory, lib_extension='.so'):
@@ -24,7 +24,8 @@ def get_lib_from(search_directory, lib_extension='.so'):
 class LibtorrentRecipe(Recipe):
     # Todo: make recipe compatible with all p4a architectures
     '''
-    .. note:: This recipe can be built only against API 21+ and arch armeabi-v7a
+    .. note:: This recipe can be built only against API 21+ and an android
+              ndk >= r19
 
     .. versionchanged:: 0.6.0
          Rewrote recipe to support clang's build and boost 1.68. The following
@@ -33,9 +34,14 @@ class LibtorrentRecipe(Recipe):
             - Bumped version number to 1.2.0
             - added python 3 compatibility
             - new system to detect/copy generated libraries
+
+    .. versionchanged:: 2019.08.09.1.dev0
+
+            - Bumped version number to 1.2.1
+            - Adapted to work with ndk-r19+
     '''
-    version = '1_2_0'
-    url = 'https://github.com/arvidn/libtorrent/archive/libtorrent_{version}.tar.gz'
+    version = '1_2_1'
+    url = 'https://github.com/arvidn/libtorrent/archive/libtorrent-{version}.tar.gz'
 
     depends = ['boost']
     opt_depends = ['openssl']
@@ -76,7 +82,7 @@ def build_arch(self, arch):
             '-j' + str(cpu_count()),
             '--debug-configuration',  # so we know if our python is detected
             # '--deprecated-functions=off',
-            'toolset=clang-arm',
+            'toolset=clang-{arch}'.format(arch=env['ARCH']),
             'abi=aapcs',
             'binary-format=elf',
             'cxxflags=-std=c++11',
@@ -105,8 +111,12 @@ def build_arch(self, arch):
         # Copy only the boost shared libraries into the libs folder. Because
         # boost build two boost_python libraries, we force to search the lib
         # into the corresponding build path.
-        b2_build_dir = 'build/clang-linux-arm/release/{encryption}/' \
-                       'lt-visibility-hidden/'.format(encryption=crypto_folder)
+        b2_build_dir = (
+            'build/clang-linux-{arch}/release/{encryption}/'
+            'lt-visibility-hidden/'.format(
+                arch=env['ARCH'], encryption=crypto_folder
+            )
+        )
         boost_libs_dir = join(env['BOOST_BUILD_PATH'], 'bin.v2/libs')
         for boost_lib in listdir(boost_libs_dir):
             lib_path = get_lib_from(join(boost_libs_dir, boost_lib, b2_build_dir))
diff --git a/pythonforandroid/recipes/libtorrent/setup-lib-name.patch b/pythonforandroid/recipes/libtorrent/setup-lib-name.patch
@@ -15,6 +15,6 @@
  setup(
 -    name='python-libtorrent',
 +    name='libtorrent',
-     version='1.2.0',
+     version='1.2.1',
      author='Arvid Norberg',
      author_email='arvid@libtorrent.org',
