Fix libtorrent+boost (update to latest versions and add python3 compatibility)

Since the openssl and ndk update the boost and libtorrent recipes weren't working. The old version for boost libraries does not support openssl 1.1, so we must update it. The old libtorrent recipe is strongly dependant on boost so we also update it. Also, before this commit, the compiler used to make those builds were gcc, now we use clang (the default compiler for the latest android ndk).

Note: We use the latest release candidate version for libtorrent because still not released the final version (and it is more prepared to be build with clang compiler). To make it work with python3 we need to apply a patch to it, which will be not necessary when released the final version (the changes made by this patch are already merged into libtorrent's master branch). Those recipes were tested for both python versions with a slight modification of the test apps and importing the libtorrent at runtime without problems, plus the generated libraries are linked with our versions of openssl and python (using the readelf command).

Author: opacam

pythonforandroid/recipes/boost/__init__.py

@@ -1,5 +1,6 @@
 from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory
 from os.path import join, exists
+from os import environ
 import sh
 
 """
@@ -9,11 +10,36 @@
 
 
 class BoostRecipe(Recipe):
-    version = '1.60.0'
-    # Don't forget to change the URL when changing the version
-    url = 'http://downloads.sourceforge.net/project/boost/boost/{version}/boost_1_60_0.tar.bz2'
-    depends = ['python2']
-    patches = ['disable-so-version.patch', 'use-android-libs.patch']
+    # Todo: make recipe compatible with all p4a architectures
+    '''
+    .. note:: This recipe can be built only against API 21+ and arch armeabi-v7a
+
+    .. versionchanged:: 0.6.0
+         Rewrote recipe to support clang's build. The following changes has
+         been made:
+
+            - Bumped version number to 1.68.0
+            - Better version handling for url
+            - Added python 3 compatibility
+            - Default compiler for ndk's toolchain set to clang
+            - Python version will be detected via user-config.jam
+            - Changed stl's lib from ``gnustl_shared`` to ``c++_shared``
+    '''
+    version = '1.68.0'
+    url = 'http://downloads.sourceforge.net/project/boost/' \
+          'boost/{version}/boost_{version_underscore}.tar.bz2'
+    depends = [('python2', 'python3')]
+    patches = ['disable-so-version.patch',
+               'use-android-libs.patch',
+               'fix-android-issues.patch']
+
+    @property
+    def versioned_url(self):
+        if self.url is None:
+            return None
+        return self.url.format(
+            version=self.version,
+            version_underscore=self.version.replace('.', '_'))
 
     def should_build(self, arch):
         return not exists(join(self.get_build_dir(arch.arch), 'b2'))
@@ -28,41 +54,50 @@ def prebuild_arch(self, arch):
                 shprint(bash, join(self.ctx.ndk_dir, 'build/tools/make-standalone-toolchain.sh'),
                         '--arch=' + env['ARCH'],
                         '--platform=android-' + str(self.ctx.android_api),
-                        '--toolchain=' + env['CROSSHOST'] + '-' + env['TOOLCHAIN_VERSION'],
+                        '--toolchain=' + env['CROSSHOST'] + '-' + self.ctx.toolchain_version + ':-llvm',
+                        '--use-llvm',
+                        '--stl=libc++',
                         '--install-dir=' + env['CROSSHOME']
-                )
+                        )
             # Set custom configuration
             shutil.copyfile(join(self.get_recipe_dir(), 'user-config.jam'),
                             join(env['BOOST_BUILD_PATH'], 'user-config.jam'))
 
     def build_arch(self, arch):
         super(BoostRecipe, self).build_arch(arch)
         env = self.get_recipe_env(arch)
+        env['PYTHON_HOST'] = self.ctx.hostpython
         with current_directory(self.get_build_dir(arch.arch)):
             # Compile Boost.Build engine with this custom toolchain
             bash = sh.Command('bash')
-            shprint(bash, 'bootstrap.sh',
-                    '--with-python=' + join(env['PYTHON_ROOT'], 'bin/python.host'),
-                    '--with-python-version=2.7',
-                    '--with-python-root=' + env['PYTHON_ROOT']
-            )  # Do not pass env
+            shprint(bash, 'bootstrap.sh')  # Do not pass env
         # Install app stl
-        shutil.copyfile(join(env['CROSSHOME'], env['CROSSHOST'], 'lib/libgnustl_shared.so'),
-                        join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so'))
+        shutil.copyfile(
+            join(self.ctx.ndk_dir, 'sources/cxx-stl/llvm-libc++/libs/'
+                                   'armeabi-v7a/libc++_shared.so'),
+            join(self.ctx.get_libs_dir(arch.arch), 'libc++_shared.so'))
 
     def select_build_arch(self, arch):
         return arch.arch.replace('eabi-v7a', '').replace('eabi', '')
 
     def get_recipe_env(self, arch):
-        env = super(BoostRecipe, self).get_recipe_env(arch)
+        # We don't use the normal env because we
+        # are building with a standalone toolchain
+        env = environ.copy()
+
         env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch)  # find user-config.jam
         env['BOOST_ROOT'] = env['BOOST_BUILD_PATH']  # find boost source
-        env['PYTHON_ROOT'] = self.ctx.get_python_install_dir()
+
+        env['PYTHON_ROOT'] = self.ctx.python_recipe.link_root(arch.arch)
+        env['PYTHON_INCLUDE'] = self.ctx.python_recipe.include_root(arch.arch)
+        env['PYTHON_MAJOR_MINOR'] = self.ctx.python_recipe.version[:3]
+        env['PYTHON_LINK_VERSION'] = self.ctx.python_recipe.major_minor_version_string
+        if 'python3' in self.ctx.python_recipe.name:
+            env['PYTHON_LINK_VERSION'] += 'm'
+
         env['ARCH'] = self.select_build_arch(arch)
-        env['ANDROIDAPI'] = str(self.ctx.android_api)
         env['CROSSHOST'] = env['ARCH'] + '-linux-androideabi'
         env['CROSSHOME'] = join(env['BOOST_ROOT'], 'standalone-' + env['ARCH'] + '-toolchain')
-        env['TOOLCHAIN_PREFIX'] = join(env['CROSSHOME'], 'bin', env['CROSSHOST'])
         return env
 
 

pythonforandroid/recipes/boost/fix-android-issues.patch

@@ -0,0 +1,68 @@
+diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/user.hpp
+--- boost_1_68_0.orig/boost/config/user.hpp	2018-08-01 22:50:46.000000000 +0200
++++ boost_1_68_0/boost/config/user.hpp	2018-08-27 15:43:38.000000000 +0200
+@@ -13,6 +13,12 @@
+ //  configuration policy:
+ //
+ 
++// Android defines
++// There is problem with std::atomic on android (and some other platforms).
++// See this link for more info:
++// https://code.google.com/p/android/issues/detail?id=42735#makechanges
++#define BOOST_ASIO_DISABLE_STD_ATOMIC 1
++
+ // define this to locate a compiler config file:
+ // #define BOOST_COMPILER_CONFIG <myheader>
+ 
+diff -u -r boost_1_68_0.orig/boost/asio/detail/config.hpp boost_1_68_0/boost/asio/detail/config.hpp
+--- boost_1_68_0.orig/boost/asio/detail/config.hpp  2018-08-01 22:50:46.000000000 +0200
++++ boost_1_68_0/boost/asio/detail/config.hpp   2018-09-19 12:39:56.000000000 +0200
+@@ -804,7 +804,11 @@
+ #  if defined(__clang__)
+ #   if (__cplusplus >= 201402)
+ #    if __has_include(<experimental/string_view>)
+-#     define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
++#     if __clang_major__ >= 7
++#      undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW
++#     else
++#      define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
++#     endif // __clang_major__ >= 7
+ #    endif // __has_include(<experimental/string_view>)
+ #   endif // (__cplusplus >= 201402)
+ #  endif // defined(__clang__)
+diff -u -r boost_1_68_0.orig/boost/system/error_code.hpp boost_1_68_0/boost/system/error_code.hpp
+--- boost_1_68_0.orig/boost/system/error_code.hpp	2018-08-01 22:50:53.000000000 +0200
++++ boost_1_68_0/boost/system/error_code.hpp	2018-08-27 15:44:29.000000000 +0200
+@@ -17,6 +17,7 @@
+ #include <boost/assert.hpp>
+ #include <boost/noncopyable.hpp>
+ #include <boost/utility/enable_if.hpp>
++#include <stdio.h>
+ #include <ostream>
+ #include <string>
+ #include <stdexcept>
+diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/libs/filesystem/src/operations.cpp
+--- boost_1_68_0.orig/libs/filesystem/src/operations.cpp	2018-08-01 22:50:47.000000000 +0200
++++ boost_1_68_0/libs/filesystem/src/operations.cpp	2018-08-27 15:47:15.000000000 +0200
+@@ -232,6 +232,21 @@
+ 
+ # if defined(BOOST_POSIX_API)
+ 
++#  if defined(__ANDROID__)
++#  define truncate libboost_truncate_wrapper
++// truncate() is present in Android libc only starting from ABI 21, so here's a simple wrapper
++static int libboost_truncate_wrapper(const char *path, off_t length)
++{
++  int fd = open(path, O_WRONLY);
++  if (fd == -1) {
++    return -1;
++  }
++  int status = ftruncate(fd, length);
++  close(fd);
++  return status;
++}
++#  endif
++
+ typedef int err_t;
+ 
+ //  POSIX uses a 0 return to indicate success

pythonforandroid/recipes/boost/user-config.jam

@@ -1,28 +1,61 @@
 import os ;
 
-local ANDROIDNDK = [ os.environ ANDROIDNDK ] ;
-local ANDROIDAPI = [ os.environ ANDROIDAPI ] ;
-local TOOLCHAIN_VERSION = [ os.environ TOOLCHAIN_VERSION ] ;
-local TOOLCHAIN_PREFIX = [ os.environ TOOLCHAIN_PREFIX ] ;
 local ARCH = [ os.environ ARCH ] ;
+local CROSSHOME = [ os.environ CROSSHOME ] ;
+local PYTHON_HOST = [ os.environ PYTHON_HOST ] ;
 local PYTHON_ROOT = [ os.environ PYTHON_ROOT ] ;
+local PYTHON_INCLUDE = [ os.environ PYTHON_INCLUDE ] ;
+local PYTHON_LINK_VERSION = [ os.environ PYTHON_LINK_VERSION ] ;
+local PYTHON_MAJOR_MINOR = [ os.environ PYTHON_MAJOR_MINOR ] ;
 
-using gcc : $(ARCH) : $(TOOLCHAIN_PREFIX)-g++ :
+using clang : $(ARCH) : $(CROSSHOME)/bin/arm-linux-androideabi-clang++ :
+<archiver>$(CROSSHOME)/bin/arm-linux-androideabi-ar
+<root>$(CROSSHOME)/sysroot
 <architecture>$(ARCH)
-<archiver>$(TOOLCHAIN_PREFIX)-ar
-<compileflags>-DBOOST_SP_USE_PTHREADS
-<compileflags>-DBOOST_AC_USE_PTHREADS
-<cxxflags>-DBOOST_SP_USE_PTHREADS
-<cxxflags>-DBOOST_AC_USE_PTHREADS
-<cxxflags>-frtti
-<cxxflags>-fexceptions
-<compileflags>-I$(ANDROIDNDK)/platforms/android-$(ANDROIDAPI)/arch-$(ARCH)/usr/include
-<compileflags>-I$(ANDROIDNDK)/sources/cxx-stl/gnu-libstdc++/$(TOOLCHAIN_VERSION)/include
-<compileflags>-I$(ANDROIDNDK)/sources/cxx-stl/gnu-libstdc++/$(TOOLCHAIN_VERSION)/libs/$(ARCH)/include
-<compileflags>-I$(PYTHON_ROOT)/include/python2.7
-<linkflags>--sysroot=$(ANDROIDNDK)/platforms/android-$(ANDROIDAPI)/arch-$(ARCH)
-<linkflags>-L$(ANDROIDNDK)/sources/cxx-stl/gnu-libstdc++/$(TOOLCHAIN_VERSION)/libs/$(ARCH)
-<linkflags>-L$(PYTHON_ROOT)/lib
-<linkflags>-lgnustl_shared
-<linkflags>-lpython2.7
+<compileflags>-fexceptions
+<compileflags>-frtti
+<compileflags>-fpic
+<compileflags>-ffunction-sections
+<compileflags>-funwind-tables
+<compileflags>-march=armv7-a
+<compileflags>-msoft-float
+<compileflags>-mfpu=neon
+<compileflags>-mthumb
+<linkflags>-march=armv7-a
+<linkflags>-Wl,--fix-cortex-a8
+<compileflags>-Os
+<compileflags>-fomit-frame-pointer
+<compileflag>-fno-strict-aliasing
+<compileflags>-DANDROID
+<compileflags>-D__ANDROID__
+<compileflags>-DANDROID_TOOLCHAIN=clang
+<compileflags>-DANDROID_ABI=armv7-a
+<compileflags>-DANDROID_STL=c++_shared
+<compileflags>-DBOOST_ALL_NO_LIB
+#<compileflags>-DNDEBUG
+<compileflags>-O2
+<compileflags>-g
+<compileflags>-fvisibility=hidden
+<compileflags>-fvisibility-inlines-hidden
+<compileflags>-fdata-sections
+<cxxflags>-D__arm__
+<cxxflags>-D_REENTRANT
+<cxxflags>-D_GLIBCXX__PTHREADS
+<compileflags>-Wno-long-long
+<compileflags>-Wno-missing-field-initializers
+<compileflags>-Wno-unused-variable
+<linkflags>-Wl,-z,relro
+<linkflags>-Wl,-z,now
+<linkflags>-lc++_shared
+<linkflags>-L$(PYTHON_ROOT)
+<linkflags>-lpython$(PYTHON_LINK_VERSION)
+<linkflags>-Wl,-O1
+<linkflags>-Wl,-Bsymbolic-functions
 ;
+
+using python : $(PYTHON_MAJOR_MINOR)
+    : $(PYTHON_host)
+    : $(PYTHON_ROOT) $(PYTHON_INCLUDE)
+    : $(PYTHON_ROOT)/libpython$(PYTHON_LINK_VERSION).so
+    : #<define>BOOST_ALL_DYN_LINK
+;