[WIP][LIBS - PART III] Rework of pyleveldb, leveldb and snappy

pythonforandroid/recipes/leveldb/__init__.py

@@ -1,47 +1,46 @@
-from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory
+from pythonforandroid.logger import shprint
+from pythonforandroid.util import current_directory
+from pythonforandroid.recipe import Recipe
+from multiprocessing import cpu_count
 from os.path import join
 import sh
 
 
 class LevelDBRecipe(Recipe):
-    version = '1.18'
-    url = 'https://github.com/google/leveldb/archive/v{version}.tar.gz'
-    opt_depends = ['snappy']
-    patches = ['disable-so-version.patch', 'find-snappy.patch']
-
-    def should_build(self, arch):
-        return not self.has_libs(arch, 'libleveldb.so', 'libgnustl_shared.so')
+    version = '1.22'
+    url = 'https://github.com/google/leveldb/archive/{version}.tar.gz'
+    depends = ['snappy']
+    built_libraries = {'libleveldb.so': '.'}
+    need_stl_shared = True
 
     def build_arch(self, arch):
-        super(LevelDBRecipe, self).build_arch(arch)
         env = self.get_recipe_env(arch)
-        with current_directory(self.get_build_dir(arch.arch)):
-            if 'snappy' in recipe.ctx.recipe_build_order:
-                # Copy source from snappy recipe
-                sh.cp('-rf', self.get_recipe('snappy', self.ctx).get_build_dir(arch.arch), 'snappy')
-            # Build
-            shprint(sh.make, _env=env)
-            # Copy the shared library
-            shutil.copyfile('libleveldb.so', join(self.ctx.get_libs_dir(arch.arch), 'libleveldb.so'))
-        # Copy stl
-        shutil.copyfile(self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so',
-                        join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so'))
-
-    def get_recipe_env(self, arch):
-        env = super(LevelDBRecipe, self).get_recipe_env(arch)
-        env['TARGET_OS'] = 'OS_ANDROID_CROSSCOMPILE'
-        if 'snappy' in recipe.ctx.recipe_build_order:
-            env['CFLAGS'] += ' -DSNAPPY' + \
-                             ' -I./snappy'
-        env['CFLAGS'] += ' -I' + self.ctx.ndk_dir + '/platforms/android-' + str(self.ctx.android_api) + '/arch-' + arch.arch.replace('eabi', '') + '/usr/include' + \
-                         ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/include' + \
-                         ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/include'
-        env['CXXFLAGS'] = env['CFLAGS']
-        env['CXXFLAGS'] += ' -frtti'
-        env['CXXFLAGS'] += ' -fexceptions'
-        env['LDFLAGS'] += ' -L' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + \
-                          ' -lgnustl_shared'
-        return env
+        source_dir = self.get_build_dir(arch.arch)
+        with current_directory(source_dir):
+            snappy_recipe = self.get_recipe('snappy', self.ctx)
+            snappy_build = snappy_recipe.get_build_dir(arch.arch)
+
+            shprint(sh.cmake, source_dir,
+                    '-DANDROID_ABI={}'.format(arch.arch),
+                    '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api),
+                    '-DANDROID_STL=' + self.stl_lib_name,
+
+                    '-DCMAKE_TOOLCHAIN_FILE={}'.format(
+                        join(self.ctx.ndk_dir, 'build', 'cmake',
+                             'android.toolchain.cmake')),
+                    '-DCMAKE_BUILD_TYPE=Release',
+
+                    '-DBUILD_SHARED_LIBS=1',
+
+                    '-DHAVE_SNAPPY=1',
+                    '-DCMAKE_CXX_FLAGS=-I{path}'.format(path=snappy_build),
+                    '-DCMAKE_SHARED_LINKER_FLAGS=-L{path} -lsnappy'.format(
+                        path=snappy_build),
+                    '-DCMAKE_EXE_LINKER_FLAGS=-L{path} -lsnappy'.format(
+                        path=snappy_build),
+
+                    _env=env)
+            shprint(sh.make, '-j' + str(cpu_count()), _env=env)
 
 
 recipe = LevelDBRecipe()

pythonforandroid/recipes/pyleveldb/__init__.py

@@ -2,12 +2,26 @@
 
 
 class PyLevelDBRecipe(CppCompiledComponentsPythonRecipe):
-    version = '0.193'
-    url = 'https://pypi.python.org/packages/source/l/leveldb/leveldb-{version}.tar.gz'
-    depends = ['snappy', 'leveldb', ('hostpython2', 'hostpython3'), 'setuptools']
+    version = '0.194'
+    url = ('https://pypi.python.org/packages/source/l/leveldb/'
+           'leveldb-{version}.tar.gz')
+    depends = ['snappy', 'leveldb', 'setuptools']
     patches = ['bindings-only.patch']
-    call_hostpython_via_targetpython = False  # Due to setuptools
     site_packages_name = 'leveldb'
 
+    def get_recipe_env(self, arch):
+        env = super(PyLevelDBRecipe, self).get_recipe_env(arch)
+
+        snappy_recipe = self.get_recipe('snappy', self.ctx)
+        leveldb_recipe = self.get_recipe('leveldb', self.ctx)
+
+        env["LDFLAGS"] += " -L" + snappy_recipe.get_build_dir(arch.arch)
+        env["LDFLAGS"] += " -L" + leveldb_recipe.get_build_dir(arch.arch)
+
+        env["SNAPPY_BUILD_PATH"] = snappy_recipe.get_build_dir(arch.arch)
+        env["LEVELDB_BUILD_PATH"] = leveldb_recipe.get_build_dir(arch.arch)
+
+        return env
+
 
 recipe = PyLevelDBRecipe()

pythonforandroid/recipes/pyleveldb/bindings-only.patch

@@ -1,103 +1,119 @@
---- pyleveldb/setup.py	2014-03-28 02:51:24.000000000 +0100
-+++ pyleveldb-patch/setup.py	2016-03-02 11:52:13.780678586 +0100
-@@ -7,41 +7,22 @@
- #
- # See LICENSE for details.
-
--import glob
--import platform
--import sys
--
+This patch force to only build the python bindings, and to do so, we modify
+the setup.py file in oder that finds our compiled libraries (libleveldb.so and
+libsnappy.so)
+--- leveldb-0.194/setup.py.orig	2016-09-17 02:05:55.000000000 +0200
++++ leveldb-0.194/setup.py	2019-02-26 16:57:40.997435911 +0100
+@@ -11,44 +11,25 @@ import platform
+ import sys
+
  from setuptools import setup, Extension
-
--system,node,release,version,machine,processor = platform.uname()
++from os import environ
+
+ system, node, release, version, machine, processor = platform.uname()
 -common_flags = [
+-    '-I./leveldb/include',
+-    '-I./leveldb',
+-    '-I./snappy',
 +extra_compile_args = [
-       '-I./leveldb/include',
-       '-I./leveldb',
--      '-I./snappy',
-+      '-I./leveldb/snappy',
-       '-I.',
--      '-fno-builtin-memcmp',
-       '-O2',
-       '-fPIC',
-       '-DNDEBUG',
-       '-DSNAPPY',
--]
--
++    '-I{}/include'.format(environ.get('LEVELDB_BUILD_PATH')),
++    '-I{}'.format(environ.get('LEVELDB_BUILD_PATH')),
++    '-I{}'.format(environ.get('SNAPPY_BUILD_PATH')),
++    '-I.',
+     '-I.',
+-    '-fno-builtin-memcmp',
+     '-O2',
+     '-fPIC',
+     '-DNDEBUG',
+     '-DSNAPPY',
++    '-pthread',
++    '-Wall',
++    '-D_REENTRANT',
++    '-DOS_ANDROID',
+ ]
+
 -if system == 'Darwin':
--  extra_compile_args = common_flags + [
--      '-DOS_MACOSX',
-+      '-Wall',
-       '-DLEVELDB_PLATFORM_POSIX',
--      '-Wno-error=unused-command-line-argument-hard-error-in-future',
--      ]
+-    extra_compile_args = common_flags + [
+-        '-DOS_MACOSX',
+-        '-DLEVELDB_PLATFORM_POSIX',
+-        '-Wno-error=unused-command-line-argument-hard-error-in-future',
+-    ]
 -elif system == 'Linux':
+-    extra_compile_args = common_flags + [
+-        '-pthread',
+-        '-Wall',
+-        '-DOS_LINUX',
+-        '-DLEVELDB_PLATFORM_POSIX',
+-    ]
+-elif system == 'SunOS':
 -  extra_compile_args = common_flags + [
 -      '-pthread',
--      '-Wall', 
--      '-DOS_LINUX',
+-      '-Wall',
+-      '-DOS_SOLARIS',
 -      '-DLEVELDB_PLATFORM_POSIX',
 -      ]
 -else:
--  print >>sys.stderr, "Don't know how to compile leveldb for %s!" % system
--  sys.exit(0)
-+      '-D_REENTRANT',
-+      '-DOS_ANDROID',
-+]
-
+-    sys.stderr.write("Don't know how to compile leveldb for %s!\n" % system)
+-    sys.exit(1)
+-
  setup(
- 	name = 'leveldb',
-@@ -75,52 +56,6 @@
- 	ext_modules = [
- 		Extension('leveldb',
- 			sources = [
--        # snappy
--        './snappy/snappy.cc',
--        './snappy/snappy-stubs-internal.cc',
--        './snappy/snappy-sinksource.cc',
--        './snappy/snappy-c.cc',
+     name = 'leveldb',
+     version = '0.194',
+@@ -81,57 +62,11 @@ setup(
+     ext_modules = [
+         Extension('leveldb',
+             sources = [
+-                # snappy
+-                './snappy/snappy.cc',
+-                './snappy/snappy-stubs-internal.cc',
+-                './snappy/snappy-sinksource.cc',
+-                './snappy/snappy-c.cc',
 -
--        #leveldb
--        'leveldb/db/builder.cc', 
--        'leveldb/db/c.cc', 
--        'leveldb/db/db_impl.cc', 
--        'leveldb/db/db_iter.cc', 
--        'leveldb/db/dbformat.cc', 
--        'leveldb/db/filename.cc', 
--        'leveldb/db/log_reader.cc', 
--        'leveldb/db/log_writer.cc', 
--        'leveldb/db/memtable.cc', 
--        'leveldb/db/repair.cc', 
--        'leveldb/db/table_cache.cc', 
--        'leveldb/db/version_edit.cc', 
--        'leveldb/db/version_set.cc', 
--        'leveldb/db/write_batch.cc', 
--        'leveldb/table/block.cc', 
--        'leveldb/table/block_builder.cc', 
--        'leveldb/table/filter_block.cc', 
--        'leveldb/table/format.cc', 
--        'leveldb/table/iterator.cc', 
--        'leveldb/table/merger.cc', 
--        'leveldb/table/table.cc', 
--        'leveldb/table/table_builder.cc', 
--        'leveldb/table/two_level_iterator.cc', 
--        'leveldb/util/arena.cc', 
--        'leveldb/util/bloom.cc', 
--        'leveldb/util/cache.cc', 
--        'leveldb/util/coding.cc', 
--        'leveldb/util/comparator.cc', 
--        'leveldb/util/crc32c.cc', 
--        'leveldb/util/env.cc', 
--        'leveldb/util/env_posix.cc', 
--        'leveldb/util/filter_policy.cc', 
--        'leveldb/util/hash.cc', 
--        'leveldb/util/histogram.cc', 
--        'leveldb/util/logging.cc', 
--        'leveldb/util/options.cc', 
--        'leveldb/util/status.cc', 
--        'leveldb/port/port_posix.cc', 
+-                #leveldb
+-                'leveldb/db/builder.cc',
+-                'leveldb/db/c.cc',
+-                'leveldb/db/db_impl.cc',
+-                'leveldb/db/db_iter.cc',
+-                'leveldb/db/dbformat.cc',
+-                'leveldb/db/filename.cc',
+-                'leveldb/db/log_reader.cc',
+-                'leveldb/db/log_writer.cc',
+-                'leveldb/db/memtable.cc',
+-                'leveldb/db/repair.cc',
+-                'leveldb/db/table_cache.cc',
+-                'leveldb/db/version_edit.cc',
+-                'leveldb/db/version_set.cc',
+-                'leveldb/db/write_batch.cc',
+-                'leveldb/table/block.cc',
+-                'leveldb/table/block_builder.cc',
+-                'leveldb/table/filter_block.cc',
+-                'leveldb/table/format.cc',
+-                'leveldb/table/iterator.cc',
+-                'leveldb/table/merger.cc',
+-                'leveldb/table/table.cc',
+-                'leveldb/table/table_builder.cc',
+-                'leveldb/table/two_level_iterator.cc',
+-                'leveldb/util/arena.cc',
+-                'leveldb/util/bloom.cc',
+-                'leveldb/util/cache.cc',
+-                'leveldb/util/coding.cc',
+-                'leveldb/util/comparator.cc',
+-                'leveldb/util/crc32c.cc',
+-                'leveldb/util/env.cc',
+-                'leveldb/util/env_posix.cc',
+-                'leveldb/util/filter_policy.cc',
+-                'leveldb/util/hash.cc',
+-                'leveldb/util/histogram.cc',
+-                'leveldb/util/logging.cc',
+-                'leveldb/util/options.cc',
+-                'leveldb/util/status.cc',
+-                'leveldb/port/port_posix.cc',
 -
- 				# python stuff
- 				'leveldb_ext.cc',
- 				'leveldb_object.cc',
+                 # python stuff
+                 'leveldb_ext.cc',
+                 'leveldb_object.cc',
+             ],
+-            libraries = ['stdc++'],
++            libraries = ['snappy', 'leveldb', 'stdc++', 'c++_shared'],
+             extra_compile_args = extra_compile_args,
+         )
+     ]

pythonforandroid/recipes/snappy/__init__.py

@@ -1,13 +1,28 @@
-from pythonforandroid.toolchain import Recipe
+from pythonforandroid.recipe import Recipe
+from pythonforandroid.logger import shprint
+from pythonforandroid.util import current_directory
+from os.path import join
+import sh
 
 
 class SnappyRecipe(Recipe):
-    version = '1.1.3'
-    url = 'https://github.com/google/snappy/releases/download/{version}/snappy-{version}.tar.gz'
+    version = '1.1.7'
+    url = 'https://github.com/google/snappy/archive/{version}.tar.gz'
+    built_libraries = {'libsnappy.so': '.'}
 
-    def should_build(self, arch):
-        # Only download to use in leveldb recipe
-        return False
+    def build_arch(self, arch):
+        env = self.get_recipe_env(arch)
+        source_dir = self.get_build_dir(arch.arch)
+        with current_directory(source_dir):
+            shprint(sh.cmake, source_dir,
+                    '-DANDROID_ABI={}'.format(arch.arch),
+                    '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api),
+                    '-DCMAKE_TOOLCHAIN_FILE={}'.format(
+                        join(self.ctx.ndk_dir, 'build', 'cmake',
+                             'android.toolchain.cmake')),
+                    '-DBUILD_SHARED_LIBS=1',
+                    _env=env)
+            shprint(sh.make, _env=env)
 
 
 recipe = SnappyRecipe()