Skip to content

Commit 8fed863

Browse files
agilewalkertshirtman
agilewalker
authored and
tshirtman
committed
Fix building ssl for python3
1 parent 74bf788 commit 8fed863

File tree

3 files changed

+318
-47
lines changed

3 files changed

+318
-47
lines changed

‎pythonforandroid/recipes/openssl/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ def build_arch(self, arch):
6060
break
6161
shprint(sh.make, 'clean', _env=env)
6262

63-
self.install_libs(arch, 'libssl' + self.version + '.so',
64-
'libcrypto' + self.version + '.so')
63+
self.install_libs(arch, 'libssl.a', 'libssl' + self.version + '.so',
64+
'libcrypto.a', 'libcrypto' + self.version + '.so')
6565

6666

6767
recipe = OpenSSLRecipe()

‎pythonforandroid/recipes/python3crystax/__init__.py

Lines changed: 227 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,63 @@
1-
21
from pythonforandroid.recipe import TargetPythonRecipe
32
from pythonforandroid.toolchain import shprint
43
from pythonforandroid.logger import info, error
54
from pythonforandroid.util import ensure_dir, temp_directory
65
from os.path import exists, join
6+
import os
7+
import glob
78
import sh
9+
from sh import Command
10+
11+
# This is the content of opensslconf.h taken from
12+
# ndkdir/build/tools/build-target-openssl.sh
13+
OPENSSLCONF = """#if defined(__ARM_ARCH_5TE__)
14+
#include "opensslconf_armeabi.h"
15+
#elif defined(__ARM_ARCH_7A__) && !defined(__ARM_PCS_VFP)
16+
#include "opensslconf_armeabi_v7a.h"
17+
#elif defined(__ARM_ARCH_7A__) && defined(__ARM_PCS_VFP)
18+
#include "opensslconf_armeabi_v7a_hard.h"
19+
#elif defined(__aarch64__)
20+
#include "opensslconf_arm64_v8a.h"
21+
#elif defined(__i386__)
22+
#include "opensslconf_x86.h"
23+
#elif defined(__x86_64__)
24+
#include "opensslconf_x86_64.h"
25+
#elif defined(__mips__) && !defined(__mips64)
26+
#include "opensslconf_mips.h"
27+
#elif defined(__mips__) && defined(__mips64)
28+
#include "opensslconf_mips64.h"
29+
#else
30+
#error "Unsupported ABI"
31+
#endif
32+
"""
33+
LATEST_FULL_VERSION = {
34+
'3.5': '3.5.1',
35+
'3.6': '3.6.4'
36+
}
37+
38+
def realpath(fname):
39+
"""
40+
Own implementation of os.realpath which may be broken in some python versions
41+
Returns: the absolute path o
42+
43+
"""
44+
45+
if not os.path.islink(fname):
46+
return os.path.abspath(fname)
847

9-
prebuilt_download_locations = {
10-
'3.6': ('https://github.com/inclement/crystax_python_builds/'
11-
'releases/download/0.1/crystax_python_3.6_armeabi_armeabi-v7a.tar.gz')}
48+
abs_path = os.path.abspath(fname).split(os.sep)[:-1]
49+
rel_path = os.readlink(fname)
50+
51+
if os.path.abspath(rel_path) == rel_path:
52+
return rel_path
53+
54+
rel_path = rel_path.split(os.sep)
55+
for folder in rel_path:
56+
if folder == '..':
57+
abs_path.pop()
58+
else:
59+
abs_path.append(folder)
60+
return os.sep.join(abs_path)
1261

1362

1463
class Python3Recipe(TargetPythonRecipe):
@@ -21,56 +70,189 @@ class Python3Recipe(TargetPythonRecipe):
2170

2271
from_crystax = True
2372

73+
def download_if_necessary(self):
74+
if 'openssl' in self.ctx.recipe_build_order or self.version == '3.6':
75+
full_version = LATEST_FULL_VERSION[self.version]
76+
Python3Recipe.url = 'https://www.python.org/ftp/python/{0}.{1}.{2}/Python-{0}.{1}.{2}.tgz'.format(*full_version.split('.'))
77+
super(Python3Recipe, self).download_if_necessary()
78+
2479
def get_dir_name(self):
2580
name = super(Python3Recipe, self).get_dir_name()
2681
name += '-version{}'.format(self.version)
2782
return name
2883

84+
def copy_include_dir(self, source, target):
85+
ensure_dir(target)
86+
for fname in os.listdir(source):
87+
sh.ln('-sf', realpath(join(source, fname)), join(target, fname))
88+
89+
def _patch_dev_defaults(self, fp, target_ver):
90+
for line in fp:
91+
if 'OPENSSL_VERSIONS=' in line:
92+
versions = line.split('"')[1].split(' ')
93+
if versions[0] == target_ver:
94+
raise ValueError('Patch not needed')
95+
96+
if target_ver in versions:
97+
versions.remove(target_ver)
98+
99+
versions.insert(0, target_ver)
100+
101+
yield 'OPENSSL_VERSIONS="{}"\n'.format(' '.join(versions))
102+
else:
103+
yield line
104+
105+
def patch_dev_defaults(self, ssl_recipe):
106+
def_fname = join(self.ctx.ndk_dir, 'build', 'tools', 'dev-defaults.sh')
107+
try:
108+
with open(def_fname, 'r') as fp:
109+
s = ''.join(self._patch_dev_defaults(fp,
110+
str(ssl_recipe.version)))
111+
with open(def_fname, 'w') as fp:
112+
fp.write(s)
113+
114+
except ValueError:
115+
pass
116+
117+
def check_for_sslso(self, ssl_recipe, arch):
118+
# type: (Recipe, str)
119+
dynlib_dir = join(self.ctx.ndk_dir, 'sources', 'python', self.version,
120+
'libs', arch.arch, 'modules')
121+
122+
if os.path.exists(join(dynlib_dir, '_ssl.so')):
123+
return 10, 'Shared object exists in ndk'
124+
125+
# find out why _ssl.so is missing
126+
127+
source_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', ssl_recipe.version)
128+
if not os.path.exists(source_dir):
129+
return 0, 'Openssl version not present'
130+
131+
# these two path checks are lifted straight from:
132+
# crystax-ndk/build/tools/build-target-python.sh
133+
if not os.path.exists(join(source_dir, 'Android.mk')):
134+
return 1.1, 'Android.mk is missing in openssl source'
135+
136+
include_dir = join(source_dir, 'include','openssl')
137+
if not os.path.exists(join(include_dir, 'opensslconf.h')):
138+
return 1.2, 'Openssl include dir missing'
139+
140+
under_scored_arch = arch.arch.replace('-', '_')
141+
if not os.path.lexists(join(include_dir,
142+
'opensslconf_{}.h'.format(under_scored_arch))):
143+
return 1.3, 'Opensslconf arch header missing from include'
144+
145+
146+
147+
# lastly a check to see whether shared objects for the correct arch
148+
# is present in the ndk
149+
if not os.path.exists(join(source_dir, 'libs', arch.arch)):
150+
return 2, 'Openssl libs for this arch is missing in ndk'
151+
152+
return 5, 'Ready to recompile python'
153+
154+
def find_Android_mk(self):
155+
openssl_dir = join(self.ctx.ndk_dir, 'sources', 'openssl')
156+
for version in os.listdir(openssl_dir):
157+
mk_path = join(openssl_dir, version, 'Android.mk')
158+
if os.path.exists(mk_path):
159+
return mk_path
160+
161+
def prebuild_arch(self, arch):
162+
super(Python3Recipe, self).prebuild_arch(arch)
163+
if self.version == '3.6':
164+
Python3Recipe.patches = ['patch_python3.6.patch']
165+
build_dir = self.get_build_dir(arch.arch)
166+
shprint(sh.ln, '-sf',
167+
realpath(join(build_dir, 'Lib/site-packages/README.txt')),
168+
join(build_dir, 'Lib/site-packages/README'))
169+
python_build_files = ['android.mk', 'config.c', 'interpreter.c']
170+
ndk_build_tools_python_dir = join(self.ctx.ndk_dir, 'build', 'tools', 'build-target-python')
171+
for python_build_file in python_build_files:
172+
shprint(sh.cp, join(ndk_build_tools_python_dir, python_build_file+'.3.5'),
173+
join(ndk_build_tools_python_dir, python_build_file+'.3.6'))
174+
ndk_sources_python_dir = join(self.ctx.ndk_dir, 'sources', 'python')
175+
if not os.path.exists(join(ndk_sources_python_dir, '3.6')):
176+
os.mkdir(join(ndk_sources_python_dir, '3.6'))
177+
sh.sed('s#3.5#3.6#',
178+
join(ndk_sources_python_dir, '3.5/Android.mk'),
179+
_out=join(ndk_sources_python_dir, '3.6/Android.mk'))
180+
29181
def build_arch(self, arch):
30-
# We don't have to actually build anything as CrystaX comes
31-
# with the necessary modules. They are included by modifying
32-
# the Android.mk in the jni folder.
33-
34-
# If the Python version to be used is not prebuilt with the CrystaX
35-
# NDK, we do have to download it.
36-
37-
crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python')
38-
if not exists(join(crystax_python_dir, self.version)):
39-
info(('The NDK does not have a prebuilt Python {}, trying '
40-
'to obtain one.').format(self.version))
41-
42-
if self.version not in prebuilt_download_locations:
43-
error(('No prebuilt version for Python {} could be found, '
44-
'the built cannot continue.'))
45-
exit(1)
46-
47-
with temp_directory() as td:
48-
self.download_file(prebuilt_download_locations[self.version],
49-
join(td, 'downloaded_python'))
50-
shprint(sh.tar, 'xf', join(td, 'downloaded_python'),
51-
'--directory', crystax_python_dir)
52-
53-
if not exists(join(crystax_python_dir, self.version)):
54-
error(('Something went wrong, the directory at {} should '
55-
'have been created but does not exist.').format(
56-
join(crystax_python_dir, self.version)))
57-
58-
if not exists(join(
59-
crystax_python_dir, self.version, 'libs', arch.arch)):
60-
error(('The prebuilt Python for version {} does not contain '
61-
'binaries for your chosen architecture "{}".').format(
62-
self.version, arch.arch))
63-
exit(1)
64-
65-
# TODO: We should have an option to build a new Python. This
66-
# would also allow linking to openssl and sqlite from CrystaX.
182+
# If openssl is needed we may have to recompile cPython to get the
183+
# ssl.py module working properly
184+
if self.from_crystax and 'openssl' in self.ctx.recipe_build_order:
185+
info('Openssl and crystax-python combination may require '
186+
'recompilation of python...')
187+
ssl_recipe = self.get_recipe('openssl', self.ctx)
188+
stage, msg = self.check_for_sslso(ssl_recipe, arch)
189+
stage = 0 if stage < 5 else stage
190+
info(msg)
191+
openssl_build_dir = ssl_recipe.get_build_dir(arch.arch)
192+
openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl',
193+
ssl_recipe.version)
194+
195+
if stage < 2:
196+
info('Copying openssl headers and Android.mk to ndk')
197+
ensure_dir(openssl_ndk_dir)
198+
if stage < 1.2:
199+
# copy include folder and Android.mk to ndk
200+
mk_path = self.find_Android_mk()
201+
if mk_path is None:
202+
raise IOError('Android.mk file could not be found in '
203+
'any versions in ndk->sources->openssl')
204+
shprint(sh.cp, mk_path, openssl_ndk_dir)
205+
206+
include_dir = join(openssl_build_dir, 'include')
207+
if stage < 1.3:
208+
ndk_include_dir = join(openssl_ndk_dir, 'include', 'openssl')
209+
self.copy_include_dir(join(include_dir, 'openssl'), ndk_include_dir)
210+
211+
target_conf = join(openssl_ndk_dir, 'include', 'openssl',
212+
'opensslconf.h')
213+
shprint(sh.rm, '-f', target_conf)
214+
# overwrite opensslconf.h
215+
with open(target_conf, 'w') as fp:
216+
fp.write(OPENSSLCONF)
217+
218+
if stage < 1.4:
219+
# move current conf to arch specific conf in ndk
220+
under_scored_arch = arch.arch.replace('-', '_')
221+
shprint(sh.ln, '-sf',
222+
realpath(join(include_dir, 'openssl', 'opensslconf.h')),
223+
join(openssl_ndk_dir, 'include', 'openssl',
224+
'opensslconf_{}.h'.format(under_scored_arch))
225+
)
67226

227+
if stage < 3:
228+
info('Copying openssl libs to ndk')
229+
arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch)
230+
ensure_dir(arch_ndk_lib)
231+
shprint(sh.ln, '-sf',
232+
realpath(join(openssl_build_dir, 'libcrypto{}.so'.format(ssl_recipe.version))),
233+
join(openssl_build_dir, 'libcrypto.so'))
234+
shprint(sh.ln, '-sf',
235+
realpath(join(openssl_build_dir, 'libssl{}.so'.format(ssl_recipe.version))),
236+
join(openssl_build_dir, 'libssl.so'))
237+
libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so']
238+
cmd = [join(openssl_build_dir, lib) for lib in libs] + [arch_ndk_lib]
239+
shprint(sh.cp, '-f', *cmd)
240+
241+
if stage < 10:
242+
info('Recompiling python-crystax')
243+
self.patch_dev_defaults(ssl_recipe)
244+
build_script = join(self.ctx.ndk_dir, 'build', 'tools',
245+
'build-target-python.sh')
246+
247+
shprint(Command(build_script),
248+
'--ndk-dir={}'.format(self.ctx.ndk_dir),
249+
'--abis={}'.format(arch.arch),
250+
'-j5', '--verbose',
251+
self.get_build_dir(arch.arch))
252+
253+
info('Extracting CrystaX python3 from NDK package')
68254
dirn = self.ctx.get_python_install_dir()
69255
ensure_dir(dirn)
70-
71-
# Instead of using a locally built hostpython, we use the
72-
# user's Python for now. They must have the right version
73-
# available. Using e.g. pyenv makes this easy.
74256
self.ctx.hostpython = 'python{}'.format(self.version)
75257

76258

‎pythonforandroid/recipes/python3crystax/patch_python3.6.patch

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
2+
--- a/Modules/expat/xmlparse.c
3+
+++ b/Modules/expat/xmlparse.c
4+
@@ -84,6 +84,8 @@
5+
# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
6+
#endif
7+
8+
+#define XML_POOR_ENTROPY 1
9+
+
10+
#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
11+
&& !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
12+
&& !defined(XML_DEV_URANDOM) \
13+
diff --git a/Modules/getpath.c b/Modules/getpath.c
14+
--- a/Modules/getpath.c
15+
+++ b/Modules/getpath.c
16+
@@ -101,8 +101,35 @@
17+
#endif
18+
19+
20+
-#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH)
21+
-#error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined"
22+
+ /* These variables were set this way in old versions of Python, but
23+
+ changed somewhere between 3.5.0 and 3.5.3. Here we just force
24+
+ the old way again. A better solution would be to work out where
25+
+ they should be defined, and make the CrystaX build scripts do
26+
+ so. */
27+
+
28+
+/* #if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) */
29+
+/* #error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined" */
30+
+/* #endif */
31+
+
32+
+#ifndef VERSION
33+
+#define VERSION "2.1"
34+
+#endif
35+
+
36+
+#ifndef VPATH
37+
+#define VPATH "."
38+
+#endif
39+
+
40+
+#ifndef PREFIX
41+
+# define PREFIX "/usr/local"
42+
+#endif
43+
+
44+
+#ifndef EXEC_PREFIX
45+
+#define EXEC_PREFIX PREFIX
46+
+#endif
47+
+
48+
+#ifndef PYTHONPATH
49+
+#define PYTHONPATH PREFIX "/lib/python" VERSION ":" \
50+
+ EXEC_PREFIX "/lib/python" VERSION "/lib-dynload"
51+
#endif
52+
53+
#ifndef LANDMARK
54+
diff --git a/Modules/timemodule.c b/Modules/timemodule.c
55+
--- a/Modules/timemodule.c
56+
+++ b/Modules/timemodule.c
57+
@@ -358,18 +358,20 @@ time_gmtime(PyObject *self, PyObject *args)
58+
#endif
59+
}
60+
61+
-#ifndef HAVE_TIMEGM
62+
-static time_t
63+
-timegm(struct tm *p)
64+
-{
65+
- /* XXX: the following implementation will not work for tm_year < 1970.
66+
- but it is likely that platforms that don't have timegm do not support
67+
- negative timestamps anyways. */
68+
- return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 +
69+
- (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 -
70+
- ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400;
71+
-}
72+
-#endif
73+
+/* In the Android build, HAVE_TIMEGM apparently should be defined but isn't. A better fix would be to work out why and fix that. */
74+
+
75+
+/* #ifndef HAVE_TIMEGM */
76+
+/* static time_t */
77+
+/* timegm(struct tm *p) */
78+
+/* { */
79+
+/* /\* XXX: the following implementation will not work for tm_year < 1970. */
80+
+/* but it is likely that platforms that don't have timegm do not support */
81+
+/* negative timestamps anyways. *\/ */
82+
+/* return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 + */
83+
+/* (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 - */
84+
+/* ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400; */
85+
+/* } */
86+
+/* #endif */
87+
88+
PyDoc_STRVAR(gmtime_doc,
89+
"gmtime([seconds]) -> (tm_year, tm_mon, tm_mday, tm_hour, tm_min,\n\

0 commit comments

Comments
 (0)