Skip to content

[WIP] Restores the ability to compile the python files into .pyo/.pyc (for both versions of python) #1601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 27, 2019
Merged
46 changes: 30 additions & 16 deletions pythonforandroid/bootstraps/common/build/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,28 @@
import jinja2


def get_bootstrap_name():
def get_dist_info_for(key):
try:
with open(join(dirname(__file__), 'dist_info.json'), 'r') as fileh:
info = json.load(fileh)
bootstrap = str(info["bootstrap"])
value = str(info[key])
except (OSError, KeyError) as e:
print("BUILD FAILURE: Couldn't extract bootstrap name " +
print("BUILD FAILURE: Couldn't extract the key `" + key + "` " +
"from dist_info.json: " + str(e))
sys.exit(1)
return bootstrap
return value


def get_hostpython():
return get_dist_info_for('hostpython')


def get_python_version():
return get_dist_info_for('python_version')


def get_bootstrap_name():
return get_dist_info_for('bootstrap')


if os.name == 'nt':
Expand All @@ -44,9 +56,9 @@ def get_bootstrap_name():

curdir = dirname(__file__)

# Try to find a host version of Python that matches our ARM version.
PYTHON = join(curdir, 'python-install', 'bin', 'python.host')
if not exists(PYTHON):
PYTHON = get_hostpython()
PYTHON_VERSION = get_python_version()
if PYTHON is not None and not exists(PYTHON):
PYTHON = None

BLACKLIST_PATTERNS = [
Expand All @@ -56,17 +68,18 @@ def get_bootstrap_name():
'^*.bzr/*',
'^*.svn/*',

# pyc/py
'*.pyc',
# '*.py',

# temp files
'~',
'*.bak',
'*.swp',
]
# pyc/py
if PYTHON is not None:
BLACKLIST_PATTERNS.append('*.py')
if PYTHON_VERSION and int(PYTHON_VERSION[0]) == 2:
# we only blacklist `.pyc` for python2 because in python3 the compiled
# extension is `.pyc` (.pyo files not exists for python >= 3.6)
BLACKLIST_PATTERNS.append('*.pyc')

WHITELIST_PATTERNS = []
if get_bootstrap_name() in ('sdl2', 'webview', 'service_only'):
Expand Down Expand Up @@ -237,14 +250,15 @@ def compile_dir(dfn, optimize_python=True):
Compile *.py in directory `dfn` to *.pyo
'''

if get_bootstrap_name() != "sdl2":
# HISTORICALLY DISABLED for other than sdl2. NEEDS REVIEW! -JonasT
return
# -OO = strip docstrings
if PYTHON is None:
return
args = [PYTHON, '-m', 'compileall', '-f', dfn]

if int(PYTHON_VERSION[0]) >= 3:
args = [PYTHON, '-m', 'compileall', '-b', '-f', dfn]
else:
args = [PYTHON, '-m', 'compileall', '-f', dfn]
if optimize_python:
# -OO = strip docstrings
args.insert(1, '-OO')
subprocess.call(args)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,11 @@ int main(int argc, char *argv[]) {
/* Get the entrypoint, search the .pyo then .py
*/
char *dot = strrchr(env_entrypoint, '.');
#if PY_MAJOR_VERSION > 2
char *ext = ".pyc";
#else
char *ext = ".pyo";
#endif
if (dot <= 0) {
LOGP("Invalid entrypoint, abort.");
return -1;
Expand All @@ -313,14 +318,14 @@ int main(int argc, char *argv[]) {
LOGP("Entrypoint path is too long, try increasing ENTRYPOINT_MAXLEN.");
return -1;
}
if (!strcmp(dot, ".pyo")) {
if (!strcmp(dot, ext)) {
if (!file_exists(env_entrypoint)) {
/* fallback on .py */
strcpy(entrypoint, env_entrypoint);
entrypoint[strlen(env_entrypoint) - 1] = '\0';
LOGP(entrypoint);
if (!file_exists(entrypoint)) {
LOGP("Entrypoint not found (.pyo, fallback on .py), abort");
LOGP("Entrypoint not found (.pyc/.pyo, fallback on .py), abort");
return -1;
}
} else {
Expand All @@ -330,7 +335,11 @@ int main(int argc, char *argv[]) {
/* if .py is passed, check the pyo version first */
strcpy(entrypoint, env_entrypoint);
entrypoint[strlen(env_entrypoint) + 1] = '\0';
#if PY_MAJOR_VERSION > 2
entrypoint[strlen(env_entrypoint)] = 'c';
#else
entrypoint[strlen(env_entrypoint)] = 'o';
#endif
if (!file_exists(entrypoint)) {
/* fallback on pure python version */
if (!file_exists(env_entrypoint)) {
Expand All @@ -340,7 +349,7 @@ int main(int argc, char *argv[]) {
strcpy(entrypoint, env_entrypoint);
}
} else {
LOGP("Entrypoint have an invalid extension (must be .py or .pyo), abort.");
LOGP("Entrypoint have an invalid extension (must be .py or .pyc/.pyo), abort.");
return -1;
}
// LOGP("Entrypoint is:");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ protected void onPostExecute(String result) {
File path = new File(getIntent().getData().getSchemeSpecificPart());

Project p = Project.scanDirectory(path);
SDLActivity.nativeSetEnv("ANDROID_ENTRYPOINT", p.dir + "/main.py");
String entry_point = getEntryPoint(p.dir);
SDLActivity.nativeSetEnv("ANDROID_ENTRYPOINT", p.dir + "/" + entry_point);
SDLActivity.nativeSetEnv("ANDROID_ARGUMENT", p.dir);
SDLActivity.nativeSetEnv("ANDROID_APP_PATH", p.dir);

Expand All @@ -171,7 +172,8 @@ protected void onPostExecute(String result) {
// pass
}
} else {
SDLActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
String entry_point = getEntryPoint(app_root_dir);
SDLActivity.nativeSetEnv("ANDROID_ENTRYPOINT", entry_point);
SDLActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
SDLActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
}
Expand Down Expand Up @@ -368,9 +370,10 @@ public static void start_service(String serviceTitle, String serviceDescription,
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
String entry_point = PythonActivity.mActivity.getEntryPoint(app_root_dir + "/service");
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("serviceEntrypoint", "service/" + entry_point);
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
Expand Down Expand Up @@ -463,6 +466,21 @@ public void run() {
});
}

public String getEntryPoint(String search_dir) {
/* Get the main file (.pyc|.pyo|.py) depending on if we
* have a compiled version or not.
*/
List<String> entryPoints = new ArrayList<String>();
entryPoints.add("main.pyo"); // python 2 compiled files
entryPoints.add("main.pyc"); // python 3 compiled files
for (String value : entryPoints) {
File mainFile = new File(search_dir + "/" + value);
if (mainFile.exists()) {
return value;
}
}
return "main.py";
}

protected void showLoadingScreen() {
// load the bitmap
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,22 @@ public String getAppRoot() {
return app_root;
}

public String getEntryPoint(String search_dir) {
/* Get the main file (.pyc|.pyo|.py) depending on if we
* have a compiled version or not.
*/
List<String> entryPoints = new ArrayList<String>();
entryPoints.add("main.pyo"); // python 2 compiled files
entryPoints.add("main.pyc"); // python 3 compiled files
for (String value : entryPoints) {
File mainFile = new File(search_dir + "/" + value);
if (mainFile.exists()) {
return value;
}
}
return "main.py";
}

public static void initialize() {
// The static nature of the singleton and Android quirkiness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
Expand Down Expand Up @@ -142,9 +158,10 @@ public void onClick(DialogInterface dialog,int id) {
// Set up the Python environment
String app_root_dir = getAppRoot();
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
String entry_point = getEntryPoint(app_root_dir);

Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", entry_point);
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
Expand Down Expand Up @@ -368,9 +385,10 @@ public static void start_service(String serviceTitle, String serviceDescription,
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
String entry_point = PythonActivity.mActivity.getEntryPoint(app_root_dir + "/service");
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("serviceEntrypoint", "service/" + entry_point);
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,22 @@ public String getAppRoot() {
return app_root;
}

public String getEntryPoint(String search_dir) {
/* Get the main file (.pyc|.pyo|.py) depending on if we
* have a compiled version or not.
*/
List<String> entryPoints = new ArrayList<String>();
entryPoints.add("main.pyo"); // python 2 compiled files
entryPoints.add("main.pyc"); // python 3 compiled files
for (String value : entryPoints) {
File mainFile = new File(search_dir + "/" + value);
if (mainFile.exists()) {
return value;
}
}
return "main.py";
}

public static void initialize() {
// The static nature of the singleton and Android quirkyness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
Expand Down Expand Up @@ -170,9 +186,10 @@ public boolean shouldOverrideUrlLoading(WebView view, String url) {
setContentView(mLayout);

String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
String entry_point = getEntryPoint(app_root_dir);

Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", entry_point);
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
Expand Down Expand Up @@ -425,9 +442,10 @@ public static void start_service(String serviceTitle, String serviceDescription,
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
String entry_point = PythonActivity.mActivity.getEntryPoint(app_root_dir + "/service");
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("serviceEntrypoint", "service/" + entry_point);
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
Expand Down
4 changes: 3 additions & 1 deletion pythonforandroid/distribution.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,9 @@ def save_info(self, dirn):
'bootstrap': self.ctx.bootstrap.name,
'archs': [arch.arch for arch in self.ctx.archs],
'ndk_api': self.ctx.ndk_api,
'recipes': self.ctx.recipe_build_order + self.ctx.python_modules},
'recipes': self.ctx.recipe_build_order + self.ctx.python_modules,
'hostpython': self.ctx.hostpython,
'python_version': self.ctx.python_recipe.major_minor_version_string},
fileh)


Expand Down
Loading