modifications for reproducible builds

* pass SOURCE_DATE_EPOCH etc. through to build env
* clean tar entries & gzip mtime
* utime() files before zipping
* sort file lists before tar/zip
* make private_version deterministic

Makefile example:

export LC_ALL             := C
export TZ                 := UTC
export SOURCE_DATE_EPOCH  := $(shell git log -1 --pretty=%ct)
export PYTHONHASHSEED     := $(SOURCE_DATE_EPOCH)
export BUILD_DATE         := $(shell date +'%b %e %Y' -d @$(SOURCE_DATE_EPOCH))
export BUILD_TIME         := $(shell date +'%H:%M:%S' -d @$(SOURCE_DATE_EPOCH))

Author: obfusk

pythonforandroid/archs.py

@@ -234,6 +234,12 @@ def get_env(self, with_flags_in_cc=True):
 
         env['PATH'] = environ['PATH']
 
+        # for reproducible builds
+        if 'SOURCE_DATE_EPOCH' in environ:
+            for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split():
+                if k in environ:
+                    env[k] = environ[k]
+
         return env
 
 

pythonforandroid/bootstraps/common/build/build.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
+from gzip import GzipFile
+import hashlib
 import json
 from os.path import (
     dirname, join, isfile, realpath,
     relpath, split, exists, basename
 )
-from os import listdir, makedirs, remove
+from os import environ, listdir, makedirs, remove
 import os
 import shlex
 import shutil
@@ -161,16 +163,25 @@ def select(fn):
             return False
         return not is_blacklist(fn)
 
+    # cleaning function (for reproducible builds)
+    def clean(tinfo):
+        tinfo.uid = tinfo.gid = 0
+        tinfo.uname = tinfo.gname = ''
+        tinfo.mtime = 0
+        return tinfo
+
     # get the files and relpath file of all the directory we asked for
     files = []
     for sd in source_dirs:
         sd = realpath(sd)
         compile_dir(sd, optimize_python=optimize_python)
         files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd)
                   if select(x)]
+    files.sort() # deterministic
 
     # create tar.gz of thoses files
-    tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT)
+    gf = GzipFile(tfn, 'wb', mtime=0) # deterministic
+    tf = tarfile.open(None, 'w', gf, format=tarfile.USTAR_FORMAT)
     dirs = []
     for fn, afn in files:
         dn = dirname(afn)
@@ -189,8 +200,9 @@ def select(fn):
                 tf.addfile(tinfo)
 
         # put the file
-        tf.add(fn, afn)
+        tf.add(fn, afn, filter=clean)
     tf.close()
+    gf.close()
 
 
 def compile_dir(dfn, optimize_python=True):
@@ -521,9 +533,18 @@ def make_package(args):
         versioned_name=versioned_name)
 
     # String resources:
+    timestamp = time.time()
+    if 'SOURCE_DATE_EPOCH' in environ:
+        # for reproducible builds
+        timestamp = int(environ['SOURCE_DATE_EPOCH'])
+    private_version = "{} {} {}".format(
+        args.version,
+        args.numeric_version,
+        timestamp
+    )
     render_args = {
         "args": args,
-        "private_version": str(time.time())
+        "private_version": hashlib.sha1(private_version.encode()).hexdigest()
     }
     if get_bootstrap_name() == "sdl2":
         render_args["url_scheme"] = url_scheme

pythonforandroid/recipes/python3/__init__.py

@@ -3,7 +3,7 @@
 import subprocess
 
 from multiprocessing import cpu_count
-from os import environ
+from os import environ, utime
 from os.path import dirname, exists, join
 from pathlib import Path
 from shutil import copy2
@@ -385,6 +385,12 @@ def create_python_bundle(self, dirn, arch):
         with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
             stdlib_filens = list(walk_valid_filens(
                 '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
+            if 'SOURCE_DATE_EPOCH' in environ:
+                # for reproducible builds
+                stdlib_filens.sort()
+                timestamp = int(environ['SOURCE_DATE_EPOCH'])
+                for filen in stdlib_filens:
+                    utime(filen, (timestamp, timestamp))
             info("Zip {} files into the bundle".format(len(stdlib_filens)))
             shprint(sh.zip, stdlib_zip, *stdlib_filens)