Cython cache diff compare

As talked about in #13425 with @gfyoung and @jreback this PR does
cython caching by comparing the pyx files and not relying on the git
history.

Author: Neil Parley <[email protected]>

Closes #13526 from nparley/pyx-diff and squashes the following commits:

8b4ad0b [Neil Parley] Remove test commit
2dc2ce2 [Neil Parley] Merge branch 'upstream' into pyx-diff
81cbb7b [Neil Parley] Reset pyx changes
79fe0f0 [Neil Parley] Remove do_not_merge
dca6e61 [Neil Parley] Test commit
30bd9da [Neil Parley] Check for the case where less cython file has been deleted
aaf4dd2 [Neil Parley] Merge branch 'upstream' into pyx-diff
6cc3d6e [Neil Parley] Diff python pyx files as a test if they have changed

Author: nparley

ci/prep_cython_cache.sh

@@ -1,43 +1,73 @@
 #!/bin/bash
 
 ls "$HOME/.cache/"
+
+PYX_CACHE_DIR="$HOME/.cache/pyxfiles"
+pyx_file_list=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"`
+pyx_cache_file_list=`find ${PYX_CACHE_DIR} -name "*.pyx"`
+
 CACHE_File="$HOME/.cache/cython_files.tar"
 
+# Clear the cython cache 0 = NO, 1 = YES
 clear_cache=0
+
+pyx_files=`echo "$pyx_file_list" | wc -l`
+pyx_cache_files=`echo "$pyx_cache_file_list" | wc -l`
+
+if [[ pyx_files -ne pyx_cache_files ]]
+then
+        echo "Different number of pyx files"
+        clear_cache=1
+fi
+
 home_dir=$(pwd)
 
-if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then
+if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then
+
+    echo "Cache available - checking pyx diff"
+
+    for i in ${pyx_file_list}
+    do
+            diff=`diff -u $i $PYX_CACHE_DIR${i}`
+            if [[ $? -eq 2 ]]
+            then
+                    echo "${i##*/} can't be diffed; probably not in cache"
+                    clear_cache=1
+            fi
+            if [[ ! -z $diff ]]
+            then
+                    echo "${i##*/} has changed:"
+                    echo $diff
+                    clear_cache=1
+            fi
+    done
 
-    echo "Cache available"
-    clear_cache=1
-    # did the last commit change cython files?
-    # go back 2 commits
     if [ "$TRAVIS_PULL_REQUEST" == "false" ]
     then
-        echo "Not a PR: checking for cython files changes from last 2 commits"
-        git diff HEAD~2 --numstat | grep -E "pyx|pxd"
-        retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l)
+        echo "Not a PR"
+        # Uncomment next 2 lines to turn off cython caching not in a PR
+        # echo "Non PR cython caching is disabled"
+        # clear_cache=1
     else
-        echo "PR: checking for any cython file changes from last 5 commits"
-        git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd"
-        retval=$(git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd"| wc -l)
-        echo "Forcing cython rebuild due to possibility of history rewritting in PR"
-        retval=-1
+        echo "In a PR"
+        # Uncomment next 2 lines to turn off cython caching in a PR
+        # echo "PR cython caching is disabled"
+        # clear_cache=1
     fi
-    echo "number of cython files changed: $retval"
+
 fi
 
-if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] && [ "$USE_CACHE" ]
+if [ $clear_cache -eq 0 ] && [ "$USE_CACHE" ]
 then
-    # nope, reuse cython files
+    # No and use_cache is set
     echo "Will reuse cached cython file"
     cd /
     tar xvmf $CACHE_File
     cd $home_dir
 else
     echo "Rebuilding cythonized files"
-    echo "Use cache = $USE_CACHE"
-    echo "Clear cache = $clear_cache"
+    echo "Use cache (Blank if not set) = $USE_CACHE"
+    echo "Clear cache (1=YES) = $clear_cache"
 fi
 
 

ci/submit_cython_cache.sh

@@ -1,17 +1,23 @@
 #!/bin/bash
 
 CACHE_File="$HOME/.cache/cython_files.tar"
+PYX_CACHE_DIR="$HOME/.cache/pyxfiles"
+pyx_file_list=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"`
+
 rm -rf $CACHE_File
+rm -rf $PYX_CACHE_DIR
 
 home_dir=$(pwd)
 
-pyx_files=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"`
+mkdir $PYX_CACHE_DIR
+rsync -Rv $pyx_file_list $PYX_CACHE_DIR
+
 echo "pyx files:"
-echo $pyx_files
+echo $pyx_file_list
 
 tar cf ${CACHE_File} --files-from /dev/null
 
-for i in ${pyx_files}
+for i in ${pyx_file_list}
 do
         f=${i%.pyx}
         ls $f.{c,cpp} | tar rf  ${CACHE_File} -T -