Fixing up more tests, making pipfile work again, adding some tests

Bringing this closer...

Author: AndrewFarley

index.js

@@ -7,7 +7,7 @@ const {addVendorHelper, removeVendorHelper, packRequirements} = require('./lib/z
 const {injectAllRequirements} = require('./lib/inject');
 const {installAllRequirements} = require('./lib/pip');
 const {pipfileToRequirements} = require('./lib/pipenv');
-const {cleanup, cleanupStaticCache} = require('./lib/clean');
+const {cleanup, cleanupCache} = require('./lib/clean');
 
 BbPromise.promisifyAll(fse);
 
@@ -32,7 +32,8 @@ class ServerlessPythonRequirements {
       dockerImage: null,
       dockerFile: null,
       useStaticCache: false,
-      staticCacheLocation: false,
+      useDownloadCache: false,
+      cacheLocation: false,
       staticCacheMaxVersions: 0,
       pipCmdExtraArgs: [],
       noDeploy: [
@@ -131,7 +132,7 @@ class ServerlessPythonRequirements {
     };
 
     const cleanStatic = () => BbPromise.bind(this)
-      .then(cleanupStaticCache);
+      .then(cleanupCache);
 
 
     this.hooks = {

lib/clean.js

@@ -2,7 +2,7 @@ const BbPromise = require('bluebird');
 const fse = require('fs-extra');
 const path = require('path');
 const glob = require('glob-all');
-const {getStaticCachePath} = require('./shared');
+const {getUserCachePath} = require('./shared');
 
 BbPromise.promisifyAll(fse);
 
@@ -34,24 +34,26 @@ function cleanup() {
   * Clean up static cache, remove all items in there
   * @return {Promise}
   */
-function cleanupStaticCache() {
-  const staticCacheLocation = getStaticCachePath(this.options);
-  if (fse.existsSync(staticCacheLocation)) {
-    this.serverless.cli.log(`Removing static caches at: ${staticCacheLocation}`);
+function cleanupCache() {
+  const cacheLocation = getUserCachePath(this.options);
+  if (fse.existsSync(cacheLocation)) {
+    if (this.serverless) {
+      this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`);
+    }
 
     // Only remove cache folders that we added, just incase someone accidentally puts a weird
     // static cache location so we don't remove a bunch of personal stuff
     promises = [];
-    glob.sync([path.join(staticCacheLocation, '*')], {mark: true, dot: false}).forEach((file) => {
-      if (file.endsWith('_slspyc/')) {
-        promises.push( fse.removeAsync(file) );
-      }
+    glob.sync([path.join(cacheLocation, '*slspyc/')], {mark: true, dot: false}).forEach((file) => {
+      promises.push( fse.removeAsync(file) );
     });
     return BbPromise.all( promises );
   } else {
-    this.serverless.cli.log(`No static cache found`);
+    if (this.serverless) {
+      this.serverless.cli.log(`No static cache found`);
+    }
     return BbPromise.resolve();
   }
 }
 
-module.exports = {cleanup, cleanupStaticCache};
+module.exports = {cleanup, cleanupCache};

lib/pip.js

@@ -7,7 +7,7 @@ const {spawnSync} = require('child_process');
 const values = require('lodash.values');
 const {buildImage, getBindPath, getDockerUid} = require('./docker');
 const {
-  checkForAndDeleteMaxCacheVersions, getRequirementsWorkingPath, md5Path,
+  checkForAndDeleteMaxCacheVersions, getRequirementsWorkingPath, getUserCachePath, md5Path,
 } = require('./shared');
 
 /**
@@ -63,6 +63,14 @@ function installRequirements(targetFolder, serverless, options) {
     pipCmd.push('-t', targetFolder);
     pipCmd.push('-r', targetRequirementsTxt);
 
+    // If we want a download cache...
+    if (options.useDownloadCache) {
+      const downloadCacheDir = path.join(getUserCachePath(options), 'downloadCacheslspyc');
+      serverless.cli.log(`Using download cache directory ${downloadCacheDir}`);
+      fse.ensureDirSync(downloadCacheDir);
+      pipCmd.push('--cache-dir', downloadCacheDir);
+    }
+
     // Check if pip has Debian's --system option and set it if so
     const pipTestRes = spawnSync(
       options.pythonBin, ['-m', 'pip', 'help', 'install']);
@@ -82,11 +90,8 @@ function installRequirements(targetFolder, serverless, options) {
   // If we are dockerizing pip
   if (options.dockerizePip) {
     cmd = 'docker';
-    // Push docker-specific paths for requirements and target directory
-    pipCmd.push('-t', '/var/task/');
-    pipCmd.push('-r', '/var/task/requirements.txt');
 
-    // Build docker image if required
+    // First, build docker image if required
     let dockerImage;
     if (options.dockerFile) {
       serverless.cli.log(`Building custom docker image from ${options.dockerFile}...`);
@@ -96,6 +101,10 @@ function installRequirements(targetFolder, serverless, options) {
     }
     serverless.cli.log(`Docker Image: ${dockerImage}`);
 
+    // Push docker-specific paths for requirements and target directory
+    pipCmd.push('-t', '/var/task/');
+    pipCmd.push('-r', '/var/task/requirements.txt');
+
     // Prepare bind path depending on os platform
     const bindPath = getBindPath(targetFolder);
 
@@ -110,6 +119,16 @@ function installRequirements(targetFolder, serverless, options) {
       cmdOptions.push('-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`);
       cmdOptions.push('-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock');
     }
+    // If we want a download cache...
+    if (options.useDownloadCache) {
+      const downloadCacheDir = path.join(getUserCachePath(options), 'downloadCacheslspyc');
+      serverless.cli.log(`Using download cache directory ${downloadCacheDir}`);
+      fse.ensureDirSync(downloadCacheDir);
+      // And now push it to a volume mount and to pip...
+      cmdOptions.push('-v', `${downloadCacheDir}:/var/useDownloadCache:z`);
+      pipCmd.push('--cache-dir', '/var/useDownloadCache');
+    }
+
     if (process.platform === 'linux') {
       // Use same user so requirements folder is not root and so --cache-dir works
       cmdOptions.push('-u', `${process.getuid()}`);
@@ -207,10 +226,7 @@ function installRequirementsIfNeeded(servicePath, modulePath, options, serverles
   // First, generate the requirements file to our local .serverless folder
   fse.ensureDirSync(path.join(servicePath, '.serverless'));
   const slsReqsTxt = path.join(servicePath, '.serverless', 'requirements.txt');
-  // Incase it's laying around from a previous package (when individually/failed packaging)
-  if (fse.existsSync(slsReqsTxt)) {
-    fse.removeSync(slsReqsTxt);
-  }
+
   installRequirementsFile(
     fileName,
     slsReqsTxt,
@@ -242,7 +258,6 @@ function installRequirementsIfNeeded(servicePath, modulePath, options, serverles
       serverless.cli.log(`Using static cache of requirements found at ${workingReqsFolder} ...`);
       // We'll "touch" the folder, as to bring it to the start of the FIFO cache
       fse.utimesSync(workingReqsFolder, new Date(), new Date());
-      fse.removeSync(slsReqsTxt);
       return workingReqsFolder;
     }
     // Remove our old folder if it didn't complete properly, but _just incase_ only remove it if named properly...
@@ -276,7 +291,6 @@ function installRequirementsIfNeeded(servicePath, modulePath, options, serverles
   if (options.useStaticCache) {
     fse.closeSync(fse.openSync(path.join(workingReqsFolder, '.completed_requirements'), 'w'));
   }
-  fse.removeSync(slsReqsTxt); // Clean up after ourselves
   return workingReqsFolder;
 }
 

lib/shared.js

@@ -19,7 +19,7 @@ function checkForAndDeleteMaxCacheVersions(options, serverless) {
       options.staticCacheMaxVersions &&
       parseInt(options.staticCacheMaxVersions) > 0) {
     // Get the list of our cache files
-    files = glob.sync([path.join(getStaticCachePath(options), '*_slspyc/')], {mark: true});
+    files = glob.sync([path.join(getUserCachePath(options), '*_slspyc/')], {mark: true});
     // Check if we have too many
     if (files.length >= options.staticCacheMaxVersions) {
       // Sort by modified time
@@ -53,7 +53,7 @@ function getRequirementsWorkingPath(subfolder, servicePath, options) {
     }
     // If we have max number of cache items...
 
-    return path.join( getStaticCachePath(options), subfolder);
+    return path.join( getUserCachePath(options), subfolder);
   }
 
   // If we don't want to use the static cache, then fallback to the way things used to work
@@ -66,10 +66,10 @@ function getRequirementsWorkingPath(subfolder, servicePath, options) {
   * @param  {Object} options
   * @return {string}
   */
-function getStaticCachePath(options) {
+function getUserCachePath(options) {
   // If we've manually set the static cache location
-  if (options && options.staticCacheLocation) {
-    return path.join(options.staticCacheLocation);
+  if (options && options.cacheLocation) {
+    return path.resolve(options.cacheLocation);
   }
 
   // Otherwise, find/use the python-ey appdirs cache location
@@ -93,6 +93,6 @@ function md5Path(fullpath) {
 module.exports = {
   checkForAndDeleteMaxCacheVersions,
   getRequirementsWorkingPath,
-  getStaticCachePath,
+  getUserCachePath,
   md5Path,
 };

test.bats

@@ -7,11 +7,15 @@ setup() {
         export LC_ALL=C.UTF-8
         export LANG=C.UTF-8
     fi
+    export USR_CACHE_DIR=`node -e 'console.log(require("./lib/shared").getUserCachePath())'`
 }
 
 teardown() {
     rm -rf puck puck2 puck3 node_modules .serverless .requirements.zip .requirements-cache
     if [ -f serverless.yml.bak ]; then mv serverless.yml.bak serverless.yml; fi
+    if [ -d "${USR_CACHE_DIR}" ] ; then
+        rm -Rf "${USR_CACHE_DIR}"
+    fi
 }
 
 @test "py3.6 can package flask with default options" {
@@ -67,13 +71,42 @@ teardown() {
     ls puck/flask
 }
 
-@test "py3.6 uses cache with dockerizePip option" {
+@test "py3.6 uses download cache with useDownloadCache option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    useDownloadCache: true/' serverless.yml
+    sls package
+    USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'`
+    ls $USR_CACHE_DIR/downloadCacheslspyc/http
+}
+
+@test "py3.6 uses download cache with useDownloadCache + cacheLocation option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    useDownloadCache: true\n    cacheLocation: .requirements-cache/' serverless.yml
+    sls package
+    ls .requirements-cache/downloadCacheslspyc/http
+}
+
+@test "py3.6 uses download cache with dockerizePip + useDownloadCache option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    useDownloadCache: true/' serverless.yml
+    sls --dockerizePip=true package
+    USR_CACHE_DIR=`node -e 'console.log(require("../../lib/shared").getUserCachePath())'`
+    ls $USR_CACHE_DIR/downloadCacheslspyc/http
+}
+
+@test "py3.6 uses download cache with dockerizePip + useDownloadCache + cacheLocation option" {
     cd tests/base
     npm i $(npm pack ../..)
     ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
-    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml
+    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    useDownloadCache: true\n    cacheLocation: .requirements-cache/' serverless.yml
     sls --dockerizePip=true package
-    ls .requirements-cache/http
+    ls .requirements-cache/downloadCacheslspyc/http
 }
 
 @test "py2.7 can package flask with default options" {