Merge branch 'master' into package_individually_like_base_sls

Author: bweigel

README.md

@@ -116,7 +116,7 @@ except ImportError:
 _Works on non 'win32' environments: Docker, WSL are included_  
 To remove the tests, information and caches from the installed packages, 
 enable the `slim` option. This will: `strip` the `.so` files, remove `__pycache__` 
-directories and `dist-info` directories.  
+and `dist-info` directories as well as `.pyc` and `.pyo` files.  
 ```yaml
 custom:
   pythonRequirements:
@@ -125,7 +125,8 @@ custom:
 #### Custom Removal Patterns  
 To specify additional directories to remove from the installed packages, 
 define a list of patterns in the serverless config using the `slimPatterns`
-option and glob syntax. Note, it matches against whole paths, so to match a file in any
+option and glob syntax. These paterns will be added to the default ones (`**/*.py[c|o]`, `**/__pycache__*`, `**/*.dist-info*`).
+Note, the glob syntax matches against whole paths, so to match a file in any
 directory, start your pattern with `**/`.
 ```yaml
 custom:
@@ -134,6 +135,15 @@ custom:
     slimPatterns:
       - "**/*.egg-info*"
 ```  
+To overwrite the default patterns set the option `slimPatternsAppendDefaults` to `false` (`true` by default).
+```yaml
+custom:
+  pythonRequirements:
+    slim: true
+    slimPatternsAppendDefaults: false
+    slimPatterns:
+      - "**/*.egg-info*"
+```  
 This will remove all folders within the installed requirements that match 
 the names in `slimPatterns`  
 ## Omitting Packages 

appveyor.yml

@@ -1,9 +1,10 @@
 version: '{build}'
 init:
-- ps: npm i -g serverless
+    - cmd: pip install pipenv
+    - ps: npm i -g serverless
 build: off
 test_script:
-- cmd: >-
-    npm i
+    - cmd: >-
+        npm i
 
-    node test.js
+        node test.js

index.js

@@ -29,6 +29,7 @@ class ServerlessPythonRequirements {
       {
         slim: false,
         slimPatterns: false,
+        slimPatternsAppendDefaults: true,
         zip: false,
         cleanupZipHelper: true,
         invalidateCaches: false,

lib/slim.js

@@ -10,7 +10,14 @@ const getStripCommand = (options, folderPath) =>
 const deleteFiles = (options, folderPath) => {
   let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*'];
   if (options.slimPatterns) {
-    patterns = patterns.concat(options.slimPatterns);
+    if (
+      options.slimPatternsAppendDefaults === false ||
+      options.slimPatternsAppendDefaults == 'false'
+    ) {
+      patterns = options.slimPatterns;
+    } else {
+      patterns = patterns.concat(options.slimPatterns);
+    }
   }
   for (const pattern of patterns) {
     for (const file of glob.sync(`${folderPath}/${pattern}`)) {

test.bats

@@ -303,7 +303,7 @@ teardown() {
     cd tests/base
     cat _slimPatterns.yml > slimPatterns.yml
     npm i $(npm pack ../..)
-    sls --runtime=python2.7 --slim=true packag
+    sls --runtime=python2.7 --slim=true package
     unzip .serverless/sls-py-req-test.zip -d puck
     ls puck/flask
     test $(find puck -name "*.pyc" | wc -l) -eq 0
@@ -468,7 +468,6 @@ teardown() {
     test $(find "puck*" -name "*.pyc" | wc -l) -eq 0
 }
 
-
 @test "py2.7 can package flask with package individually option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -494,7 +493,6 @@ teardown() {
     test $(find puck* -name "*.pyc" | wc -l) -eq 0
 }
 
-
 @test "py3.6 can package only requirements of module" {
     cd tests/individually
     npm i $(npm pack ../..)

test.js

@@ -3,7 +3,8 @@ const deasync = require('deasync-promise');
 const glob = require('glob-all');
 const JSZip = require('jszip');
 const tape = require('tape');
-const { removeSync, readFileSync, pathExistsSync } = require('fs-extra');
+
+const { removeSync, readFileSync, pathExistsSync, copySync } = require('fs-extra');
 const { sep } = require('path');
 
 const { getUserCachePath } = require('./lib/shared');
@@ -63,8 +64,8 @@ const teardown = () => {
   removeSync('tests/base with a space');
 };
 
-const test = (desc, func) =>
-  tape.test(desc, t => {
+const test = (desc, func, opts = {}) =>
+  tape.test(desc, opts, t => {
     setup();
     try {
       func(t);
@@ -89,6 +90,16 @@ const listRequirementsZipFiles = filename => {
   return Object.keys(reqsZip.files);
 };
 
+const canUseDocker = () => {
+  let result;
+  try {
+    result = crossSpawn.sync('docker', ['ps']);
+  } catch (e) {
+    return false;
+  }
+  return result.status === 0;
+};
+
 test('default pythonBin can package flask with default options', t => {
   process.chdir('tests/base');
   const path = npm(['pack', '../..']);
@@ -530,6 +541,7 @@ test('py3.6 uses download cache with cacheLocation option', t => {
   t.end();
 });
 
+
 /*
  * News tests not in test.bats
  */
@@ -569,3 +581,131 @@ test("py3.6 doesn't package bottle with zip option", t => {
   );
   t.end();
 });
+
+test('py3.6 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', t => {
+  process.chdir('tests/base');
+  copySync('_slimPatterns.yml', 'slimPatterns.yml');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']);
+
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.true(
+    zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+    'pyc files are packaged'
+  );
+  t.deepEqual(
+    zipfiles.filter(filename => filename.includes('.egg-info')),
+    [],
+    '.egg-info folders are not packaged'
+  );
+  t.end();
+});
+
+test(
+  'py3.6 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options',
+  t => {
+    process.chdir('tests/base');
+    copySync('_slimPatterns.yml', 'slimPatterns.yml');
+    const path = npm(['pack', '../..']);
+    npm(['i', path]);
+    sls([
+      '--dockerizePip=true',
+      '--slim=true',
+      '--slimPatternsAppendDefaults=false',
+      'package'
+    ]);
+
+    const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+    t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+    t.true(
+      zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+      'pyc files are packaged'
+    );
+    t.deepEqual(
+      zipfiles.filter(filename => filename.includes('.egg-infooo')),
+      [],
+      '.egg-info folders are not packaged'
+    );
+    t.end();
+  },
+  { skip: !canUseDocker() }
+);
+
+test('py2.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false options', t => {
+  process.chdir('tests/base');
+  copySync('_slimPatterns.yml', 'slimPatterns.yml');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls([
+    '--runtime=python2.7',
+    '--slim=true',
+    '--slimPatternsAppendDefaults=false',
+    'package'
+  ]);
+
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.true(
+    zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+    'pyc files are packaged'
+  );
+  t.deepEqual(
+    zipfiles.filter(filename => filename.includes('.egg-info')),
+    [],
+    '.egg-info folders are not packaged'
+  );
+  t.end();
+});
+
+test(
+  'py2.7 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options',
+  t => {
+    process.chdir('tests/base');
+    copySync('_slimPatterns.yml', 'slimPatterns.yml');
+    const path = npm(['pack', '../..']);
+    npm(['i', path]);
+    sls([
+      '--dockerizePip=true',
+      '--runtime=python2.7',
+      '--slim=true',
+      '--slimPatternsAppendDefaults=false',
+      'package'
+    ]);
+    const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+    t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+    t.true(
+      zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+      'pyc files are packaged'
+    );
+    t.deepEqual(
+      zipfiles.filter(filename => filename.includes('.egg-info')),
+      [],
+      '.egg-info folders are not packaged'
+    );
+    t.end();
+  },
+  { skip: !canUseDocker() }
+);
+
+test('pipenv py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false  option', t => {
+  process.chdir('tests/pipenv');
+  copySync('_slimPatterns.yml', 'slimPatterns.yml');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+
+  sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.true(
+    zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+    'pyc files are packaged'
+  );
+  t.deepEqual(
+    zipfiles.filter(filename => filename.includes('.egg-info')),
+    [],
+    '.egg-info folders are not packaged'
+  );
+  t.end();
+});

tests/base/serverless.yml

@@ -12,6 +12,7 @@ custom:
     dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip}
     slim: ${opt:slim, self:custom.defaults.slim}
     slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns}
+    slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults}
     vendor: ${opt:vendor, ''}
     fileName: ${opt:fileName, 'requirements.txt'}
     IndividuallyMoveUpModules: ${opt:moveup, self:custom.defaults.IndividuallyMoveUpModules}
@@ -21,6 +22,7 @@ custom:
   defaults:
     slim: false
     slimPatterns: false
+    slimPatternsAppendDefaults: true
     zip: false
     dockerizePip: false
     individually: false

tests/pipenv/serverless.yml

@@ -11,10 +11,12 @@ custom:
     zip: ${opt:zip, self:custom.defaults.zip}
     slim: ${opt:slim, self:custom.defaults.slim}
     slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns}
+    slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults}
     dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip}
   defaults:
     zip: false
     slimPatterns: false
+    slimPatternsAppendDefaults: true
     slim: false
     dockerizePip: false