feat: Add support to build automatically npm dependencies (#293)

Co-authored-by: Massimo Maino <[email protected]>

Author: antonbabenko

README.md

@@ -403,6 +403,11 @@ source_path = [
       "!vendor/colorful-.+.dist-info/.*",
       "!vendor/colorful/__pycache__/?.*",
     ]
+  }, {
+    path             = "src/nodejs14.x-app1",
+    npm_requirements = true,
+    npm_tmp_dir      = "/tmp/dir/location"
+    prefix_in_zip    = "foo/bar1",
   }, {
     path     = "src/python3.8-app3",
     commands = [
@@ -424,8 +429,9 @@ source_path = [
 ]
 ```
 
-Few notes:
+*Few notes:*
 
+- If you specify a source path as a string that references a folder and the runtime begins with `python` or `nodejs`, the build process will automatically build python and nodejs dependencies if `requirements.txt` or `package.json` file will be found in the source folder. If you want to customize this behavior, please use the object notation as explained below.
 - All arguments except `path` are optional.
 - `patterns` - List of Python regex filenames should satisfy. Default value is "include everything" which is equal to `patterns = [".*"]`. This can also be specified as multiline heredoc string (no comments allowed). Some examples of valid patterns:
 
@@ -442,10 +448,12 @@ Few notes:
     !abc/def/hgk/.*    # Filter out again in abc/def/hgk sub folder
 ```
 
-- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements`.
+- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements` and `npm_requirements`.
   - `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument).
 - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead.
 - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds.
+- `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead.
+- `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds.
 - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive.
 
 ### Building in Docker
@@ -455,7 +463,7 @@ If your Lambda Function or Layer uses some dependencies you can build them in Do
     build_in_docker   = true
     docker_file       = "src/python3.8-app1/docker/Dockerfile"
     docker_build_root = "src/python3.8-app1/docker"
-    docker_image      = "lambci/lambda:build-python3.8"
+    docker_image      = "public.ecr.aws/sam/build-python3.8"
     runtime           = "python3.8"    # Setting runtime is required when building package in Docker and Lambda Layer resource.
 
 Using this module you can install dependencies from private hosts. To do this, you need for forward SSH agent:

examples/build-package/README.md

@@ -38,11 +38,14 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_lambda_layer_pip_requirements"></a> [lambda\_layer\_pip\_requirements](#module\_lambda\_layer\_pip\_requirements) | ../.. | n/a |
 | <a name="module_package_dir"></a> [package\_dir](#module\_package\_dir) | ../../ | n/a |
 | <a name="module_package_dir_pip_dir"></a> [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a |
+| <a name="module_package_dir_with_npm_install"></a> [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a |
+| <a name="module_package_dir_without_npm_install"></a> [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a |
 | <a name="module_package_dir_without_pip_install"></a> [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a |
 | <a name="module_package_file"></a> [package\_file](#module\_package\_file) | ../../ | n/a |
 | <a name="module_package_file_with_pip_requirements"></a> [package\_file\_with\_pip\_requirements](#module\_package\_file\_with\_pip\_requirements) | ../../ | n/a |
 | <a name="module_package_with_commands_and_patterns"></a> [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a |
 | <a name="module_package_with_docker"></a> [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a |
+| <a name="module_package_with_npm_requirements_in_docker"></a> [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a |
 | <a name="module_package_with_patterns"></a> [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a |
 | <a name="module_package_with_pip_requirements_in_docker"></a> [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a |
 

examples/build-package/main.tf

@@ -220,7 +220,45 @@ module "package_with_docker" {
   docker_with_ssh_agent = true
   #  docker_file           = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile"
   docker_build_root = "${path.module}/../../docker"
-  docker_image      = "lambci/lambda:build-python3.8"
+  docker_image      = "public.ecr.aws/sam/build-python3.8"
+}
+
+# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime)
+module "package_dir_with_npm_install" {
+  source = "../../"
+
+  create_function = false
+
+  runtime     = "nodejs14.x"
+  source_path = "${path.module}/../fixtures/nodejs14.x-app1"
+}
+
+# Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime)
+module "package_dir_without_npm_install" {
+  source = "../../"
+
+  create_function = false
+
+  runtime = "nodejs14.x"
+  source_path = [
+    {
+      path             = "${path.module}/../fixtures/nodejs14.x-app1"
+      npm_requirements = false
+      # npm_requirements = true  # Will run "npm install" with package.json
+    }
+  ]
+}
+
+# Create zip-archive of a single directory where "npm install" will also be executed using docker
+module "package_with_npm_requirements_in_docker" {
+  source = "../../"
+
+  create_function = false
+
+  runtime         = "nodejs14.x"
+  source_path     = "${path.module}/../fixtures/nodejs14.x-app1"
+  build_in_docker = true
+  hash_extra      = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
 }
 
 ################################
@@ -240,6 +278,7 @@ module "lambda_layer" {
 
   build_in_docker = true
   runtime         = "python3.8"
+  docker_image    = "public.ecr.aws/sam/build-python3.8"
   docker_file     = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile"
 }
 

examples/fixtures/nodejs14.x-app1/index.js

@@ -0,0 +1,16 @@
+'use strict';
+
+module.exports.hello = async (event) => {
+  console.log(event);
+  return {
+    statusCode: 200,
+    body: JSON.stringify(
+      {
+        message: `Go Serverless.tf! Your Nodejs function executed successfully!`,
+        input: event,
+      },
+      null,
+      2
+    ),
+  };
+};

examples/fixtures/nodejs14.x-app1/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "nodejs14.x-app1",
+  "version": "1.0.0",
+  "main": "index.js",
+  "dependencies": {
+    "requests": "^0.3.0"
+  }
+}

package.py

@@ -660,6 +660,18 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
                 step('pip', runtime, requirements, prefix, tmp_dir)
                 hash(requirements)
 
+        def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
+            requirements = path
+            if os.path.isdir(path):
+                requirements = os.path.join(path, 'package.json')
+            if not os.path.isfile(requirements):
+                if required:
+                    raise RuntimeError(
+                        'File not found: {}'.format(requirements))
+            else:
+                step('npm', runtime, requirements, prefix, tmp_dir)
+                hash(requirements)
+
         def commands_step(path, commands):
             if not commands:
                 return
@@ -717,6 +729,9 @@ def commands_step(path, commands):
                 if runtime.startswith('python'):
                     pip_requirements_step(
                         os.path.join(path, 'requirements.txt'))
+                elif runtime.startswith('nodejs'):
+                    npm_requirements_step(
+                        os.path.join(path, 'package.json'))
                 step('zip', path, None)
                 hash(path)
 
@@ -731,6 +746,7 @@ def commands_step(path, commands):
                 else:
                     prefix = claim.get('prefix_in_zip')
                     pip_requirements = claim.get('pip_requirements')
+                    npm_requirements = claim.get('npm_package_json')
                     runtime = claim.get('runtime', query.runtime)
 
                     if pip_requirements and runtime.startswith('python'):
@@ -740,6 +756,13 @@ def commands_step(path, commands):
                             pip_requirements_step(pip_requirements, prefix,
                                                   required=True, tmp_dir=claim.get('pip_tmp_dir'))
 
+                    if npm_requirements and runtime.startswith('nodejs'):
+                        if isinstance(npm_requirements, bool) and path:
+                            npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir'))
+                        else:
+                            npm_requirements_step(npm_requirements, prefix,
+                                                  required=True, tmp_dir=claim.get('npm_tmp_dir'))
+
                     if path:
                         step('zip', path, prefix)
                         if patterns:
@@ -793,6 +816,16 @@ def execute(self, build_plan, zip_stream, query):
                         else:
                             # XXX: timestamp=0 - what actually do with it?
                             zs.write_dirs(rd, prefix=prefix, timestamp=0)
+            elif cmd == 'npm':
+                runtime, npm_requirements, prefix, tmp_dir = action[1:]
+                with install_npm_requirements(query, npm_requirements, tmp_dir) as rd:
+                    if rd:
+                        if pf:
+                            self._zip_write_with_filter(zs, pf, rd, prefix,
+                                                        timestamp=0)
+                        else:
+                            # XXX: timestamp=0 - what actually do with it?
+                            zs.write_dirs(rd, prefix=prefix, timestamp=0)
             elif cmd == 'sh':
                 r, w = os.pipe()
                 side_ch = os.fdopen(r)
@@ -934,6 +967,89 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
             yield temp_dir
 
 
+@contextmanager
+def install_npm_requirements(query, requirements_file, tmp_dir):
+    # TODO:
+    #  1. Emit files instead of temp_dir
+
+    if not os.path.exists(requirements_file):
+        yield
+        return
+
+    runtime = query.runtime
+    artifacts_dir = query.artifacts_dir
+    temp_dir = query.temp_dir
+    docker = query.docker
+    docker_image_tag_id = None
+
+    if docker:
+        docker_file = docker.docker_file
+        docker_image = docker.docker_image
+        docker_build_root = docker.docker_build_root
+
+        if docker_image:
+            ok = False
+            while True:
+                output = check_output(docker_image_id_command(docker_image))
+                if output:
+                    docker_image_tag_id = output.decode().strip()
+                    log.debug("DOCKER TAG ID: %s -> %s",
+                              docker_image, docker_image_tag_id)
+                    ok = True
+                if ok:
+                    break
+                docker_cmd = docker_build_command(
+                    build_root=docker_build_root,
+                    docker_file=docker_file,
+                    tag=docker_image,
+                )
+                check_call(docker_cmd)
+                ok = True
+        elif docker_file or docker_build_root:
+            raise ValueError('docker_image must be specified '
+                             'for a custom image future references')
+
+    log.info('Installing npm requirements: %s', requirements_file)
+    with tempdir(tmp_dir) as temp_dir:
+        requirements_filename = os.path.basename(requirements_file)
+        target_file = os.path.join(temp_dir, requirements_filename)
+        shutil.copyfile(requirements_file, target_file)
+
+        subproc_env = None
+        if not docker and OSX:
+            subproc_env = os.environ.copy()
+
+        # Install dependencies into the temporary directory.
+        with cd(temp_dir):
+            npm_command = ['npm', 'install']
+            if docker:
+                with_ssh_agent = docker.with_ssh_agent
+                chown_mask = '{}:{}'.format(os.getuid(), os.getgid())
+                shell_command = [shlex_join(npm_command), '&&',
+                                    shlex_join(['chown', '-R',
+                                                chown_mask, '.'])]
+                shell_command = [' '.join(shell_command)]
+                check_call(docker_run_command(
+                    '.', shell_command, runtime,
+                    image=docker_image_tag_id,
+                    shell=True, ssh_agent=with_ssh_agent
+                ))
+            else:
+                cmd_log.info(shlex_join(npm_command))
+                log_handler and log_handler.flush()
+                try:
+                    check_call(npm_command, env=subproc_env)
+                except FileNotFoundError as e:
+                    raise RuntimeError(
+                        "Nodejs interpreter version equal "
+                        "to defined lambda runtime ({}) should be "
+                        "available in system PATH".format(runtime)
+                    ) from e
+
+            os.remove(target_file)
+            yield temp_dir
+
+
 def docker_image_id_command(tag):
     """"""
     docker_cmd = ['docker', 'images', '--format={{.ID}}', tag]
@@ -1011,7 +1127,7 @@ def docker_run_command(build_root, command, runtime,
             ])
 
     if not image:
-        image = 'lambci/lambda:build-{}'.format(runtime)
+        image = 'public.ecr.aws/sam/build-{}'.format(runtime)
 
     docker_cmd.append(image)
 
@@ -1128,7 +1244,7 @@ def prepare_command(args):
 def build_command(args):
     """
     Builds a zip file from the source_dir or source_file.
-    Installs dependencies with pip automatically.
+    Installs dependencies with pip or npm automatically.
     """
 
     log = logging.getLogger('build')