feat: use build_dir in esbuild workflow to support building in source (#437)

Author: torresxb1

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -81,11 +81,10 @@ class NodejsNpmInstallAction(BaseAction):
     DESCRIPTION = "Installing dependencies from NPM"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, artifacts_dir, subprocess_npm):
+    def __init__(self, install_dir, subprocess_npm):
         """
-        :type artifacts_dir: str
-        :param artifacts_dir: an existing (writable) directory with project source files.
-            Dependencies will be installed in this directory.
+        :type install_dir: str
+        :param install_dir: Dependencies will be installed in this directory.
 
         :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
         :param subprocess_npm: An instance of the NPM process wrapper
@@ -95,7 +94,7 @@ def __init__(self, artifacts_dir, subprocess_npm):
         """
 
         super(NodejsNpmInstallAction, self).__init__()
-        self.artifacts_dir = artifacts_dir
+        self.install_dir = install_dir
         self.subprocess_npm = subprocess_npm
 
     def execute(self):
@@ -105,10 +104,10 @@ def execute(self):
         :raises lambda_builders.actions.ActionFailedError: when NPM execution fails
         """
         try:
-            LOG.debug("NODEJS installing in: %s", self.artifacts_dir)
+            LOG.debug("NODEJS installing in: %s", self.install_dir)
 
             self.subprocess_npm.run(
-                ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.artifacts_dir
+                ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.install_dir
             )
 
         except NpmExecutionError as ex:
@@ -128,18 +127,17 @@ class NodejsNpmCIAction(BaseAction):
     DESCRIPTION = "Installing dependencies from NPM using the CI method"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, artifacts_dir, subprocess_npm):
+    def __init__(self, install_dir, subprocess_npm):
         """
-        :type artifacts_dir: str
-        :param artifacts_dir: an existing (writable) directory with project source files.
-            Dependencies will be installed in this directory.
+        :type install_dir: str
+        :param install_dir: Dependencies will be installed in this directory.
 
         :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
         :param subprocess_npm: An instance of the NPM process wrapper
         """
 
         super(NodejsNpmCIAction, self).__init__()
-        self.artifacts_dir = artifacts_dir
+        self.install_dir = install_dir
         self.subprocess_npm = subprocess_npm
 
     def execute(self):
@@ -150,9 +148,9 @@ def execute(self):
         """
 
         try:
-            LOG.debug("NODEJS installing ci in: %s", self.artifacts_dir)
+            LOG.debug("NODEJS installing ci in: %s", self.install_dir)
 
-            self.subprocess_npm.run(["ci"], cwd=self.artifacts_dir)
+            self.subprocess_npm.run(["ci"], cwd=self.install_dir)
 
         except NpmExecutionError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -148,15 +148,15 @@ def get_resolvers(self):
         return [PathResolver(runtime=self.runtime, binary="npm")]
 
     @staticmethod
-    def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options):
+    def get_install_action(source_dir, install_dir, subprocess_npm, osutils, build_options):
         """
         Get the install action used to install dependencies at artifacts_dir
 
         :type source_dir: str
         :param source_dir: an existing (readable) directory containing source files
 
-        :type artifacts_dir: str
-        :param artifacts_dir: Dependencies will be installed in this directory.
+        :type install_dir: str
+        :param install_dir: Dependencies will be installed in this directory.
 
         :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
         :param osutils: An instance of OS Utilities for file manipulation
@@ -181,6 +181,6 @@ def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build
             npm_ci_option = build_options.get("use_npm_ci", False)
 
         if (osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path)) and npm_ci_option:
-            return NodejsNpmCIAction(artifacts_dir, subprocess_npm=subprocess_npm)
+            return NodejsNpmCIAction(install_dir=install_dir, subprocess_npm=subprocess_npm)
 
-        return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm)
+        return NodejsNpmInstallAction(install_dir=install_dir, subprocess_npm=subprocess_npm)

aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py

@@ -42,7 +42,7 @@ class NodejsNpmEsbuildWorkflow(BaseWorkflow):
     CONFIG_PROPERTY = "aws_sam"
 
     DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH
-    BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED
+    BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED
 
     def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs):
 
@@ -77,23 +77,26 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
                 "dependencies directory was not provided and downloading dependencies is disabled."
             )
 
-        self.actions = [
-            CopySourceAction(source_dir=self.source_dir, dest_dir=self.scratch_dir, excludes=self.EXCLUDED_FILES)
-        ]
+        # if we're building in the source directory, we don't have to copy the source code
+        self.actions = (
+            []
+            if self.build_dir == self.source_dir
+            else [CopySourceAction(source_dir=self.source_dir, dest_dir=self.build_dir, excludes=self.EXCLUDED_FILES)]
+        )
 
         if self.download_dependencies:
             self.actions.append(
                 NodejsNpmWorkflow.get_install_action(
                     source_dir=source_dir,
-                    artifacts_dir=self.scratch_dir,
+                    install_dir=self.build_dir,
                     subprocess_npm=self.subprocess_npm,
                     osutils=self.osutils,
                     build_options=self.options,
                 )
             )
 
         bundle_action = EsbuildBundleAction(
-            working_directory=self.scratch_dir,
+            working_directory=self.build_dir,
             output_directory=self.artifacts_dir,
             bundler_config=bundler_config,
             osutils=self.osutils,
@@ -103,7 +106,9 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
         )
 
         # If there's no dependencies_dir, just bundle and we're done.
-        if not self.dependencies_dir:
+        # Same thing if we're building in the source directory (since the dependencies persist in
+        # the source directory, we don't want to move them or symlink them back to the source)
+        if not self.dependencies_dir or self.build_dir == self.source_dir:
             self.actions.append(bundle_action)
             return
 

tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py

@@ -20,6 +20,7 @@ class TestNodejsNpmWorkflowWithEsbuild(TestCase):
     TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
 
     def setUp(self):
+        self.source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild")
         self.artifacts_dir = tempfile.mkdtemp()
         self.scratch_dir = tempfile.mkdtemp()
         self.dependencies_dir = tempfile.mkdtemp()
@@ -39,6 +40,11 @@ def tearDown(self):
         shutil.rmtree(self.artifacts_dir)
         shutil.rmtree(self.scratch_dir)
 
+        # clean up dependencies that were installed in source dir
+        source_dependencies = os.path.join(self.source_dir, "node_modules")
+        if os.path.exists(source_dependencies):
+            shutil.rmtree(source_dependencies)
+
     @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)])
     def test_builds_javascript_project_with_dependencies(self, runtime):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild")
@@ -407,3 +413,29 @@ def test_esbuild_produces_mjs_output_files(self, runtime):
         expected_files = {"included.mjs", "included.mjs.map"}
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
+
+    @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)])
+    def test_esbuild_can_build_in_source(self, runtime):
+        options = {"entry_points": ["included.js"]}
+
+        self.builder.build(
+            self.source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(self.source_dir, "package.json"),
+            runtime=runtime,
+            options=options,
+            executable_search_paths=[self.binpath],
+            build_in_source=True,
+        )
+
+        # dependencies installed in source folder
+        self.assertIn("node_modules", os.listdir(self.source_dir))
+
+        # dependencies not in scratch
+        self.assertNotIn("node_modules", os.listdir(self.scratch_dir))
+
+        # bundle is in artifacts
+        expected_files = {"included.js"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEqual(expected_files, output_files)

tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py

@@ -313,7 +313,7 @@ def test_workflow_uses_production_npm_version(self, get_workflow_mock):
         self.assertIsInstance(workflow.actions[2], EsbuildBundleAction)
 
         get_workflow_mock.get_install_action.assert_called_with(
-            source_dir="source", artifacts_dir="scratch_dir", subprocess_npm=ANY, osutils=ANY, build_options=None
+            source_dir="source", install_dir="scratch_dir", subprocess_npm=ANY, osutils=ANY, build_options=None
         )
 
     @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.workflow.SubprocessNpm")
@@ -342,3 +342,21 @@ def test_no_download_dependencies_and_no_dependencies_dir_fails(self):
                 osutils=self.osutils,
                 download_dependencies=False,
             )
+
+    def test_build_in_source(self):
+        source_dir = "source"
+        workflow = NodejsNpmEsbuildWorkflow(
+            source_dir=source_dir,
+            artifacts_dir="artifacts",
+            scratch_dir="scratch_dir",
+            manifest_path="manifest",
+            osutils=self.osutils,
+            build_in_source=True,
+        )
+
+        self.assertEqual(len(workflow.actions), 2)
+
+        self.assertIsInstance(workflow.actions[0], NodejsNpmInstallAction)
+        self.assertEqual(workflow.actions[0].install_dir, source_dir)
+        self.assertIsInstance(workflow.actions[1], EsbuildBundleAction)
+        self.assertEqual(workflow.actions[1]._working_directory, source_dir)