chore: add memory profiling in streams examples (#946)

Author: josecorella

stream_mem_stress_test/.gitignore

@@ -0,0 +1,6 @@
+# Node modules
+node_modules/
+
+# Large files
+mem_leak_data1/*
+mem_leak_data_default/*

stream_mem_stress_test/README.md

@@ -0,0 +1,25 @@
+# Streams - Memory Stress Test and Memory Profiling
+This directory contains test code for running a memory profiler when using the `encryptStream` and `decryptUnsignedMessageStream`.
+This directory contains everything you need to run a memory profiler on these two operations.
+
+## Requirements
+- Node >= 12
+- Chrome Browser
+
+## How to run the application and memory profiler
+1. For easier debugging open two chrome windows
+  1. One where you can look at the profiler
+  2. One where you can navigate through the application paths.
+1. On Chrome, navigate to: `chrome://inspect/#devices`
+1. Make sure you are in the `stream_mem_stress_test` directory.
+1. Start debugger and server by running `npm run start`
+1. On the devices page click on `inspect` for the remote target that just appeared
+  1. Navigate to the Memory tab. You will have three options:
+    1. Heap snapshot
+        - Useful to focus on a specific action during runtime.
+    1. Allocation instrumentation on timeline
+        - Works better for our stress tests since we can see memory allocation and
+        garbage collection during runtime
+    1. Allocation Sampling (not useful for our test)
+1. Navigate to any of the provided paths and watch memory allocation and garbage collection in
+  real time 🍿

stream_mem_stress_test/index.ts

@@ -0,0 +1,88 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import express from 'express'
+import { kmsDecryptStream, kmsEncryptStream } from './memory'
+
+const app  = express()
+const leak = []
+
+app.get('/', (req, res) => {
+    // Quick little Test so we can check both in browser and in terminal
+    res.write("Test")
+    res.send()
+    console.log("Back in home")
+})
+
+/** This path is here to see how a memory leak would look like in the memory profiler.
+ * To test it you can run `npm run load-mem` to see memory get allocated but never
+ * garbage collected
+ */
+app.get('/now', (req, res) => {
+    let resp = JSON.stringify({ now: new Date() })
+    leak.push(JSON.parse(resp))
+    res.writeHead(200, { 'Content-Type': 'application/json' })
+    res.write(resp)
+    res.end()
+})
+
+/** Path to read 1mb file, parameter to readFile takes any file in the current directory
+ * so you can test with any file you'd like. We supply 2 test files for you to use.
+*/
+app.get('/readRandom_1mb', (req, res) => {
+    res.write("Attempt to encrypt 1mb random")
+    // you can optionaly pass in a frame size, otherwise uses the default frame size of 4096 bytes
+    readFile('./random_1mb.txt')
+    res.send()
+})
+
+// Path to decrypt 1mb encrypted file
+app.get('/readRandom_1mbEnc', (req, res) => {
+    res.write("Attempt to decrypt ./random_1mb.txt.encrypted")
+    readEncryptedFile('./random_1mb.txt.encrypted')
+    res.send()
+
+})
+
+// Path to encrypt 5mb file
+app.get('/readRandom_5mb', (req, res) => {
+    res.write("Attempt to encrypt 5mb random")
+    readFile('./random_5mb.txt', 1)
+    res.send()
+})
+
+// Path to decrypt 5mb encrypted file
+app.get('/readRandom_5mbEnc', (req, res) => {
+    res.write("Attempt to decrypt ./random_5mb.txt.encrypted")
+    readEncryptedFile('./random_5mb.txt.encrypted')
+    res.send()
+
+})
+
+/** Path to encrypt 1Gb file, not included in this directory because size exceeds GitHub transfer size limit.
+ * If you want to create a 1Gb file of random data, you can do so by running
+ * `dd if=/dev/urandom of=rand_1gb.txt bs=1024 count=1024000` on a linux system
+*/
+app.get('/readRandom_1gb', (req, res) => {
+    res.write("Attempt to encrypt large random")
+    readFile('./rand_1gb.txt')
+    res.send()
+})
+
+// Path to decrypt 1mb encrypted file
+app.get('/readRandom_1gbEnc', (req, res) => {
+    res.write("Attempt to encrypt large random")
+    kmsDecryptStream('./rand_1gb.txt.encrypted')
+    res.send()
+})
+
+app.listen(3000, () => {
+    console.log("Listening on port 3000");
+})
+
+async function readFile(filename: string, framesize?: number) {
+    await kmsEncryptStream(filename, framesize);
+}
+
+async function readEncryptedFile(filename: string) {
+    await kmsDecryptStream(filename)
+}

stream_mem_stress_test/memory.ts

@@ -0,0 +1,101 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import {
+    KmsKeyringNode,
+    buildClient,
+    CommitmentPolicy
+} from '@aws-crypto/client-node'
+import {AlgorithmSuiteIdentifier} from '@aws-crypto/material-management'
+
+/* This builds client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy.
+ * REQUIRE_ENCRYPT_REQUIRE_DECRYPT enforces that this client only encrypts using committing algorithm suites
+ * and enforces that this client will only decrypt encrypted messages
+ * that were created with a committing algorithm suite.
+ * This is the default if you build `buildClient()`.
+ */
+const { encryptStream, decryptUnsignedMessageStream } = buildClient(
+    CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT
+)
+
+import { finished } from 'stream'
+import { createReadStream, createWriteStream } from 'fs'
+import { promisify } from 'util'
+const finishedAsync = promisify(finished)
+
+/* A KMS CMK is required to generate the data key.
+ * You need kms:GenerateDataKey permission on the CMK in generatorKeyId.
+ * This key is public, DO NOT USE in production environment.
+ */
+const generatorKeyId = 
+    'arn:aws:kms:us-west-2:658956600833:alias/EncryptDecrypt'
+
+// configure keyring with CMK(s) you wish to work with
+const keyring = new KmsKeyringNode({ generatorKeyId })
+
+/**
+ * Encryption Context is very useful if you want to assert things about the encrypted data
+ * See: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#encryption-context
+ */
+const context = {
+    stage: 'demo',
+    purpose: 'streaming memory stress test',
+    origin: 'us-west-2',
+}
+
+/**
+ * kmsEncryptStream will use the encryptStream function and create a pipeline to encrypt a stream of data
+ *  from a file and write it to destination `./{filename}.encrypted`
+ * @param filename string of file name you wish to encrypt
+ * @param framesize optional parameter to determine frame size; default is 4096 bytes
+ */
+export async function kmsEncryptStream(filename: string, framesize?: number) {
+    const readable = createReadStream(filename)
+    const encFile = filename + '.encrypted'
+    const writeable = createWriteStream(encFile)
+    
+    // pipeline of read stream 
+    readable.pipe(
+        encryptStream(keyring, {
+            /**
+             * Since we are streaming, and assuming that the encryption and decryption contexts
+             * are equally trusted, using an unsigned algorithm suite is faster and avoids
+             * the possibility of processing plaintext before the signature is verified.
+             */
+            suiteId:
+                AlgorithmSuiteIdentifier.ALG_AES256_GCM_IV12_TAG16_HKDF_SHA512_COMMIT_KEY,
+            encryptionContext: context,
+            frameLength: framesize
+        })
+    ).pipe(writeable.on('finish', () => {
+        // output from encryptStream will get piped to writeable
+        console.log(`The new file name is ${encFile}.`);
+    }))   
+    
+    await finishedAsync(writeable)
+    console.log("Finished Encrypting");
+    
+}
+/**
+ * kmsDecryptStream will take a filename and create a decryption stream to decrypt contents of file
+ * and write it to destination `./{filename}.decrypted
+ * @param filename string of file name you wish to encrypt
+ */
+export async function kmsDecryptStream(filename: string) {
+    const readable = createReadStream(filename)
+    const decFile = filename + '.decrypted'
+    const writeable = createWriteStream(decFile)
+    
+    readable.pipe(
+        /** 
+         * decryptUnsignedMessageStream is recommended when streaming if you don't need
+         * digital signatures.
+         */
+        decryptUnsignedMessageStream(keyring)
+    ).pipe(writeable.on('finish', () => {
+        console.log(`The new file name is ${decFile}.`);
+    }))
+
+    await finishedAsync(writeable)
+    console.log("Finished Decrypting")
+}
+