feat(proxy): Save request payload in proxy server

.gitignore

@@ -122,9 +122,15 @@ dist
 # Stores VSCode versions used for testing VSCode extensions
 .vscode-test
 
+# JetBrains
+.idea
+
 # yarn v2
 .yarn/cache
 .yarn/unplugged
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# Mac
+.DS_Store

README.md

@@ -3,5 +3,5 @@
 _This repository is a work in progress_
 
 The main purpose of this repository is to visualize the differences between the
-[Sentry JS SDK](https://github.com/getsentry/sentry-javascript) version 7 and version 8. Those example applications can
-also be used as a reference for using the JS SDKs.
+[Sentry JS SDK](https://github.com/getsentry/sentry-javascript) version 7 and version 8. Those
+example applications can also be used as a reference for using the JS SDKs.

package.json

@@ -8,12 +8,14 @@
   },
   "packageManager": "[email protected]",
   "scripts": {
+    "start:proxy-server": "yarn workspace event-proxy-server run start",
     "start:express": "yarn workspace express-test-application run start",
     "fix:prettier": "prettier . --write",
     "fix:lint": "yarn run eslint --fix",
     "lint": "yarn run eslint"
   },
   "workspaces": [
+    "utils/event-proxy-server",
     "apps/express"
   ],
   "devDependencies": {
@@ -32,7 +34,7 @@
   },
   "prettier": {
     "arrowParens": "avoid",
-    "printWidth": 120,
+    "printWidth": 100,
     "proseWrap": "always",
     "singleQuote": true,
     "trailingComma": "all"

utils/event-proxy-server/package.json

@@ -0,0 +1,25 @@
+{
+  "private": true,
+  "version": "8.0.0-alpha.7",
+  "name": "event-proxy-server",
+  "author": "Sentry",
+  "license": "MIT",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=14.18"
+  },
+  "scripts": {
+    "start": "ts-node start-event-proxy.ts",
+    "fix": "eslint . --format stylish --fix",
+    "lint": "eslint . --format stylish",
+    "build:dev": "yarn build",
+    "clean": "rimraf -g ./node_modules ./build"
+  },
+  "dependencies": {
+    "@sentry/types": "7.109.0",
+    "@sentry/utils": "7.109.0"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

utils/event-proxy-server/payload-files/readme.md

@@ -0,0 +1,2 @@
+When running the event-proxy-server, the request are saved in a json file. This folder is where all
+the generated files go.

utils/event-proxy-server/src/event-proxy-server.ts

@@ -0,0 +1,256 @@
+import * as fs from 'fs';
+import * as http from 'http';
+import * as https from 'https';
+import type { AddressInfo } from 'net';
+import * as os from 'os';
+import * as path from 'path';
+import * as util from 'util';
+import * as zlib from 'zlib';
+import type { Envelope, EnvelopeItem } from '@sentry/types';
+import { parseEnvelope } from '@sentry/utils';
+
+const readFile = util.promisify(fs.readFile);
+const writeFile = util.promisify(fs.writeFile);
+const unlink = util.promisify(fs.unlink);
+
+interface EventProxyServerOptions {
+  /** Port to start the event proxy server at. */
+  port: number;
+  /** The name for the proxy server used for referencing it with listener functions */
+  proxyServerName: string;
+}
+
+interface SentryRequestCallbackData {
+  envelope: Envelope;
+  rawProxyRequestBody: string;
+  rawSentryResponseBody: string;
+  sentryResponseStatusCode?: number;
+}
+
+const TEMPORARY_FILE_PATH = 'payload-files/temporary.json';
+
+function isDateLikeString(str: string): boolean {
+  // matches strings in the format "YYYY-MM-DD"
+  const datePattern = /^\d{4}-\d{2}-\d{2}/;
+  return datePattern.test(str);
+}
+
+function extractPathFromUrl(url: string): string {
+  const localhost = 'http://localhost:3030/';
+  return url.replace(localhost, '');
+}
+
+function addCommaAfterEachLine(data: string): string {
+  const jsonData = data.trim().split('\n');
+
+  const jsonDataWithCommas = jsonData.map((item, index) =>
+    index < jsonData.length - 1 ? item + ',' : item,
+  );
+
+  return jsonDataWithCommas.join('\n');
+}
+
+let idCounter = 1;
+const idMap = new Map();
+
+function recursivelyReplaceData(obj: any) {
+  for (let key in obj) {
+    if (typeof obj[key] === 'string' && isDateLikeString(obj[key])) {
+      obj[key] = `[[ISODateString]]`;
+    } else if (key.includes('timestamp')) {
+      obj[key] = `[[timestamp]]`;
+    } else if (typeof obj[key] === 'number' && obj[key] > 1000) {
+      obj[key] = `[[highNumber]]`;
+    } else if (key.includes('_id')) {
+      if (idMap.has(obj[key])) {
+        // give the same ID replacement to the same value
+        obj[key] = idMap.get(obj[key]);
+      } else {
+        const newId = `[[ID${idCounter++}]]`;
+        idMap.set(obj[key], newId);
+        obj[key] = newId;
+      }
+    } else if (typeof obj[key] === 'object' && obj[key] !== null) {
+      recursivelyReplaceData(obj[key]);
+    }
+  }
+}
+
+function replaceDynamicValues(data: string): string[] {
+  const jsonData = JSON.parse(data);
+
+  recursivelyReplaceData(jsonData);
+
+  // change remaining dynamic values
+  jsonData.forEach((item: any) => {
+    if (item.trace?.public_key) {
+      item.trace.public_key = '[[publicKey]]';
+    }
+  });
+
+  return jsonData;
+}
+
+/** This function transforms all dynamic data (like timestamps) from the temporarily saved file.
+ *  The new content is saved into a new file with the url as the filename.
+ *  The temporary file is deleted in the end.
+ */
+async function transformSavedJSON() {
+  try {
+    const data = await readFile(TEMPORARY_FILE_PATH, 'utf8');
+
+    const jsonData = addCommaAfterEachLine(data);
+    const transformedJSON = replaceDynamicValues(jsonData);
+    const objWithReq = transformedJSON[2] as unknown as { request: { url: string } };
+
+    if ('request' in objWithReq) {
+      const url = objWithReq.request.url;
+      const filepath = `payload-files/${extractPathFromUrl(url)}.json`;
+
+      writeFile(filepath, JSON.stringify(transformedJSON, null, 2)).then(() => {
+        console.log(`Successfully replaced data and saved file in ${filepath}`);
+
+        unlink(TEMPORARY_FILE_PATH).then(() =>
+          console.log(`Successfully deleted ${TEMPORARY_FILE_PATH}`),
+        );
+      });
+    }
+  } catch (err) {
+    console.error('Error', err);
+  }
+}
+
+/**
+ * Starts an event proxy server that will proxy events to sentry when the `tunnel` option is used. Point the `tunnel`
+ * option to this server (like this `tunnel: http://localhost:${port option}/`).
+ *
+ */
+export async function startEventProxyServer(options: EventProxyServerOptions): Promise<void> {
+  const eventCallbackListeners: Set<(data: string) => void> = new Set();
+
+  console.log(`Proxy server "${options.proxyServerName}" running. Waiting for events...`);
+
+  const proxyServer = http.createServer((proxyRequest, proxyResponse) => {
+    const proxyRequestChunks: Uint8Array[] = [];
+
+    proxyRequest.addListener('data', (chunk: Buffer) => {
+      proxyRequestChunks.push(chunk);
+    });
+
+    proxyRequest.addListener('error', err => {
+      throw err;
+    });
+
+    proxyRequest.addListener('end', () => {
+      const proxyRequestBody =
+        proxyRequest.headers['content-encoding'] === 'gzip'
+          ? zlib.gunzipSync(Buffer.concat(proxyRequestChunks)).toString()
+          : Buffer.concat(proxyRequestChunks).toString();
+
+      // save the JSON payload into a file
+      try {
+        writeFile(TEMPORARY_FILE_PATH, `[${proxyRequestBody}]`).then(() => {
+          transformSavedJSON();
+        });
+      } catch (err) {
+        console.error(`Error writing file ${TEMPORARY_FILE_PATH}`, err);
+      }
+
+      const envelopeHeader: EnvelopeItem[0] = JSON.parse(proxyRequestBody.split('\n')[0]);
+
+      if (!envelopeHeader.dsn) {
+        throw new Error(
+          '[event-proxy-server] No dsn on envelope header. Please set tunnel option.',
+        );
+      }
+
+      const { origin, pathname, host } = new URL(envelopeHeader.dsn as string);
+
+      const projectId = pathname.substring(1);
+      const sentryIngestUrl = `${origin}/api/${projectId}/envelope/`;
+
+      proxyRequest.headers.host = host;
+
+      const sentryResponseChunks: Uint8Array[] = [];
+
+      const sentryRequest = https.request(
+        sentryIngestUrl,
+        { headers: proxyRequest.headers, method: proxyRequest.method },
+        sentryResponse => {
+          sentryResponse.addListener('data', (chunk: Buffer) => {
+            proxyResponse.write(chunk, 'binary');
+            sentryResponseChunks.push(chunk);
+          });
+
+          sentryResponse.addListener('end', () => {
+            eventCallbackListeners.forEach(listener => {
+              const rawSentryResponseBody = Buffer.concat(sentryResponseChunks).toString();
+
+              const data: SentryRequestCallbackData = {
+                envelope: parseEnvelope(proxyRequestBody, new TextEncoder(), new TextDecoder()),
+                rawProxyRequestBody: proxyRequestBody,
+                rawSentryResponseBody,
+                sentryResponseStatusCode: sentryResponse.statusCode,
+              };
+
+              listener(Buffer.from(JSON.stringify(data)).toString('base64'));
+            });
+            proxyResponse.end();
+          });
+
+          sentryResponse.addListener('error', err => {
+            throw err;
+          });
+
+          proxyResponse.writeHead(sentryResponse.statusCode || 500, sentryResponse.headers);
+        },
+      );
+
+      sentryRequest.write(Buffer.concat(proxyRequestChunks), 'binary');
+      sentryRequest.end();
+    });
+  });
+
+  const proxyServerStartupPromise = new Promise<void>(resolve => {
+    proxyServer.listen(options.port, () => {
+      resolve();
+    });
+  });
+
+  const eventCallbackServer = http.createServer((eventCallbackRequest, eventCallbackResponse) => {
+    eventCallbackResponse.statusCode = 200;
+    eventCallbackResponse.setHeader('connection', 'keep-alive');
+
+    const callbackListener = (data: string): void => {
+      eventCallbackResponse.write(data.concat('\n'), 'utf8');
+    };
+
+    eventCallbackListeners.add(callbackListener);
+
+    eventCallbackRequest.on('close', () => {
+      eventCallbackListeners.delete(callbackListener);
+    });
+
+    eventCallbackRequest.on('error', () => {
+      eventCallbackListeners.delete(callbackListener);
+    });
+  });
+
+  const eventCallbackServerStartupPromise = new Promise<void>(resolve => {
+    eventCallbackServer.listen(0, () => {
+      const port = String((eventCallbackServer.address() as AddressInfo).port);
+      void registerCallbackServerPort(options.proxyServerName, port).then(resolve);
+    });
+  });
+
+  await eventCallbackServerStartupPromise;
+  await proxyServerStartupPromise;
+  return;
+}
+
+const TEMP_FILE_PREFIX = 'event-proxy-server-';
+
+async function registerCallbackServerPort(serverName: string, port: string): Promise<void> {
+  const tmpFilePath = path.join(os.tmpdir(), `${TEMP_FILE_PREFIX}${serverName}`);
+  await writeFile(tmpFilePath, port, { encoding: 'utf8' });
+}

utils/event-proxy-server/src/index.ts

@@ -0,0 +1 @@
+export { startEventProxyServer } from './event-proxy-server';

utils/event-proxy-server/start-event-proxy.ts

@@ -0,0 +1,6 @@
+import { startEventProxyServer } from './src/event-proxy-server';
+
+startEventProxyServer({
+  port: 3031,
+  proxyServerName: 'event-proxy-server',
+});

utils/event-proxy-server/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {},
+  "include": ["src/**/*.ts"]
+}

yarn.lock

@@ -1259,6 +1259,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"event-proxy-server@workspace:utils/event-proxy-server":
+  version: 0.0.0-use.local
+  resolution: "event-proxy-server@workspace:utils/event-proxy-server"
+  dependencies:
+    "@sentry/types": "npm:7.109.0"
+    "@sentry/utils": "npm:7.109.0"
+  languageName: unknown
+  linkType: soft
+
 "exponential-backoff@npm:^3.1.1":
   version: 3.1.1
   resolution: "exponential-backoff@npm:3.1.1"