save payload file

Author: s1gr1d

utils/event-proxy-server/payload-files/readme.md

@@ -0,0 +1,2 @@
+When running the event-proxy-server, the request are saved in a json file. This folder is where all
+the generated files go.

utils/event-proxy-server/src/event-proxy-server.ts

@@ -6,7 +6,7 @@ import * as os from 'os';
 import * as path from 'path';
 import * as util from 'util';
 import * as zlib from 'zlib';
-import type { Envelope } from '@sentry/types';
+import type { Envelope, EnvelopeItem } from '@sentry/types';
 import { parseEnvelope } from '@sentry/utils';
 
 const writeFile = util.promisify(fs.writeFile);
@@ -25,6 +25,98 @@ interface SentryRequestCallbackData {
   sentryResponseStatusCode?: number;
 }
 
+const TEMPORARY_FILE_PATH = 'payload-files/temporary.json';
+
+function isDateLikeString(str: string): boolean {
+  // matches strings in the format "YYYY-MM-DD"
+  const datePattern = /^\d{4}-\d{2}-\d{2}/;
+  return datePattern.test(str);
+}
+
+function extractPathFromUrl(url: string): string {
+  const localhost = 'http://localhost:3030/';
+  return url.replace(localhost, '');
+}
+
+function addCommaAfterEachLine(data: string): string {
+  const jsonData = data.trim().split('\n');
+
+  const jsonDataWithCommas = jsonData.map((item, index) =>
+    index < jsonData.length - 1 ? item + ',' : item,
+  );
+
+  return jsonDataWithCommas.join('\n');
+}
+
+function recursivelyReplaceData(obj: any) {
+  for (let key in obj) {
+    if (typeof obj[key] === 'string' && isDateLikeString(obj[key])) {
+      obj[key] = `[[ISODateString]]`;
+    } else if (key.includes('timestamp')) {
+      obj[key] = `[[timestamp]]`;
+    } else if (key.includes('_id')) {
+      obj[key] = `[[ID]]`;
+    } else if (typeof obj[key] === 'number' && obj[key] > 1000) {
+      obj[key] = `[[highNumber]]`;
+    } else if (typeof obj[key] === 'object' && obj[key] !== null) {
+      recursivelyReplaceData(obj[key]);
+    }
+  }
+}
+
+function replaceDynamicValues(data: string): string[] {
+  const jsonData = JSON.parse(data);
+
+  recursivelyReplaceData(jsonData);
+
+  // change remaining dynamic values
+  jsonData.forEach((item: any) => {
+    if (item.trace?.public_key) {
+      item.trace.public_key = '[[publicKey]]';
+    }
+  });
+
+  return jsonData;
+}
+
+/** This function transforms all dynamic data (like timestamps) from the temporarily saved file.
+ *  The new content is saved into a new file with the url as the filename.
+ *  The temporary file is deleted in the end.
+ */
+function transformSavedJSON() {
+  fs.readFile(TEMPORARY_FILE_PATH, 'utf8', (err, data) => {
+    if (err) {
+      console.error('Error reading file', err);
+      return;
+    }
+
+    const jsonData = addCommaAfterEachLine(data);
+    const transformedJSON = replaceDynamicValues(jsonData);
+    const objWithReq = transformedJSON[2] as unknown as { request: { url: string } };
+
+    if ('request' in objWithReq) {
+      const url = objWithReq.request.url;
+      const filepath = `payload-files/${extractPathFromUrl(url)}.json`;
+
+      fs.writeFile(filepath, JSON.stringify(transformedJSON, null, 2), err => {
+        if (err) {
+          console.error('Error writing file', err);
+        } else {
+          console.log(`Successfully modified timestamp in ${filepath}`);
+        }
+      });
+    }
+  });
+
+  fs.unlink(TEMPORARY_FILE_PATH, err => {
+    if (err) {
+      console.error('Error deleting file', err);
+    } else {
+      console.log(`Successfully deleted ${TEMPORARY_FILE_PATH}`);
+    }
+  });
+}
+
 /**
  * Starts an event proxy server that will proxy events to sentry when the `tunnel` option is used. Point the `tunnel`
  * option to this server (like this `tunnel: http://localhost:${port option}/`).
@@ -33,6 +125,8 @@ interface SentryRequestCallbackData {
 export async function startEventProxyServer(options: EventProxyServerOptions): Promise<void> {
   const eventCallbackListeners: Set<(data: string) => void> = new Set();
 
+  console.log(`Proxy server "${options.proxyServerName}" running. Waiting for events...`);
+
   const proxyServer = http.createServer((proxyRequest, proxyResponse) => {
     const proxyRequestChunks: Uint8Array[] = [];
 
@@ -50,15 +144,25 @@ export async function startEventProxyServer(options: EventProxyServerOptions): P
           ? zlib.gunzipSync(Buffer.concat(proxyRequestChunks)).toString()
           : Buffer.concat(proxyRequestChunks).toString();
 
-      let envelopeHeader = JSON.parse(proxyRequestBody.split('\n')[0]);
+      fs.writeFile(TEMPORARY_FILE_PATH, `[${proxyRequestBody}]`, err => {
+        if (err) {
+          console.error(`Error writing file ${TEMPORARY_FILE_PATH}`, err);
+        } else {
+          console.log(`Successfully wrote to ${TEMPORARY_FILE_PATH}`);
+        }
+      });
+
+      transformSavedJSON();
+
+      const envelopeHeader: EnvelopeItem[0] = JSON.parse(proxyRequestBody.split('\n')[0]);
 
       if (!envelopeHeader.dsn) {
         throw new Error(
           '[event-proxy-server] No dsn on envelope header. Please set tunnel option.',
         );
       }
 
-      const { origin, pathname, host } = new URL(envelopeHeader.dsn);
+      const { origin, pathname, host } = new URL(envelopeHeader.dsn as string);
 
       const projectId = pathname.substring(1);
       const sentryIngestUrl = `${origin}/api/${projectId}/envelope/`;