feat: Send transactions in envelopes

The new Envelope endpoint and format is what we want to use for
transactions going forward.

Author: rhcarvalho

packages/browser/src/integrations/breadcrumbs.ts

@@ -170,7 +170,7 @@ export class Breadcrumbs implements Integration {
     const client = getCurrentHub().getClient<BrowserClient>();
     const dsn = client && client.getDsn();
     if (this._options.sentry && dsn) {
-      const filterUrl = new API(dsn).getStoreEndpoint();
+      const filterUrl = new API(dsn).getBaseApiEndpoint();
       // if Sentry key appears in URL, don't capture it as a request
       // but rather as our own 'sentry' type breadcrumb
       if (

packages/browser/src/transports/base.ts

@@ -5,15 +5,20 @@ import { PromiseBuffer, SentryError } from '@sentry/utils';
 /** Base Transport class implementation */
 export abstract class BaseTransport implements Transport {
   /**
-   * @inheritDoc
+   * @deprecated
    */
   public url: string;
 
+  /** Helper to get Sentry API endpoints. */
+  protected readonly _api: API;
+
   /** A simple buffer holding all requests. */
   protected readonly _buffer: PromiseBuffer<Response> = new PromiseBuffer(30);
 
   public constructor(public options: TransportOptions) {
-    this.url = new API(this.options.dsn).getStoreEndpointWithUrlEncodedAuth();
+    this._api = new API(this.options.dsn);
+    // tslint:disable-next-line:deprecation
+    this.url = this._api.getStoreEndpointWithUrlEncodedAuth();
   }
 
   /**

packages/browser/src/transports/fetch.ts

@@ -1,3 +1,4 @@
+import { eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status } from '@sentry/types';
 import { getGlobalObject, logger, parseRetryAfterHeader, supportsReferrerPolicy, SyncPromise } from '@sentry/utils';
 
@@ -22,8 +23,10 @@ export class FetchTransport extends BaseTransport {
       });
     }
 
-    const defaultOptions: RequestInit = {
-      body: JSON.stringify(event),
+    const sentryReq = eventToSentryRequest(event, this._api);
+
+    const options: RequestInit = {
+      body: sentryReq.body,
       method: 'POST',
       // Despite all stars in the sky saying that Edge supports old draft syntax, aka 'never', 'always', 'origin' and 'default
       // https://caniuse.com/#feat=referrer-policy
@@ -33,13 +36,13 @@ export class FetchTransport extends BaseTransport {
     };
 
     if (this.options.headers !== undefined) {
-      defaultOptions.headers = this.options.headers;
+      options.headers = this.options.headers;
     }
 
     return this._buffer.add(
       new SyncPromise<Response>((resolve, reject) => {
         global
-          .fetch(this.url, defaultOptions)
+          .fetch(sentryReq.url, options)
           .then(response => {
             const status = Status.fromHttpCode(response.status);
 

packages/browser/src/transports/xhr.ts

@@ -1,3 +1,4 @@
+import { eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status } from '@sentry/types';
 import { logger, parseRetryAfterHeader, SyncPromise } from '@sentry/utils';
 
@@ -20,6 +21,8 @@ export class XHRTransport extends BaseTransport {
       });
     }
 
+    const sentryReq = eventToSentryRequest(event, this._api);
+
     return this._buffer.add(
       new SyncPromise<Response>((resolve, reject) => {
         const request = new XMLHttpRequest();
@@ -45,13 +48,13 @@ export class XHRTransport extends BaseTransport {
           reject(request);
         };
 
-        request.open('POST', this.url);
+        request.open('POST', sentryReq.url);
         for (const header in this.options.headers) {
           if (this.options.headers.hasOwnProperty(header)) {
             request.setRequestHeader(header, this.options.headers[header]);
           }
         }
-        request.send(JSON.stringify(event));
+        request.send(sentryReq.body);
       }),
     );
   }

packages/browser/test/unit/transports/base.test.ts

@@ -19,6 +19,7 @@ describe('BaseTransport', () => {
 
   it('has correct endpoint url', () => {
     const transport = new SimpleTransport({ dsn: testDsn });
+    // tslint:disable-next-line:deprecation
     expect(transport.url).equal('https://sentry.io/api/42/store/?sentry_key=123&sentry_version=7');
   });
 });

packages/browser/test/unit/transports/fetch.test.ts

@@ -28,6 +28,7 @@ describe('FetchTransport', () => {
   });
 
   it('inherits composeEndpointUrl() implementation', () => {
+    // tslint:disable-next-line:deprecation
     expect(transport.url).equal(transportUrl);
   });
 

packages/browser/test/unit/transports/xhr.test.ts

@@ -28,6 +28,7 @@ describe('XHRTransport', () => {
   });
 
   it('inherits composeEndpointUrl() implementation', () => {
+    // tslint:disable-next-line:deprecation
     expect(transport.url).equal(transportUrl);
   });
 

packages/core/src/api.ts

@@ -17,29 +17,58 @@ export class API {
     return this._dsnObject;
   }
 
-  /** Returns a string with auth headers in the url to the store endpoint. */
+  /** Returns the prefix to construct Sentry ingestion API endpoints. */
+  public getBaseApiEndpoint(): string {
+    const dsn = this._dsnObject;
+    const protocol = dsn.protocol ? `${dsn.protocol}:` : '';
+    const port = dsn.port ? `:${dsn.port}` : '';
+    return `${protocol}//${dsn.host}${port}${dsn.path ? `/${dsn.path}` : ''}/api/`;
+  }
+
+  /** Returns the store endpoint URL. */
   public getStoreEndpoint(): string {
-    return `${this._getBaseUrl()}${this.getStoreEndpointPath()}`;
+    return this._getIngestEndpoint('store');
+  }
+
+  /** Returns the envelope endpoint URL. */
+  public getEnvelopeEndpoint(): string {
+    return this._getIngestEndpoint('envelope');
+  }
+
+  /** Returns the ingest API endpoint for target. */
+  private _getIngestEndpoint(target: string): string {
+    const base = this.getBaseApiEndpoint();
+    const dsn = this._dsnObject;
+    return `${base}${dsn.projectId}/${target}/`;
   }
 
-  /** Returns the store endpoint with auth added in url encoded. */
+  /**
+   * Returns the store endpoint URL with auth in the query string.
+   *
+   * Sending auth as part of the query string and not as custom HTTP headers avoids CORS preflight requests.
+   */
   public getStoreEndpointWithUrlEncodedAuth(): string {
+    return `${this.getStoreEndpoint()}?${this._encodedAuth()}`;
+  }
+
+  /**
+   * Returns the envelope endpoint URL with auth in the query string.
+   *
+   * Sending auth as part of the query string and not as custom HTTP headers avoids CORS preflight requests.
+   */
+  public getEnvelopeEndpointWithUrlEncodedAuth(): string {
+    return `${this.getEnvelopeEndpoint()}?${this._encodedAuth()}`;
+  }
+
+  /** Returns a URL-encoded string with auth config suitable for a query string. */
+  private _encodedAuth(): string {
     const dsn = this._dsnObject;
     const auth = {
-      sentry_key: dsn.user, // sentry_key is currently used in tracing integration to identify internal sentry requests
+      sentry_key: dsn.user,
       sentry_version: SENTRY_API_VERSION,
+      // REVIEW: don't we send sentry_client (with SDK info) when using this?! Compare with `getRequestHeaders`.
     };
-    // Auth is intentionally sent as part of query string (NOT as custom HTTP header)
-    // to avoid preflight CORS requests
-    return `${this.getStoreEndpoint()}?${urlEncode(auth)}`;
-  }
-
-  /** Returns the base path of the url including the port. */
-  private _getBaseUrl(): string {
-    const dsn = this._dsnObject;
-    const protocol = dsn.protocol ? `${dsn.protocol}:` : '';
-    const port = dsn.port ? `:${dsn.port}` : '';
-    return `${protocol}//${dsn.host}${port}`;
+    return urlEncode(auth);
   }
 
   /** Returns only the path component for the store endpoint. */
@@ -48,7 +77,11 @@ export class API {
     return `${dsn.path ? `/${dsn.path}` : ''}/api/${dsn.projectId}/store/`;
   }
 
-  /** Returns an object that can be used in request headers. */
+  /**
+   * Returns an object that can be used in request headers.
+   *
+   * @deprecated in favor of `getStoreEndpointWithUrlEncodedAuth` and `getEnvelopeEndpointWithUrlEncodedAuth`.
+   */
   public getRequestHeaders(clientName: string, clientVersion: string): { [key: string]: string } {
     const dsn = this._dsnObject;
     const header = [`Sentry sentry_version=${SENTRY_API_VERSION}`];
@@ -71,7 +104,7 @@ export class API {
     } = {},
   ): string {
     const dsn = this._dsnObject;
-    const endpoint = `${this._getBaseUrl()}${dsn.path ? `/${dsn.path}` : ''}/api/embed/error-page/`;
+    const endpoint = `${this.getBaseApiEndpoint()}embed/error-page/`;
 
     const encodedOptions = [];
     encodedOptions.push(`dsn=${dsn.toString()}`);

packages/core/src/index.ts

@@ -16,6 +16,7 @@ export { addGlobalEventProcessor, getCurrentHub, getHubFromCarrier, Hub, Scope }
 export { API } from './api';
 export { BaseClient } from './baseclient';
 export { BackendClass, BaseBackend } from './basebackend';
+export { eventToSentryRequest } from './request';
 export { initAndBind, ClientClass } from './sdk';
 export { NoopTransport } from './transports/noop';
 

packages/core/src/request.ts

@@ -0,0 +1,61 @@
+import { Event } from '@sentry/types';
+
+import { API } from './api';
+
+/** A generic client request. */
+interface SentryRequest {
+  body: string;
+  url: string;
+  // headers would contain auth & content-type headers for @sentry/node, but
+  // since @sentry/browser avoids custom headers to prevent CORS preflight
+  // requests, we can use the same approach for @sentry/browser and @sentry/node
+  // for simplicity -- no headers involved.
+  // headers: { [key: string]: string };
+}
+
+/** Creates a SentryRequest from an event. */
+export function eventToSentryRequest(event: Event, api: API): SentryRequest {
+  const useEnvelope = event.type === 'transaction';
+
+  const req: SentryRequest = {
+    body: JSON.stringify(event),
+    url: useEnvelope ? api.getEnvelopeEndpointWithUrlEncodedAuth() : api.getStoreEndpointWithUrlEncodedAuth(),
+  };
+
+  // https://docs.sentry.io/development/sdk-dev/envelopes/
+
+  // Since we don't need to manipulate envelopes nor store them, there is no
+  // exported concept of an Envelope with operations including serialization and
+  // deserialization. Instead, we only implement a minimal subset of the spec to
+  // serialize events inline here.
+  if (useEnvelope) {
+    const envelopeHeaders = JSON.stringify({
+      event_id: event.event_id,
+      sent_at: new Date().toISOString(),
+    });
+    const itemHeaders = JSON.stringify({
+      type: event.type,
+      // The content-type is assumed to be 'application/json' and not part of
+      // the current spec for transaction items, so we don't bloat the request
+      // body with it.
+      //
+      // content_type: 'application/json',
+      //
+      // The length is optional. It must be the number of bytes in req.Body
+      // encoded as UTF-8. Since the server can figure this out and would
+      // otherwise refuse events that report the length incorrectly, we decided
+      // not to send the length to avoid problems related to reporting the wrong
+      // size and to reduce request body size.
+      //
+      // length: new TextEncoder().encode(req.body).length,
+    });
+    // The trailing newline is optional. We intentionally don't send it to avoid
+    // sending unnecessary bytes.
+    //
+    // const envelope = `${envelopeHeaders}\n${itemHeaders}\n${req.body}\n`;
+    const envelope = `${envelopeHeaders}\n${itemHeaders}\n${req.body}`;
+    req.body = envelope;
+  }
+
+  return req;
+}

packages/core/test/lib/api.test.ts

@@ -16,11 +16,13 @@ describe('API', () => {
   });
 
   test('getRequestHeaders', () => {
+    // tslint:disable-next-line:deprecation
     expect(new API(dsnPublic).getRequestHeaders('a', '1.0')).toMatchObject({
       'Content-Type': 'application/json',
       'X-Sentry-Auth': expect.stringMatching(/^Sentry sentry_version=\d, sentry_client=a\/1\.0, sentry_key=abc$/),
     });
 
+    // tslint:disable-next-line:deprecation
     expect(new API(legacyDsn).getRequestHeaders('a', '1.0')).toMatchObject({
       'Content-Type': 'application/json',
       'X-Sentry-Auth': expect.stringMatching(

packages/node/src/transports/base.ts

@@ -1,12 +1,12 @@
-import { API } from '@sentry/core';
+import { API, eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status, Transport, TransportOptions } from '@sentry/types';
-import { logger, parseRetryAfterHeader, PromiseBuffer, SentryError, isString } from '@sentry/utils';
+import { logger, parseRetryAfterHeader, PromiseBuffer, SentryError } from '@sentry/utils';
 import * as fs from 'fs';
 import * as http from 'http';
 import * as https from 'https';
 import * as url from 'url';
 
-import { SDK_NAME, SDK_VERSION } from '../version';
+// import { SDK_NAME, SDK_VERSION } from '../version';
 
 /**
  * Internal used interface for typescript.
@@ -23,7 +23,9 @@ export interface HTTPRequest {
     callback?: (res: http.IncomingMessage) => void,
   ): http.ClientRequest;
 
-  // This is the new type for versions that handle URL argument correctly, but it's most likely not needed here just yet
+  // This is the type for nodejs versions that handle the URL argument
+  // (v10.9.0+), but we do not use it just yet because we support older node
+  // versions:
 
   // request(
   //   url: string | url.URL,
@@ -55,23 +57,22 @@ export abstract class BaseTransport implements Transport {
   }
 
   /** Returns a build request option object used by request */
-  protected _getRequestOptions(address: string | url.URL): http.RequestOptions | https.RequestOptions {
-    if (!isString(address) || !(address instanceof url.URL)) {
-      throw new SentryError(`Incorrect transport url: ${address}`);
-    }
-
+  protected _getRequestOptions(uri: url.URL): http.RequestOptions | https.RequestOptions {
     const headers = {
-      ...this._api.getRequestHeaders(SDK_NAME, SDK_VERSION),
+      // The auth headers are not included because auth is done via query string to match @sentry/browser.
+      //
+      // ...this._api.getRequestHeaders(SDK_NAME, SDK_VERSION)
       ...this.options.headers,
     };
-    const addr = address instanceof url.URL ? address : new url.URL(address);
-    const { hostname, pathname: path, port, protocol } = addr;
+    const { hostname, port, protocol } = uri;
+    // See https://github.com/nodejs/node/blob/38146e717fed2fabe3aacb6540d839475e0ce1c6/lib/internal/url.js#L1268-L1290
+    const path = `${uri.pathname || ''}${uri.search || ''}`;
 
     return {
       agent: this.client,
-      method: 'POST',
       headers,
       hostname,
+      method: 'POST',
       path,
       port,
       protocol,
@@ -92,7 +93,10 @@ export abstract class BaseTransport implements Transport {
     }
     return this._buffer.add(
       new Promise<Response>((resolve, reject) => {
-        const req = httpModule.request(this._getRequestOptions('http://foo.com/123'), (res: http.IncomingMessage) => {
+        const sentryReq = eventToSentryRequest(event, this._api);
+        const options = this._getRequestOptions(new url.URL(sentryReq.url));
+
+        const req = httpModule.request(options, (res: http.IncomingMessage) => {
           const statusCode = res.statusCode || 500;
           const status = Status.fromHttpCode(statusCode);
 
@@ -126,7 +130,7 @@ export abstract class BaseTransport implements Transport {
           });
         });
         req.on('error', reject);
-        req.end(JSON.stringify(event));
+        req.end(sentryReq.body);
       }),
     );
   }