feat: Auto-enabling integrations behind feature flag (#625)

 This was asked for in the context of APM where people would have to enable a lot of small integrations to get a meaningful span tree. Generally it's nice if people have to think about as little as possible (so, not about which integrations are necessary) when enabling the SDK.

The semver compatibility is another thing. Yes, you could upgrade to a new version of the SDK and just get more integrations enabled automatically. Similar effects were already observable to some degree as we added more features to integrations that the user already explicitly enabled (breadcrumbs for django sql queries for example). I think we will just avoid this problem and make sure that new integrations or changes to existing ones don't break fundamental things like grouping, ever (like we already do)

Author: untitaker

docs-requirements.txt

@@ -1,3 +1,4 @@
 sphinx==2.3.1
 sphinx-rtd-theme
 sphinx-autodoc-typehints[type_comments]>=1.8.0
+typing-extensions

sentry_sdk/client.py

@@ -106,6 +106,9 @@ def _init_impl(self):
             self.integrations = setup_integrations(
                 self.options["integrations"],
                 with_defaults=self.options["default_integrations"],
+                with_auto_enabling_integrations=self.options["_experiments"].get(
+                    "auto_enabling_integrations", False
+                ),
             )
         finally:
             _client_init_debug.set(old_debug)

sentry_sdk/consts.py

@@ -9,12 +9,27 @@
     from typing import Dict
     from typing import Any
     from typing import Sequence
+    from typing_extensions import TypedDict
 
     from sentry_sdk.transport import Transport
     from sentry_sdk.integrations import Integration
 
     from sentry_sdk._types import Event, EventProcessor, BreadcrumbProcessor
 
+    # Experiments are feature flags to enable and disable certain unstable SDK
+    # functionality. Changing them from the defaults (`None`) in production
+    # code is highly discouraged. They are not subject to any stability
+    # guarantees such as the ones from semantic versioning.
+    Experiments = TypedDict(
+        "Experiments",
+        {
+            "max_spans": Optional[int],
+            "record_sql_params": Optional[bool],
+            "auto_enabling_integrations": Optional[bool],
+        },
+        total=False,
+    )
+
 
 # This type exists to trick mypy and PyCharm into thinking `init` and `Client`
 # take these arguments (even though they take opaque **kwargs)
@@ -49,7 +64,7 @@ def __init__(
         # DO NOT ENABLE THIS RIGHT NOW UNLESS YOU WANT TO EXCEED YOUR EVENT QUOTA IMMEDIATELY
         traces_sample_rate=0.0,  # type: float
         traceparent_v2=False,  # type: bool
-        _experiments={},  # type: Dict[str, Any]  # noqa: B006
+        _experiments={},  # type: Experiments  # noqa: B006
     ):
         # type: (...) -> None
         pass

sentry_sdk/integrations/__init__.py

@@ -9,53 +9,85 @@
 from sentry_sdk._types import MYPY
 
 if MYPY:
-    from typing import Iterator
+    from typing import Callable
     from typing import Dict
+    from typing import Iterator
     from typing import List
     from typing import Set
+    from typing import Tuple
     from typing import Type
-    from typing import Callable
 
 
 _installer_lock = Lock()
 _installed_integrations = set()  # type: Set[str]
 
 
-def _generate_default_integrations_iterator(*import_strings):
-    # type: (*str) -> Callable[[], Iterator[Type[Integration]]]
-    def iter_default_integrations():
-        # type: () -> Iterator[Type[Integration]]
+def _generate_default_integrations_iterator(integrations, auto_enabling_integrations):
+    # type: (Tuple[str, ...], Tuple[str, ...]) -> Callable[[bool], Iterator[Type[Integration]]]
+
+    def iter_default_integrations(with_auto_enabling_integrations):
+        # type: (bool) -> Iterator[Type[Integration]]
         """Returns an iterator of the default integration classes:
         """
         from importlib import import_module
 
-        for import_string in import_strings:
-            module, cls = import_string.rsplit(".", 1)
-            yield getattr(import_module(module), cls)
+        if with_auto_enabling_integrations:
+            all_import_strings = integrations + auto_enabling_integrations
+        else:
+            all_import_strings = integrations
+
+        for import_string in all_import_strings:
+            try:
+                module, cls = import_string.rsplit(".", 1)
+                yield getattr(import_module(module), cls)
+            except (DidNotEnable, SyntaxError) as e:
+                logger.debug(
+                    "Did not import default integration %s: %s", import_string, e
+                )
 
     if isinstance(iter_default_integrations.__doc__, str):
-        for import_string in import_strings:
+        for import_string in integrations:
             iter_default_integrations.__doc__ += "\n- `{}`".format(import_string)
 
     return iter_default_integrations
 
 
+_AUTO_ENABLING_INTEGRATIONS = (
+    "sentry_sdk.integrations.django.DjangoIntegration",
+    "sentry_sdk.integrations.flask.FlaskIntegration",
+    "sentry_sdk.integrations.bottle.BottleIntegration",
+    "sentry_sdk.integrations.falcon.FalconIntegration",
+    "sentry_sdk.integrations.sanic.SanicIntegration",
+    "sentry_sdk.integrations.celery.CeleryIntegration",
+    "sentry_sdk.integrations.rq.RqIntegration",
+    "sentry_sdk.integrations.aiohttp.AioHttpIntegration",
+    "sentry_sdk.integrations.tornado.TornadoIntegration",
+    "sentry_sdk.integrations.sqlalchemy.SqlalchemyIntegration",
+)
+
+
 iter_default_integrations = _generate_default_integrations_iterator(
-    "sentry_sdk.integrations.logging.LoggingIntegration",
-    "sentry_sdk.integrations.stdlib.StdlibIntegration",
-    "sentry_sdk.integrations.excepthook.ExcepthookIntegration",
-    "sentry_sdk.integrations.dedupe.DedupeIntegration",
-    "sentry_sdk.integrations.atexit.AtexitIntegration",
-    "sentry_sdk.integrations.modules.ModulesIntegration",
-    "sentry_sdk.integrations.argv.ArgvIntegration",
-    "sentry_sdk.integrations.threading.ThreadingIntegration",
+    integrations=(
+        # stdlib/base runtime integrations
+        "sentry_sdk.integrations.logging.LoggingIntegration",
+        "sentry_sdk.integrations.stdlib.StdlibIntegration",
+        "sentry_sdk.integrations.excepthook.ExcepthookIntegration",
+        "sentry_sdk.integrations.dedupe.DedupeIntegration",
+        "sentry_sdk.integrations.atexit.AtexitIntegration",
+        "sentry_sdk.integrations.modules.ModulesIntegration",
+        "sentry_sdk.integrations.argv.ArgvIntegration",
+        "sentry_sdk.integrations.threading.ThreadingIntegration",
+    ),
+    auto_enabling_integrations=_AUTO_ENABLING_INTEGRATIONS,
 )
 
 del _generate_default_integrations_iterator
 
 
-def setup_integrations(integrations, with_defaults=True):
-    # type: (List[Integration], bool) -> Dict[str, Integration]
+def setup_integrations(
+    integrations, with_defaults=True, with_auto_enabling_integrations=False
+):
+    # type: (List[Integration], bool, bool) -> Dict[str, Integration]
     """Given a list of integration instances this installs them all.  When
     `with_defaults` is set to `True` then all default integrations are added
     unless they were already provided before.
@@ -66,11 +98,17 @@ def setup_integrations(integrations, with_defaults=True):
 
     logger.debug("Setting up integrations (with default = %s)", with_defaults)
 
+    # Integrations that are not explicitly set up by the user.
+    used_as_default_integration = set()
+
     if with_defaults:
-        for integration_cls in iter_default_integrations():
+        for integration_cls in iter_default_integrations(
+            with_auto_enabling_integrations
+        ):
             if integration_cls.identifier not in integrations:
                 instance = integration_cls()
                 integrations[instance.identifier] = instance
+                used_as_default_integration.add(instance.identifier)
 
     for identifier, integration in iteritems(integrations):
         with _installer_lock:
@@ -90,6 +128,14 @@ def setup_integrations(integrations, with_defaults=True):
                         integration.install()
                     else:
                         raise
+                except DidNotEnable as e:
+                    if identifier not in used_as_default_integration:
+                        raise
+
+                    logger.debug(
+                        "Did not enable default integration %s: %s", identifier, e
+                    )
+
                 _installed_integrations.add(identifier)
 
     for identifier in integrations:
@@ -98,6 +144,16 @@ def setup_integrations(integrations, with_defaults=True):
     return integrations
 
 
+class DidNotEnable(Exception):
+    """
+    The integration could not be enabled due to a trivial user error like
+    `flask` not being installed for the `FlaskIntegration`.
+
+    This exception is silently swallowed for default integrations, but reraised
+    for explicitly enabled integrations.
+    """
+
+
 class Integration(object):
     """Baseclass for all integrations.
 

sentry_sdk/integrations/aiohttp.py

@@ -3,7 +3,7 @@
 
 from sentry_sdk._compat import reraise
 from sentry_sdk.hub import Hub
-from sentry_sdk.integrations import Integration
+from sentry_sdk.integrations import Integration, DidNotEnable
 from sentry_sdk.integrations.logging import ignore_logger
 from sentry_sdk.integrations._wsgi_common import (
     _filter_headers,
@@ -18,8 +18,13 @@
     AnnotatedValue,
 )
 
-import asyncio
-from aiohttp.web import Application, HTTPException, UrlDispatcher
+try:
+    import asyncio
+
+    from aiohttp import __version__ as AIOHTTP_VERSION
+    from aiohttp.web import Application, HTTPException, UrlDispatcher
+except ImportError:
+    raise DidNotEnable("AIOHTTP not installed")
 
 from sentry_sdk._types import MYPY
 
@@ -43,6 +48,15 @@ class AioHttpIntegration(Integration):
     @staticmethod
     def setup_once():
         # type: () -> None
+
+        try:
+            version = tuple(map(int, AIOHTTP_VERSION.split(".")))
+        except (TypeError, ValueError):
+            raise DidNotEnable("AIOHTTP version unparseable: {}".format(version))
+
+        if version < (3, 4):
+            raise DidNotEnable("AIOHTTP 3.4 or newer required.")
+
         if not HAS_REAL_CONTEXTVARS:
             # We better have contextvars or we're going to leak state between
             # requests.

sentry_sdk/integrations/bottle.py

@@ -6,7 +6,7 @@
     event_from_exception,
     transaction_from_function,
 )
-from sentry_sdk.integrations import Integration
+from sentry_sdk.integrations import Integration, DidNotEnable
 from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
 from sentry_sdk.integrations._wsgi_common import RequestExtractor
 
@@ -22,7 +22,16 @@
 
     from sentry_sdk._types import EventProcessor
 
-from bottle import Bottle, Route, request as bottle_request, HTTPResponse
+try:
+    from bottle import (
+        Bottle,
+        Route,
+        request as bottle_request,
+        HTTPResponse,
+        __version__ as BOTTLE_VERSION,
+    )
+except ImportError:
+    raise DidNotEnable("Bottle not installed")
 
 
 class BottleIntegration(Integration):
@@ -32,6 +41,7 @@ class BottleIntegration(Integration):
 
     def __init__(self, transaction_style="endpoint"):
         # type: (str) -> None
+
         TRANSACTION_STYLE_VALUES = ("endpoint", "url")
         if transaction_style not in TRANSACTION_STYLE_VALUES:
             raise ValueError(
@@ -44,6 +54,14 @@ def __init__(self, transaction_style="endpoint"):
     def setup_once():
         # type: () -> None
 
+        try:
+            version = tuple(map(int, BOTTLE_VERSION.split(".")))
+        except (TypeError, ValueError):
+            raise DidNotEnable("Unparseable Bottle version: {}".format(version))
+
+        if version < (0, 12):
+            raise DidNotEnable("Bottle 0.12 or newer required.")
+
         # monkey patch method Bottle.__call__
         old_app = Bottle.__call__
 

sentry_sdk/integrations/celery.py

@@ -3,18 +3,11 @@
 import functools
 import sys
 
-from celery.exceptions import (  # type: ignore
-    SoftTimeLimitExceeded,
-    Retry,
-    Ignore,
-    Reject,
-)
-
 from sentry_sdk.hub import Hub
 from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
 from sentry_sdk.tracing import Span
 from sentry_sdk._compat import reraise
-from sentry_sdk.integrations import Integration
+from sentry_sdk.integrations import Integration, DidNotEnable
 from sentry_sdk.integrations.logging import ignore_logger
 from sentry_sdk._types import MYPY
 
@@ -29,6 +22,18 @@
     F = TypeVar("F", bound=Callable[..., Any])
 
 
+try:
+    from celery import VERSION as CELERY_VERSION  # type: ignore
+    from celery.exceptions import (  # type: ignore
+        SoftTimeLimitExceeded,
+        Retry,
+        Ignore,
+        Reject,
+    )
+except ImportError:
+    raise DidNotEnable("Celery not installed")
+
+
 CELERY_CONTROL_FLOW_EXCEPTIONS = (Retry, Ignore, Reject)
 
 
@@ -42,6 +47,9 @@ def __init__(self, propagate_traces=True):
     @staticmethod
     def setup_once():
         # type: () -> None
+        if CELERY_VERSION < (3,):
+            raise DidNotEnable("Celery 3 or newer required.")
+
         import celery.app.trace as trace  # type: ignore
 
         old_build_tracer = trace.build_tracer