feat(workflow): Add new experiment for issue alert fallback (#46124)

Author: scttcper

src/sentry/conf/server.py

@@ -1207,6 +1207,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME():
     "organizations:invite-members": True,
     # Enable rate limits for inviting members.
     "organizations:invite-members-rate-limits": True,
+    # Test 10 member fallback vs 10 members
+    "organizations:issue-alert-fallback-experiment": False,
     # Enable new issue alert "issue owners" fallback
     "organizations:issue-alert-fallback-targeting": False,
     # Enable removing issue from issue list if action taken.

src/sentry/features/__init__.py

@@ -82,6 +82,7 @@
 default_manager.add("organizations:higher-ownership-limit", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:invite-members", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:invite-members-rate-limits", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
+default_manager.add("organizations:issue-alert-fallback-experiment", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:issue-alert-fallback-targeting", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:issue-alert-incompatible-rules", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:issue-alert-preview", OrganizationFeature, FeatureHandlerStrategy.REMOTE)

src/sentry/mail/analytics.py

@@ -12,6 +12,8 @@ class EmailNotificationSent(analytics.Event):
         analytics.Attribute("actor_id"),
         analytics.Attribute("user_id", required=False),
         analytics.Attribute("group_id", required=False),
+        # Remove after IssueAlertFallbackExperiment
+        analytics.Attribute("fallback_experiment", required=False),
     )
 
 

src/sentry/notifications/notifications/rules.py

@@ -26,7 +26,11 @@
     has_alert_integration,
     has_integrations,
 )
-from sentry.notifications.utils.participants import get_owner_reason, get_send_to
+from sentry.notifications.utils.participants import (
+    get_owner_reason,
+    get_send_to,
+    should_use_smaller_issue_alert_fallback,
+)
 from sentry.plugins.base.structs import Notification
 from sentry.services.hybrid_cloud.actor import ActorType, RpcActor
 from sentry.types.integrations import ExternalProviders
@@ -108,12 +112,19 @@ def get_context(self) -> MutableMapping[str, Any]:
             event=self.event,
             fallthrough_choice=self.fallthrough_choice,
         )
+        fallback_params: MutableMapping[str, str] = {}
+        # Piggybacking off of notification_reason that already determines if we're using the fallback
+        if notification_reason and self.fallthrough_choice == FallthroughChoiceType.ACTIVE_MEMBERS:
+            _, fallback_experiment = should_use_smaller_issue_alert_fallback(org=self.organization)
+            fallback_params = {"ref_fallback": fallback_experiment}
 
         context = {
             "project_label": self.project.get_full_name(),
             "group": self.group,
             "event": self.event,
-            "link": get_group_settings_link(self.group, environment, rule_details),
+            "link": get_group_settings_link(
+                self.group, environment, rule_details, None, **fallback_params
+            ),
             "rules": rule_details,
             "has_integrations": has_integrations(self.organization, self.project),
             "enhanced_privacy": enhanced_privacy,
@@ -214,8 +225,10 @@ def send(self) -> None:
             notify(provider, self, participants, shared_context)
 
     def get_log_params(self, recipient: RpcActor) -> Mapping[str, Any]:
+        _, fallback_experiment = should_use_smaller_issue_alert_fallback(org=self.organization)
         return {
             "target_type": self.target_type,
             "target_identifier": self.target_identifier,
+            "fallback_experiment": fallback_experiment,
             **super().get_log_params(recipient),
         }

src/sentry/notifications/utils/participants.py

@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Sequence, Tuple
 
 from sentry import features
+from sentry.experiments import manager as expt_manager
 from sentry.models import (
     ActorTuple,
     Group,
@@ -51,7 +52,8 @@
     ExternalProviders.SLACK,
 }
 
-FALLTHROUGH_NOTIFICATION_LIMIT_EA = 20
+FALLTHROUGH_NOTIFICATION_LIMIT_EA = 10
+FALLTHROUGH_NOTIFICATION_LIMIT = 20
 
 
 class ParticipantMap:
@@ -385,6 +387,25 @@ def get_send_to(
     return get_recipients_by_provider(project, recipients, notification_type)
 
 
+def should_use_smaller_issue_alert_fallback(org: Organization) -> Tuple[bool, str]:
+    """
+    Remove after IssueAlertFallbackExperiment experiment
+    Returns a tuple of (enabled, analytics_label)
+    """
+    if not org.flags.early_adopter.is_set:
+        # Not early access, not in experiment
+        return (False, "ga")
+
+    # Disabled
+    if not features.has("organizations:issue-alert-fallback-experiment", org, actor=None):
+        return (False, "disabled")
+
+    org_exposed = expt_manager.get("IssueAlertFallbackExperiment", org=org) == 1
+    if org_exposed:
+        return (True, "expt")
+    return (False, "ctrl")
+
+
 def get_fallthrough_recipients(
     project: Project, fallthrough_choice: FallthroughChoiceType | None
 ) -> Iterable[RpcUser]:
@@ -408,13 +429,19 @@ def get_fallthrough_recipients(
         )
 
     elif fallthrough_choice == FallthroughChoiceType.ACTIVE_MEMBERS:
+        use_smaller_limit, _ = should_use_smaller_issue_alert_fallback(org=project.organization)
+        limit = (
+            FALLTHROUGH_NOTIFICATION_LIMIT_EA
+            if use_smaller_limit
+            else FALLTHROUGH_NOTIFICATION_LIMIT
+        )
         return user_service.get_many(
             filter={
                 "user_ids": project.member_set.order_by("-user__last_active").values_list(
                     "user_id", flat=True
                 )
             }
-        )[:FALLTHROUGH_NOTIFICATION_LIMIT_EA]
+        )[:limit]
 
     raise NotImplementedError(f"Unknown fallthrough choice: {fallthrough_choice}")
 

tests/sentry/notifications/utils/test_participants.py

@@ -1,4 +1,5 @@
 from typing import Iterable, Mapping, Optional, Sequence, Set, Union
+from unittest import mock
 
 import pytest
 
@@ -15,6 +16,7 @@
     NotificationSettingTypes,
 )
 from sentry.notifications.utils.participants import (
+    FALLTHROUGH_NOTIFICATION_LIMIT,
     FALLTHROUGH_NOTIFICATION_LIMIT_EA,
     get_fallthrough_recipients,
     get_owner_reason,
@@ -902,7 +904,7 @@ def test_fallthrough_admin_or_recent_under_20(self):
     @with_feature("organizations:issue-alert-fallback-targeting")
     def test_fallthrough_admin_or_recent_over_20(self):
         notifiable_users = [self.user, self.user2]
-        for i in range(FALLTHROUGH_NOTIFICATION_LIMIT_EA + 5):
+        for i in range(FALLTHROUGH_NOTIFICATION_LIMIT + 5):
             new_user = self.create_user(email=f"user_{i}@example.com", is_active=True)
             self.create_member(
                 user=new_user, organization=self.organization, role="owner", teams=[self.team2]
@@ -923,5 +925,35 @@ def test_fallthrough_admin_or_recent_over_20(self):
             event, self.project, FallthroughChoiceType.ACTIVE_MEMBERS
         )[ExternalProviders.EMAIL]
 
-        assert len(notified_users) == FALLTHROUGH_NOTIFICATION_LIMIT_EA
+        assert len(notified_users) == FALLTHROUGH_NOTIFICATION_LIMIT
         assert notified_users.issubset(expected_notified_users)
+
+    @mock.patch("sentry.experiments.manager.get", return_value=1)
+    @with_feature("organizations:issue-alert-fallback-targeting")
+    @with_feature("organizations:issue-alert-fallback-experiment")
+    def test_fallthrough_ea_experiment_limit(self, mock_func):
+        self.organization.flags.early_adopter = True
+
+        notifiable_users = [self.user, self.user2]
+        for i in range(FALLTHROUGH_NOTIFICATION_LIMIT_EA + 5):
+            new_user = self.create_user(email=f"user_{i}@example.com", is_active=True)
+            self.create_member(
+                user=new_user, organization=self.organization, role="owner", teams=[self.team2]
+            )
+            notifiable_users.append(new_user)
+
+        for user in notifiable_users:
+            NotificationSetting.objects.update_settings(
+                ExternalProviders.SLACK,
+                NotificationSettingTypes.ISSUE_ALERTS,
+                NotificationSettingOptionValues.NEVER,
+                user=user,
+            )
+
+        event = self.store_event("admin.lol", self.project)
+        notified_users = self.get_send_to_fallthrough(
+            event, self.project, FallthroughChoiceType.ACTIVE_MEMBERS
+        )[ExternalProviders.EMAIL]
+
+        # Check that we notify all possible folks with the EA experiment limit.
+        assert len(notified_users) == FALLTHROUGH_NOTIFICATION_LIMIT_EA