Skip to content

Commit

Permalink
feat(severity): Use new escalation logic for archived issues <1 day o…
Browse files Browse the repository at this point in the history
…ld (#61396)
  • Loading branch information
isabellaenriquez authored Dec 8, 2023
1 parent 0d27c7f commit d2f369c
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 12 deletions.
20 changes: 15 additions & 5 deletions src/sentry/tasks/post_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -874,9 +874,16 @@ def process_snoozes(job: PostProcessJob) -> None:
if not group.issue_type.should_detect_escalation(group.organization):
return

# groups less than a day old should use the new -> escalating logic
group_age_hours = (timezone.now() - group.first_seen).total_seconds() / 3600
should_use_new_escalation_logic = (
group_age_hours < MAX_NEW_ESCALATION_AGE_HOURS
and features.has("projects:first-event-severity-new-escalation", group.project)
)
# Check if group is escalating
if (
features.has("organizations:escalating-issues", group.organization)
not should_use_new_escalation_logic
and features.has("organizations:escalating-issues", group.organization)
and group.status == GroupStatus.IGNORED
and group.substatus == GroupSubStatus.UNTIL_ESCALATING
):
Expand Down Expand Up @@ -1357,6 +1364,7 @@ def detect_new_escalation(job: PostProcessJob):
"""
from sentry.issues.issue_velocity import get_latest_threshold
from sentry.models.activity import Activity
from sentry.models.group import GroupStatus
from sentry.models.grouphistory import GroupHistoryStatus, record_group_history
from sentry.models.groupinbox import GroupInboxReason, add_group_to_inbox
from sentry.types.activity import ActivityType
Expand All @@ -1366,8 +1374,11 @@ def detect_new_escalation(job: PostProcessJob):
"projects:first-event-severity-new-escalation", job["event"].project
):
return
group_age_hours = (datetime.now() - group.first_seen).total_seconds() / 3600
if group_age_hours >= MAX_NEW_ESCALATION_AGE_HOURS or group.substatus != GroupSubStatus.NEW:
group_age_hours = (timezone.now() - group.first_seen).total_seconds() / 3600
has_valid_status = group.substatus == GroupSubStatus.NEW or (
group.status == GroupStatus.IGNORED and group.substatus == GroupSubStatus.UNTIL_ESCALATING
)
if group_age_hours >= MAX_NEW_ESCALATION_AGE_HOURS or not has_valid_status:
return
# Get escalation lock for this group. If we're unable to acquire this lock, another process is handling
# this group at the same time. In that case, just exit early, no need to retry.
Expand All @@ -1384,8 +1395,7 @@ def detect_new_escalation(job: PostProcessJob):
# a rate of 0 means there was no threshold that could be calculated
if project_escalation_rate > 0 and group_hourly_event_rate > project_escalation_rate:
job["has_escalated"] = True
group.update(substatus=GroupSubStatus.ESCALATING)

group.update(status=GroupStatus.UNRESOLVED, substatus=GroupSubStatus.ESCALATING)
# TODO(snigdha): reuse manage_issue_states when we allow escalating from other statuses
add_group_to_inbox(group, GroupInboxReason.ESCALATING)
record_group_history(group, GroupHistoryStatus.ESCALATING)
Expand Down
108 changes: 101 additions & 7 deletions tests/sentry/tasks/test_post_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -1748,6 +1748,28 @@ def test_forecast_in_activity(self, mock_is_escalating):
data={"event_id": event.event_id, "forecast": 0},
).exists()

@with_feature("projects:first-event-severity-new-escalation")
@with_feature("organizations:escalating-issues")
@patch("sentry.issues.escalating.is_escalating")
def test_skip_escalation_logic_for_new_groups(self, mock_is_escalating):
"""
Test that we skip checking escalation in the process_snoozes job if the group is less than
a day old.
"""
event = self.create_event(data={"message": "testing"}, project_id=self.project.id)
group = event.group
group.status = GroupStatus.IGNORED
group.substatus = GroupSubStatus.UNTIL_ESCALATING
group.save()
self.call_post_process_group(
is_new=True,
is_regression=False,
is_new_group_environment=True,
event=event,
)

mock_is_escalating.assert_not_called()


@patch("sentry.utils.sdk_crashes.sdk_crash_detection.sdk_crash_detection")
class SDKCrashMonitoringTestMixin(BasePostProgressGroupMixin):
Expand Down Expand Up @@ -1978,7 +2000,7 @@ class DetectNewEscalationTestMixin(BasePostProgressGroupMixin):
def test_has_escalated(self, mock_run_post_process_job):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(hours=1), times_seen=10000)
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=10000)
event.group = Group.objects.get(id=group.id)

with self.feature("projects:first-event-severity-new-escalation"):
Expand All @@ -1999,7 +2021,7 @@ def test_has_escalated(self, mock_run_post_process_job):
def test_has_escalated_no_flag(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(hours=1), times_seen=10000)
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=10000)

self.call_post_process_group(
is_new=True,
Expand All @@ -2017,7 +2039,7 @@ def test_has_escalated_no_flag(self, mock_run_post_process_job, mock_threshold):
def test_has_escalated_old(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(days=2), times_seen=10000)
group.update(first_seen=django_timezone.now() - timedelta(days=2), times_seen=10000)

with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
Expand All @@ -2036,7 +2058,7 @@ def test_has_escalated_old(self, mock_run_post_process_job, mock_threshold):
def test_has_not_escalated(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(hours=1), times_seen=1)
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=1)

with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
Expand All @@ -2055,7 +2077,7 @@ def test_has_not_escalated(self, mock_run_post_process_job, mock_threshold):
def test_has_escalated_locked(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(hours=1), times_seen=10000)
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=10000)
lock = locks.get(f"detect_escalation:{group.id}", duration=10, name="detect_escalation")
with self.feature("projects:first-event-severity-new-escalation"), lock.acquire():
self.call_post_process_group(
Expand All @@ -2081,7 +2103,7 @@ def test_has_escalated_already_escalated(self, mock_run_post_process_job, mock_t
event=event,
)
group.update(
first_seen=datetime.now() - timedelta(hours=1),
first_seen=django_timezone.now() - timedelta(hours=1),
times_seen=10000,
substatus=GroupSubStatus.ESCALATING,
)
Expand All @@ -2097,12 +2119,84 @@ def test_has_escalated_already_escalated(self, mock_run_post_process_job, mock_t
group.refresh_from_db()
assert group.substatus == GroupSubStatus.ESCALATING

@patch("sentry.issues.issue_velocity.get_latest_threshold", return_value=1)
@patch("sentry.tasks.post_process.run_post_process_job", side_effect=run_post_process_job)
def test_has_escalated_archived(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=10000)
group.status = GroupStatus.IGNORED
group.substatus = GroupSubStatus.UNTIL_ESCALATING
group.save()

with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
is_new=False,
is_regression=False,
is_new_group_environment=False,
event=event,
)
mock_threshold.assert_called() # ensures we escalate from the new logic
job = mock_run_post_process_job.call_args[0][0]
assert job["has_escalated"]
group.refresh_from_db()
assert group.status == GroupStatus.UNRESOLVED
assert group.substatus == GroupSubStatus.ESCALATING

@patch("sentry.issues.issue_velocity.get_latest_threshold", return_value=1)
@patch("sentry.tasks.post_process.run_post_process_job", side_effect=run_post_process_job)
def test_has_escalated_archived_old(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=django_timezone.now() - timedelta(days=2), times_seen=10000)
group.status = GroupStatus.IGNORED
group.substatus = GroupSubStatus.UNTIL_ESCALATING
group.save()

with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
is_new=False,
is_regression=False,
is_new_group_environment=False,
event=event,
)
mock_threshold.assert_not_called()
job = mock_run_post_process_job.call_args[0][0]
assert not job["has_escalated"]
group.refresh_from_db()
assert group.status == GroupStatus.IGNORED
assert group.substatus == GroupSubStatus.UNTIL_ESCALATING

@patch("sentry.issues.issue_velocity.get_latest_threshold", return_value=1)
@patch("sentry.tasks.post_process.run_post_process_job", side_effect=run_post_process_job)
def test_has_escalated_ignored_not_archived(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=django_timezone.now() - timedelta(days=1), times_seen=10000)
group.status = GroupStatus.IGNORED
group.substatus = GroupSubStatus.UNTIL_CONDITION_MET
group.save()

with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
is_new=False,
is_regression=False,
is_new_group_environment=False,
event=event,
)
mock_threshold.assert_not_called()
job = mock_run_post_process_job.call_args[0][0]
assert not job["has_escalated"]
group.refresh_from_db()
assert group.status == GroupStatus.IGNORED
assert group.substatus == GroupSubStatus.UNTIL_CONDITION_MET

@patch("sentry.issues.issue_velocity.get_latest_threshold", return_value=0)
@patch("sentry.tasks.post_process.run_post_process_job", side_effect=run_post_process_job)
def test_zero_escalation_rate(self, mock_run_post_process_job, mock_threshold):
event = self.create_event(data={}, project_id=self.project.id)
group = event.group
group.update(first_seen=datetime.now() - timedelta(hours=1), times_seen=10000)
group.update(first_seen=django_timezone.now() - timedelta(hours=1), times_seen=10000)
with self.feature("projects:first-event-severity-new-escalation"):
self.call_post_process_group(
is_new=True,
Expand Down

0 comments on commit d2f369c

Please sign in to comment.