Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(issue-search): support IN for semver release search #76313

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/sentry/models/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def filter_by_semver_build(
self,
organization_id: int,
operator: str,
build: str,
build: str | Sequence[str],
project_ids: Sequence[int] | None = None,
negated: bool = False,
) -> models.QuerySet:
Expand Down
49 changes: 41 additions & 8 deletions src/sentry/models/releases/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def filter_by_semver_build(
self,
organization_id: int,
operator: str,
build: str,
build: str | Sequence[str],
project_ids: Sequence[int] | None = None,
negated: bool = False,
) -> Self:
Expand All @@ -80,16 +80,49 @@ def filter_by_semver_build(
"release_id", flat=True
)
)
if isinstance(build, str):
if build.isdecimal() and validate_bigint(int(build)):
qs = getattr(qs, query_func)(**{f"build_number__{operator}": int(build)})
else:
if not build or build.endswith("*"):
qs = getattr(qs, query_func)(build_code__startswith=build[:-1])
else:
qs = getattr(qs, query_func)(build_code=build)

if build.isdecimal() and validate_bigint(int(build)):
qs = getattr(qs, query_func)(**{f"build_number__{operator}": int(build)})
return qs
else:
if not build or build.endswith("*"):
qs = getattr(qs, query_func)(build_code__startswith=build[:-1])
build_number_filters = Q()
build_code_filters = Q()
for b in build:
if b.isdecimal() and validate_bigint(int(b)):
build_number_filters |= Q(**{f"build_number__{operator}": [int(b)]})
else:
if not b or b.endswith("*"):
build_code_filters |= Q(build_code__startswith=b[:-1])
else:
build_code_filters |= Q(build_code=b)

if build_number_filters:
qs = getattr(qs, query_func)(build_number_filters)
if build_code_filters:
qs = getattr(qs, query_func)(build_code_filters)

# Handle the case where operator is 'in' separately
if operator == "in":
return qs.filter(
Q(
build_number__in=[
int(b) for b in build if b.isdecimal() and validate_bigint(int(b))
]
)
| Q(
build_code__in=[
b for b in build if not b.isdecimal() or not validate_bigint(int(b))
]
)
)
else:
qs = getattr(qs, query_func)(build_code=build)

return qs
return qs

def filter_by_semver(
self,
Expand Down
9 changes: 8 additions & 1 deletion src/sentry/search/events/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,14 @@ class ThresholdDict(TypedDict):
"IN": "NOT IN",
"NOT IN": "IN",
}
OPERATOR_TO_DJANGO = {">=": "gte", "<=": "lte", ">": "gt", "<": "lt", "=": "exact"}
OPERATOR_TO_DJANGO = {
">=": "gte",
"<=": "lte",
">": "gt",
"<": "lt",
"=": "exact",
"IN": "in",
}

MAX_SEARCH_RELEASES = 1000
SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____"
Expand Down
4 changes: 2 additions & 2 deletions src/sentry/search/events/datasets/filter_aliases.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from functools import reduce

from snuba_sdk import Column, Condition, Function, Op
Expand Down Expand Up @@ -280,7 +280,7 @@ def semver_build_filter_converter(
"""
if builder.params.organization is None:
raise ValueError("organization is a required param")
build: str = search_filter.value.raw_value
build: str | Sequence[str] = search_filter.value.raw_value

operator, negated = handle_operator_negation(search_filter.operator)
try:
Expand Down
92 changes: 49 additions & 43 deletions src/sentry/search/events/filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,52 +385,58 @@ def _semver_filter_converter(
organization_id: int = params["organization_id"]
project_ids: list[int] | None = params.get("project_id")
# We explicitly use `raw_value` here to avoid converting wildcards to shell values
version: str = search_filter.value.raw_value
version: str | Sequence[str] = search_filter.value.raw_value
operator: str = search_filter.operator

# Note that we sort this such that if we end up fetching more than
# MAX_SEMVER_SEARCH_RELEASES, we will return the releases that are closest to
# the passed filter.
order_by = Release.SEMVER_COLS
if operator.startswith("<"):
order_by = list(map(_flip_field_sort, order_by))
qs = (
Release.objects.filter_by_semver(
organization_id,
parse_semver(version, operator),
project_ids=project_ids,
)
.values_list("version", flat=True)
.order_by(*order_by)[:MAX_SEARCH_RELEASES]
)
versions = list(qs)
final_operator = "IN"
if len(versions) == MAX_SEARCH_RELEASES:
# We want to limit how many versions we pass through to Snuba. If we've hit
# the limit, make an extra query and see whether the inverse has fewer ids.
# If so, we can do a NOT IN query with these ids instead. Otherwise, we just
# do our best.
operator = OPERATOR_NEGATION_MAP[operator]
# Note that the `order_by` here is important for index usage. Postgres seems
# to seq scan with this query if the `order_by` isn't included, so we
# include it even though we don't really care about order for this query
JoshFerge marked this conversation as resolved.
Show resolved Hide resolved
qs_flipped = (
Release.objects.filter_by_semver(organization_id, parse_semver(version, operator))
.order_by(*map(_flip_field_sort, order_by))
.values_list("version", flat=True)[:MAX_SEARCH_RELEASES]
def get_versions(v: str, op: str) -> tuple[str, list[str]]:
# Note that we sort this such that if we end up fetching more than
# MAX_SEMVER_SEARCH_RELEASES, we will return the releases that are closest to
# the passed filter.
order_by = Release.SEMVER_COLS
if op.startswith("<"):
order_by = list(map(_flip_field_sort, order_by))
qs = (
Release.objects.filter_by_semver(
organization_id,
parse_semver(v, op),
project_ids=project_ids,
)
.values_list("version", flat=True)
.order_by(*order_by)[:MAX_SEARCH_RELEASES]
)

exclude_versions = list(qs_flipped)
if exclude_versions and len(exclude_versions) < len(versions):
# Do a negative search instead
final_operator = "NOT IN"
versions = exclude_versions

if not versions:
# XXX: Just return a filter that will return no results if we have no versions
versions = [SEMVER_EMPTY_RELEASE]

return ["release", final_operator, versions]
versions = list(qs)
final_operator = "IN"
if len(versions) == MAX_SEARCH_RELEASES:
# We want to limit how many versions we pass through to Snuba. If we've hit
# the limit, make an extra query and see whether the inverse has fewer ids.
# If so, we can do a NOT IN query with these ids instead. Otherwise, we just
# do our best.
op = OPERATOR_NEGATION_MAP[op]
# Note that the `order_by` here is important for index usage. Postgres seems
# to seq scan with this query if the `order_by` isn't included, so we
# include it even though we don't really care about order for this query
qs_flipped = (
Release.objects.filter_by_semver(organization_id, parse_semver(v, op))
.order_by(*map(_flip_field_sort, order_by))
.values_list("version", flat=True)[:MAX_SEARCH_RELEASES]
)
exclude_versions = list(qs_flipped)
if exclude_versions and len(exclude_versions) < len(versions):
final_operator = "NOT IN"
versions = exclude_versions
if not versions:
versions = [SEMVER_EMPTY_RELEASE]
return final_operator, versions

if isinstance(version, str):
final_operator, versions = get_versions(version, operator)
return ["release", final_operator, versions]
else:
semver_filters = []
for v in version:
_, versions = get_versions(v, operator)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not really sure get_versions is necessary for IN. Since in will just be doing exact matches, all the sorting junk that we do isn't really relevant.

semver_filters.extend(versions)
return ["release", "IN", semver_filters]


def _semver_package_filter_converter(
Expand Down
62 changes: 60 additions & 2 deletions tests/sentry/issues/endpoints/test_organization_group_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -1376,6 +1376,7 @@ def test_semver(self, _: MagicMock) -> None:
release_1 = self.create_release(version="test@1.2.3")
release_2 = self.create_release(version="test@1.2.4")
release_3 = self.create_release(version="test@1.2.5")
release_4 = self.create_release(version="test@2.0.0")

release_1_g_1 = self.store_event(
data={
Expand Down Expand Up @@ -1425,6 +1426,14 @@ def test_semver(self, _: MagicMock) -> None:
},
project_id=self.project.id,
).group.id
release_4_g_1 = self.store_event(
data={
"timestamp": iso_format(before_now(minutes=7)),
"fingerprint": ["group-7"],
"release": release_4.version,
},
project_id=self.project.id,
).group.id
self.login_as(user=self.user)
response = self.get_response(sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:>1.2.3")
assert response.status_code == 200, response.content
Expand All @@ -1433,6 +1442,7 @@ def test_semver(self, _: MagicMock) -> None:
release_2_g_2,
release_3_g_1,
release_3_g_2,
release_4_g_1,
]

response = self.get_response(sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:>=1.2.3")
Expand All @@ -1444,6 +1454,7 @@ def test_semver(self, _: MagicMock) -> None:
release_2_g_2,
release_3_g_1,
release_3_g_2,
release_4_g_1,
]

response = self.get_response(sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:<1.2.4")
Expand All @@ -1461,6 +1472,46 @@ def test_semver(self, _: MagicMock) -> None:
release_1_g_2,
release_3_g_1,
release_3_g_2,
release_4_g_1,
]

# Test multiple semver in same filter
response = self.get_response(
sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:[1.2.3,1.2.5]"
)
assert response.status_code == 200, response.content
assert [int(r["id"]) for r in response.json()] == [
release_1_g_1,
release_1_g_2,
release_2_g_1,
release_2_g_2,
release_3_g_1,
release_3_g_2,
]

response = self.get_response(
sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:[>1.2.3,<2.0.0]"
)
assert response.status_code == 200, response.content
assert [int(r["id"]) for r in response.json()] == [
release_2_g_1,
release_2_g_2,
release_3_g_1,
release_3_g_2,
]

response = self.get_response(
sort_by="date", limit=10, query=f"{SEMVER_ALIAS}:[1.2.3,>2.0.0]"
)
assert response.status_code == 200, response.content
assert [int(r["id"]) for r in response.json()] == [
release_1_g_1,
release_1_g_2,
release_2_g_1,
release_2_g_2,
release_3_g_1,
release_3_g_2,
release_4_g_1,
]

def test_release_stage(self, _: MagicMock) -> None:
Expand Down Expand Up @@ -1653,8 +1704,15 @@ def test_semver_build(self, _: MagicMock) -> None:
release_2_g_1,
]

response = self.get_response(sort_by="date", limit=10, query=f"{SEMVER_BUILD_ALIAS}:[124]")
assert response.status_code == 400, response.content
response = self.get_response(
sort_by="date", limit=10, query=f"{SEMVER_BUILD_ALIAS}:[123,124]"
)
assert response.status_code == 200, response.content
assert [int(r["id"]) for r in response.json()] == [
release_1_g_1,
release_1_g_2,
release_2_g_1,
]

def test_aggregate_stats_regression_test(self, _: MagicMock) -> None:
self.store_event(
Expand Down
25 changes: 13 additions & 12 deletions tests/sentry/search/events/test_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,13 @@ def test(self):
self.run_test("<=", "1.2.3", "IN", [release.version])
self.run_test("=", "1.2.4", "IN", [release_2.version])

def test_semver_in_operator(self):
release = self.create_release(version="test@1.2.3")
release_2 = self.create_release(version="test@1.2.4")
self.run_test("IN", ["1.2.3", "1.2.4"], "IN", [release.version, release_2.version])
# TODO: support not in operator
# self.run_test("NOT IN", ["1.2.3", "1.2.4"], "NOT IN", [release.version, release_2.version])

def test_invert_query(self):
# Tests that flipping the query works and uses a NOT IN. Test all operators to
# make sure the inversion works correctly.
Expand Down Expand Up @@ -436,12 +443,6 @@ def test_invalid_params(self):
with pytest.raises(ValueError, match="organization_id is a required param"):
_semver_filter_converter(filter, key, {"something": 1})

filter = SearchFilter(SearchKey(key), "IN", SearchValue("sentry"))
with pytest.raises(
InvalidSearchQuery, match="Invalid operation 'IN' for semantic version filter."
):
_semver_filter_converter(filter, key, {"organization_id": 1})

def test_empty(self):
self.run_test("=", "test", "IN", [SEMVER_EMPTY_RELEASE])

Expand All @@ -456,7 +457,7 @@ def test(self):


class ParseSemverTest(unittest.TestCase):
def run_test(self, version: str, operator: str, expected: SemverFilter):
def run_test(self, version: str | list[str], operator: str, expected: SemverFilter):
semver_filter = parse_semver(version, operator)
assert semver_filter == expected

Expand All @@ -471,11 +472,6 @@ def test_invalid(self):
match=INVALID_SEMVER_MESSAGE,
):
assert parse_semver("hello", ">") is None
with pytest.raises(
InvalidSearchQuery,
match="Invalid operation 'IN' for semantic version filter.",
):
assert parse_semver("1.2.3.4", "IN") is None

def test_normal(self):
self.run_test("1", ">", SemverFilter("gt", [1, 0, 0, 0, 1, ""]))
Expand All @@ -493,6 +489,11 @@ def test_wildcard(self):
self.run_test("sentry@1.2.3.*", "=", SemverFilter("exact", [1, 2, 3], "sentry"))
self.run_test("1.X", "=", SemverFilter("exact", [1]))

def test_in(self):
self.run_test("1.2.3.4", "IN", SemverFilter("in", [1, 2, 3, 4, 1, ""]))
# TODO: support not in operator
# self.run_test("1.2.3.4", "NOT IN", SemverFilter("not in", [1, 2, 3, 4, 1, ""]))


def _cond(lhs, op, rhs):
return Condition(lhs=Column(name=lhs), op=op, rhs=rhs)
Expand Down
Loading