Merge pull request #4090 from Zac-HD/test-cleanups

Various test cleanups for Crosshair integration

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch contains some internal code cleanup.  There is no user-visible change.

hypothesis-python/setup.py

@@ -60,7 +60,7 @@ def local_file(name):
     "pytest": ["pytest>=4.6"],
     "dpcontracts": ["dpcontracts>=0.4"],
     "redis": ["redis>=3.0.0"],
-    "crosshair": ["hypothesis-crosshair>=0.0.13", "crosshair-tool>=0.0.68"],
+    "crosshair": ["hypothesis-crosshair>=0.0.13", "crosshair-tool>=0.0.70"],
     # zoneinfo is an odd one: every dependency is conditional, because they're
     # only necessary on old versions of Python or Windows systems or emscripten.
     "zoneinfo": [

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1935,14 +1935,6 @@ def permitted(f):
 }
 
 
-# eventually we'll want to expose this publicly, but for now it lives as psuedo-internal.
-def realize(value: object) -> object:
-    from hypothesis.control import current_build_context
-
-    context = current_build_context()
-    return context.data.provider.realize(value)
-
-
 class ConjectureData:
     @classmethod
     def for_buffer(

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -1112,7 +1112,7 @@ def generate_mutations_from(
                 (start1, end1), (start2, end2) = self.random.sample(sorted(group), 2)
                 if (start1 <= start2 <= end2 <= end1) or (
                     start2 <= start1 <= end1 <= end2
-                ):
+                ):  # pragma: no cover  # flaky on conjecture-cover tests
                     # one example entirely contains the other. give up.
                     # TODO use more intelligent mutation for containment, like
                     # replacing child with parent or vice versa. Would allow for

hypothesis-python/tests/array_api/test_arrays.py

@@ -187,11 +187,7 @@ def test_minimize_large_uint_arrays(xp, xps):
     example."""
     if not hasattr(xp, "nonzero"):
         pytest.skip("optional API")
-    smallest = minimal(
-        xps.arrays(xp.uint8, 100),
-        lambda x: xp.any(x) and not xp.all(x),
-        timeout_after=60,
-    )
+    smallest = minimal(xps.arrays(xp.uint8, 100), lambda x: xp.any(x) and not xp.all(x))
     assert xp.all(xp.logical_or(smallest == 0, smallest == 1))
     idx = xp.nonzero(smallest)[0]
     assert idx.size in (1, smallest.size - 1)

hypothesis-python/tests/common/debug.py

@@ -25,30 +25,21 @@ class Timeout(BaseException):
     pass
 
 
-def minimal(definition, condition=lambda x: True, settings=None, timeout_after=10):
+def minimal(definition, condition=lambda x: True, settings=None):
     definition.validate()
-    runtime = None
     result = None
 
+    def wrapped_condition(x):
+        # This sure seems pointless, but `test_sum_of_pair` fails otherwise...
+        return condition(x)
+
     if (
         context := _current_build_context.value
     ) and context.data.provider.avoid_realization:
         raise SkipTest("`minimal()` helper not supported under symbolic execution")
 
-    def wrapped_condition(x):
-        nonlocal runtime
-        if timeout_after is not None:
-            if runtime:
-                runtime += TIME_INCREMENT
-                if runtime >= timeout_after:
-                    raise Timeout
-        result = condition(x)
-        if result and not runtime:
-            runtime = 0.0
-        return result
-
     if settings is None:
-        settings = Settings(max_examples=50000, phases=(Phase.generate, Phase.shrink))
+        settings = Settings(max_examples=500, phases=(Phase.generate, Phase.shrink))
 
     verbosity = settings.verbosity
     if verbosity == Verbosity.normal:

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -18,14 +18,14 @@
 import pytest
 
 from hypothesis import given, settings, strategies as st
+from hypothesis.control import current_build_context
 from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.errors import Flaky, HypothesisException, InvalidArgument
 from hypothesis.internal.compat import int_to_bytes
 from hypothesis.internal.conjecture.data import (
     AVAILABLE_PROVIDERS,
     ConjectureData,
     PrimitiveProvider,
-    realize,
 )
 from hypothesis.internal.conjecture.engine import ConjectureRunner
 from hypothesis.internal.floats import SIGNALING_NAN
@@ -412,7 +412,7 @@ def test_realize():
         @given(st.integers())
         @settings(backend="realize")
         def test_function(n):
-            values.append(realize(n))
+            values.append(current_build_context().data.provider.realize(n))
 
         test_function()
 

hypothesis-python/tests/cover/test_core.py

@@ -80,18 +80,20 @@ def inner(x):
 
 
 def test_pytest_skip_skips_shrinking():
-    values = []
+    seen_large = False
 
     @settings(derandomize=True, max_examples=100)
     @given(s.integers())
     def inner(x):
-        values.append(x)
+        nonlocal seen_large
         if x > 100:
+            if seen_large:
+                raise Exception("Should never replay a skipped test!")
+            seen_large = True
             pytest.skip(f"{x=} is too big!")
 
     with pytest.raises(Skipped):
         inner()
-    assert len([x for x in values if x > 100]) == 1
 
 
 def test_can_find_with_db_eq_none():

hypothesis-python/tests/cover/test_datetimes.py

@@ -73,7 +73,7 @@ def test_bordering_on_a_leap_year():
             dt.datetime.min.replace(year=2003), dt.datetime.max.replace(year=2005)
         ),
         lambda x: x.month == 2 and x.day == 29,
-        timeout_after=60,
+        settings=settings(max_examples=1200),
     )
     assert x.year == 2004
 

hypothesis-python/tests/cover/test_deadline.py

@@ -53,7 +53,7 @@ def test_slow_with_none_deadline(i):
 def test_raises_flaky_if_a_test_becomes_fast_on_rerun():
     once = [True]
 
-    @settings(deadline=500)
+    @settings(deadline=500, backend="hypothesis")
     @given(st.integers())
     def test_flaky_slow(i):
         if once[0]:
@@ -82,11 +82,10 @@ def test_keeps_you_well_above_the_deadline():
     seen = set()
     failed_once = [False]
 
-    @settings(deadline=100)
+    @settings(deadline=100, backend="hypothesis")
     @given(st.integers(0, 2000))
     def slow(i):
-        # Make sure our initial failure isn't something that immediately goes
-        # flaky.
+        # Make sure our initial failure isn't something that immediately goes flaky.
         if not failed_once[0]:
             if i * 0.9 <= 100:
                 return
@@ -107,7 +106,7 @@ def slow(i):
 def test_gives_a_deadline_specific_flaky_error_message():
     once = [True]
 
-    @settings(deadline=100)
+    @settings(deadline=100, backend="hypothesis")
     @given(st.integers())
     def slow_once(i):
         if once[0]:

hypothesis-python/tests/cover/test_health_checks.py

@@ -8,13 +8,13 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import re
 import time
 
 import pytest
-from pytest import raises
 
 from hypothesis import HealthCheck, given, settings, strategies as st
-from hypothesis.control import assume
+from hypothesis.control import assume, current_build_context
 from hypothesis.errors import FailedHealthCheck, InvalidArgument
 from hypothesis.internal.compat import int_from_bytes
 from hypothesis.stateful import (
@@ -28,26 +28,28 @@
 
 from tests.common.utils import no_shrink
 
-HEALTH_CHECK_SETTINGS = settings(max_examples=11, database=None)
+HEALTH_CHECK_SETTINGS = settings(
+    max_examples=11, database=None, suppress_health_check=()
+)
 
 
 def test_slow_generation_fails_a_health_check():
-    @HEALTH_CHECK_SETTINGS
+    @settings(HEALTH_CHECK_SETTINGS, deadline=200)
     @given(st.integers().map(lambda x: time.sleep(0.2)))
     def test(x):
         pass
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test()
 
 
 def test_slow_generation_inline_fails_a_health_check():
-    @HEALTH_CHECK_SETTINGS
+    @settings(HEALTH_CHECK_SETTINGS, deadline=200)
     @given(st.data())
     def test(data):
         data.draw(st.integers().map(lambda x: time.sleep(0.2)))
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test()
 
 
@@ -75,7 +77,7 @@ def unhealthy_filter(x):
     def test1(x):
         raise ValueError
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test1()
 
     forbidden = set()
@@ -85,7 +87,7 @@ def test1(x):
     def test2(x):
         raise ValueError
 
-    with raises(ValueError):
+    with pytest.raises(ValueError):
         test2()
 
 
@@ -95,9 +97,8 @@ def test_filtering_everything_fails_a_health_check():
     def test(x):
         pass
 
-    with raises(FailedHealthCheck) as e:
+    with pytest.raises(FailedHealthCheck, match="filter"):
         test()
-    assert "filter" in e.value.args[0]
 
 
 class fails_regularly(SearchStrategy):
@@ -109,21 +110,21 @@ def do_draw(self, data):
 
 def test_filtering_most_things_fails_a_health_check():
     @given(fails_regularly())
-    @settings(database=None, phases=no_shrink)
+    @settings(database=None, phases=no_shrink, suppress_health_check=())
     def test(x):
-        pass
+        if current_build_context().data.provider.avoid_realization:
+            pytest.skip("symbolic backends can filter efficiently!")
 
-    with raises(FailedHealthCheck) as e:
+    with pytest.raises(FailedHealthCheck, match="filter"):
         test()
-    assert "filter" in e.value.args[0]
 
 
 def test_returning_non_none_is_forbidden():
     @given(st.integers())
     def a(x):
         return 1
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         a()
 
 
@@ -153,19 +154,18 @@ def test(self, _):
 
 def test_differing_executors_fails_health_check():
     sample_test_runner().test()
-    with pytest.raises(FailedHealthCheck) as exc:
+    msg = re.escape(str(HealthCheck.differing_executors))
+    with pytest.raises(FailedHealthCheck, match=msg):
         sample_test_runner().test()
 
-    assert str(HealthCheck.differing_executors) in str(exc.value)
-
 
 def test_it_is_an_error_to_suppress_non_iterables():
-    with raises(InvalidArgument):
+    with pytest.raises(InvalidArgument):
         settings(suppress_health_check=1)
 
 
 def test_it_is_an_error_to_suppress_non_healthchecks():
-    with raises(InvalidArgument):
+    with pytest.raises(InvalidArgument):
         settings(suppress_health_check=[1])
 
 

hypothesis-python/tests/cover/test_targeting.py

@@ -13,6 +13,7 @@
 import pytest
 
 from hypothesis import example, given, strategies as st, target
+from hypothesis.control import current_build_context
 from hypothesis.errors import InvalidArgument
 
 
@@ -92,6 +93,8 @@ def test_cannot_target_outside_test():
 
 @given(st.none())
 def test_cannot_target_same_label_twice(_):
+    if current_build_context().data.provider.avoid_realization:
+        pytest.skip("target() is a noop to avoid realizing arguments")
     target(0.0, label="label")
     with pytest.raises(InvalidArgument):
         target(1.0, label="label")

hypothesis-python/tests/nocover/test_randomization.py

@@ -8,7 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-from pytest import raises
+import pytest
 
 from hypothesis import Verbosity, core, find, given, settings, strategies as st
 
@@ -35,7 +35,7 @@ def test_blah(x, rnd):
         def test_nest(y):
             assert y < x
 
-        with raises(AssertionError):
+        with pytest.raises(AssertionError):
             test_nest()
 
     test_blah()

hypothesis-python/tests/nocover/test_recursive.py

@@ -34,7 +34,6 @@ def flatten(x):
             max_leaves=size * 2,
         ),
         lambda x: isinstance(x, list) and len(flatten(x)) >= size,
-        timeout_after=None,
     )
     assert flatten(xs) == [0] * size
 
@@ -46,11 +45,7 @@ def depth(x):
         else:
             return 1
 
-    xs = minimal(
-        st.recursive(st.integers(), st.lists),
-        lambda x: depth(x) > 1,
-        timeout_after=None,
-    )
+    xs = minimal(st.recursive(st.integers(), st.lists), lambda x: depth(x) > 1)
     assert xs in ([0], [[]])
 
 
@@ -67,7 +62,6 @@ def breadth(x):
         st.recursive(st.booleans(), lambda x: st.lists(x, max_size=target // 2)),
         lambda x: breadth(x) >= target,
         settings=settings(max_examples=10000),
-        timeout_after=None,
     )
     assert breadth(broad) == target
 
@@ -79,7 +73,7 @@ def test_drawing_many_near_boundary():
         lambda x: st.lists(x, min_size=2 * (size - 1), max_size=2 * size).map(tuple),
         max_leaves=2 * size - 1,
     )
-    ls = minimal(st.lists(elems), lambda x: len(set(x)) >= size, timeout_after=None)
+    ls = minimal(st.lists(elems), lambda x: len(set(x)) >= size)
     assert len(ls) == size
 
 
@@ -117,7 +111,7 @@ def test_can_form_sets_of_recursive_data():
             max_leaves=20,
         )
     )
-    xs = minimal(trees, lambda x: len(x) >= size, timeout_after=None)
+    xs = minimal(trees, lambda x: len(x) >= size)
     assert len(xs) == size