diff --git a/codecov.yml b/codecov.yml index c2441c970..6ca30f640 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,23 +1,25 @@ ignore: - - "newrelic/packages/**/*" - - "newrelic/packages/*" - "newreilc/hooks/component_sentry.py" - - "newrelic/hooks/adapter_meinheld.py" + - "newrelic/admin/*" + - "newrelic/console.py" - "newrelic/hooks/adapter_flup.py" + - "newrelic/hooks/adapter_meinheld.py" - "newrelic/hooks/adapter_paste.py" - "newrelic/hooks/component_piston.py" + - "newrelic/hooks/database_oursql.py" + - "newrelic/hooks/database_psycopg2ct.py" + - "newrelic/hooks/datastore_aioredis.py" + - "newrelic/hooks/datastore_aredis.py" + - "newrelic/hooks/datastore_motor.py" - "newrelic/hooks/datastore_pyelasticsearch.py" - - "newrelic/hooks/external_pywapi.py" + - "newrelic/hooks/datastore_umemcache.py" - "newrelic/hooks/external_dropbox.py" - "newrelic/hooks/external_facepy.py" + - "newrelic/hooks/external_pywapi.py" - "newrelic/hooks/external_xmlrpclib.py" - "newrelic/hooks/framework_pylons.py" - "newrelic/hooks/framework_web2py.py" - - "newrelic/hooks/middleware_weberror.py" - "newrelic/hooks/framework_webpy.py" - - "newrelic/hooks/datastore_motor.py" - - "newrelic/hooks/database_oursql.py" - - "newrelic/hooks/database_psycopg2ct.py" - - "newrelic/hooks/datastore_umemcache.py" - - "newrelic/admin/*" - - "newrelic/console.py" + - "newrelic/hooks/middleware_weberror.py" + - "newrelic/packages/*" + - "newrelic/packages/**/*" diff --git a/newrelic/config.py b/newrelic/config.py index c72b7fdf9..f1304d7b3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2690,12 +2690,14 @@ def _process_module_builtin_defaults(): "aioredis.connection", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_connection" ) + # Redis v4.2+ _process_module_definition( - "redis.asyncio.client", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client" + "redis.asyncio.client", "newrelic.hooks.datastore_redis", "instrument_asyncio_redis_client" ) + # Redis v4.2+ _process_module_definition( - "redis.asyncio.commands", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client" + "redis.asyncio.commands", "newrelic.hooks.datastore_redis", "instrument_asyncio_redis_client" ) _process_module_definition( diff --git a/newrelic/hooks/datastore_aioredis.py b/newrelic/hooks/datastore_aioredis.py index 47c987971..df79acd75 100644 --- a/newrelic/hooks/datastore_aioredis.py +++ b/newrelic/hooks/datastore_aioredis.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from newrelic.api.datastore_trace import DatastoreTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction @@ -60,7 +59,13 @@ def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): # Method will return synchronously without executing, # it will be added to the command stack and run later. aioredis_version = get_package_version_tuple("aioredis") - if aioredis_version and aioredis_version < (2,): + + # This conditional is for versions of aioredis that are outside + # New Relic's supportability window but will still work. New + # Relic does not provide testing/support for this. In order to + # keep functionality without affecting coverage metrics, this + # segment is excluded from coverage analysis. + if aioredis_version and aioredis_version < (2,): # pragma: no cover # AioRedis v1 uses a RedisBuffer instead of a real connection for queueing up pipeline commands from aioredis.commands.transaction import _RedisBuffer @@ -72,8 +77,6 @@ def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): # AioRedis v2 uses a Pipeline object for a client and internally queues up pipeline commands if aioredis_version: from aioredis.client import Pipeline - else: - from redis.asyncio.client import Pipeline if isinstance(instance, Pipeline): return wrapped(*args, **kwargs) @@ -137,7 +140,12 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): return await wrapped(*args, **kwargs) -def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): +# This wrapper is for versions of aioredis that are outside +# New Relic's supportability window but will still work. New +# Relic does not provide testing/support for this. In order to +# keep functionality without affecting coverage metrics, this +# segment is excluded from coverage analysis. +def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): # pragma: no cover # RedisConnection in aioredis v1 returns a future instead of using coroutines transaction = current_transaction() if not transaction: @@ -205,6 +213,11 @@ def instrument_aioredis_connection(module): if hasattr(module.Connection, "send_command"): wrap_function_wrapper(module, "Connection.send_command", wrap_Connection_send_command) - if hasattr(module, "RedisConnection"): + # This conditional is for versions of aioredis that are outside + # New Relic's supportability window but will still work. New + # Relic does not provide testing/support for this. In order to + # keep functionality without affecting coverage metrics, this + # segment is excluded from coverage analysis. + if hasattr(module, "RedisConnection"): # pragma: no cover if hasattr(module.RedisConnection, "execute"): wrap_function_wrapper(module, "RedisConnection.execute", wrap_RedisConnection_execute) diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index 26ab2f5c7..0f1c522b7 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -16,12 +16,67 @@ from newrelic.api.datastore_trace import DatastoreTrace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper -_redis_client_methods = { +_redis_client_sync_methods = { + "acl_dryrun", + "auth", + "bgrewriteaof", + "bitfield", + "blmpop", + "bzmpop", + "client", + "command", + "command_docs", + "command_getkeysandflags", + "command_info", + "debug_segfault", + "expiretime", + "failover", + "hello", + "latency_doctor", + "latency_graph", + "latency_histogram", + "lcs", + "lpop", + "lpos", + "memory_doctor", + "memory_help", + "monitor", + "pexpiretime", + "psetex", + "psync", + "pubsub", + "renamenx", + "rpop", + "script_debug", + "sentinel_ckquorum", + "sentinel_failover", + "sentinel_flushconfig", + "sentinel_get_master_addr_by_name", + "sentinel_master", + "sentinel_masters", + "sentinel_monitor", + "sentinel_remove", + "sentinel_reset", + "sentinel_sentinels", + "sentinel_set", + "sentinel_slaves", + "shutdown", + "sort", + "sort_ro", + "spop", + "srandmember", + "unwatch", + "watch", + "zlexcount", + "zrevrangebyscore", +} + + +_redis_client_async_methods = { "acl_cat", "acl_deluser", - "acl_dryrun", "acl_genpass", "acl_getuser", "acl_help", @@ -50,11 +105,8 @@ "arrlen", "arrpop", "arrtrim", - "auth", - "bgrewriteaof", "bgsave", "bitcount", - "bitfield", "bitfield_ro", "bitop_and", "bitop_not", @@ -63,13 +115,11 @@ "bitop", "bitpos", "blmove", - "blmpop", "blpop", "brpop", "brpoplpush", "byrank", "byrevrank", - "bzmpop", "bzpopmax", "bzpopmin", "card", @@ -90,7 +140,6 @@ "client_trackinginfo", "client_unblock", "client_unpause", - "client", "cluster_add_slots", "cluster_addslots", "cluster_count_failure_report", @@ -112,15 +161,13 @@ "cluster_reset", "cluster_save_config", "cluster_set_config_epoch", + "client_setinfo", "cluster_setslot", "cluster_slaves", "cluster_slots", "cluster", "command_count", - "command_docs", "command_getkeys", - "command_getkeysandflags", - "command_info", "command_list", "command", "commit", @@ -136,7 +183,6 @@ "createrule", "dbsize", "debug_object", - "debug_segfault", "debug_sleep", "debug", "decr", @@ -159,10 +205,8 @@ "exists", "expire", "expireat", - "expiretime", "explain_cli", "explain", - "failover", "fcall_ro", "fcall", "flushall", @@ -176,6 +220,7 @@ "function_load", "function_restore", "function_stats", + "gears_refresh_cluster", "geoadd", "geodist", "geohash", @@ -191,7 +236,6 @@ "getrange", "getset", "hdel", - "hello", "hexists", "hget", "hgetall", @@ -219,13 +263,9 @@ "insertnx", "keys", "lastsave", - "latency_doctor", - "latency_graph", - "latency_histogram", "latency_history", "latency_latest", "latency_reset", - "lcs", "lindex", "linsert", "list", @@ -234,8 +274,6 @@ "lmpop", "loadchunk", "lolwut", - "lpop", - "lpos", "lpush", "lpushx", "lrange", @@ -244,8 +282,6 @@ "ltrim", "madd", "max", - "memory_doctor", - "memory_help", "memory_malloc_stats", "memory_purge", "memory_stats", @@ -260,7 +296,6 @@ "module_load", "module_loadex", "module_unload", - "monitor", "move", "mrange", "mrevrange", @@ -276,21 +311,19 @@ "persist", "pexpire", "pexpireat", - "pexpiretime", "pfadd", "pfcount", "pfmerge", "ping", "profile", - "psetex", "psubscribe", - "psync", "pttl", "publish", "pubsub_channels", "pubsub_numpat", "pubsub_numsub", - "pubsub", + "pubsub_shardchannels", + "pubsub_shardnumsub", "punsubscribe", "quantile", "query", @@ -302,7 +335,6 @@ "readonly", "readwrite", "rename", - "renamenx", "replicaof", "reserve", "reset", @@ -311,7 +343,6 @@ "revrange", "revrank", "role", - "rpop", "rpoplpush", "rpush", "rpushx", @@ -321,7 +352,6 @@ "scan", "scandump", "scard", - "script_debug", "script_exists", "script_flush", "script_kill", @@ -330,24 +360,11 @@ "sdiffstore", "search", "select", - "sentinel_ckquorum", - "sentinel_failover", - "sentinel_flushconfig", - "sentinel_get_master_addr_by_name", - "sentinel_master", - "sentinel_masters", - "sentinel_monitor", - "sentinel_remove", - "sentinel_reset", - "sentinel_sentinels", - "sentinel_set", - "sentinel_slaves", "set", "setbit", "setex", "setnx", "setrange", - "shutdown", "sinter", "sintercard", "sinterstore", @@ -360,11 +377,8 @@ "smembers", "smismember", "smove", - "sort_ro", - "sort", "spellcheck", - "spop", - "srandmember", + "spublish", "srem", "sscan_iter", "sscan", @@ -384,6 +398,11 @@ "syndump", "synupdate", "tagvals", + "tfcall", + "tfcall_async", + "tfunction_delete", + "tfunction_list", + "tfunction_load", "time", "toggle", "touch", @@ -392,10 +411,8 @@ "type", "unlink", "unsubscribe", - "unwatch", "wait", "waitaof", - "watch", "xack", "xadd", "xautoclaim", @@ -431,7 +448,6 @@ "zinter", "zintercard", "zinterstore", - "zlexcount", "zmpop", "zmscore", "zpopmax", @@ -448,7 +464,6 @@ "zremrangebyscore", "zrevrange", "zrevrangebylex", - "zrevrangebyscore", "zrevrank", "zscan_iter", "zscan", @@ -457,6 +472,8 @@ "zunionstore", } +_redis_client_methods = _redis_client_sync_methods.union(_redis_client_async_methods) + _redis_multipart_commands = set(["client", "cluster", "command", "config", "debug", "sentinel", "slowlog", "script"]) _redis_operation_re = re.compile(r"[-\s]+") @@ -504,6 +521,29 @@ def _nr_wrapper_Redis_method_(wrapped, instance, args, kwargs): wrap_function_wrapper(module, name, _nr_wrapper_Redis_method_) +def _wrap_asyncio_Redis_method_wrapper(module, instance_class_name, operation): + @function_wrapper + async def _nr_wrapper_asyncio_Redis_async_method_(wrapped, instance, args, kwargs): + transaction = current_transaction() + if transaction is None: + return await wrapped(*args, **kwargs) + + with DatastoreTrace(product="Redis", target=None, operation=operation): + return await wrapped(*args, **kwargs) + + def _nr_wrapper_asyncio_Redis_method_(wrapped, instance, args, kwargs): + from redis.asyncio.client import Pipeline + + if isinstance(instance, Pipeline): + return wrapped(*args, **kwargs) + + # Method should be run when awaited, therefore we wrap in an async wrapper. + return _nr_wrapper_asyncio_Redis_async_method_(wrapped)(*args, **kwargs) + + name = "%s.%s" % (instance_class_name, operation) + wrap_function_wrapper(module, name, _nr_wrapper_asyncio_Redis_method_) + + def _nr_Connection_send_command_wrapper_(wrapped, instance, args, kwargs): transaction = current_transaction() @@ -565,6 +605,14 @@ def instrument_redis_client(module): _wrap_Redis_method_wrapper_(module, "Redis", name) +def instrument_asyncio_redis_client(module): + if hasattr(module, "Redis"): + class_ = getattr(module, "Redis") + for operation in _redis_client_async_methods: + if hasattr(class_, operation): + _wrap_asyncio_Redis_method_wrapper(module, "Redis", operation) + + def instrument_redis_commands_core(module): _instrument_redis_commands_module(module, "CoreCommands") @@ -597,6 +645,10 @@ def instrument_redis_commands_bf_commands(module): _instrument_redis_commands_module(module, "TOPKCommands") +def instrument_redis_commands_cluster(module): + _instrument_redis_commands_module(module, "RedisClusterCommands") + + def _instrument_redis_commands_module(module, class_name): for name in _redis_client_methods: if hasattr(module, class_name): diff --git a/setup.cfg b/setup.cfg index 453a10eeb..006265c36 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,4 +5,4 @@ license_files = [flake8] max-line-length = 120 -extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504 +extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504,E731 diff --git a/tests/datastore_aioredis/test_execute_command.py b/tests/datastore_aioredis/test_execute_command.py index 54851a659..b600abea5 100644 --- a/tests/datastore_aioredis/test_execute_command.py +++ b/tests/datastore_aioredis/test_execute_command.py @@ -13,8 +13,6 @@ # limitations under the License. import pytest - -# import aioredis from conftest import AIOREDIS_VERSION, loop # noqa # pylint: disable=E0611,W0611 from testing_support.db_settings import redis_settings from testing_support.fixtures import override_application_settings diff --git a/tests/datastore_redis/test_asyncio.py b/tests/datastore_redis/test_asyncio.py new file mode 100644 index 000000000..f17d6b2e0 --- /dev/null +++ b/tests/datastore_redis/test_asyncio.py @@ -0,0 +1,114 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import pytest +from testing_support.db_settings import redis_settings +from testing_support.fixture.event_loop import event_loop as loop # noqa: F401 +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +# Settings + +DB_SETTINGS = redis_settings()[0] +REDIS_PY_VERSION = get_package_version_tuple("redis") + +# Metrics + +_base_scoped_metrics = [("Datastore/operation/Redis/publish", 3)] + +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_rollup_metrics = [ + ("Datastore/all", datastore_all_metric_count), + ("Datastore/allOther", datastore_all_metric_count), + ("Datastore/Redis/all", datastore_all_metric_count), + ("Datastore/Redis/allOther", datastore_all_metric_count), + ("Datastore/operation/Redis/publish", 3), + ( + "Datastore/instance/Redis/%s/%s" % (instance_hostname(DB_SETTINGS["host"]), DB_SETTINGS["port"]), + datastore_all_metric_count, + ), +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +# Tests + + +@pytest.fixture() +def client(loop): # noqa + import redis.asyncio + + return loop.run_until_complete(redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="This functionality exists in Redis 4.2+") +@validate_transaction_metrics("test_asyncio:test_async_pipeline", background_task=True) +@background_task() +def test_async_pipeline(client, loop): # noqa + async def _test_pipeline(client): + async with client.pipeline(transaction=True) as pipe: + await pipe.set("key1", "value1") + await pipe.execute() + + loop.run_until_complete(_test_pipeline(client)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="This functionality exists in Redis 4.2+") +@validate_transaction_metrics( + "test_asyncio:test_async_pubsub", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_base_rollup_metrics, + background_task=True, +) +@background_task() +def test_async_pubsub(client, loop): # noqa + messages_received = [] + + async def reader(pubsub): + while True: + message = await pubsub.get_message(ignore_subscribe_messages=True) + if message: + messages_received.append(message["data"].decode()) + if message["data"].decode() == "NOPE": + break + + async def _test_pubsub(): + async with client.pubsub() as pubsub: + await pubsub.psubscribe("channel:*") + + future = asyncio.create_task(reader(pubsub)) + + await client.publish("channel:1", "Hello") + await client.publish("channel:2", "World") + await client.publish("channel:1", "NOPE") + + await future + + loop.run_until_complete(_test_pubsub()) + assert messages_received == ["Hello", "World", "NOPE"] diff --git a/tests/datastore_redis/test_custom_conn_pool.py b/tests/datastore_redis/test_custom_conn_pool.py index 156c9ce31..3da24f454 100644 --- a/tests/datastore_redis/test_custom_conn_pool.py +++ b/tests/datastore_redis/test_custom_conn_pool.py @@ -12,23 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -''' The purpose of these tests is to confirm that using a non-standard +""" The purpose of these tests is to confirm that using a non-standard connection pool that does not have a `connection_kwargs` attribute will not result in an error. -''' +""" import pytest import redis - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") class FakeConnectionPool(object): @@ -43,112 +45,117 @@ def get_connection(self, name, *keys, **options): def release(self, connection): self.connection.disconnect() + # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics # We don't record instance metrics when using redis blaster, # so we just check for base metrics. - -_base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) - -_base_rollup_metrics = ( - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Redis/all', 3), - ('Datastore/Redis/allOther', 3), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) - -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) - -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] - -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) - -_enable_rollup_metrics.append( - (_instance_metric_name, 3) -) - -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_scoped_metrics = [ + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +_base_rollup_metrics = [ + ("Datastore/all", datastore_all_metric_count), + ("Datastore/allOther", datastore_all_metric_count), + ("Datastore/Redis/all", datastore_all_metric_count), + ("Datastore/Redis/allOther", datastore_all_metric_count), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) + +instance_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_enable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, instance_metric_count)) + +_disable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, None)) # Operations + def exercise_redis(client): - client.set('key', 'value') - client.get('key') - client.execute_command('CLIENT', 'LIST', parse='LIST') + client.set("key", "value") + client.get("key") + client.execute_command("CLIENT", "LIST", parse="LIST") + # Tests -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), - reason='Client list command introduced in 2.7') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), reason="Client list command introduced in 2.7") @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_custom_conn_pool:test_fake_conn_pool_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_custom_conn_pool:test_fake_conn_pool_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_fake_conn_pool_enable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) # Get a real connection - conn = client.connection_pool.get_connection('GET') + conn = client.connection_pool.get_connection("GET") # Replace the original connection pool with one that doesn't # have the `connection_kwargs` attribute. fake_pool = FakeConnectionPool(conn) client.connection_pool = fake_pool - assert not hasattr(client.connection_pool, 'connection_kwargs') + assert not hasattr(client.connection_pool, "connection_kwargs") exercise_redis(client) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), - reason='Client list command introduced in 2.7') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), reason="Client list command introduced in 2.7") @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_custom_conn_pool:test_fake_conn_pool_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_custom_conn_pool:test_fake_conn_pool_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_fake_conn_pool_disable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) # Get a real connection - conn = client.connection_pool.get_connection('GET') + conn = client.connection_pool.get_connection("GET") # Replace the original connection pool with one that doesn't # have the `connection_kwargs` attribute. fake_pool = FakeConnectionPool(conn) client.connection_pool = fake_pool - assert not hasattr(client.connection_pool, 'connection_kwargs') + assert not hasattr(client.connection_pool, "connection_kwargs") exercise_redis(client) diff --git a/tests/datastore_redis/test_execute_command.py b/tests/datastore_redis/test_execute_command.py index 747588072..99a5ed22f 100644 --- a/tests/datastore_redis/test_execute_command.py +++ b/tests/datastore_redis/test_execute_command.py @@ -14,165 +14,174 @@ import pytest import redis - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") + # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics -_base_scoped_metrics = ( - ('Datastore/operation/Redis/client_list', 1), -) +_base_scoped_metrics = [ + ("Datastore/operation/Redis/client_list", 1), +] +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) -_base_rollup_metrics = ( - ('Datastore/all', 1), - ('Datastore/allOther', 1), - ('Datastore/Redis/all', 1), - ('Datastore/Redis/allOther', 1), - ('Datastore/operation/Redis/client_list', 1), -) +_base_rollup_metrics = [ + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Redis/all", 3), + ("Datastore/Redis/allOther", 3), + ("Datastore/operation/Redis/client_list", 1), +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) +_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] +instance_metric_count = 3 if REDIS_PY_VERSION >= (5, 0) else 1 -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_enable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, instance_metric_count)) -_enable_rollup_metrics.append( - (_instance_metric_name, 1) -) +_disable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, None)) -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) def exercise_redis_multi_args(client): - client.execute_command('CLIENT', 'LIST', parse='LIST') + client.execute_command("CLIENT", "LIST", parse="LIST") + def exercise_redis_single_arg(client): - client.execute_command('CLIENT LIST') + client.execute_command("CLIENT LIST") + @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_strict_redis_execute_command_two_args_enable', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_execute_command:test_strict_redis_execute_command_two_args_enable", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_execute_command_two_args_enable(): - r = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_multi_args(r) + @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_strict_redis_execute_command_two_args_disabled', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_execute_command:test_strict_redis_execute_command_two_args_disabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_execute_command_two_args_disabled(): - r = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_multi_args(r) + @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_redis_execute_command_two_args_enable', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_execute_command:test_redis_execute_command_two_args_enable", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_execute_command_two_args_enable(): - r = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_multi_args(r) + @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_redis_execute_command_two_args_disabled', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_execute_command:test_redis_execute_command_two_args_disabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_execute_command_two_args_disabled(): - r = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_multi_args(r) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), - reason='This command is not implemented yet') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), reason="This command is not implemented yet") @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_strict_redis_execute_command_as_one_arg_enable', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_execute_command:test_strict_redis_execute_command_as_one_arg_enable", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_execute_command_as_one_arg_enable(): - r = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_single_arg(r) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), - reason='This command is not implemented yet') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), reason="This command is not implemented yet") @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_strict_redis_execute_command_as_one_arg_disabled', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_execute_command:test_strict_redis_execute_command_as_one_arg_disabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_execute_command_as_one_arg_disabled(): - r = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_single_arg(r) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), - reason='This command is not implemented yet') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), reason="This command is not implemented yet") @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_redis_execute_command_as_one_arg_enable', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_execute_command:test_redis_execute_command_as_one_arg_enable", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_execute_command_as_one_arg_enable(): - r = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_single_arg(r) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), - reason='This command is not implemented yet') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10), reason="This command is not implemented yet") @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_execute_command:test_redis_execute_command_as_one_arg_disabled', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_execute_command:test_redis_execute_command_as_one_arg_disabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_execute_command_as_one_arg_disabled(): - r = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + r = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis_single_arg(r) diff --git a/tests/datastore_redis/test_get_and_set.py b/tests/datastore_redis/test_get_and_set.py index 0e2df4bb1..92e6ef945 100644 --- a/tests/datastore_redis/test_get_and_set.py +++ b/tests/datastore_redis/test_get_and_set.py @@ -13,112 +13,112 @@ # limitations under the License. import redis - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task DB_SETTINGS = redis_settings()[0] # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics _base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), ) _base_rollup_metrics = ( - ('Datastore/all', 2), - ('Datastore/allOther', 2), - ('Datastore/Redis/all', 2), - ('Datastore/Redis/allOther', 2), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + ("Datastore/Redis/all", 2), + ("Datastore/Redis/allOther", 2), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), ) -_disable_scoped_metrics = list(_base_scoped_metrics) _disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) _enable_rollup_metrics = list(_base_rollup_metrics) -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) -_enable_rollup_metrics.append( - (_instance_metric_name, 2) -) +_enable_rollup_metrics.append((_instance_metric_name, 2)) -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) +_disable_rollup_metrics.append((_instance_metric_name, None)) # Operations + def exercise_redis(client): - client.set('key', 'value') - client.get('key') + client.set("key", "value") + client.get("key") + # Tests + @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_get_and_set:test_strict_redis_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_get_and_set:test_strict_redis_operation_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_operation_enable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis(client) + @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_get_and_set:test_strict_redis_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_get_and_set:test_strict_redis_operation_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_strict_redis_operation_disable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis(client) + @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_get_and_set:test_redis_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_get_and_set:test_redis_operation_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_operation_enable_instance(): - client = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis(client) + @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_get_and_set:test_redis_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_get_and_set:test_redis_operation_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_operation_disable_instance(): - client = redis.Redis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) exercise_redis(client) diff --git a/tests/datastore_redis/test_instance_info.py b/tests/datastore_redis/test_instance_info.py index b3e9a0d5d..211e96169 100644 --- a/tests/datastore_redis/test_instance_info.py +++ b/tests/datastore_redis/test_instance_info.py @@ -15,9 +15,10 @@ import pytest import redis +from newrelic.common.package_version_utils import get_package_version_tuple from newrelic.hooks.datastore_redis import _conn_attrs_to_dict, _instance_info -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") _instance_info_tests = [ ((), {}, ("localhost", "6379", "0")), diff --git a/tests/datastore_redis/test_multiple_dbs.py b/tests/datastore_redis/test_multiple_dbs.py index 15777cc38..7d3dc75cb 100644 --- a/tests/datastore_redis/test_multiple_dbs.py +++ b/tests/datastore_redis/test_multiple_dbs.py @@ -14,106 +14,130 @@ import pytest import redis - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_MULTIPLE_SETTINGS = redis_settings() +REDIS_PY_VERSION = get_package_version_tuple("redis") + # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics -_base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) +_base_scoped_metrics = [ + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_rollup_metrics = [ + ("Datastore/all", datastore_all_metric_count), + ("Datastore/allOther", datastore_all_metric_count), + ("Datastore/Redis/all", datastore_all_metric_count), + ("Datastore/Redis/allOther", datastore_all_metric_count), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] + +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) -_base_rollup_metrics = ( - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Redis/all', 3), - ('Datastore/Redis/allOther', 3), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) - -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) if len(DB_MULTIPLE_SETTINGS) > 1: redis_1 = DB_MULTIPLE_SETTINGS[0] redis_2 = DB_MULTIPLE_SETTINGS[1] - host_1 = instance_hostname(redis_1['host']) - port_1 = redis_1['port'] + host_1 = instance_hostname(redis_1["host"]) + port_1 = redis_1["port"] - host_2 = instance_hostname(redis_2['host']) - port_2 = redis_2['port'] + host_2 = instance_hostname(redis_2["host"]) + port_2 = redis_2["port"] - instance_metric_name_1 = 'Datastore/instance/Redis/%s/%s' % (host_1, port_1) - instance_metric_name_2 = 'Datastore/instance/Redis/%s/%s' % (host_2, port_2) + instance_metric_name_1 = "Datastore/instance/Redis/%s/%s" % (host_1, port_1) + instance_metric_name_2 = "Datastore/instance/Redis/%s/%s" % (host_2, port_2) - _enable_rollup_metrics.extend([ - (instance_metric_name_1, 2), - (instance_metric_name_2, 1), - ]) + instance_metric_name_1_count = 2 if REDIS_PY_VERSION >= (5, 0) else 2 + instance_metric_name_2_count = 3 if REDIS_PY_VERSION >= (5, 0) else 1 - _disable_rollup_metrics.extend([ + _enable_rollup_metrics = _base_rollup_metrics.extend( + [ + (instance_metric_name_1, instance_metric_name_1_count), + (instance_metric_name_2, instance_metric_name_2_count), + ] + ) + + _disable_rollup_metrics = _base_rollup_metrics.extend( + [ (instance_metric_name_1, None), (instance_metric_name_2, None), - ]) + ] + ) + def exercise_redis(client_1, client_2): - client_1.set('key', 'value') - client_1.get('key') + client_1.set("key", "value") + client_1.get("key") - client_2.execute_command('CLIENT', 'LIST', parse='LIST') + client_2.execute_command("CLIENT", "LIST", parse="LIST") -@pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, - reason='Test environment not configured with multiple databases.') + +@pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, reason="Test environment not configured with multiple databases.") @override_application_settings(_enable_instance_settings) -@validate_transaction_metrics('test_multiple_dbs:test_multiple_datastores_enabled', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) +@validate_transaction_metrics( + "test_multiple_dbs:test_multiple_datastores_enabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_multiple_datastores_enabled(): redis1 = DB_MULTIPLE_SETTINGS[0] redis2 = DB_MULTIPLE_SETTINGS[1] - client_1 = redis.StrictRedis(host=redis1['host'], port=redis1['port'], db=0) - client_2 = redis.StrictRedis(host=redis2['host'], port=redis2['port'], db=1) + client_1 = redis.StrictRedis(host=redis1["host"], port=redis1["port"], db=0) + client_2 = redis.StrictRedis(host=redis2["host"], port=redis2["port"], db=1) exercise_redis(client_1, client_2) -@pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, - reason='Test environment not configured with multiple databases.') + +@pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, reason="Test environment not configured with multiple databases.") @override_application_settings(_disable_instance_settings) -@validate_transaction_metrics('test_multiple_dbs:test_multiple_datastores_disabled', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) +@validate_transaction_metrics( + "test_multiple_dbs:test_multiple_datastores_disabled", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_multiple_datastores_disabled(): redis1 = DB_MULTIPLE_SETTINGS[0] redis2 = DB_MULTIPLE_SETTINGS[1] - client_1 = redis.StrictRedis(host=redis1['host'], port=redis1['port'], db=0) - client_2 = redis.StrictRedis(host=redis2['host'], port=redis2['port'], db=1) + client_1 = redis.StrictRedis(host=redis1["host"], port=redis1["port"], db=0) + client_2 = redis.StrictRedis(host=redis2["host"], port=redis2["port"], db=1) exercise_redis(client_1, client_2) diff --git a/tests/datastore_redis/test_rb.py b/tests/datastore_redis/test_rb.py index 5678c2787..8915b4e96 100644 --- a/tests/datastore_redis/test_rb.py +++ b/tests/datastore_redis/test_rb.py @@ -12,34 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -''' The purpose of these tests is to confirm that we will record +""" The purpose of these tests is to confirm that we will record record instance info for Redis Blaster commands that go through redis.Connection:send_command(). Commands that don't use send_command, like the one that use the fanout client, won't have instance info. -''' +""" import pytest -import redis import six - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics @@ -48,90 +49,78 @@ # so we just check for base metrics. _base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), ) _base_rollup_metrics = ( - ('Datastore/all', 2), - ('Datastore/allOther', 2), - ('Datastore/Redis/all', 2), - ('Datastore/Redis/allOther', 2), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + ("Datastore/Redis/all", 2), + ("Datastore/Redis/allOther", 2), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), ) -_disable_scoped_metrics = list(_base_scoped_metrics) _disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) _enable_rollup_metrics = list(_base_rollup_metrics) -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) -_enable_rollup_metrics.append( - (_instance_metric_name, 2) -) +_enable_rollup_metrics.append((_instance_metric_name, 2)) -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) +_disable_rollup_metrics.append((_instance_metric_name, None)) -# Operations +# Operations def exercise_redis(routing_client): - routing_client.set('key', 'value') - routing_client.get('key') + routing_client.set("key", "value") + routing_client.get("key") + def exercise_fanout(cluster): - with cluster.fanout(hosts='all') as client: - client.execute_command('CLIENT', 'LIST') + with cluster.fanout(hosts="all") as client: + client.execute_command("CLIENT", "LIST") -# Tests -@pytest.mark.skipif(six.PY3, reason='Redis Blaster is Python 2 only.') -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10, 2), - reason='Redis Blaster requires redis>=2.10.2') +# Tests +@pytest.mark.skipif(six.PY3, reason="Redis Blaster is Python 2 only.") +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10, 2), reason="Redis Blaster requires redis>=2.10.2") @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_rb:test_redis_blaster_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_rb:test_redis_blaster_operation_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_blaster_operation_enable_instance(): from rb import Cluster - cluster = Cluster( - hosts={0: {'port': DB_SETTINGS['port']}}, - host_defaults={'host': DB_SETTINGS['host']} - ) + cluster = Cluster(hosts={0: {"port": DB_SETTINGS["port"]}}, host_defaults={"host": DB_SETTINGS["host"]}) exercise_fanout(cluster) client = cluster.get_routing_client() exercise_redis(client) -@pytest.mark.skipif(six.PY3, reason='Redis Blaster is Python 2 only.') -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10,2 ), - reason='Redis Blaster requires redis>=2.10.2') +@pytest.mark.skipif(six.PY3, reason="Redis Blaster is Python 2 only.") +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10, 2), reason="Redis Blaster requires redis>=2.10.2") @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_rb:test_redis_blaster_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_rb:test_redis_blaster_operation_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_redis_blaster_operation_disable_instance(): from rb import Cluster - cluster = Cluster( - hosts={0: {'port': DB_SETTINGS['port']}}, - host_defaults={'host': DB_SETTINGS['host']} - ) + cluster = Cluster(hosts={0: {"port": DB_SETTINGS["port"]}}, host_defaults={"host": DB_SETTINGS["host"]}) exercise_fanout(cluster) client = cluster.get_routing_client() diff --git a/tests/datastore_rediscluster/conftest.py b/tests/datastore_rediscluster/conftest.py new file mode 100644 index 000000000..fe53f1fe2 --- /dev/null +++ b/tests/datastore_rediscluster/conftest.py @@ -0,0 +1,32 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_redis)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py new file mode 100644 index 000000000..ae211aa31 --- /dev/null +++ b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py @@ -0,0 +1,168 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import redis +from testing_support.db_settings import redis_cluster_settings + +DB_CLUSTER_SETTINGS = redis_cluster_settings()[0] + +# Set socket_timeout to 5s for fast fail, otherwise the default is to wait forever. +client = redis.RedisCluster(host=DB_CLUSTER_SETTINGS["host"], port=DB_CLUSTER_SETTINGS["port"], socket_timeout=5) + +IGNORED_METHODS = { + "MODULE_CALLBACKS", + "MODULE_VERSION", + "NAME", + "add_edge", + "add_node", + "append_bucket_size", + "append_capacity", + "append_error", + "append_expansion", + "append_items_and_increments", + "append_items", + "append_max_iterations", + "append_no_create", + "append_no_scale", + "append_values_and_weights", + "append_weights", + "batch_indexer", + "BatchIndexer", + "bulk", + "call_procedure", + "client_tracking_off", + "client_tracking_on", + "client", + "close", + "commandmixin", + "connection_pool", + "connection", + "debug_segfault", + "edges", + "execute_command", + "flush", + "from_url", + "get_connection_kwargs", + "get_encoder", + "get_label", + "get_params_args", + "get_property", + "get_relation", + "get_retry", + "hscan_iter", + "index_name", + "labels", + "list_keys", + "load_document", + "load_external_module", + "lock", + "name", + "nodes", + "parse_response", + "pipeline", + "property_keys", + "register_script", + "relationship_types", + "response_callbacks", + "RESPONSE_CALLBACKS", + "sentinel", + "set_file", + "set_path", + "set_response_callback", + "set_retry", + "transaction", + "version", + "ALL_NODES", + "CLUSTER_COMMANDS_RESPONSE_CALLBACKS", + "COMMAND_FLAGS", + "DEFAULT_NODE", + "ERRORS_ALLOW_RETRY", + "NODE_FLAGS", + "PRIMARIES", + "RANDOM", + "REPLICAS", + "RESULT_CALLBACKS", + "RedisClusterRequestTTL", + "SEARCH_COMMANDS", + "client_no_touch", + "cluster_addslotsrange", + "cluster_bumpepoch", + "cluster_delslotsrange", + "cluster_error_retry_attempts", + "cluster_flushslots", + "cluster_links", + "cluster_myid", + "cluster_myshardid", + "cluster_replicas", + "cluster_response_callbacks", + "cluster_setslot_stable", + "cluster_shards", + "command_flags", + "commands_parser", + "determine_slot", + "disconnect_connection_pools", + "encoder", + "get_default_node", + "get_node", + "get_node_from_key", + "get_nodes", + "get_primaries", + "get_random_node", + "get_redis_connection", + "get_replicas", + "keyslot", + "mget_nonatomic", + "monitor", + "mset_nonatomic", + "node_flags", + "nodes_manager", + "on_connect", + "pubsub", + "read_from_replicas", + "reinitialize_counter", + "reinitialize_steps", + "replace_default_node", + "result_callbacks", + "set_default_node", + "user_on_connect_func", +} + +REDIS_MODULES = { + "bf", + "cf", + "cms", + "ft", + "graph", + "json", + "tdigest", + "topk", + "ts", +} + +IGNORED_METHODS |= REDIS_MODULES + + +def test_uninstrumented_methods(): + methods = {m for m in dir(client) if not m[0] == "_"} + is_wrapped = lambda m: hasattr(getattr(client, m), "__wrapped__") + uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)} + + for module in REDIS_MODULES: + if hasattr(client, module): + module_client = getattr(client, module)() + module_methods = {m for m in dir(module_client) if not m[0] == "_"} + is_wrapped = lambda m: hasattr(getattr(module_client, m), "__wrapped__") + uninstrumented |= {m for m in module_methods - IGNORED_METHODS if not is_wrapped(m)} + + assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented) diff --git a/tox.ini b/tox.ini index b1ea001e3..a1084d982 100644 --- a/tox.ini +++ b/tox.ini @@ -96,11 +96,8 @@ envlist = mssql-datastore_pymssql-{py37,py38,py39,py310,py311}, mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, - redis-datastore_redis-{py27,py37,py38,pypy27,pypy38}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest}, - redis-datastore_aioredis-{py37,py38,py39,py310,pypy38}-aioredislatest, - redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy38}-redislatest, - redis-datastore_aredis-{py37,py38,py39,pypy38}-aredislatest, + rediscluster-datastore_rediscluster-{py37,py311,pypy38}-redis{latest}, python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, @@ -135,14 +132,11 @@ envlist = python-framework_flask-{pypy27,py27}-flask0012, python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy38}-flask0101, ; temporarily disabling flaskmaster tests - python-framework_flask-{py37,py38,py39,py310,py311,pypy38}-flask{latest}, + python-framework_flask-{py37,py38,py39,py310,py311,pypy38}-flasklatest, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy38}-graphene{0200,0201}, - python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-graphql02, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy38}-graphql03, + python-framework_graphql-{py37,py38,py39,py310,py311,pypy38}-graphqllatest, ; temporarily disabling graphqlmaster tests - python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + python-framework_graphql-py37-graphql{0300,0301,0302}, grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, @@ -264,12 +258,8 @@ deps = datastore_pymysql: PyMySQL<0.11 datastore_pysolr: pysolr<4.0 datastore_redis-redislatest: redis + datastore_rediscluster-redislatest: redis datastore_redis-redis0400: redis<4.1 - datastore_redis-redis03: redis<4.0 - datastore_redis-{py27,pypy27}: rb - datastore_aioredis-redislatest: redis - datastore_aioredis-aioredislatest: aioredis - datastore_aredis-aredislatest: aredis external_boto3-boto01: boto3<2.0 external_boto3-boto01: moto<2.0 external_boto3-py27: rsa<4.7.1 @@ -325,12 +315,7 @@ deps = framework_flask-flaskmaster: https://github.com/pallets/werkzeug/archive/main.zip framework_flask-flaskmaster: https://github.com/pallets/flask/archive/main.zip#egg=flask[async] framework_graphene-graphenelatest: graphene - framework_graphene-graphene0200: graphene<2.1 - framework_graphene-graphene0201: graphene<2.2 - framework_graphql-graphql02: graphql-core<3 - framework_graphql-graphql03: graphql-core<4 - framework_graphql-graphql0202: graphql-core<2.3 - framework_graphql-graphql0203: graphql-core<2.4 + framework_graphql-graphqllatest: graphql-core<4 framework_graphql-graphql0300: graphql-core<3.1 framework_graphql-graphql0301: graphql-core<3.2 framework_graphql-graphql0302: graphql-core<3.3 @@ -468,8 +453,7 @@ changedir = datastore_pymysql: tests/datastore_pymysql datastore_pysolr: tests/datastore_pysolr datastore_redis: tests/datastore_redis - datastore_aioredis: tests/datastore_aioredis - datastore_aredis: tests/datastore_aredis + datastore_rediscluster: tests/datastore_rediscluster datastore_sqlite: tests/datastore_sqlite external_boto3: tests/external_boto3 external_botocore: tests/external_botocore @@ -505,6 +489,7 @@ changedir = template_jinja2: tests/template_jinja2 template_mako: tests/template_mako + [pytest] usefixtures = collector_available_fixture @@ -513,10 +498,10 @@ usefixtures = [coverage:run] branch = True disable_warnings = couldnt-parse -source = newrelic +source = newrelic [coverage:paths] -source = +source = newrelic/ .tox/**/site-packages/newrelic/ /__w/**/site-packages/newrelic/ @@ -525,4 +510,4 @@ source = directory = ${TOX_ENV_DIR-.}/htmlcov [coverage:xml] -output = ${TOX_ENV_DIR-.}/coverage.xml +output = ${TOX_ENV_DIR-.}/coverage.xml \ No newline at end of file