Skip to content

Commit

Permalink
Infinite Tracing Supportability Feature Toggle Metrics (#769)
Browse files Browse the repository at this point in the history
* Add 8T feature toggle supportability metrics

* Remove supportability metrics when 8t is disabled.

* Formatting
  • Loading branch information
TimPansino committed Feb 15, 2023
1 parent 919754e commit 8e1820e
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 16 deletions.
42 changes: 30 additions & 12 deletions newrelic/core/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -520,24 +520,26 @@ def connect_to_data_collector(self, activate_agent):

self._global_events_account = 0

# Record metrics for how long it took us to connect and how
# many attempts we made. Also record metrics for the final
# successful attempt. If we went through multiple attempts,
# individual details of errors before the final one that
# worked are not recorded as recording them all in the
# initial harvest would possibly skew first harvest metrics
# and cause confusion as we cannot properly mark the time over
# which they were recorded. Make sure we do this before we
# mark the session active so we don't have to grab a lock on
# merging the internal metrics.

with InternalTraceContext(internal_metrics):
# Record metrics for how long it took us to connect and how
# many attempts we made. Also record metrics for the final
# successful attempt. If we went through multiple attempts,
# individual details of errors before the final one that
# worked are not recorded as recording them all in the
# initial harvest would possibly skew first harvest metrics
# and cause confusion as we cannot properly mark the time over
# which they were recorded. Make sure we do this before we
# mark the session active so we don't have to grab a lock on
# merging the internal metrics.

internal_metric(
"Supportability/Python/Application/Registration/Duration", self._period_start - connect_start
)
internal_metric("Supportability/Python/Application/Registration/Attempts", connect_attempts)

# Logging feature toggle supportability metrics
# Record metrics for feature toggles from settings

# Logging feature toggle metrics
application_logging_metrics = (
configuration.application_logging.enabled and configuration.application_logging.metrics.enabled
)
Expand All @@ -562,6 +564,22 @@ def connect_to_data_collector(self, activate_agent):
1,
)

# Infinite tracing feature toggle metrics
infinite_tracing = configuration.infinite_tracing.enabled # Property that checks trace observer host
if infinite_tracing:
infinite_tracing_batching = configuration.infinite_tracing.batching
infinite_tracing_compression = configuration.infinite_tracing.compression
internal_metric(
"Supportability/InfiniteTracing/gRPC/Batching/%s"
% ("enabled" if infinite_tracing_batching else "disabled"),
1,
)
internal_metric(
"Supportability/InfiniteTracing/gRPC/Compression/%s"
% ("enabled" if infinite_tracing_compression else "disabled"),
1,
)

self._stats_engine.merge_custom_metrics(internal_metrics.metrics())

# Update the active session in this object. This will the
Expand Down
51 changes: 47 additions & 4 deletions tests/agent_streaming/test_infinite_tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,14 @@

import pytest
from testing_support.fixtures import override_generic_settings
from testing_support.validators.validate_metric_payload import validate_metric_payload
from testing_support.util import conditional_decorator
from testing_support.validators.validate_metric_payload import validate_metric_payload

from newrelic.common.streaming_utils import StreamBuffer
from newrelic.core.agent_streaming import StreamingRpc
from newrelic.core.application import Application
from newrelic.core.config import global_settings
from newrelic.core.infinite_tracing_pb2 import AttributeValue, Span

from newrelic.packages import six

settings = global_settings()
Expand All @@ -44,7 +43,7 @@ def app():
app.internal_agent_shutdown(restart=False)
except:
pass
if active_session:
if active_session and active_session._rpc is not None:
assert not active_session._rpc.response_processing_thread.is_alive()
assert not active_session._rpc.channel

Expand Down Expand Up @@ -331,7 +330,9 @@ def connect_complete():
_test()


@conditional_decorator(condition=six.PY2, decorator=pytest.mark.xfail(reason="Test frequently times out on Py2.", strict=False))
@conditional_decorator(
condition=six.PY2, decorator=pytest.mark.xfail(reason="Test frequently times out on Py2.", strict=False)
)
def test_no_data_loss_on_reconnect(mock_grpc_server, app, buffer_empty_event, batching, spans_processed_event):
"""
Test for data loss when channel is closed by the server while waiting for more data in a request iterator.
Expand Down Expand Up @@ -464,3 +465,45 @@ def _test():
app.harvest()

_test()


@pytest.mark.parametrize("trace_observer_host", ["localhost", None])
@pytest.mark.parametrize("batching", [True, False])
@pytest.mark.parametrize("compression", [True, False])
def test_settings_supportability_metrics(mock_grpc_server, app, trace_observer_host, batching, compression):
connect_event = threading.Event()

enabled = bool(trace_observer_host)

metrics = [
("Supportability/InfiniteTracing/gRPC/Batching/enabled", 1 if enabled and batching else None),
("Supportability/InfiniteTracing/gRPC/Batching/disabled", 1 if enabled and not batching else None),
("Supportability/InfiniteTracing/gRPC/Compression/enabled", 1 if enabled and compression else None),
("Supportability/InfiniteTracing/gRPC/Compression/disabled", 1 if enabled and not compression else None),
]

@override_generic_settings(
settings,
{
"distributed_tracing.enabled": True,
"span_events.enabled": True,
"infinite_tracing.trace_observer_host": trace_observer_host,
"infinite_tracing.trace_observer_port": mock_grpc_server,
"infinite_tracing.ssl": False,
"infinite_tracing.batching": batching,
"infinite_tracing.compression": compression,
},
)
@validate_metric_payload(metrics)
def _test():
def connect_complete():
connect_event.set()

app.connect_to_data_collector(connect_complete)

assert connect_event.wait(timeout=5)
connect_event.clear()

app.harvest()

_test()

0 comments on commit 8e1820e

Please sign in to comment.