Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Infinite Tracing Supportability Feature Toggle Metrics #769

Merged
merged 3 commits into from
Feb 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions newrelic/core/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -520,24 +520,26 @@ def connect_to_data_collector(self, activate_agent):

self._global_events_account = 0

# Record metrics for how long it took us to connect and how
# many attempts we made. Also record metrics for the final
# successful attempt. If we went through multiple attempts,
# individual details of errors before the final one that
# worked are not recorded as recording them all in the
# initial harvest would possibly skew first harvest metrics
# and cause confusion as we cannot properly mark the time over
# which they were recorded. Make sure we do this before we
# mark the session active so we don't have to grab a lock on
# merging the internal metrics.

with InternalTraceContext(internal_metrics):
# Record metrics for how long it took us to connect and how
# many attempts we made. Also record metrics for the final
# successful attempt. If we went through multiple attempts,
# individual details of errors before the final one that
# worked are not recorded as recording them all in the
# initial harvest would possibly skew first harvest metrics
# and cause confusion as we cannot properly mark the time over
# which they were recorded. Make sure we do this before we
# mark the session active so we don't have to grab a lock on
# merging the internal metrics.

internal_metric(
"Supportability/Python/Application/Registration/Duration", self._period_start - connect_start
)
internal_metric("Supportability/Python/Application/Registration/Attempts", connect_attempts)

# Logging feature toggle supportability metrics
# Record metrics for feature toggles from settings

# Logging feature toggle metrics
application_logging_metrics = (
configuration.application_logging.enabled and configuration.application_logging.metrics.enabled
)
Expand All @@ -562,6 +564,22 @@ def connect_to_data_collector(self, activate_agent):
1,
)

# Infinite tracing feature toggle metrics
infinite_tracing = configuration.infinite_tracing.enabled # Property that checks trace observer host
if infinite_tracing:
infinite_tracing_batching = configuration.infinite_tracing.batching
infinite_tracing_compression = configuration.infinite_tracing.compression
internal_metric(
"Supportability/InfiniteTracing/gRPC/Batching/%s"
% ("enabled" if infinite_tracing_batching else "disabled"),
1,
)
internal_metric(
"Supportability/InfiniteTracing/gRPC/Compression/%s"
% ("enabled" if infinite_tracing_compression else "disabled"),
1,
)

self._stats_engine.merge_custom_metrics(internal_metrics.metrics())

# Update the active session in this object. This will the
Expand Down
51 changes: 47 additions & 4 deletions tests/agent_streaming/test_infinite_tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,14 @@

import pytest
from testing_support.fixtures import override_generic_settings
from testing_support.validators.validate_metric_payload import validate_metric_payload
from testing_support.util import conditional_decorator
from testing_support.validators.validate_metric_payload import validate_metric_payload

from newrelic.common.streaming_utils import StreamBuffer
from newrelic.core.agent_streaming import StreamingRpc
from newrelic.core.application import Application
from newrelic.core.config import global_settings
from newrelic.core.infinite_tracing_pb2 import AttributeValue, Span

from newrelic.packages import six

settings = global_settings()
Expand All @@ -44,7 +43,7 @@ def app():
app.internal_agent_shutdown(restart=False)
except:
pass
if active_session:
if active_session and active_session._rpc is not None:
assert not active_session._rpc.response_processing_thread.is_alive()
assert not active_session._rpc.channel

Expand Down Expand Up @@ -331,7 +330,9 @@ def connect_complete():
_test()


@conditional_decorator(condition=six.PY2, decorator=pytest.mark.xfail(reason="Test frequently times out on Py2.", strict=False))
@conditional_decorator(
condition=six.PY2, decorator=pytest.mark.xfail(reason="Test frequently times out on Py2.", strict=False)
)
def test_no_data_loss_on_reconnect(mock_grpc_server, app, buffer_empty_event, batching, spans_processed_event):
"""
Test for data loss when channel is closed by the server while waiting for more data in a request iterator.
Expand Down Expand Up @@ -464,3 +465,45 @@ def _test():
app.harvest()

_test()


@pytest.mark.parametrize("trace_observer_host", ["localhost", None])
@pytest.mark.parametrize("batching", [True, False])
@pytest.mark.parametrize("compression", [True, False])
def test_settings_supportability_metrics(mock_grpc_server, app, trace_observer_host, batching, compression):
connect_event = threading.Event()

enabled = bool(trace_observer_host)

metrics = [
("Supportability/InfiniteTracing/gRPC/Batching/enabled", 1 if enabled and batching else None),
("Supportability/InfiniteTracing/gRPC/Batching/disabled", 1 if enabled and not batching else None),
("Supportability/InfiniteTracing/gRPC/Compression/enabled", 1 if enabled and compression else None),
("Supportability/InfiniteTracing/gRPC/Compression/disabled", 1 if enabled and not compression else None),
]

@override_generic_settings(
settings,
{
"distributed_tracing.enabled": True,
"span_events.enabled": True,
"infinite_tracing.trace_observer_host": trace_observer_host,
"infinite_tracing.trace_observer_port": mock_grpc_server,
"infinite_tracing.ssl": False,
"infinite_tracing.batching": batching,
"infinite_tracing.compression": compression,
},
)
@validate_metric_payload(metrics)
def _test():
def connect_complete():
connect_event.set()

app.connect_to_data_collector(connect_complete)

assert connect_event.wait(timeout=5)
connect_event.clear()

app.harvest()

_test()