diff --git a/newrelic/config.py b/newrelic/config.py index 7c3fa2279..5602efb3c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -320,6 +320,8 @@ def _process_configuration(section): _process_setting(section, "api_key", "get", None) _process_setting(section, "host", "get", None) _process_setting(section, "port", "getint", None) + _process_setting(section, "otlp_host", "get", None) + _process_setting(section, "otlp_port", "getint", None) _process_setting(section, "ssl", "getboolean", None) _process_setting(section, "proxy_scheme", "get", None) _process_setting(section, "proxy_host", "get", None) diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index f1861a09a..6bb196df3 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -38,6 +38,7 @@ global_settings_dump, ) from newrelic.core.internal_metrics import internal_count_metric +from newrelic.core.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode from newrelic.network.exceptions import ( DiscardDataForRequest, ForceAgentDisconnect, @@ -45,7 +46,7 @@ NetworkInterfaceException, RetryDataForRequest, ) -from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, Resource, create_key_values_from_iterable, otlp_encode +from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode _logger = logging.getLogger(__name__) @@ -528,31 +529,15 @@ def connect( class OtlpProtocol(AgentProtocol): - def __init__(self, settings, host=None, resource=None, client_cls=ApplicationModeClient): - self.HOST_MAP = { - "collector.newrelic.com": "otlp.nr-data.net", - "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", - "gov-collector.newrelic.com": "gov-otlp.nr-data.net", - "staging-collector.newrelic.com": "staging-otlp.nr-data.net", - "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", - "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", - "fake-collector.newrelic.com": "fake-otlp.nr-data.net", - } - + def __init__(self, settings, host=None, client_cls=ApplicationModeClient): if settings.audit_log_file: audit_log_fp = open(settings.audit_log_file, "a") else: audit_log_fp = None - otlp_host = self.HOST_MAP.get(host or settings.host, None) - if not otlp_host: - default = self.HOST_MAP["collector.newrelic.com"] - _logger.warn("Unable to find corresponding OTLP host using default %s" % default) - otlp_host = default - self.client = client_cls( - host=otlp_host, - port=4318, + host=host or settings.otlp_host, + port=settings.otlp_port or 4318, proxy_scheme=settings.proxy_scheme, proxy_host=settings.proxy_host, proxy_port=settings.proxy_port, @@ -561,19 +546,18 @@ def __init__(self, settings, host=None, resource=None, client_cls=ApplicationMod timeout=settings.agent_limits.data_collector_timeout, ca_bundle_path=settings.ca_bundle_path, disable_certificate_validation=settings.debug.disable_certificate_validation, - default_content_encoding_header=None, compression_threshold=settings.agent_limits.data_compression_threshold, compression_level=settings.agent_limits.data_compression_level, compression_method=settings.compressed_content_encoding, max_payload_size_in_bytes=1000000, audit_log_fp=audit_log_fp, + default_content_encoding_header=None, ) self._params = {} self._headers = { "api-key": settings.license_key, } - self._resource = resource # In Python 2, the JSON is loaded with unicode keys and values; # however, the header name must be a non-unicode value when given to @@ -606,9 +590,7 @@ def connect( settings, client_cls=ApplicationModeClient, ): - resource = Resource(attributes=create_key_values_from_iterable({"service.name": app_name})) - - with cls(settings, resource=resource, client_cls=client_cls) as protocol: + with cls(settings, client_cls=client_cls) as protocol: pass return protocol diff --git a/newrelic/core/config.py b/newrelic/core/config.py index ccd9a6132..55b359174 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -104,6 +104,7 @@ def create_settings(nested): class TopLevelSettings(Settings): _host = None + _otlp_host = None @property def host(self): @@ -115,6 +116,16 @@ def host(self): def host(self, value): self._host = value + @property + def otlp_host(self): + if self._otlp_host: + return self._otlp_host + return default_otlp_host(self.host) + + @otlp_host.setter + def otlp_host(self, value): + self._otlp_host = value + class AttributesSettings(Settings): pass @@ -560,6 +571,24 @@ def default_host(license_key): return host +def default_otlp_host(host): + HOST_MAP = { + "collector.newrelic.com": "otlp.nr-data.net", + "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", + "gov-collector.newrelic.com": "gov-otlp.nr-data.net", + "staging-collector.newrelic.com": "staging-otlp.nr-data.net", + "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", + "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", + "fake-collector.newrelic.com": "fake-otlp.nr-data.net", + } + otlp_host = HOST_MAP.get(host, None) + if not otlp_host: + default = HOST_MAP["collector.newrelic.com"] + _logger.warn("Unable to find corresponding OTLP host using default %s" % default) + otlp_host = default + return otlp_host + + _LOG_LEVEL = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, @@ -585,7 +614,9 @@ def default_host(license_key): _settings.ssl = _environ_as_bool("NEW_RELIC_SSL", True) _settings.host = os.environ.get("NEW_RELIC_HOST") +_settings.otlp_host = os.environ.get("NEW_RELIC_OTLP_HOST") _settings.port = int(os.environ.get("NEW_RELIC_PORT", "0")) +_settings.otlp_port = int(os.environ.get("NEW_RELIC_OTLP_PORT", "0")) _settings.agent_run_id = None _settings.entity_guid = None diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 2fdff4e43..eb5e75b83 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -36,6 +36,8 @@ _logger = logging.getLogger(__name__) +DIMENSIONAL_METRIC_DATA_TEMP = [] # TODO: REMOVE THIS + class Session(object): PROTOCOL = AgentProtocol diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py new file mode 100644 index 000000000..6a44cb4e3 --- /dev/null +++ b/newrelic/core/otlp_utils.py @@ -0,0 +1,107 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module provides common utilities for interacting with OTLP protocol buffers.""" + +import logging + +_logger = logging.getLogger(__name__) + +try: + from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue + from newrelic.packages.opentelemetry_proto.logs_pb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, + ) + from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( + AggregationTemporality, + Metric, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, + ) + from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource + + AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + ValueAtQuantile = SummaryDataPoint.ValueAtQuantile + + otlp_encode = lambda payload: payload.SerializeToString() + OTLP_CONTENT_TYPE = "application/x-protobuf" + +except ImportError: + from newrelic.common.encoding_utils import json_encode + + def otlp_encode(*args, **kwargs): + _logger.warn( + "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." + ) + return json_encode(*args, **kwargs) + + Resource = dict + ValueAtQuantile = dict + AnyValue = dict + KeyValue = dict + NumberDataPoint = dict + SummaryDataPoint = dict + Sum = dict + Summary = dict + Metric = dict + MetricsData = dict + ScopeMetrics = dict + ResourceMetrics = dict + AGGREGATION_TEMPORALITY_DELTA = 1 + ResourceLogs = dict + ScopeLogs = dict + LogRecord = dict + OTLP_CONTENT_TYPE = "application/json" + + +def create_key_value(key, value): + if isinstance(value, bool): + return KeyValue(key=key, value=AnyValue(bool_value=value)) + elif isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + elif isinstance(value, float): + return KeyValue(key=key, value=AnyValue(double_value=value)) + elif isinstance(value, str): + return KeyValue(key=key, value=AnyValue(string_value=value)) + # Technically AnyValue accepts array, kvlist, and bytes however, since + # those are not valid custom attribute types according to our api spec, + # we will not bother to support them here either. + else: + _logger.warn("Unsupported attribute value type %s: %s." % (key, value)) + + +def create_key_values_from_iterable(iterable): + if isinstance(iterable, dict): + iterable = iterable.items() + + # The create_key_value list may return None if the value is an unsupported type + # so filter None values out before returning. + return list( + filter( + lambda i: i is not None, + (create_key_value(key, value) for key, value in iterable), + ) + ) + + +def create_resource(attributes=None): + attributes = attributes or {"instrumentation.provider": "nr_performance_monitoring"} + return Resource(attributes=create_key_values_from_iterable(attributes)) diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py index 5df69d71e..547a0eeb6 100644 --- a/tests/agent_features/test_configuration.py +++ b/tests/agent_features/test_configuration.py @@ -577,6 +577,8 @@ def test_translate_deprecated_ignored_params_with_new_setting(): ("agent_run_id", None), ("entity_guid", None), ("distributed_tracing.exclude_newrelic_header", False), + ("otlp_host", "otlp.nr-data.net"), + ("otlp_port", 0), ), ) def test_default_values(name, expected_value): diff --git a/tests/agent_unittests/test_utilization_settings.py b/tests/agent_unittests/test_utilization_settings.py index 8af4bcbf1..96cf47669 100644 --- a/tests/agent_unittests/test_utilization_settings.py +++ b/tests/agent_unittests/test_utilization_settings.py @@ -118,6 +118,22 @@ def reset(wrapped, instance, args, kwargs): return reset +@reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) +def test_otlp_host_port_default(): + settings = global_settings() + assert settings.otlp_host == "otlp.nr-data.net" + assert settings.otlp_port == 0 + + +@reset_agent_config( + INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_OTLP_HOST": "custom-otlp.nr-data.net", "NEW_RELIC_OTLP_PORT": 443} +) +def test_otlp_port_override(): + settings = global_settings() + assert settings.otlp_host == "custom-otlp.nr-data.net" + assert settings.otlp_port == 443 + + @reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) def test_heroku_default(): settings = global_settings()