Only send 1 event per prediction & fix tests (#867)

* Only send 1 event per prediction This changes from sending 1 event per feature/label value to 1 event per prediction. This change is referred to as a change in "schema". * Replace feature_name & feature_value and label_name & label_value to feature.<name> = <value> and label.<name> = <value>. * Remove feature_type and label_type. * Add new_relic_data_schema_version * Remove pypy3.8-scipy will not compile * Remove scipy compile deps * Set ci build back to latest
newrelic · Aug 2, 2023 · c043782 · c043782
1 parent c77adce
commit c043782
Show file tree

Hide file tree

Showing 6 changed files with 152 additions and 443 deletions.
diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile
@@ -26,7 +26,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         freetds-common \
         freetds-dev \
         gcc \
-        gfortran \
         git \
         libbz2-dev \
         libcurl4-openssl-dev \
@@ -44,7 +43,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         make \
         odbc-postgresql \
         openssl \
-        pkg-config \
         python2-dev \
         python3-dev \
         python3-pip \

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -119,7 +119,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -164,7 +164,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -209,7 +209,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -269,7 +269,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -332,7 +332,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -395,7 +395,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -453,7 +453,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -513,7 +513,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -571,7 +571,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -630,7 +630,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -700,7 +700,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -758,7 +758,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -818,7 +818,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -879,7 +879,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30

diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py
@@ -96,8 +96,7 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs):
         if method in ("predict", "fit_predict"):
             training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown")
             inference_id = uuid.uuid4()
-            create_feature_event(transaction, class_, inference_id, instance, args, kwargs)
-            create_label_event(transaction, class_, inference_id, instance, return_val)
+            create_prediction_event(transaction, class_, inference_id, instance, args, kwargs, return_val)
             return PredictReturnTypeProxy(return_val, model_name=class_, training_step=training_step)
         return return_val
 
@@ -170,53 +169,6 @@ def _calc_prediction_label_stats(labels, class_, label_column_names, tags):
     _record_stats(labels, label_column_names, class_, "Label", tags)
 
 
-def create_label_event(transaction, class_, inference_id, instance, return_val):
-    model_name = getattr(instance, "_nr_wrapped_name", class_)
-    model_version = getattr(instance, "_nr_wrapped_version", "0.0.0")
-    label_names = getattr(instance, "_nr_wrapped_label_names", None)
-
-    settings = transaction.settings if transaction.settings is not None else global_settings()
-    if return_val is not None:
-        import numpy as np
-
-        if not hasattr(return_val, "__iter__"):
-            labels = np.array([return_val])
-        else:
-            labels = np.array(return_val)
-        if len(labels.shape) == 1:
-            labels = np.reshape(labels, (len(labels) // 1, 1))
-
-        label_names_list = _get_label_names(label_names, labels)
-        _calc_prediction_label_stats(
-            labels,
-            class_,
-            label_names_list,
-            tags={
-                "inference_id": inference_id,
-                "model_version": model_version,
-                # The following are used for entity synthesis.
-                "modelName": model_name,
-            },
-        )
-        for prediction in labels:
-            for index, value in enumerate(prediction):
-                python_value_type = str(type(value))
-                value_type = str(categorize_data_type(python_value_type))
-
-                event = {
-                    "inference_id": inference_id,
-                    "model_version": model_version,
-                    "label_name": str(label_names_list[index]),
-                    "label_type": value_type,
-                    # The following are used for entity synthesis.
-                    "modelName": model_name,
-                }
-                # Don't include the raw value when inference_event_value is disabled.
-                if settings and settings.machine_learning.inference_events_value.enabled:
-                    event["label_value"] = str(value)
-                transaction.record_ml_event("InferenceData", event)
-
-
 def _get_label_names(user_defined_label_names, prediction_array):
     import numpy as np
 
@@ -283,15 +235,38 @@ def bind_predict(X, *args, **kwargs):
     return X
 
 
-def create_feature_event(transaction, class_, inference_id, instance, args, kwargs):
+def create_prediction_event(transaction, class_, inference_id, instance, args, kwargs, return_val):
     import numpy as np
 
     data_set = bind_predict(*args, **kwargs)
     model_name = getattr(instance, "_nr_wrapped_name", class_)
     model_version = getattr(instance, "_nr_wrapped_version", "0.0.0")
     user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None)
+    label_names = getattr(instance, "_nr_wrapped_label_names", None)
     settings = transaction.settings if transaction.settings is not None else global_settings()
 
+    labels = []
+    if return_val is not None:
+        if not hasattr(return_val, "__iter__"):
+            labels = np.array([return_val])
+        else:
+            labels = np.array(return_val)
+        if len(labels.shape) == 1:
+            labels = np.reshape(labels, (len(labels) // 1, 1))
+
+        label_names_list = _get_label_names(label_names, labels)
+        _calc_prediction_label_stats(
+            labels,
+            class_,
+            label_names_list,
+            tags={
+                "inference_id": inference_id,
+                "model_version": model_version,
+                # The following are used for entity synthesis.
+                "modelName": model_name,
+            },
+        )
+
     final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set)
     np_casted_data_set = np.array(data_set)
     _calc_prediction_feature_stats(
@@ -305,21 +280,30 @@ def create_feature_event(transaction, class_, inference_id, instance, args, kwar
             "modelName": model_name,
         },
     )
-    for col_index, feature in enumerate(np_casted_data_set):
-        for row_index, value in enumerate(feature):
-            value_type = find_type_category(data_set, row_index, col_index)
-            event = {
-                "inference_id": inference_id,
-                "model_version": model_version,
-                "feature_name": str(final_feature_names[row_index]),
-                "feature_type": value_type,
-                # The following are used for entity synthesis.
-                "modelName": model_name,
-            }
-            # Don't include the raw value when inference_event_value is disabled.
-            if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled:
-                event["feature_value"] = str(value)
-            transaction.record_ml_event("InferenceData", event)
+    features, predictions = np_casted_data_set.shape
+    for prediction_index, prediction in enumerate(np_casted_data_set):
+        event = {
+            "inference_id": inference_id,
+            "model_version": model_version,
+            "new_relic_data_schema_version": 2,
+            # The following are used for entity synthesis.
+            "modelName": model_name,
+        }
+        # Don't include the raw value when inference_event_value is disabled.
+        if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled:
+            event.update(
+                {
+                    "feature.%s" % str(final_feature_names[feature_col_index]): str(value)
+                    for feature_col_index, value in enumerate(prediction)
+                }
+            )
+            event.update(
+                {
+                    "label.%s" % str(label_names_list[index]): str(value)
+                    for index, value in enumerate(labels[prediction_index])
+                }
+            )
+        transaction.record_ml_event("InferenceData", event)
 
 
 def _nr_instrument_model(module, model_class):