Send x-datadog-test-session-token metric and send metrics to request-…

…replayer (#2802) This allows for clear test session separation without interference from other processes.
DataDog · Aug 21, 2024 · e79d9c3 · e79d9c3
1 parent 14de5d5
commit e79d9c3
Show file tree

Hide file tree

Showing 9 changed files with 137 additions and 72 deletions.
diff --git a/Makefile b/Makefile
@@ -489,7 +489,7 @@ cores:
 # TESTS
 ########################################################################################################################
 TRACER_SOURCES_INI := -d datadog.trace.sources_path=$(TRACER_SOURCE_DIR)
-ENV_OVERRIDE := $(shell [ -n "${DD_TRACE_DOCKER_DEBUG}" ] && echo DD_AUTOLOAD_NO_COMPILE=true DD_TRACE_SOURCES_PATH=$(TRACER_SOURCE_DIR)) DD_DOGSTATSD_URL=http://127.0.0.1:9876 DD_TRACE_CLI_ENABLED=true DD_TRACE_GIT_METADATA_ENABLED=false
+ENV_OVERRIDE := $(shell [ -n "${DD_TRACE_DOCKER_DEBUG}" ] && echo DD_AUTOLOAD_NO_COMPILE=true DD_TRACE_SOURCES_PATH=$(TRACER_SOURCE_DIR)) DD_DOGSTATSD_URL=http://request-replayer:80 DD_TRACE_CLI_ENABLED=true DD_TRACE_GIT_METADATA_ENABLED=false
 TEST_EXTRA_INI ?=
 TEST_EXTRA_ENV ?=
 

diff --git a/dockerfiles/services/request-replayer/linux.Dockerfile b/dockerfiles/services/request-replayer/linux.Dockerfile
@@ -9,6 +9,7 @@ WORKDIR /var/www
 COPY src /var/www
 
 EXPOSE 80
+EXPOSE 80/udp
 
 RUN composer install
 

diff --git a/dockerfiles/services/request-replayer/src/index.php b/dockerfiles/services/request-replayer/src/index.php
@@ -12,9 +12,61 @@
     exit;
 }
 
+function decodeDogStatsDMetrics($metrics)
+{
+    $metrics = array_filter($metrics);
+
+    // Format of DogStatsD metrics: metric_name:value|type|#tag1:value1,tag2:value2
+    // Parts:                      |-> 0             |-> 1|-> 2
+    $decodedMetrics = [];
+    foreach ($metrics as $metric) {
+        $parts = explode('|', $metric);
+
+        $nameAndValue = explode(':', $parts[0]);
+        $metricName = $nameAndValue[0];
+        $value = $nameAndValue[1];
+
+        $type = $parts[1];
+
+        $tags = [];
+        if (count($parts) > 2) {
+            $parts[2] = substr($parts[2], 1); // Remove leading #
+            $tags = explode(',', $parts[2]);
+            $tags = array_map(function ($tag) {
+                return explode(':', $tag);
+            }, $tags);
+            $tags = array_combine(array_column($tags, 0), array_column($tags, 1));
+        }
+        $decodedMetrics[] = [
+            'name' => $metricName,
+            'value' => $value,
+            'type' => $type,
+            'tags' => $tags,
+        ];
+    }
+    return $decodedMetrics;
+}
+
+$uri = explode("?", $_SERVER['REQUEST_URI'])[0];
+
 $temp_location = sys_get_temp_dir();
 
-if ("" != $token = ($_SERVER["HTTP_X_DATADOG_TEST_SESSION_TOKEN"] ?? "")) {
+$metricsServerPid = "$temp_location/metrics-server.pid";
+if (!file_exists($metricsServerPid)) {
+    shell_exec("nohup bash -c 'php metricsserver.php & pid=$!; echo \$pid > $metricsServerPid; wait \$pid; rm $metricsServerPid' > /dev/null 2>&1 &");
+}
+
+$token = $_SERVER["HTTP_X_DATADOG_TEST_SESSION_TOKEN"] ?? "";
+
+if ($uri === "/metrics") {
+    $decodedMetrics = decodeDogStatsDMetrics(explode("\n", trim($_GET["metrics"], "\n")));
+    if (isset($decodedMetrics[0]["tags"]["x-datadog-test-session-token"])) {
+        $token = $decodedMetrics[0]["tags"]["x-datadog-test-session-token"];
+        unset($decodedMetrics[0]["tags"]["x-datadog-test-session-token"]);
+    }
+}
+
+if ($token != "") {
     $token = str_replace("/", "-", $token);
     $temp_location .= "/token-$token";
     @mkdir($temp_location);
@@ -24,6 +76,8 @@
 define('REQUEST_NEXT_RESPONSE_FILE', getenv('REQUEST_NEXT_RESPONSE_FILE') ?: ("$temp_location/response.json"));
 define('REQUEST_LOG_FILE', getenv('REQUEST_LOG_FILE') ?: ("$temp_location/requests-log.txt"));
 define('REQUEST_RC_CONFIGS_FILE', getenv('REQUEST_RC_CONFIGS_FILE') ?: ("$temp_location/rc_configs.json"));
+define('REQUEST_METRICS_FILE', getenv('REQUEST_METRICS_FILE') ?: ("$temp_location/metrics.json"));
+define('REQUEST_METRICS_LOG_FILE', getenv('REQUEST_METRICS_LOG_FILE') ?: ("$temp_location/metrics-log.txt"));
 
 function logRequest($message, $data = '')
 {
@@ -37,13 +91,13 @@ function logRequest($message, $data = '')
 }
 
 set_error_handler(function ($number, $message, $errfile, $errline) {
-    logRequest("Triggered error $number $message in $errfile on line $errline");
+    logRequest("Triggered error $number $message in $errfile on line $errline: " . (new \Exception)->getTraceAsString());
     trigger_error($message, $number);
 });
 
 $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : [];
 
-switch (explode("?", $_SERVER['REQUEST_URI'])[0]) {
+switch ($uri) {
     case '/replay':
         if (!file_exists(REQUEST_LATEST_DUMP_FILE)) {
             logRequest('Cannot replay last request; request log does not exist');
@@ -55,8 +109,19 @@ function logRequest($message, $data = '')
         unlink(REQUEST_LOG_FILE);
         logRequest('Returned last request and deleted request log', $request);
         break;
+    case '/replay-metrics':
+        if (!file_exists(REQUEST_METRICS_FILE)) {
+            logRequest('Cannot replay last request; metrics log does not exist');
+            break;
+        }
+        $request = file_get_contents(REQUEST_METRICS_FILE);
+        echo $request;
+        unlink(REQUEST_METRICS_FILE);
+        unlink(REQUEST_METRICS_LOG_FILE);
+        logRequest('Returned last metrics and deleted metrics log', $request);
+        break;
     case '/clear-dumped-data':
-        if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) {
+        if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_METRICS_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) {
             logRequest('Cannot delete request log; request log does not exist');
             break;
         }
@@ -67,6 +132,10 @@ function logRequest($message, $data = '')
             unlink(REQUEST_LATEST_DUMP_FILE);
             unlink(REQUEST_LOG_FILE);
         }
+        if (file_exists(REQUEST_METRICS_FILE)) {
+            unlink(REQUEST_METRICS_FILE);
+            unlink(REQUEST_METRICS_LOG_FILE);
+        }
         if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) {
             unlink(REQUEST_NEXT_RESPONSE_FILE);
         }
@@ -127,6 +196,21 @@ function logRequest($message, $data = '')
         $response["targets"] = base64_encode(json_encode($response["targets"], JSON_UNESCAPED_SLASHES));
         echo json_encode($response, JSON_UNESCAPED_SLASHES);
         break;
+    case "/metrics":
+        $_SERVER['REQUEST_URI'] = $uri;
+        logRequest('Logged new metrics', json_encode($decodedMetrics));
+        foreach ($decodedMetrics as $metric) {
+            file_put_contents(REQUEST_METRICS_LOG_FILE, json_encode($metric) . "\n", FILE_APPEND);
+
+            if (file_exists(REQUEST_METRICS_FILE)) {
+                $allMetrics = json_decode(file_get_contents(REQUEST_METRICS_FILE), true);
+            } else {
+                $allMetrics = [];
+            }
+            $allMetrics[] = $metric;
+            file_put_contents(REQUEST_METRICS_FILE, json_encode($allMetrics));
+        }
+        break;
     default:
         $headers = getallheaders();
         if (isset($headers['X-Datadog-Diagnostic-Check']) || isset($headers['x-datadog-diagnostic-check'])) {

diff --git a/dockerfiles/services/request-replayer/src/metricsserver.php b/dockerfiles/services/request-replayer/src/metricsserver.php
@@ -0,0 +1,8 @@
+<?php
+
+$server = stream_socket_server("udp://0.0.0.0:80", $err, $errstr, STREAM_SERVER_BIND);
+
+while (true) {
+    $buf = stream_socket_recvfrom($server, 2048);
+    file_get_contents("http://localhost/metrics?metrics=" . urlencode($buf));
+}
diff --git a/dockerfiles/services/request-replayer/windows.Dockerfile b/dockerfiles/services/request-replayer/windows.Dockerfile
@@ -5,6 +5,7 @@ WORKDIR /var/www
 COPY src /var/www
 
 EXPOSE 80
+EXPOSE 80/udp
 
 RUN composer install
 

diff --git a/ext/sidecar.c b/ext/sidecar.c
@@ -202,6 +202,10 @@ static void ddtrace_sidecar_dogstatsd_push_tags(ddog_Vec_Tag *vec, zval *tags) {
     }
     zend_string_release(version);
 
+    if (ZSTR_LEN(get_DD_TRACE_AGENT_TEST_SESSION_TOKEN())) {
+        ddtrace_sidecar_dogstatsd_push_tag(vec, DDOG_CHARSLICE_C("x-datadog-test-session-token"), dd_zend_string_to_CharSlice(get_DD_TRACE_AGENT_TEST_SESSION_TOKEN()));
+    }
+
     // Specific tags
     if (!tags || Z_TYPE_P(tags) != IS_ARRAY) {
         return;

diff --git a/tests/Common/SnapshotTestTrait.php b/tests/Common/SnapshotTestTrait.php
@@ -10,10 +10,6 @@
 trait SnapshotTestTrait
 {
     protected static $testAgentUrl = 'http://test-agent:9126';
-    protected static $dogstatsdAddr = '127.0.0.1';
-    protected static $dogstatsdPort = 9876;
-    /** @var UDPServer */
-    protected $server;
     protected $logFileSize = 0;
 
     private function decamelize($string): string
@@ -55,11 +51,6 @@ private function generateToken(): string
         return $class . '.' . $function;
     }
 
-    private function startMetricsSnapshotSession()
-    {
-        $this->server = new UDPServer(self::$dogstatsdAddr, self::$dogstatsdPort);
-    }
-
     /**
      * Start a snapshotting session associated with a given token.
      *
@@ -68,9 +59,8 @@ private function startMetricsSnapshotSession()
      * @param string $token The token to associate with the snapshotting session
      * @return void
      */
-    private function startSnapshotSession(string $token, $snapshotMetrics = false, $logsFile = null)
+    private function startSnapshotSession(string $token, $logsFile = null)
     {
-
         $url = self::$testAgentUrl . '/test/session/start?test_session_token=' . $token;
 
         $ch = curl_init($url);
@@ -81,10 +71,6 @@ private function startSnapshotSession(string $token, $snapshotMetrics = false, $
             TestCase::fail('Error starting snapshot session: ' . $response);
         }
 
-        if ($snapshotMetrics) {
-            $this->startMetricsSnapshotSession();
-        }
-
         if ($logsFile) {
             if (file_exists($logsFile)) {
                 $this->logFileSize = (int)filesize($logsFile);
@@ -214,8 +200,10 @@ private function stopAndCompareMetricsSnapshotSession(
         string $token,
         array $fieldsToIgnore = ['openai.request.duration']
     ) {
-        $receivedMetrics = $this->server->dump();
-        $this->server->close();
+        $receivedMetrics = $this->retrieveDumpedMetrics(function($metrics) {
+            return $metrics["name"] == "tracer-snapshot-end";
+        });
+        array_pop($receivedMetrics);
 
         $basePath = implode('/', array_slice(explode('/', getcwd()), 0, 4)); // /home/circleci/[app|datadog]
         $expectedMetricsFile = $basePath . '/tests/snapshots/metrics/' . $token . '.txt';
@@ -230,10 +218,7 @@ private function stopAndCompareMetricsSnapshotSession(
     private function compareMetrics($expectedMetrics, $receivedMetrics, $fieldsToIgnore)
     {
         $expectedMetrics = explode("\n", $expectedMetrics);
-        $receivedMetrics = explode("\n", $receivedMetrics);
-
         $expectedMetrics = $this->decodeDogStatsDMetrics($expectedMetrics);
-        $receivedMetrics = $this->decodeDogStatsDMetrics($receivedMetrics);
 
         $this->compareMetricsArrays($expectedMetrics, $receivedMetrics, $fieldsToIgnore);
     }
@@ -345,10 +330,11 @@ public function isolateTracerSnapshot(
         self::putEnv('DD_TRACE_AGENT_RETRIES=3');
 
         $token = $this->generateToken();
-        $this->startSnapshotSession($token, $snapshotMetrics, $logsFile);
+        $this->startSnapshotSession($token, $logsFile);
         $originalToken = ini_get("datadog.trace.agent_test_session_token");
         update_test_agent_session_token($token);
 
+        $this->resetRequestDumper();
         $this->resetTracer($tracer, $config);
 
         $tracer = GlobalTracer::get();
@@ -357,15 +343,15 @@ public function isolateTracerSnapshot(
         }
         $fn($tracer);
 
+        if ($snapshotMetrics) {
+            \DDTrace\dogstatsd_count("tracer-snapshot-end", 1);
+        }
+
         $traces = $this->flushAndGetTraces($tracer);
         if (!empty($traces)) {
             $this->sendTracesToTestAgent($traces);
         }
 
-        update_test_agent_session_token($originalToken);
-        self::putEnv('DD_TRACE_SHUTDOWN_TIMEOUT');
-        self::putEnv('DD_TRACE_AGENT_RETRIES');
-
         $this->stopAndCompareSnapshotSession(
             $token,
             $fieldsToIgnore,
@@ -375,5 +361,9 @@ public function isolateTracerSnapshot(
             $logsFile,
             $fieldsToIgnoreLogs
         );
+
+        update_test_agent_session_token($originalToken);
+        self::putEnv('DD_TRACE_SHUTDOWN_TIMEOUT');
+        self::putEnv('DD_TRACE_AGENT_RETRIES');
     }
 }
diff --git a/tests/Common/TracerTestTrait.php b/tests/Common/TracerTestTrait.php
@@ -444,13 +444,31 @@ public function retrieveDumpedData(callable $until = null, $throw = false)
             };
         }
 
+        return $this->retrieveAnyDumpedData($until, $throw);
+    }
+
+    /**
+     * Returns the raw response body, if any, or null otherwise.
+     */
+    public function retrieveDumpedMetrics(callable $until = null, $throw = false)
+    {
+        if (!$until) {
+            $until = function ($request) {
+                return (strpos($request["uri"] ?? "", "/telemetry/") !== 0);
+            };
+        }
+
+        return $this->retrieveAnyDumpedData($until, $throw, true);
+    }
+
+    public function retrieveAnyDumpedData(callable $until, $throw, $metrics = false) {
         $allResponses = [];
 
         // When tests run with the background sender enabled, there might be some delay between when a trace is flushed
         // and actually sent. While we should find a smart way to tackle this, for now we do it quick and dirty, in a
         // for loop.
         for ($attemptNumber = 1; $attemptNumber <= 50; $attemptNumber++) {
-            $curl = curl_init(self::$agentRequestDumperUrl . '/replay');
+            $curl = curl_init(self::$agentRequestDumperUrl . '/replay' . ($metrics ? '-metrics' : ''));
             curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
             curl_setopt($curl, CURLOPT_HTTPHEADER, ['x-datadog-test-session-token: ' . ini_get("datadog.trace.agent_test_session_token")]);
             // Retrieving data

diff --git a/tests/Common/UDPServer.php b/tests/Common/UDPServer.php