Merge pull request #3 from mezmo/edge1

Edge1
mezmo · Feb 14, 2024 · a049b0c · a049b0c
2 parents 7ddc8c2 + 596ba22
commit a049b0c
Show file tree

Hide file tree

Showing 34 changed files with 3,181 additions and 1,016 deletions.
diff --git a/.github/workflows/integration-test-mezmo.yml b/.github/workflows/integration-test-mezmo.yml
@@ -5,6 +5,11 @@ on:
   pull_request:
     branches-ignore:
       - 'dependabot/**'
+    paths-ignore:
+      - 'deployment/**'
+      - 'test/data/**'
+      - 'docs/**'
+
   push:
     branches:
       - master
@@ -64,16 +69,12 @@ jobs:
           # - test: 'nats'
           # - test: 'nginx'
           - test: 'prometheus'
+          # - test: 'postgres'
           # - test: 'pulsar'
           # - test: 'redis'
           # - test: 'shutdown'
+          # - test: 'splunk'  This started breaking again for no apparent reason
           # - test: 'sumo-logic'
-          # We should re-enable splunk tests in LOG-18162
-          # - test: 'postgres'
-          # - test: 'splunk'
-          #   env:
-          #     SPLUNK_VERSION: 7.3.9
-          # - test: 'splunk'
     steps:
       - uses: actions/checkout@v3
       - run: git config --global url."https://${GITHUB_TOKEN}@github.com".insteadOf ssh://git@github.com

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -122,7 +122,7 @@ members = [
 ]
 
 [workspace.dependencies]
-vrl = { version = "0.8.1", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.11.0", features = ["cli", "test", "test_framework", "arbitrary"] }
+vrl = { version = "0.8.1", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.14.0", features = ["cli", "test", "test_framework", "arbitrary"] }
 pin-project = { version = "1.1.3", default-features = false }
 
 [dependencies]
@@ -228,8 +228,8 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std"
 ratatui = { version = "0.24.0", optional = true, default-features = false, features = ["crossterm"] }
 
 # Datadog Pipelines
-# datadog-filter = { package = "datadog-filter", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.11.0" }
-# datadog-search-syntax = { package = "datadog-search-syntax", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.11.0" }
+# datadog-filter = { package = "datadog-filter", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.14.0" }
+# datadog-search-syntax = { package = "datadog-search-syntax", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.14.0" }
 
 hex = { version = "0.4.3", default-features = false } # Do not do `optional`
 sha2 = { version = "0.10.8", default-features = false } # Do not do `optional`
@@ -307,6 +307,7 @@ rdkafka = { version = "0.34.0", default-features = false, features = ["tokio", "
 redis = { version = "0.23.3", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true }
 regex = { version = "1.10.2", default-features = false, features = ["std", "perf"] }
 roaring = { version = "0.10.2", default-features = false, optional = true }
+rocksdb = { version = "0.21", optional = true }
 seahash = { version = "4.1.0", default-features = false }
 semver = { version = "1.0.20", default-features = false, features = ["serde", "std"], optional = true }
 smallvec = { version = "1", default-features = false, features = ["union", "serde"] }
@@ -380,7 +381,7 @@ tokio-test = "0.4.3"
 tower-test = "0.4.0"
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl", "test"] }
 snap = "1"
-vrl = { version = "0.8.1", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.11.0",features = ["cli", "test", "test_framework", "arbitrary"] }
+vrl = { version = "0.8.1", git = "ssh://git@github.com/mezmo/vrl.git", rev = "v0.14.0",features = ["cli", "test", "test_framework", "arbitrary"] }
 
 wiremock = "0.5.21"
 zstd = { version = "0.13.0", default-features = false }
@@ -478,6 +479,9 @@ enrichment-tables = ["enrichment-tables-geoip", "enrichment-tables-state_variabl
 enrichment-tables-geoip = ["dep:maxminddb"]
 enrichment-tables-state_variables = ["dep:tokio-postgres"]
 
+# Component state persistence
+component-persistence = ["dep:rocksdb"]
+
 # Sources (Upstream)
 # sources = ["sources-logs", "sources-metrics"]
 
@@ -639,7 +643,6 @@ transforms-logs-mezmo = [
 ]
 transforms-metrics = [
   "transforms-aggregate",
-  "transforms-sliding_aggregate",
   "transforms-filter",
   "transforms-log_to_metric",
   "transforms-lua",
@@ -654,15 +657,15 @@ transforms-metrics-mezmo = [
   "transforms-remap",
   "transforms-throttle",
   "transforms-aggregate",
-  "transforms-sliding_aggregate",
+  "transforms-mezmo_aggregate_v2",
   "transforms-mezmo_aggregate",
   "transforms-mezmo_tag_cardinality_limit",
   "transforms-mezmo_log_to_metric",
   "transforms-protobuf_to_metric",
 ]
 
 transforms-aggregate = []
-transforms-sliding_aggregate = ["transforms-remap"]
+transforms-mezmo_aggregate_v2 = ["transforms-remap", "component-persistence"]
 transforms-aws_ec2_metadata = ["dep:arc-swap"]
 transforms-dedupe = ["dep:lru"]
 transforms-filter = []

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -45,13 +45,14 @@ pipeline {
   options {
     timestamps()
     disableConcurrentBuilds()
-    timeout time: 1, unit: 'HOURS'
+    timeout time: 90, unit: 'MINUTES'
     ansiColor 'xterm'
     withCredentials(CREDS)
   }
   environment {
     ENVIRONMENT_AUTOBUILD = 'false'
     ENVIRONMENT_TTY = 'false'
+    CI = 'true'
   }
   post {
     always {
@@ -132,9 +133,7 @@ pipeline {
         }
         stage('Test build container image') {
           when {
-            not {
-              branch pattern: "(^main|^master|v\\d+\\.\\d+.\\d+.\\d+(-[a-z_\\-0-9]+)?)", comparator: "REGEXP"
-            }
+            changeRequest() // Only do this during PRs because it can take 30 minutes
           }
           steps {
             script {