feat(lookup): exposing vector lookup logic through wasm

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @answerbook/pipeline

.releaserc.json

@@ -0,0 +1,25 @@
+{
+  "ci": false,
+  "branches": [
+    "main"
+  ],
+  "extends": [
+    "@answerbook/release-config-logdna"
+  ],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    "@semantic-release/changelog",
+    ["@semantic-release/exec", {
+      "prepareCmd": "semantic-release-cargo prepare ${nextRelease.version}; cargo update --workspace; sleep 2"
+    }],
+    "@semantic-release/github",
+    ["@semantic-release/git", {
+      "assets": [
+        "CHANGELOG.md",
+        "**/Cargo.toml",
+        "Cargo.lock"
+      ]
+    }]
+  ]
+}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vrl"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false
@@ -52,11 +52,11 @@ bytes = "1.4.0"
 compiler = { package = "vrl-compiler", path = "lib/compiler", default-features = false }
 diagnostic = { package = "vrl-diagnostic", path = "lib/diagnostic" }
 indoc = "2"
-lookup = { path = "lib/lookup" }
+lookup = { path = "lib/lookup" , version = "0.3.0" }
 ordered-float = "3"
 parser = { package = "vrl-parser", path = "lib/parser" }
-value = { path = "lib/value", default-features = false }
-vrl-core = { path = "lib/core" }
+value = { path = "lib/value", default-features = false , version = "0.3.0" }
+vrl-core = { path = "lib/core" , version = "0.3.0" }
 
 [dev-dependencies]
 criterion = "0.4"

Dockerfile

@@ -0,0 +1,49 @@
+FROM rust:1.68.2 AS base
+
+WORKDIR /opt/app
+
+COPY Cargo.toml /opt/app/
+COPY benches /opt/app/benches
+COPY lib /opt/app/lib
+COPY src /opt/app/src
+
+RUN cargo check
+RUN cargo test
+
+FROM rust:1.68.2 AS wasm-base
+
+WORKDIR /opt/app
+
+COPY lib/lookup/Cargo.toml lib/lookup/build.rs ./
+COPY lib/lookup/src ./src
+COPY lib/lookup/tests ./tests
+COPY lib/lookup/benches ./benches
+
+RUN apt-get update && apt-get install nodejs --yes
+RUN curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+# Run both rust unit tests and wasm unit tests
+RUN wasm-pack test --node -- --features=path-parsing-wasm
+
+# Then build the final artifact
+RUN wasm-pack build -t nodejs --release --scope answerbook -- --features=path-parsing-wasm
+
+FROM us.gcr.io/logdna-k8s/node-bullseye:16 AS wasm-publish
+ARG GITHUB_TOKEN
+ARG DRY_RUN=false
+
+WORKDIR /opt/app
+
+COPY --from=wasm-base /opt/app/ .
+RUN touch pkg/.npmrc \
+  && echo "@answerbook:registry=https://npm.pkg.github.com/" >> pkg/.npmrc \
+  && echo '//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}' >> pkg/.npmrc
+
+RUN if [ ${DRY_RUN} = true ]; then \
+        echo 'Dry running publish...' && \
+        cd pkg && npm publish --dry-run; \
+    else \
+        echo 'Publishing...' && \
+        cd pkg && npm publish; \
+    fi
+RUN rm pkg/.npmrc

Jenkinsfile

@@ -0,0 +1,175 @@
+library "magic-butler-catalogue"
+def PROJECT_NAME = "vrl"
+def DEFAULT_BRANCH = "main"
+def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null}
+def DRY_RUN = CURRENT_BRANCH != DEFAULT_BRANCH
+
+pipeline {
+    agent {
+        node {
+            label "ec2-fleet"
+            customWorkspace("/tmp/workspace/${env.BUILD_TAG}")
+        }
+    }
+
+    parameters {
+        string(name: "SANITY_BUILD", defaultValue: "", description: "Is this a scheduled sanity build that skips releasing?")
+    }
+
+    triggers {
+        parameterizedCron(
+            // Cron hours are in GMT, so this is roughly 12-3am EST, depending on DST
+            env.BRANCH_NAME == DEFAULT_BRANCH ? "H H(5-6) * * * % SANITY_BUILD=true" : ""
+        )
+    }
+
+    options {
+        timeout time: 1, unit: "HOURS"
+        timestamps()
+        ansiColor "xterm"
+    }
+
+    environment {
+        GITHUB_TOKEN = credentials("github-api-token")
+        NPM_CONFIG_CACHE = ".npm"
+        NPM_CONFIG_USERCONFIG = ".npm/rc"
+        SPAWN_WRAP_SHIM_ROOT = ".npm"
+        RUSTUP_HOME = "/opt/rust/cargo"
+        CARGO_HOME = "/opt/rust/cargo"
+        PATH = """${sh(
+           returnStdout: true,
+           script: 'echo /opt/rust/cargo/bin:\$PATH'
+        )}
+        """
+        // for the semantic-release-rust executable, we must have this set even when not publishing the crate directly
+        CARGO_REGISTRY_TOKEN = "not-in-use"
+    }
+
+    post {
+        always {
+            script {
+                jiraSendBuildInfo site: "logdna.atlassian.net"
+
+                if (env.SANITY_BUILD == "true") {
+                    notifySlack(
+                        currentBuild.currentResult,
+                        [
+                            channel: "#pipeline-bots",
+                            tokenCredentialId: "qa-slack-token"
+                        ],
+                        "`${PROJECT_NAME}` sanity build took ${currentBuild.durationString.replaceFirst(' and counting', '')}."
+                    )
+                }
+            }
+        }
+    }
+
+    stages {
+        stage("Validate") {
+            tools {
+                nodejs "NodeJS 18"
+            }
+
+            steps {
+                script {
+                    sh "mkdir -p ${NPM_CONFIG_CACHE}"
+                    npm.auth token: GITHUB_TOKEN
+                    sh "npx @answerbook/commitlint-config-logdna"
+                }
+            }
+        }
+
+        stage("Test") {
+            when {
+                beforeAgent true
+                not {
+                    changelog "\\[skip ci\\]"
+                }
+            }
+
+            parallel {
+                stage("Rust Unit Tests") {
+                    steps {
+                        script {
+                            sh "docker build --progress=plain --target base ."
+                        }
+                    }
+                }
+
+                stage("Wasm Unit Tests") {
+                    steps {
+                        script {
+                            sh "docker build --progress=plain --target wasm-base ."
+                        }
+                    }
+                }
+
+                stage("Release Test") {
+                    when {
+                        beforeAgent true
+                        not {
+                            branch DEFAULT_BRANCH
+                        }
+                    }
+
+                    environment {
+                        GIT_BRANCH = "${CURRENT_BRANCH}"
+                        BRANCH_NAME = "${CURRENT_BRANCH}"
+                        CHANGE_ID = ""
+                    }
+
+                    tools {
+                        nodejs 'NodeJS 18'
+                    }
+
+                    steps {
+                        script {
+                            sh "mkdir -p ${NPM_CONFIG_CACHE}"
+                            npm.auth token: GITHUB_TOKEN
+                            sh "cargo install semantic-release-cargo --version 2.1.92"
+                            sh "npm install -G semantic-release@^19.0.0 @semantic-release/git@10.0.1 @semantic-release/changelog@6.0.3 @semantic-release/exec@6.0.3 @answerbook/release-config-logdna@2.0.0"
+                            sh 'npx semantic-release --dry-run --no-ci --branches=${BRANCH_NAME:-main}'
+                        }
+                    }
+                }
+            }
+        }
+
+        stage("Release") {
+            when {
+                beforeAgent true
+                branch DEFAULT_BRANCH
+                not {
+                    allOf {
+                        changelog "\\[skip ci\\]"
+                        environment name: "SANITY_BUILD", value: "true"
+                    }
+                }
+            }
+
+            tools {
+                nodejs "NodeJS 18"
+            }
+
+            steps {
+                script {
+                    sh "mkdir -p ${NPM_CONFIG_CACHE}"
+                    npm.auth token: GITHUB_TOKEN
+                    sh "cargo install semantic-release-cargo --version 2.1.92"
+                    sh "npm install -G semantic-release@^19.0.0 @semantic-release/git@10.0.1 @semantic-release/changelog@6.0.3 @semantic-release/exec@6.0.3 @answerbook/release-config-logdna@2.0.0"
+                    sh "npx semantic-release"
+
+                    def RELEASE_VERSION = sh(
+                        returnStdout: true,
+                        script: "cargo metadata -q --no-deps --format-version 1 | jq -r \'.packages[0].version\'"
+                    ).trim()
+
+                    // Only the wasm module needs to be published. The rest of this repo is used as a cargo dependency
+                    // and that does not require publishing the library anywhere. Cargo just uses git urls to find the
+                    // dependency.
+                    sh "docker build --progress=plain --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} --build-arg DRY_RUN=${DRY_RUN} --target wasm-publish ."
+                }
+            }
+        }
+    }
+}

lib/cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vrl-cli"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false
@@ -20,11 +20,11 @@ regex = { version = "1", default-features = false, optional = true, features = [
 rustyline = { version = "11", default-features = false, optional = true }
 serde_json = "1"
 thiserror = "1"
-vrl = { path = "../..", default-features = false }
+vrl = { path = "../..", default-features = false , version = "0.3.0" }
 core = { package = "vrl-core", path = "../core", default-features = false }
-value = { path = "../value", default-features = false, features = [] }
+value = { path = "../value", default-features = false, features = [] , version = "0.3.0" }
 webbrowser = { version = "0.8", default-features = false, optional = true }
-lookup = { package = "lookup", path = "../lookup" }
+lookup = { package = "lookup", path = "../lookup" , version = "0.3.0" }
 
 [dependencies.stdlib]
 package = "vrl-stdlib"

lib/compiler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vrl-compiler"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false
@@ -31,8 +31,8 @@ test = []
 core = { package = "vrl-core", path = "../core", default-features = false }
 diagnostic = { package = "vrl-diagnostic", path = "../diagnostic" }
 parser = { package = "vrl-parser", path = "../parser" }
-lookup = { path = "../lookup" }
-value = { path = "../value" }
+lookup = { path = "../lookup" , version = "0.3.0" }
+value = { path = "../value" , version = "0.3.0" }
 getrandom = { version = "0.2", features = ["js"] }
 
 bytes = { version = "1.4.0", default-features = false }

lib/core/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "vrl-core"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false
 
 [dependencies]
-lookup = { path = "../lookup" }
-value = { path = "../value", features = ["json"] }
+lookup = { path = "../lookup" , version = "0.3.0" }
+value = { path = "../value", features = ["json"] , version = "0.3.0" }
 derivative = "2.1.3"
 diagnostic = { package = "vrl-diagnostic", path = "../diagnostic" }
 chrono-tz = { version = "0.8.1", default-features = false }

lib/datadog/filter/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "datadog-filter"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false
 license = "MPL-2.0"
 
 [dependencies]
-datadog-search-syntax = { path = "../search-syntax" }
+datadog-search-syntax = { path = "../search-syntax" , version = "0.3.0" }
 
 regex = "1"
 dyn-clone = { version = "1.0.11", default-features = false }

lib/datadog/grok/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datadog-grok"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 build = "build.rs" # LALRPOP preprocessing
@@ -22,9 +22,9 @@ thiserror = { version = "1", default-features = false }
 tracing = { version = "0.1.34", default-features = false }
 
 # Internal
-lookup = { path = "../../lookup" }
-value = {path = "../../value", features = ["json", "test"]}
-vrl-compiler = { path = "../../compiler" }
+lookup = { path = "../../lookup" , version = "0.3.0" }
+value = {path = "../../value", features = ["json", "test"], version = "0.3.0" }
+vrl-compiler = { path = "../../compiler" , version = "0.3.0" }
 
 [dev-dependencies]
 vrl-compiler = { path = "../../compiler" }

lib/datadog/search-syntax/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datadog-search-syntax"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false

lib/diagnostic/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vrl-diagnostic"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Vector Contributors <vector@datadoghq.com>"]
 edition = "2021"
 publish = false