From f13bebe42b8ff5e94f9890d3bb445f55bc8a888d Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 17 Sep 2019 16:20:06 -0700 Subject: [PATCH] Wasm dockerfile (#174) (#108) * Add Dockerfile for the C++ SDK. Signed-off-by: John Plevyak --- WASM.md | 61 +--------- api/wasm/cpp/Dockerfile-sdk | 7 ++ api/wasm/cpp/Makefile.base | 6 +- api/wasm/cpp/Makefile.base_lite | 4 + api/wasm/cpp/README.md | 199 +++++++++++++++++++++++++++++++- api/wasm/cpp/build_wasm.sh | 4 + api/wasm/cpp/sdk_container.sh | 54 +++++++++ api/wasm/rust/README.md | 8 ++ 8 files changed, 281 insertions(+), 62 deletions(-) create mode 100644 api/wasm/cpp/Dockerfile-sdk create mode 100755 api/wasm/cpp/build_wasm.sh create mode 100755 api/wasm/cpp/sdk_container.sh create mode 100644 api/wasm/rust/README.md diff --git a/WASM.md b/WASM.md index 95729d0b64ea..20b34dcd668a 100644 --- a/WASM.md +++ b/WASM.md @@ -1,65 +1,14 @@ WebAssembly Extension Support -# Build Dependencies +# C++ SDK -## glib2.0 +A C++ SDK is available including a docker image build environment. See api/wasm/cpp/README.md. -Note: this may be required on Debian/Ubuntu. +# Rust SDK -apt-get install libglib2.0-dev +The Rust SDK is a WIP. See api/wasm/rust/README.md. -## emscripten - -git clone https://github.com/emscripten-core/emsdk.git -cd emsdk -./emsdk install sdk-1.38.25-64bit -./emsdk activate sdk-1.38.25-64bit - -. ./emsdk\_env.sh - -It is possible later versions will work, e.g. - -./emsdk install latest -./emsdk activate latest - -However 1.38.25 is known to work. - -## clang-7 or clang-8 - -export CC=clang -export CXX=clang++ - -Note: ensure that you have clang in your path (e.g. /usr/lib/llvm-7/bin). - -## protobuf v3.6.1 - -git clone https://github.com/protocolbuffers/protobuf -cd protobuf -git checkout v3.6.1 -git submodule update --init --recursive -./autogen.sh -./configure -make -make check -sudo make install - -# Dependencies for regenerating test modules - -## WAVM binaries if you want to rebuild the c++ WebAssembly tests - -git clone git@github.com:WAVM/WAVM.git -cd WAVM -cmake "." -make -sudo make install - -Note: ensure /usr/local/bin is in your path - -## rust if you want to use it or rebuild the rust WebAssembly tests - -curl https://sh.rustup.rs -sSf | sh - -# Building +# Building in WebAssembly support Building with WebAssembly support requires enabling one or more WebAssembly runtime via the "wasm" define: diff --git a/api/wasm/cpp/Dockerfile-sdk b/api/wasm/cpp/Dockerfile-sdk new file mode 100644 index 000000000000..dd4451fb4445 --- /dev/null +++ b/api/wasm/cpp/Dockerfile-sdk @@ -0,0 +1,7 @@ +FROM ubuntu:bionic + +COPY ./sdk_container.sh / +COPY ./build_wasm.sh / +COPY *.cc *.h *.js *.proto Makefile* *.bc /sdk/ + +RUN ./sdk_container.sh diff --git a/api/wasm/cpp/Makefile.base b/api/wasm/cpp/Makefile.base index 480104f8e9d4..530279ccce90 100644 --- a/api/wasm/cpp/Makefile.base +++ b/api/wasm/cpp/Makefile.base @@ -1,4 +1,8 @@ -CPP_API:=$(shell git rev-parse --show-toplevel)/api/wasm/cpp +ifdef DOCKER_SDK + CPP_API:=${DOCKER_SDK} +else + CPP_API:=$(shell git rev-parse --show-toplevel)/api/wasm/cpp +endif ifdef NO_CONTEXT CPP_CONTEXT_LIB = diff --git a/api/wasm/cpp/Makefile.base_lite b/api/wasm/cpp/Makefile.base_lite index 7349f799b6e7..a8ace429e221 100644 --- a/api/wasm/cpp/Makefile.base_lite +++ b/api/wasm/cpp/Makefile.base_lite @@ -1,4 +1,8 @@ +ifdef DOCKER_SDK +CPP_API:=${DOCKER_SDK} +else CPP_API:=$(shell git rev-parse --show-toplevel)/api/wasm/cpp +endif ifdef NO_CONTEXT CPP_CONTEXT_LIB = diff --git a/api/wasm/cpp/README.md b/api/wasm/cpp/README.md index 53d48ad23108..18ad5e23a6ef 100644 --- a/api/wasm/cpp/README.md +++ b/api/wasm/cpp/README.md @@ -1,7 +1,196 @@ -Dependencies for building WASM modules: +# Compiling C++ to .wasm files using the SDK -- You must install the version of protobuf on your build system that matches the libprotobuf.bc files (without any patches) so that the generated code matches the .bc library. - Currently this is based on tag v3.6.1 of https://github.com/protocolbuffers/protobuf. +The SDK has dependencies on specific versions of emscription and the protobuf library, therefor use of a Docker image is recommended. + +## Docker + +A Dockerfile for the C++ SDK is provided in Dockerfile-sdk. + +It can built in this directory by: + +docker build -t wasmsdk:v1 -f Dockerfile-sdk . + +The docker image can be used for compiling wasm files. + +### Creating a project for use with the Docker build image + +Create a directory parallel to envoy with your source files and a Makefile: + +``` +DOCKER_SDK=/sdk + +all: myproject.wasm + +include ${DOCKER_SDK}/Makefile.base_lite +``` + +Source file (myproject.cc): + +``` +#include +#include + +#include "proxy_wasm_intrinsics.h" + +class ExampleContext : public Context { +public: + explicit ExampleContext(uint32_t id, RootContext* root) : Context(id, root) {} + + FilterHeadersStatus onRequestHeaders() override; + FilterDataStatus onRequestBody(size_t body_buffer_length, bool end_of_stream) override; + void onDone() override; +}; +static RegisterContextFactory register_ExampleContext(CONTEXT_FACTORY(ExampleContext)); + +FilterHeadersStatus ExampleContext::onRequestHeaders() { + logInfo(std::string("onRequestHeaders ") + std::to_string(id())); + auto path = getRequestHeader(":path"); + logInfo(std::string("header path ") + std::string(path->view())); + return FilterHeadersStatus::Continue; +} + +void ExampleContext::onDone() { logInfo("onDone " + std::to_string(id())); } +``` + +### Compiling with the Docker build image + +Run docker: + +```bash +docker run -v $PWD:/work -w /work wasmsdk:v1 bash /build_wasm.sh +``` + +### Caching the standard libraries + +The first time that emscripten runs it will generate the standard libraries. To cache these in the docker image, +after the first successful compilation (e.g myproject.cc above), commit the image with the standard libraries: + +```bash +docker commit `docker ps -l | grep wasmsdk:v1 | awk '{print $1}'` wasmsdk:v1 +``` + +This will save time on subsequent compiles. + +### Using the SDK from a newer/specific version of Envoy + +To use a newer/specific version of the SDK (e.g. from the version of Enovy you are going to deploy the WebAssembly module to) bind that volume and use it in the Makefile. + +Makefile referencing the SDK in the /work directory: + +``` +DOCKER_SDK=/work/sdk + +all: myproject.wasm + +include ${DOCKER_SDK}/Makefile.base_lite +``` + +Run docker pointing to Envoy sources in a directory parallel (at the same level) as your project directory: + +```bash +docker run -v $PWD:/work -v $PWD/../envoy/api/wasm/cpp:/work/sdk -w /work wasmsdk:v1 bash /build_wasm.sh +``` + +### Using abseil form the image + +Abseil (optionally) is built in /root/abseil and can be used by customizing the Makefile e.g.: + +``` +DOCKER_SDK=/sdk +CPP_API:=${DOCKER_SDK} +CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc +ABSL = /root/abseil-cpp +ABSL_CPP = ${ABSL}/absl/strings/str_cat.cc ${ABSL}/absl/strings/str_split.cc ${ABSL}/absl/strings/numbers.cc ${ABSL}/absl/strings/ascii.cc + +all: plugin.wasm + +%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB} + ls /root + em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.bc -o $*.js +``` + +Precompiled abseil libraries are also available, so the above can also be done as: + +``` +DOCKER_SDK=/sdk +CPP_API:=${DOCKER_SDK} +CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc +ABSL = /root/abseil-cpp +ABSL_LIBS = ${ABSL}/absl/strings/libabsl_strings.a ${ABSL}/absl/strings/libabsl_strings_internal.a ${ABSL}/absl/strings/libabsl_str_format_internal.a + +all: plugin.wasm + +%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB} + ls /root + em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.bc ${ABSL_LIBS} -o $*.js +``` + +### Ownership of the resulting .wasm files + +The compiled files may be owned by root. To chown them add a line in the Makefile where ID is the desired user id (e.g. the result of "id -u"): + +``` +DOCKER_SDK=/work/sdk + +all: myproject.wasm + chown ID.ID $^ + +include ${DOCKER_SDK}/Makefile.base_lite +``` + +## Dependencies for building WASM modules: + +If you do not wish to use the Docker file, the dependencies can be installed by script (sdk\_container.sh), or by hand. + +### protobuf v3.6.1 + +You must install the version of protobuf on your build system that matches the libprotobuf.bc files (without any patches) so that the generated code matches the .bc library. Currently this is based on tag v3.6.1 of https://github.com/protocolbuffers/protobuf. + +```bash +git clone https://github.com/protocolbuffers/protobuf +cd protobuf +git checkout v3.6.1 +git submodule update --init --recursive +./autogen.sh +./configure +make +make check +sudo make install +``` + +### rebulding the protobuf.bc files + +If want to rebuild the .bc files or use a different version see the instructions at https://github.com/kwonoj/protobuf-wasm (note: this is pinned to git tag v3.6.1). A pre-patched repo is available at https://github.com/jplevyak/protobuf branch envoy-wasm. + +### emscripten + +```bash +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install sdk-1.38.42-64bit +./emsdk activate sdk-1.38.42-64bit + +. ./emsdk\_env.sh +``` + +It is possible later versions will work, e.g. + +```bash +./emsdk install latest +./emsdk activate latest +``` + +However 1.38.42 is known to work. + +### WAVM binaries + +```bash +git clone git@github.com:WAVM/WAVM.git +cd WAVM +cmake "." +make +sudo make install +``` + +Note: ensure /usr/local/bin is in your path -- If want to rebuild the .bc files or use a different version see the instructions at https://github.com/kwonoj/protobuf-wasm (note: this is pinned to git tag v3.6.1) - A pre-patched repo is available at https://github.com/jplevyak/protobuf branch envoy-wasm diff --git a/api/wasm/cpp/build_wasm.sh b/api/wasm/cpp/build_wasm.sh new file mode 100755 index 000000000000..440e8ee40d29 --- /dev/null +++ b/api/wasm/cpp/build_wasm.sh @@ -0,0 +1,4 @@ +#!/bin/bash +source /root/emsdk/emsdk_env.sh +export PATH=/usr/local/bin:$PATH +make diff --git a/api/wasm/cpp/sdk_container.sh b/api/wasm/cpp/sdk_container.sh new file mode 100755 index 000000000000..280e423c6e2d --- /dev/null +++ b/api/wasm/cpp/sdk_container.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# basics +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get upgrade -y +apt-get install -y --no-install-recommends apt-utils ca-certificates +apt-get autoremove -y +apt-get clean +apt-get install -y --no-install-recommends software-properties-common apt-transport-https git wget curl libglib2.0-dev autoconf autotools-dev automake libtool cmake python + +# gcc-7 +apt-get install -y --no-install-recommends gcc-7 g++-7 cpp-7 +export CC=gcc-7 +export CXX=g++-7 +export CPP=cpp-7 + +# specific version of protobufs to match the pre-compiled support libraries +git clone https://github.com/protocolbuffers/protobuf +cd protobuf +git checkout v3.6.1 -b v3.6.1 +git submodule update --init --recursive +./autogen.sh +./configure +make +make check +make install +cd + +# emscripten +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install 1.38.42 +./emsdk activate 1.38.42 +source ./emsdk_env.sh +cd + +# abseil (optional) +git clone https://github.com/abseil/abseil-cpp +cd abseil-cpp +git checkout 14550beb3b7b97195e483fb74b5efb906395c31e -b Jul302019 # Jul 30 2019 +emconfigure cmake -DCMAKE_CXX_STANDARD=17 "." +emmake make +cd + +# WAVM (optional) +apt-get install -y --no-install-recommends llvm-6.0-dev +git clone https://github.com/WAVM/WAVM +cd WAVM +git checkout 5e69711c074d16e717e476e3e7365505ebb0c42b -b Aug292019 # Aug 29 2019 +cmake "." +make +make install +cd diff --git a/api/wasm/rust/README.md b/api/wasm/rust/README.md new file mode 100644 index 000000000000..5d45f4d83264 --- /dev/null +++ b/api/wasm/rust/README.md @@ -0,0 +1,8 @@ +# Compiling rust to .wasm files using the SDK + +The rust SDK is a WIP. + +## rust if you want to use it or rebuild the rust WebAssembly tests + +curl https://sh.rustup.rs -sSf | sh +