Merge latest changes from master branch (#217)

* Not call report if decodeHeaders is not called. (#150) * Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again * Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 * Uses a specific version of the Shared Pipeline lib (#158) * Update lyft/envoy commit Id to latest. (#161) * Update lyft/envoy commit Id to latest. * Remove the comment about pull request * Add new line - will delete in next commit. * Update repositories.bzl (#169) * Always set response latency (#172) * Update mixerclient to sync_transport change. (#178) * Use opaque config to turn on/off forward attribute and mixer filter (#179) * Modify mixer filter * Swap defaults * Make the filter decoder only * cache mixer disabled decision * Fix a bug in opaque config change and test it out (#182) * Fix a bug and test it out * Update filter type * Update README.md * Update mixer client to mixer api with gogoproto. (#184) * Move .bazelrc to tools/bazel.rc (#186) * Move .bazelrc to tools/bazel.rc * Update Jenkinsfile with latest version of pipeline * Support apikey based traffic restriction (#189) * b/36368559 support apikey based traffic restriction * Fixed code formatting * Fix crash in unreachable/overloaded RDS (#190) * Add mixer client end to end integration test. (#177) * Add mixer client end to end integration test. * Split some repositories into a separate file. * use real mixer for fake mixer_server. * Test repository * use mixer bzl file. * Use mixer repositories * Not to use mixer repository. * Add return line at the end of WORKSPACE. * Fix broken link (#193) * Make quota call (#192) * hookup quota call * Make quota call. * Update indent. * Update envoy and update configs (#195) * Update envoy and update configs * Use gcc-4.9 for travis * Use bazel 0.4.5 * Fix SHA of lightstep-tracer-common * Enable check cache and refactory mixer config loading (#197) * Refactory the mixer config loading. * fix format * Add integration test. * updated README.md * s/send/sent/ * Split into separate tests. (#201) * Update README on how to enable check cache. (#204) * Update README on how to enable check cache. * Update the comment. * build: support Envoy native Bazel build. (#210) * build: support Envoy native Bazel build. This patch switches the Envoy build from src/envoy/repositories.bzl to using the upstream native build. See envoyproxy/envoy#663 for the corresponding changes on the Envoy side. * Use Envoy master with BUILD.wip rename merged. * Fix clang-format issues. * Fixes bazel.rc issues (#212) * Fixes bazel rc issues * Update Jenkins to latest pipeline version
istio · Apr 3, 2017 · d63d7d0 · d63d7d0
1 parent 495ed68
commit d63d7d0
Show file tree

Hide file tree

Showing 35 changed files with 2,130 additions and 512 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,25 +1,27 @@
 sudo: required
-dist: xenial
+
+dist: trusty
+
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - gcc-4.9
+    - g++-4.9
+    - wget
 
 branches:
   except:
   - stable
 
-lang: go
-
-go:
-  - 1.7.x
+language: cpp
 
 jdk:
   - oraclejdk8
 
 env:
-  - BAZEL_VERSION=0.4.3
-
-addons:
-  apt:
-    packages:
-      - wget
+  - BAZEL_VERSION=0.4.5
 
 cache:
   directories:
@@ -35,12 +37,12 @@ before_install:
   - sudo dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb
   - sudo apt-get -f install -qqy uuid-dev
   - cd ${TRAVIS_BUILD_DIR}
-  - mv .bazelrc .bazelrc.orig
-  - cat .bazelrc.travis .bazelrc.orig > .bazelrc
+  - mv tools/bazel.rc tools/bazel.rc.orig
+  - cat tools/bazel.rc.travis tools/bazel.rc.orig > tools/bazel.rc
 
 script:
   - script/check-style
-  - bazel --output_base=${HOME}/bazel/outbase test //...
+  - CC=/usr/bin/gcc-4.9 CXX=/usr/bin/g++-4.9 bazel --output_base=${HOME}/bazel/outbase test //...
 
 notifications:
   slack: istio-dev:wEEEbaabdP5ieCgDOFetA9nX
diff --git a/BUILD b/BUILD
@@ -24,3 +24,7 @@ config_setting(
     },
     visibility = ["//visibility:public"],
 )
+
+load("@io_bazel_rules_go//go:def.bzl", "go_prefix")
+
+go_prefix("istio.io/proxy")
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,6 @@
 #!groovy
 
-@Library('testutils')
+@Library('testutils@stable-3e4d089')
 
 import org.istio.testutils.Utilities
 import org.istio.testutils.GitUtilities
@@ -12,20 +12,17 @@ def utils = new Utilities()
 def bazel = new Bazel()
 
 mainFlow(utils) {
-  pullRequest(utils) {
-    node {
-      gitUtils.initialize()
-      // Proxy does build work correctly with Hazelcast.
-      // Must use .bazelrc.jenkins
-      bazel.setVars('', '')
-    }
-
-    if (utils.runStage('PRESUBMIT')) {
-      presubmit(gitUtils, bazel)
-    }
-    if (utils.runStage('POSTSUBMIT')) {
-      postsubmit(gitUtils, bazel, utils)
-    }
+  node {
+    gitUtils.initialize()
+    // Proxy does build work correctly with Hazelcast.
+    // Must use .bazelrc.jenkins
+    bazel.setVars('', '')
+  }
+  if (utils.runStage('PRESUBMIT')) {
+    presubmit(gitUtils, bazel)
+  }
+  if (utils.runStage('POSTSUBMIT')) {
+    postsubmit(gitUtils, bazel, utils)
   }
 }
 
@@ -62,4 +59,4 @@ def postsubmit(gitUtils, bazel, utils) {
       utils.publishDockerImages(images, tags, 'release')
     }
   }
-}
+}
diff --git a/WORKSPACE b/WORKSPACE
@@ -96,3 +96,19 @@ http_file(
     url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz",
     sha256 = DEBUG_BASE_IMAGE_SHA,
 )
+
+# Following go repositories are for building go integration test for mixer filter.
+git_repository(
+    name = "io_bazel_rules_go",
+    commit = "9496d79880a7d55b8e4a96f04688d70a374eaaf4", # Mar 3, 2017 (v0.4.1)
+    remote = "https://github.com/bazelbuild/rules_go.git",
+)
+
+git_repository(
+    name = "org_pubref_rules_protobuf",
+    commit = "d42e895387c658eda90276aea018056fcdcb30e4", # Mar 07 2017 (gogo* support)
+    remote = "https://github.com/pubref/rules_protobuf",
+)
+
+load("//src/envoy/mixer/integration_test:repositories.bzl", "go_mixer_repositories")
+go_mixer_repositories()
diff --git a/src/envoy/BUILD b/src/envoy/BUILD
@@ -1,34 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-################################################################################
-#
-cc_test(
-    name = "envoy_test",
-    data = [
-        "@envoy_git//:envoy-testdata",
-    ],
-    copts = [
-        "-include ./external/envoy_git/test/precompiled/precompiled_test.h",
-    ],
-    deps = [
-        "@envoy_git//:envoy-test-lib",
-        "//external:googletest_main",
-    ],
-    args = [
-        #TODO: Make all test pass
-        "--gtest_filter=RouterTest.*",
-    ],
-    linkstatic=1,
-)

diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD
@@ -15,7 +15,6 @@
 ################################################################################
 #
 
-
 load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build")
 load("@protobuf_git//:protobuf.bzl", "cc_proto_library")
@@ -31,7 +30,8 @@ cc_proto_library(
 cc_library(
     name = "filter_lib",
     srcs = [
-        "forward_attribute_filter.cc",
+        "config.cc",
+        "config.h",
         "http_control.cc",
         "http_control.h",
         "http_filter.cc",
@@ -41,17 +41,19 @@ cc_library(
     deps = [
         ":string_map_proto",
         "//external:mixer_client_lib",
-        "@envoy_git//:envoy-common",
+        "@envoy//source/exe:envoy_common_lib",
     ],
     alwayslink = 1,
 )
 
 cc_binary(
     name = "envoy",
     linkstatic = 1,
+    linkopts = ["-lrt"],
+    visibility = [":__subpackages__"],
     deps = [
         ":filter_lib",
-        "@envoy_git//:envoy-main",
+        "@envoy//source/exe:envoy_main_lib",
     ],
 )
 
@@ -81,10 +83,6 @@ pkg_tar(
 )
 
 proxy_docker_build(
-    images = [
-        {"name": "proxy",       "base": "@docker_ubuntu//:xenial"},
-        {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"},
-    ],
     entrypoint = [
         "/usr/local/bin/start_envoy",
         "-e",
@@ -94,6 +92,16 @@ proxy_docker_build(
         "-t",
         "/etc/opt/proxy/envoy.conf.template",
     ],
+    images = [
+        {
+            "name": "proxy",
+            "base": "@docker_ubuntu//:xenial",
+        },
+        {
+            "name": "proxy_debug",
+            "base": "@ubuntu_xenial_debug//file",
+        },
+    ],
     ports = ["9090"],
     repository = "istio",
     tags = ["manual"],

diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md
@@ -3,7 +3,7 @@ This Proxy will use Envoy and talk to Mixer server.
 
 ## Build Mixer server
 
-* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via:
+* Follow https://github.com/istio/mixer/blob/master/doc/dev/development.md to set up environment, and build via:
 
 ```
   cd $(ISTIO)/mixer
@@ -46,54 +46,98 @@ This Proxy will use Envoy and talk to Mixer server.
 * Then issue HTTP request to proxy.
 
 ```
+  # request to server-side proxy
   curl http://localhost:9090/echo -d "hello world"
+  # request to client-side proxy that gets sent to server-side proxy
+  curl http://localhost:7070/echo -d "hello world"
 ```
 
-## How to configurate HTTP filters
-
-This module has two HTTP filters:
-1. mixer filter: intercept all HTTP requests, call the mixer.
-2. forward_attribute filter: Forward attributes to the upstream istio/proxy.
-
-### *mixer* filter:
+## How to configurate HTTP Mixer filters
 
 This filter will intercept all HTTP requests and call Mixer. Here is its config:
 
 ```
    "filters": [
-      "type": "both",
+      "type": "decoder",
       "name": "mixer",
       "config": {
          "mixer_server": "${MIXER_SERVER}",
-         "attributes" : {
+         "mixer_attributes" : {
+            "attribute_name1": "attribute_value1",
+            "attribute_name2": "attribute_value2",
+            "quota.name": "RequestCount"
+         },
+         "forward_attributes" : {
             "attribute_name1": "attribute_value1",
             "attribute_name2": "attribute_value2"
-         }
+         },
+         "quota_name": "RequestCount",
+         "quota_amount": "1",
+         "check_cache_expiration_in_seconds": "600",
+         "check_cache_keys": [
+              "request.host",
+              "request.path",
+              "origin.user"
+         ]
     }
 ```
 
 Notes:
 * mixer_server is required
-* attributes: these attributes will be send to the mixer
-
-### *forward_attribute* HTTP filter:
+* mixer_attributes: these attributes will be sent to the mixer in both Check and Report calls.
+* forward_attributes: these attributes will be forwarded to the upstream istio/proxy. It will send them to mixer in Check and Report calls.
+* quota_name, quota_amount are used for making quota call. quota_amount defaults to 1.
+* check_cache_keys is to cache check calls. If missing or empty, check calls are not cached.
 
-This filer will forward attributes to the upstream istio/proxy.
+## HTTP Route opaque config
+By default, the mixer filter only forwards attributes and does not call mixer server. This behavior can be changed per HTTP route by supplying an opaque config:
 
 ```
-   "filters": [
-      "type": "decoder",
-      "name": "forward_attribute",
-      "config": {
-         "attributes": {
-            "attribute_name1": "attribute_value1",
-            "attribute_name2": "attribute_value2"
- 	    }
-    }
+ "routes": [
+   {
+     "timeout_ms": 0,
+     "prefix": "/",
+     "cluster": "service1",
+     "opaque_config": {
+      "mixer_control": "on",
+      "mixer_forward": "off"
+     }
+   }
 ```
 
-Notes:
-* attributes: these attributes will be forwarded to the upstream istio/proxy.
+This route opaque config reverts the behavior by sending requests to mixer server but not forwarding any attributes.
 
 
+## How to enable quota (rate limiting)
+
+Quota (rate limiting) is enforced by the mixer. Mixer needs to be configured with Quota in its global config and service config. Its quota config will have
+"quota name", its limit within a window.  If "Quota" is added but param is missing, the default config is: quota name is "RequestCount", the limit is 10 with 1 second window. Essentially, it is imposing 10 qps rate limiting.
+
+Mixer client can be configured to make Quota call for all requests.  If "quota_name" is specified in the mixer filter config, mixer client will call Quota with the specified quota name.  If "quota_amount" is specified, it will call with that amount, otherwise the used amount is 1.
+
+
+## How to pass some attributes from client proxy to mixer.
+
+Usually client proxy is not configured to call mixer (it can be enabled in the route opaque_config). Client proxy can pass some attributes to mixer by using "forward_attributes" field.  Its attributes will be sent to the upstream proxy (the server proxy). If the server proxy is calling mixer, these attributes will be sent to the mixer.
+
+
+## How to enable cache for Check calls
+
+Check calls can be cached. By default, it is not enabled. It can be enabled by supplying non-empty "check_cache_keys" string list in the mixer filter config. Only these attributes in the Check request, their keys and values, are used to calculate the key for the cache lookup. If it is a cache hit, the cached response will be used.
+The cached response will be expired in 5 minutes by default. It can be overrided by supplying "check_cache_expiration_in_seconds" in the mixer filter config. The Check response from the mixer has an expiration field. If it is filled, it will be used. By design, the mixer will control the cache expiration time.
+
+Following is a sample mixer filter config to enable the Check call cache:
+```
+         "check_cache_expiration_in_seconds": "600",
+         "check_cache_keys": [
+              "request.host",
+              "request.path",
+              "source.labels",
+              "request.headers/:method",
+              "origin.user"
+         ]
+```
 
+For the string map attributes in the above example:
+1) "request.headers" attribute is a string map, "request.headers/:method" cache key means only its ":method" key and value are used for cache key.
+2) "source.labels" attribute is a string map, "source.labels" cache key means all key value pairs for the string map will be used.