diff --git a/Documentation/op-guide/container.md b/Documentation/op-guide/container.md index 99d410efb424..bed9c08f28e6 100644 --- a/Documentation/op-guide/container.md +++ b/Documentation/op-guide/container.md @@ -79,14 +79,16 @@ export NODE1=192.168.1.21 Run the latest version of etcd: ``` -docker run --net=host \ - --volume=${DATA_DIR}:/etcd-data \ - --name etcd quay.io/coreos/etcd:latest \ - /usr/local/bin/etcd \ - --data-dir=/etcd-data --name node1 \ - --initial-advertise-peer-urls http://${NODE1}:2380 --listen-peer-urls http://${NODE1}:2380 \ - --advertise-client-urls http://${NODE1}:2379 --listen-client-urls http://${NODE1}:2379 \ - --initial-cluster node1=http://${NODE1}:2380 +docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --volume=${DATA_DIR}:/etcd-data \ + --name etcd quay.io/coreos/etcd:latest \ + /usr/local/bin/etcd \ + --data-dir=/etcd-data --name node1 \ + --initial-advertise-peer-urls http://${NODE1}:2380 --listen-peer-urls http://${NODE1}:2380 \ + --advertise-client-urls http://${NODE1}:2379 --listen-client-urls http://${NODE1}:2379 \ + --initial-cluster node1=http://${NODE1}:2380 ``` List the cluster member: @@ -114,41 +116,47 @@ DATA_DIR=/var/lib/etcd # For node 1 THIS_NAME=${NAME_1} THIS_IP=${HOST_1} -docker run --net=host \ - --volume=${DATA_DIR}:/etcd-data \ - --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ - /usr/local/bin/etcd \ - --data-dir=/etcd-data --name ${THIS_NAME} \ - --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ - --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ - --initial-cluster ${CLUSTER} \ - --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} +docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --volume=${DATA_DIR}:/etcd-data \ + --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ + /usr/local/bin/etcd \ + --data-dir=/etcd-data --name ${THIS_NAME} \ + --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ + --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ + --initial-cluster ${CLUSTER} \ + --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} # For node 2 THIS_NAME=${NAME_2} THIS_IP=${HOST_2} -docker run --net=host \ - --volume=${DATA_DIR}:/etcd-data \ - --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ - /usr/local/bin/etcd \ - --data-dir=/etcd-data --name ${THIS_NAME} \ - --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ - --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ - --initial-cluster ${CLUSTER} \ - --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} +docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --volume=${DATA_DIR}:/etcd-data \ + --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ + /usr/local/bin/etcd \ + --data-dir=/etcd-data --name ${THIS_NAME} \ + --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ + --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ + --initial-cluster ${CLUSTER} \ + --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} # For node 3 THIS_NAME=${NAME_3} THIS_IP=${HOST_3} -docker run --net=host \ - --volume=${DATA_DIR}:/etcd-data \ - --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ - /usr/local/bin/etcd \ - --data-dir=/etcd-data --name ${THIS_NAME} \ - --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ - --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ - --initial-cluster ${CLUSTER} \ - --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} +docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --volume=${DATA_DIR}:/etcd-data \ + --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \ + /usr/local/bin/etcd \ + --data-dir=/etcd-data --name ${THIS_NAME} \ + --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \ + --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \ + --initial-cluster ${CLUSTER} \ + --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN} ``` To run `etcdctl` using API version 3: @@ -170,17 +178,19 @@ rkt run \ --volume etcd-ssl-certs-bundle,kind=host,source=/etc/ssl/certs/ca-certificates.crt \ --mount volume=etcd-ssl-certs-bundle,target=/etc/ssl/certs/ca-certificates.crt \ quay.io/coreos/etcd:latest -- --name my-name \ - --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \ - --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \ - --discovery https://discovery.etcd.io/c11fbcdc16972e45253491a24fcf45e1 + --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \ + --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \ + --discovery https://discovery.etcd.io/c11fbcdc16972e45253491a24fcf45e1 ``` ``` docker run \ - --volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt \ - quay.io/coreos/etcd:latest \ - /usr/local/bin/etcd --name my-name \ - --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \ - --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \ - --discovery https://discovery.etcd.io/86a9ff6c8cb8b4c4544c1a2f88f8b801 + -p 2379:2379 \ + -p 2380:2380 \ + --volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt \ + quay.io/coreos/etcd:latest \ + /usr/local/bin/etcd --name my-name \ + --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \ + --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \ + --discovery https://discovery.etcd.io/86a9ff6c8cb8b4c4544c1a2f88f8b801 ``` diff --git a/Documentation/op-guide/etcd3_alert.rules b/Documentation/op-guide/etcd3_alert.rules index 90c3770e8023..e25cbc4c2b27 100644 --- a/Documentation/op-guide/etcd3_alert.rules +++ b/Documentation/op-guide/etcd3_alert.rules @@ -76,7 +76,7 @@ LABELS { } ANNOTATIONS { summary = "slow gRPC requests", - description = "on etcd instance {{ $labels.instance }} gRPC requests to {{ $label.grpc_method }} are slow", + description = "on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method }} are slow", } # HTTP requests alerts @@ -117,7 +117,7 @@ LABELS { } ANNOTATIONS { summary = "slow HTTP requests", - description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $label.method }} are slow", + description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow", } # file descriptor alerts @@ -161,7 +161,7 @@ LABELS { } ANNOTATIONS { summary = "etcd member communication is slow", - description = "etcd instance {{ $labels.instance }} member communication with {{ $label.To }} is slow", + description = "etcd instance {{ $labels.instance }} member communication with {{ $labels.To }} is slow", } # etcd proposal alerts diff --git a/Documentation/upgrades/upgrade_3_0.md b/Documentation/upgrades/upgrade_3_0.md index 6531b472fea7..57c117911c7c 100644 --- a/Documentation/upgrades/upgrade_3_0.md +++ b/Documentation/upgrades/upgrade_3_0.md @@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this #### Upgrade requirements -To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.0) before upgrading to 3.0. +To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.8) before upgrading to 3.0. Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl cluster-health` command before proceeding. @@ -52,7 +52,7 @@ member 8211f1d0f64f3269 is healthy: got healthy result from http://localhost:123 cluster is healthy $ curl http://localhost:2379/version -{"etcdserver":"2.3.x","etcdcluster":"2.3.0"} +{"etcdserver":"2.3.x","etcdcluster":"2.3.8"} ``` #### 2. Stop the existing etcd process diff --git a/Documentation/upgrades/upgrade_3_1.md b/Documentation/upgrades/upgrade_3_1.md index e5fe06b51772..7eabd9ab4cf1 100644 --- a/Documentation/upgrades/upgrade_3_1.md +++ b/Documentation/upgrades/upgrade_3_1.md @@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this #### Upgrade requirements -To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please upgrade to [3.0](https://github.com/coreos/etcd/releases/tag/v3.0.16) before upgrading to 3.1. +To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please [upgrade to 3.0](upgrade_3_0.md) before upgrading to 3.1. Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding. diff --git a/Documentation/upgrades/upgrade_3_2.md b/Documentation/upgrades/upgrade_3_2.md index 47c5bef2bffb..1252ed361e16 100644 --- a/Documentation/upgrades/upgrade_3_2.md +++ b/Documentation/upgrades/upgrade_3_2.md @@ -50,7 +50,7 @@ clientv3yaml.NewConfig #### Upgrade requirements -To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please upgrade to [3.1](https://github.com/coreos/etcd/releases/tag/v3.1.7) before upgrading to 3.2. +To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please [upgrade to 3.1](upgrade_3_1.md) before upgrading to 3.2. Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding. diff --git a/Documentation/v2/etcd_alert.rules b/Documentation/v2/etcd_alert.rules index 1793cad49adc..5493c56b6630 100644 --- a/Documentation/v2/etcd_alert.rules +++ b/Documentation/v2/etcd_alert.rules @@ -62,7 +62,7 @@ ALERT HTTPRequestsSlow } ANNOTATIONS { summary = "slow HTTP requests", - description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $label.method }} are slow", + description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow", } ### File descriptor alerts ### diff --git a/NEWS b/NEWS index b11975f3b84a..771966e7b31c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,60 @@ +etcd v3.2.0 (2017-06-09) +- improved backend read concurrency +- embedded etcd + - Etcd.Peers field is now []*peerListener +- RPCs + - add Election, Lock service +- native client etcdserver/api/v3client + - client "embedded" in the server +- v3 client + - LeaseTimeToLive returns TTL=-1 resp on lease not found + - clientv3.NewFromConfigFile is moved to clientv3/yaml.NewConfig + - STM prefetching + - add namespace feature + - concurrency package's elections updated to match RPC interfaces + - let client dial endpoints not in the balancer + - add ErrOldCluster with server version checking + - translate WithPrefix() into WithFromKey() for empty key +- v3 etcdctl + - add check perf command + - add --from-key flag to role grant-permission command + - lock command takes an optional command to execute +- etcd flags + - add --enable-v2 flag to configure v2 backend (enabled by default) + - add --auth-token flag +- gRPC proxy + - proxy endpoint discovery + - namespaces + - coalesce lease requests +- gateway + - support DNS SRV priority +- auth + - support Watch API + - JWT tokens +- logging, monitoring + - server warns large snapshot operations + - add 'etcd_debugging_server_lease_expired_total' metrics +- security + - deny incoming peer certs with wrong IP SAN + - resolve TLS DNSNames when SAN checking + - reload TLS certificates on every client connection +- release + - annotate acbuild with supports-systemd-notify + - add nsswitch.conf to Docker container image + - add ppc64le, arm64(experimental) builds + - Go 1.8.3 + - gRPC v1.2.1 + - grpc-gateway to v1.2.0 +- v2 + - allow snapshot over 512MB + +etcd v3.1.9 (2017-06-09) +- allow v2 snapshot over 512MB + +etcd v3.1.8 (2017-05-19) + +etcd v3.1.7 (2017-04-28) + etcd v3.1.6 (2017-04-19) - remove auth check in Status API - fill in Auth API response header diff --git a/bill-of-materials.json b/bill-of-materials.json index d0fd91605df2..0a0f98a99519 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -1,207 +1,379 @@ [ { "project": "bitbucket.org/ww/goautoneg", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 1 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 1 + } + ] }, { "project": "github.com/beorn7/perks/quantile", - "license": "MIT License", - "confidence": 0.989 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9891304347826086 + } + ] }, { "project": "github.com/bgentry/speakeasy", - "license": "MIT License", - "confidence": 0.944 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9441624365482234 + } + ] }, { "project": "github.com/boltdb/bolt", - "license": "MIT License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] }, { "project": "github.com/cockroachdb/cmux", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/coreos/etcd", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/coreos/go-semver/semver", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/coreos/go-systemd", - "license": "Apache License 2.0", - "confidence": 0.997 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9966703662597114 + } + ] }, { "project": "github.com/coreos/pkg", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/cpuguy83/go-md2man/md2man", - "license": "MIT License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] }, { "project": "github.com/dgrijalva/jwt-go", - "license": "MIT License", - "confidence": 0.989 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9891304347826086 + } + ] }, { "project": "github.com/dustin/go-humanize", - "license": "MIT License", - "confidence": 0.969 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.96875 + } + ] }, { "project": "github.com/ghodss/yaml", - "license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 1 + } + ] }, { "project": "github.com/gogo/protobuf/proto", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.909 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9090909090909091 + } + ] }, { "project": "github.com/golang/groupcache/lru", - "license": "Apache License 2.0", - "confidence": 0.997 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9966703662597114 + } + ] }, { "project": "github.com/golang/protobuf", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.92 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.92 + } + ] }, { "project": "github.com/google/btree", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/grpc-ecosystem/go-grpc-prometheus", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/grpc-ecosystem/grpc-gateway", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.979 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.979253112033195 + } + ] }, { "project": "github.com/inconshreveable/mousetrap", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 1 + }, + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/jonboulle/clockwork", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/mattn/go-runewidth", - "license": "MIT License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] }, { "project": "github.com/matttproud/golang_protobuf_extensions/pbutil", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/olekukonko/tablewriter", - "license": "MIT License", - "confidence": 0.989 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9891304347826086 + } + ] }, { "project": "github.com/prometheus/client_golang/prometheus", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/prometheus/client_model/go", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/prometheus/common", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/prometheus/procfs", - "license": "Apache License 2.0", - "confidence": 1 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] }, { "project": "github.com/russross/blackfriday", - "license": "BSD 2-clause \"Simplified\" License", - "confidence": 0.963 + "licenses": [ + { + "type": "BSD 2-clause \"Simplified\" License", + "confidence": 0.9626168224299065 + } + ] }, { "project": "github.com/spf13/cobra", - "license": "Apache License 2.0", - "confidence": 0.957 + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9573241061130334 + } + ] }, { "project": "github.com/spf13/pflag", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.966 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] }, { "project": "github.com/ugorji/go/codec", - "license": "MIT License", - "confidence": 0.995 + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9946524064171123 + } + ] }, { "project": "github.com/urfave/cli", - "license": "MIT License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] }, { "project": "github.com/xiang90/probing", - "license": "MIT License", - "confidence": 1 + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] }, { "project": "golang.org/x/crypto", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.966 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] }, { "project": "golang.org/x/net", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.966 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] }, { "project": "golang.org/x/text", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.966 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] }, { "project": "golang.org/x/time/rate", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.966 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] }, { "project": "google.golang.org/grpc", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.979 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.979253112033195 + } + ] }, { "project": "gopkg.in/cheggaaa/pb.v1", - "license": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.992 + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9916666666666667 + } + ] }, { "project": "gopkg.in/yaml.v2", - "license": "Apache License 2.0 and MIT License", - "confidence": 1 + "licenses": [ + { + "type": "The Unlicense", + "confidence": 0.35294117647058826 + }, + { + "type": "MIT License", + "confidence": 0.8975609756097561 + } + ] } ] diff --git a/bill-of-materials.override.json b/bill-of-materials.override.json index 914cb5c90486..34de90e7142d 100644 --- a/bill-of-materials.override.json +++ b/bill-of-materials.override.json @@ -1,18 +1,26 @@ [ { "project": "bitbucket.org/ww/goautoneg", - "license": "BSD 3-clause \"New\" or \"Revised\" License" + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License" + } + ] }, { "project": "github.com/ghodss/yaml", - "license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License" + "licenses": [ + { + "type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License" + } + ] }, { "project": "github.com/inconshreveable/mousetrap", - "license": "Apache License 2.0" - }, - { - "project": "gopkg.in/yaml.v2", - "license": "Apache License 2.0 and MIT License" + "licenses": [ + { + "type": "Apache License 2.0" + } + ] } ] diff --git a/cmd/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go b/cmd/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go index b8cea1c73ea7..292fca343bc4 100644 --- a/cmd/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go +++ b/cmd/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go @@ -118,11 +118,24 @@ func (r *roffRenderer) Paragraph(out *bytes.Buffer, text func() bool) { } } -// TODO: This might now work func (r *roffRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) { - out.WriteString(".TS\nallbox;\n") - + out.WriteString("\n.TS\nallbox;\n") + + max_delims := 0 + lines := strings.Split(strings.TrimRight(string(header), "\n")+"\n"+strings.TrimRight(string(body), "\n"), "\n") + for _, w := range lines { + cur_delims := strings.Count(w, "\t") + if cur_delims > max_delims { + max_delims = cur_delims + } + } + out.Write([]byte(strings.Repeat("l ", max_delims+1) + "\n")) + out.Write([]byte(strings.Repeat("l ", max_delims+1) + ".\n")) out.Write(header) + if len(header) > 0 { + out.Write([]byte("\n")) + } + out.Write(body) out.WriteString("\n.TE\n") } @@ -132,24 +145,30 @@ func (r *roffRenderer) TableRow(out *bytes.Buffer, text []byte) { out.WriteString("\n") } out.Write(text) - out.WriteString("\n") } func (r *roffRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, align int) { if out.Len() > 0 { - out.WriteString(" ") + out.WriteString("\t") } - out.Write(text) - out.WriteString(" ") + if len(text) == 0 { + text = []byte{' '} + } + out.Write([]byte("\\fB\\fC" + string(text) + "\\fR")) } -// TODO: This is probably broken func (r *roffRenderer) TableCell(out *bytes.Buffer, text []byte, align int) { if out.Len() > 0 { out.WriteString("\t") } + if len(text) > 30 { + text = append([]byte("T{\n"), text...) + text = append(text, []byte("\nT}")...) + } + if len(text) == 0 { + text = []byte{' '} + } out.Write(text) - out.WriteString("\t") } func (r *roffRenderer) Footnotes(out *bytes.Buffer, text func() bool) { diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/decode.go b/cmd/vendor/github.com/gogo/protobuf/proto/decode.go index 0d6634cc085b..737f2731d45d 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/decode.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/decode.go @@ -61,7 +61,6 @@ var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. func DecodeVarint(buf []byte) (x uint64, n int) { - // x, n already 0 for shift := uint(0); shift < 64; shift += 7 { if n >= len(buf) { return 0, 0 @@ -78,13 +77,7 @@ func DecodeVarint(buf []byte) (x uint64, n int) { return 0, 0 } -// DecodeVarint reads a varint-encoded integer from the Buffer. -// This is the format for the -// int32, int64, uint32, uint64, bool, and enum -// protocol buffer types. -func (p *Buffer) DecodeVarint() (x uint64, err error) { - // x, err already 0 - +func (p *Buffer) decodeVarintSlow() (x uint64, err error) { i := p.index l := len(p.buf) @@ -107,6 +100,107 @@ func (p *Buffer) DecodeVarint() (x uint64, err error) { return } +// DecodeVarint reads a varint-encoded integer from the Buffer. +// This is the format for the +// int32, int64, uint32, uint64, bool, and enum +// protocol buffer types. +func (p *Buffer) DecodeVarint() (x uint64, err error) { + i := p.index + buf := p.buf + + if i >= len(buf) { + return 0, io.ErrUnexpectedEOF + } else if buf[i] < 0x80 { + p.index++ + return uint64(buf[i]), nil + } else if len(buf)-i < 10 { + return p.decodeVarintSlow() + } + + var b uint64 + // we already checked the first byte + x = uint64(buf[i]) - 0x80 + i++ + + b = uint64(buf[i]) + i++ + x += b << 7 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 7 + + b = uint64(buf[i]) + i++ + x += b << 14 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 14 + + b = uint64(buf[i]) + i++ + x += b << 21 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 21 + + b = uint64(buf[i]) + i++ + x += b << 28 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 28 + + b = uint64(buf[i]) + i++ + x += b << 35 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 35 + + b = uint64(buf[i]) + i++ + x += b << 42 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 42 + + b = uint64(buf[i]) + i++ + x += b << 49 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 49 + + b = uint64(buf[i]) + i++ + x += b << 56 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 56 + + b = uint64(buf[i]) + i++ + x += b << 63 + if b&0x80 == 0 { + goto done + } + // x -= 0x80 << 63 // Always zero. + + return 0, errOverflow + +done: + p.index = i + return x, nil +} + // DecodeFixed64 reads a 64-bit integer from the Buffer. // This is the format for the // fixed64, sfixed64, and double protocol buffer types. @@ -340,6 +434,8 @@ func (p *Buffer) DecodeGroup(pb Message) error { // Buffer and places the decoded result in pb. If the struct // underlying pb does not match the data in the buffer, the results can be // unpredictable. +// +// Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. func (p *Buffer) Unmarshal(pb Message) error { // If the object can unmarshal itself, let it. if u, ok := pb.(Unmarshaler); ok { diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/decode_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/decode_gogo.go index ecc63873e479..6fb74de4cc94 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/decode_gogo.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/decode_gogo.go @@ -98,7 +98,7 @@ func setPtrCustomType(base structPointer, f field, v interface{}) { if v == nil { return } - structPointer_SetStructPointer(base, f, structPointer(reflect.ValueOf(v).Pointer())) + structPointer_SetStructPointer(base, f, toStructPointer(reflect.ValueOf(v))) } func setCustomType(base structPointer, f field, value interface{}) { @@ -165,7 +165,8 @@ func (o *Buffer) dec_custom_slice_bytes(p *Properties, base structPointer) error } newBas := appendStructPointer(base, p.field, p.ctype) - setCustomType(newBas, 0, custom) + var zero field + setCustomType(newBas, zero, custom) return nil } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/duration.go b/cmd/vendor/github.com/gogo/protobuf/proto/duration.go new file mode 100644 index 000000000000..93464c91cffb --- /dev/null +++ b/cmd/vendor/github.com/gogo/protobuf/proto/duration.go @@ -0,0 +1,100 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +// This file implements conversions between google.protobuf.Duration +// and time.Duration. + +import ( + "errors" + "fmt" + "time" +) + +const ( + // Range of a Duration in seconds, as specified in + // google/protobuf/duration.proto. This is about 10,000 years in seconds. + maxSeconds = int64(10000 * 365.25 * 24 * 60 * 60) + minSeconds = -maxSeconds +) + +// validateDuration determines whether the Duration is valid according to the +// definition in google/protobuf/duration.proto. A valid Duration +// may still be too large to fit into a time.Duration (the range of Duration +// is about 10,000 years, and the range of time.Duration is about 290). +func validateDuration(d *duration) error { + if d == nil { + return errors.New("duration: nil Duration") + } + if d.Seconds < minSeconds || d.Seconds > maxSeconds { + return fmt.Errorf("duration: %#v: seconds out of range", d) + } + if d.Nanos <= -1e9 || d.Nanos >= 1e9 { + return fmt.Errorf("duration: %#v: nanos out of range", d) + } + // Seconds and Nanos must have the same sign, unless d.Nanos is zero. + if (d.Seconds < 0 && d.Nanos > 0) || (d.Seconds > 0 && d.Nanos < 0) { + return fmt.Errorf("duration: %#v: seconds and nanos have different signs", d) + } + return nil +} + +// DurationFromProto converts a Duration to a time.Duration. DurationFromProto +// returns an error if the Duration is invalid or is too large to be +// represented in a time.Duration. +func durationFromProto(p *duration) (time.Duration, error) { + if err := validateDuration(p); err != nil { + return 0, err + } + d := time.Duration(p.Seconds) * time.Second + if int64(d/time.Second) != p.Seconds { + return 0, fmt.Errorf("duration: %#v is out of range for time.Duration", p) + } + if p.Nanos != 0 { + d += time.Duration(p.Nanos) + if (d < 0) != (p.Nanos < 0) { + return 0, fmt.Errorf("duration: %#v is out of range for time.Duration", p) + } + } + return d, nil +} + +// DurationProto converts a time.Duration to a Duration. +func durationProto(d time.Duration) *duration { + nanos := d.Nanoseconds() + secs := nanos / 1e9 + nanos -= secs * 1e9 + return &duration{ + Seconds: secs, + Nanos: int32(nanos), + } +} diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/duration_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/duration_gogo.go new file mode 100644 index 000000000000..18e2a5f77654 --- /dev/null +++ b/cmd/vendor/github.com/gogo/protobuf/proto/duration_gogo.go @@ -0,0 +1,203 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2016, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "reflect" + "time" +) + +var durationType = reflect.TypeOf((*time.Duration)(nil)).Elem() + +type duration struct { + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` + Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"` +} + +func (m *duration) Reset() { *m = duration{} } +func (*duration) ProtoMessage() {} +func (*duration) String() string { return "duration" } + +func init() { + RegisterType((*duration)(nil), "gogo.protobuf.proto.duration") +} + +func (o *Buffer) decDuration() (time.Duration, error) { + b, err := o.DecodeRawBytes(true) + if err != nil { + return 0, err + } + dproto := &duration{} + if err := Unmarshal(b, dproto); err != nil { + return 0, err + } + return durationFromProto(dproto) +} + +func (o *Buffer) dec_duration(p *Properties, base structPointer) error { + d, err := o.decDuration() + if err != nil { + return err + } + word64_Set(structPointer_Word64(base, p.field), o, uint64(d)) + return nil +} + +func (o *Buffer) dec_ref_duration(p *Properties, base structPointer) error { + d, err := o.decDuration() + if err != nil { + return err + } + word64Val_Set(structPointer_Word64Val(base, p.field), o, uint64(d)) + return nil +} + +func (o *Buffer) dec_slice_duration(p *Properties, base structPointer) error { + d, err := o.decDuration() + if err != nil { + return err + } + newBas := appendStructPointer(base, p.field, reflect.SliceOf(reflect.PtrTo(durationType))) + var zero field + setPtrCustomType(newBas, zero, &d) + return nil +} + +func (o *Buffer) dec_slice_ref_duration(p *Properties, base structPointer) error { + d, err := o.decDuration() + if err != nil { + return err + } + structPointer_Word64Slice(base, p.field).Append(uint64(d)) + return nil +} + +func size_duration(p *Properties, base structPointer) (n int) { + structp := structPointer_GetStructPointer(base, p.field) + if structPointer_IsNil(structp) { + return 0 + } + dur := structPointer_Interface(structp, durationType).(*time.Duration) + d := durationProto(*dur) + size := Size(d) + return size + sizeVarint(uint64(size)) + len(p.tagcode) +} + +func (o *Buffer) enc_duration(p *Properties, base structPointer) error { + structp := structPointer_GetStructPointer(base, p.field) + if structPointer_IsNil(structp) { + return ErrNil + } + dur := structPointer_Interface(structp, durationType).(*time.Duration) + d := durationProto(*dur) + data, err := Marshal(d) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + return nil +} + +func size_ref_duration(p *Properties, base structPointer) (n int) { + dur := structPointer_InterfaceAt(base, p.field, durationType).(*time.Duration) + d := durationProto(*dur) + size := Size(d) + return size + sizeVarint(uint64(size)) + len(p.tagcode) +} + +func (o *Buffer) enc_ref_duration(p *Properties, base structPointer) error { + dur := structPointer_InterfaceAt(base, p.field, durationType).(*time.Duration) + d := durationProto(*dur) + data, err := Marshal(d) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + return nil +} + +func size_slice_duration(p *Properties, base structPointer) (n int) { + pdurs := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(reflect.PtrTo(durationType))).(*[]*time.Duration) + durs := *pdurs + for i := 0; i < len(durs); i++ { + if durs[i] == nil { + return 0 + } + dproto := durationProto(*durs[i]) + size := Size(dproto) + n += len(p.tagcode) + size + sizeVarint(uint64(size)) + } + return n +} + +func (o *Buffer) enc_slice_duration(p *Properties, base structPointer) error { + pdurs := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(reflect.PtrTo(durationType))).(*[]*time.Duration) + durs := *pdurs + for i := 0; i < len(durs); i++ { + if durs[i] == nil { + return errRepeatedHasNil + } + dproto := durationProto(*durs[i]) + data, err := Marshal(dproto) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + } + return nil +} + +func size_slice_ref_duration(p *Properties, base structPointer) (n int) { + pdurs := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(durationType)).(*[]time.Duration) + durs := *pdurs + for i := 0; i < len(durs); i++ { + dproto := durationProto(durs[i]) + size := Size(dproto) + n += len(p.tagcode) + size + sizeVarint(uint64(size)) + } + return n +} + +func (o *Buffer) enc_slice_ref_duration(p *Properties, base structPointer) error { + pdurs := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(durationType)).(*[]time.Duration) + durs := *pdurs + for i := 0; i < len(durs); i++ { + dproto := durationProto(durs[i]) + data, err := Marshal(dproto) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + } + return nil +} diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/encode.go b/cmd/vendor/github.com/gogo/protobuf/proto/encode.go index 8c1b8fd1f683..2b30f84626ad 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/encode.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/encode.go @@ -234,10 +234,6 @@ func Marshal(pb Message) ([]byte, error) { } p := NewBuffer(nil) err := p.Marshal(pb) - var state errorState - if err != nil && !state.shouldContinue(err, nil) { - return nil, err - } if p.buf == nil && err == nil { // Return a non-nil slice on success. return []byte{}, nil @@ -266,11 +262,8 @@ func (p *Buffer) Marshal(pb Message) error { // Can the object marshal itself? if m, ok := pb.(Marshaler); ok { data, err := m.Marshal() - if err != nil { - return err - } p.buf = append(p.buf, data...) - return nil + return err } t, base, err := getbase(pb) @@ -282,7 +275,7 @@ func (p *Buffer) Marshal(pb Message) error { } if collectStats { - stats.Encode++ + (stats).Encode++ // Parens are to work around a goimports bug. } if len(p.buf) > maxMarshalSize { @@ -309,7 +302,7 @@ func Size(pb Message) (n int) { } if collectStats { - stats.Size++ + (stats).Size++ // Parens are to work around a goimports bug. } return @@ -1014,7 +1007,6 @@ func size_slice_struct_message(p *Properties, base structPointer) (n int) { if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, _ := m.Marshal() - n += len(p.tagcode) n += sizeRawBytes(data) continue } @@ -1083,10 +1075,17 @@ func (o *Buffer) enc_map(p *Properties, base structPointer) error { func (o *Buffer) enc_exts(p *Properties, base structPointer) error { exts := structPointer_Extensions(base, p.field) - if err := encodeExtensions(exts); err != nil { + + v, mu := exts.extensionsRead() + if v == nil { + return nil + } + + mu.Lock() + defer mu.Unlock() + if err := encodeExtensionsMap(v); err != nil { return err } - v, _ := exts.extensionsRead() return o.enc_map_body(v) } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/encode_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/encode_gogo.go index 66e7e16303bd..32111b7f41d7 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/encode_gogo.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/encode_gogo.go @@ -196,12 +196,10 @@ func size_ref_struct_message(p *Properties, base structPointer) int { // Encode a slice of references to message struct pointers ([]struct). func (o *Buffer) enc_slice_ref_struct_message(p *Properties, base structPointer) error { var state errorState - ss := structPointer_GetStructPointer(base, p.field) - ss1 := structPointer_GetRefStructPointer(ss, field(0)) - size := p.stype.Size() - l := structPointer_Len(base, p.field) + ss := structPointer_StructRefSlice(base, p.field, p.stype.Size()) + l := ss.Len() for i := 0; i < l; i++ { - structp := structPointer_Add(ss1, field(uintptr(i)*size)) + structp := ss.Index(i) if structPointer_IsNil(structp) { return errRepeatedHasNil } @@ -233,13 +231,11 @@ func (o *Buffer) enc_slice_ref_struct_message(p *Properties, base structPointer) //TODO this is only copied, please fix this func size_slice_ref_struct_message(p *Properties, base structPointer) (n int) { - ss := structPointer_GetStructPointer(base, p.field) - ss1 := structPointer_GetRefStructPointer(ss, field(0)) - size := p.stype.Size() - l := structPointer_Len(base, p.field) + ss := structPointer_StructRefSlice(base, p.field, p.stype.Size()) + l := ss.Len() n += l * len(p.tagcode) for i := 0; i < l; i++ { - structp := structPointer_Add(ss1, field(uintptr(i)*size)) + structp := ss.Index(i) if structPointer_IsNil(structp) { return // return the size up to this point } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/equal.go b/cmd/vendor/github.com/gogo/protobuf/proto/equal.go index 8b16f951c712..2ed1cf596664 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/equal.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/equal.go @@ -54,13 +54,17 @@ Equality is defined in this way: in a proto3 .proto file, fields are not "set"; specifically, zero length proto3 "bytes" fields are equal (nil == {}). - Two repeated fields are equal iff their lengths are the same, - and their corresponding elements are equal (a "bytes" field, - although represented by []byte, is not a repeated field) + and their corresponding elements are equal. Note a "bytes" field, + although represented by []byte, is not a repeated field and the + rule for the scalar fields described above applies. - Two unset fields are equal. - Two unknown field sets are equal if their current encoded state is equal. - Two extension sets are equal iff they have corresponding elements that are pairwise equal. + - Two map fields are equal iff their lengths are the same, + and they contain the same set of elements. Zero-length map + fields are equal. - Every other combination of things are not equal. The return value is undefined if a and b are not protocol buffers. diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/extensions.go b/cmd/vendor/github.com/gogo/protobuf/proto/extensions.go index f7384baa8795..0dfcb538e8f1 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/extensions.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/extensions.go @@ -167,6 +167,7 @@ type ExtensionDesc struct { Field int32 // field number Name string // fully-qualified name of extension, for text formatting Tag string // protobuf tag style + Filename string // name of the file in which the extension is defined } func (ed *ExtensionDesc) repeated() bool { @@ -587,6 +588,9 @@ func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { registeredExtensions := RegisteredExtensions(pb) emap, mu := epb.extensionsRead() + if emap == nil { + return nil, nil + } mu.Lock() defer mu.Unlock() extensions := make([]*ExtensionDesc, 0, len(emap)) diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/lib.go b/cmd/vendor/github.com/gogo/protobuf/proto/lib.go index 2c30d7095144..7580bb45c617 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/lib.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/lib.go @@ -308,7 +308,7 @@ func GetStats() Stats { return stats } // temporary Buffer and are fine for most applications. type Buffer struct { buf []byte // encode/decode byte stream - index int // write point + index int // read point // pools of basic types to amortize allocation. bools []bool diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/pointer_reflect_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/pointer_reflect_gogo.go new file mode 100644 index 000000000000..1763a5f227a0 --- /dev/null +++ b/cmd/vendor/github.com/gogo/protobuf/proto/pointer_reflect_gogo.go @@ -0,0 +1,85 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2016, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +build appengine js + +package proto + +import ( + "reflect" +) + +func structPointer_FieldPointer(p structPointer, f field) structPointer { + panic("not implemented") +} + +func appendStructPointer(base structPointer, f field, typ reflect.Type) structPointer { + panic("not implemented") +} + +func structPointer_InterfaceAt(p structPointer, f field, t reflect.Type) interface{} { + panic("not implemented") +} + +func structPointer_InterfaceRef(p structPointer, f field, t reflect.Type) interface{} { + panic("not implemented") +} + +func structPointer_GetRefStructPointer(p structPointer, f field) structPointer { + panic("not implemented") +} + +func structPointer_Add(p structPointer, size field) structPointer { + panic("not implemented") +} + +func structPointer_Len(p structPointer, f field) int { + panic("not implemented") +} + +func structPointer_GetSliceHeader(p structPointer, f field) *reflect.SliceHeader { + panic("not implemented") +} + +func structPointer_Copy(oldptr structPointer, newptr structPointer, size int) { + panic("not implemented") +} + +func structPointer_StructRefSlice(p structPointer, f field, size uintptr) *structRefSlice { + panic("not implemented") +} + +type structRefSlice struct{} + +func (v *structRefSlice) Len() int { + panic("not implemented") +} + +func (v *structRefSlice) Index(i int) structPointer { + panic("not implemented") +} diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go index ad7c851793e2..f156a29f0e83 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go @@ -26,7 +26,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build !appengine +// +build !appengine,!js // This file contains the implementation of the proto field accesses using package unsafe. @@ -105,3 +105,24 @@ func structPointer_Add(p structPointer, size field) structPointer { func structPointer_Len(p structPointer, f field) int { return len(*(*[]interface{})(unsafe.Pointer(structPointer_GetRefStructPointer(p, f)))) } + +func structPointer_StructRefSlice(p structPointer, f field, size uintptr) *structRefSlice { + return &structRefSlice{p: p, f: f, size: size} +} + +// A structRefSlice represents a slice of structs (themselves submessages or groups). +type structRefSlice struct { + p structPointer + f field + size uintptr +} + +func (v *structRefSlice) Len() int { + return structPointer_Len(v.p, v.f) +} + +func (v *structRefSlice) Index(i int) structPointer { + ss := structPointer_GetStructPointer(v.p, v.f) + ss1 := structPointer_GetRefStructPointer(ss, 0) + return structPointer_Add(ss1, field(uintptr(i)*v.size)) +} diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/properties.go b/cmd/vendor/github.com/gogo/protobuf/proto/properties.go index 3e4cad038132..44b332052ef5 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/properties.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/properties.go @@ -190,10 +190,11 @@ type Properties struct { proto3 bool // whether this is known to be a proto3 field; set for []byte only oneof bool // whether this is a oneof field - Default string // default value - HasDefault bool // whether an explicit default was provided - CustomType string - def_uint64 uint64 + Default string // default value + HasDefault bool // whether an explicit default was provided + CustomType string + StdTime bool + StdDuration bool enc encoder valEnc valueEncoder // set for bool and numeric types only @@ -340,6 +341,10 @@ func (p *Properties) Parse(s string) { p.OrigName = strings.Split(f, "=")[1] case strings.HasPrefix(f, "customtype="): p.CustomType = strings.Split(f, "=")[1] + case f == "stdtime": + p.StdTime = true + case f == "stdduration": + p.StdDuration = true } } } @@ -355,11 +360,22 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock p.enc = nil p.dec = nil p.size = nil - if len(p.CustomType) > 0 { + isMap := typ.Kind() == reflect.Map + if len(p.CustomType) > 0 && !isMap { p.setCustomEncAndDec(typ) p.setTag(lockGetProp) return } + if p.StdTime && !isMap { + p.setTimeEncAndDec(typ) + p.setTag(lockGetProp) + return + } + if p.StdDuration && !isMap { + p.setDurationEncAndDec(typ) + p.setTag(lockGetProp) + return + } switch t1 := typ; t1.Kind() { default: fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) @@ -630,6 +646,10 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock // so we need encoders for the pointer to this type. vtype = reflect.PtrTo(vtype) } + + p.mvalprop.CustomType = p.CustomType + p.mvalprop.StdDuration = p.StdDuration + p.mvalprop.StdTime = p.StdTime p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) } p.setTag(lockGetProp) @@ -920,7 +940,15 @@ func RegisterType(x Message, name string) { } // MessageName returns the fully-qualified proto name for the given message type. -func MessageName(x Message) string { return revProtoTypes[reflect.TypeOf(x)] } +func MessageName(x Message) string { + type xname interface { + XXX_MessageName() string + } + if m, ok := x.(xname); ok { + return m.XXX_MessageName() + } + return revProtoTypes[reflect.TypeOf(x)] +} // MessageType returns the message type (pointer to struct) for a named message. func MessageType(name string) reflect.Type { return protoTypes[name] } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/properties_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/properties_gogo.go index 4607a975410a..b6b7176c5656 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/properties_gogo.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/properties_gogo.go @@ -51,6 +51,51 @@ func (p *Properties) setCustomEncAndDec(typ reflect.Type) { } } +func (p *Properties) setDurationEncAndDec(typ reflect.Type) { + if p.Repeated { + if typ.Elem().Kind() == reflect.Ptr { + p.enc = (*Buffer).enc_slice_duration + p.dec = (*Buffer).dec_slice_duration + p.size = size_slice_duration + } else { + p.enc = (*Buffer).enc_slice_ref_duration + p.dec = (*Buffer).dec_slice_ref_duration + p.size = size_slice_ref_duration + } + } else if typ.Kind() == reflect.Ptr { + p.enc = (*Buffer).enc_duration + p.dec = (*Buffer).dec_duration + p.size = size_duration + } else { + p.enc = (*Buffer).enc_ref_duration + p.dec = (*Buffer).dec_ref_duration + p.size = size_ref_duration + } +} + +func (p *Properties) setTimeEncAndDec(typ reflect.Type) { + if p.Repeated { + if typ.Elem().Kind() == reflect.Ptr { + p.enc = (*Buffer).enc_slice_time + p.dec = (*Buffer).dec_slice_time + p.size = size_slice_time + } else { + p.enc = (*Buffer).enc_slice_ref_time + p.dec = (*Buffer).dec_slice_ref_time + p.size = size_slice_ref_time + } + } else if typ.Kind() == reflect.Ptr { + p.enc = (*Buffer).enc_time + p.dec = (*Buffer).dec_time + p.size = size_time + } else { + p.enc = (*Buffer).enc_ref_time + p.dec = (*Buffer).dec_ref_time + p.size = size_ref_time + } + +} + func (p *Properties) setSliceOfNonPointerStructs(typ reflect.Type) { t2 := typ.Elem() p.sstype = typ diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/text.go b/cmd/vendor/github.com/gogo/protobuf/proto/text.go index b3e12e268445..d63732fcbda5 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/text.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/text.go @@ -51,6 +51,7 @@ import ( "sort" "strings" "sync" + "time" ) var ( @@ -181,7 +182,93 @@ type raw interface { Bytes() []byte } -func writeStruct(w *textWriter, sv reflect.Value) error { +func requiresQuotes(u string) bool { + // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. + for _, ch := range u { + switch { + case ch == '.' || ch == '/' || ch == '_': + continue + case '0' <= ch && ch <= '9': + continue + case 'A' <= ch && ch <= 'Z': + continue + case 'a' <= ch && ch <= 'z': + continue + default: + return true + } + } + return false +} + +// isAny reports whether sv is a google.protobuf.Any message +func isAny(sv reflect.Value) bool { + type wkt interface { + XXX_WellKnownType() string + } + t, ok := sv.Addr().Interface().(wkt) + return ok && t.XXX_WellKnownType() == "Any" +} + +// writeProto3Any writes an expanded google.protobuf.Any message. +// +// It returns (false, nil) if sv value can't be unmarshaled (e.g. because +// required messages are not linked in). +// +// It returns (true, error) when sv was written in expanded format or an error +// was encountered. +func (tm *TextMarshaler) writeProto3Any(w *textWriter, sv reflect.Value) (bool, error) { + turl := sv.FieldByName("TypeUrl") + val := sv.FieldByName("Value") + if !turl.IsValid() || !val.IsValid() { + return true, errors.New("proto: invalid google.protobuf.Any message") + } + + b, ok := val.Interface().([]byte) + if !ok { + return true, errors.New("proto: invalid google.protobuf.Any message") + } + + parts := strings.Split(turl.String(), "/") + mt := MessageType(parts[len(parts)-1]) + if mt == nil { + return false, nil + } + m := reflect.New(mt.Elem()) + if err := Unmarshal(b, m.Interface().(Message)); err != nil { + return false, nil + } + w.Write([]byte("[")) + u := turl.String() + if requiresQuotes(u) { + writeString(w, u) + } else { + w.Write([]byte(u)) + } + if w.compact { + w.Write([]byte("]:<")) + } else { + w.Write([]byte("]: <\n")) + w.ind++ + } + if err := tm.writeStruct(w, m.Elem()); err != nil { + return true, err + } + if w.compact { + w.Write([]byte("> ")) + } else { + w.ind-- + w.Write([]byte(">\n")) + } + return true, nil +} + +func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { + if tm.ExpandAny && isAny(sv) { + if canExpand, err := tm.writeProto3Any(w, sv); canExpand { + return err + } + } st := sv.Type() sprops := GetProperties(st) for i := 0; i < sv.NumField(); i++ { @@ -234,10 +321,10 @@ func writeStruct(w *textWriter, sv reflect.Value) error { continue } if len(props.Enum) > 0 { - if err := writeEnum(w, v, props); err != nil { + if err := tm.writeEnum(w, v, props); err != nil { return err } - } else if err := writeAny(w, v, props); err != nil { + } else if err := tm.writeAny(w, v, props); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -279,7 +366,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if err := writeAny(w, key, props.mkeyprop); err != nil { + if err := tm.writeAny(w, key, props.mkeyprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -296,7 +383,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if err := writeAny(w, val, props.mvalprop); err != nil { + if err := tm.writeAny(w, val, props.mvalprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -368,10 +455,10 @@ func writeStruct(w *textWriter, sv reflect.Value) error { } if len(props.Enum) > 0 { - if err := writeEnum(w, fv, props); err != nil { + if err := tm.writeEnum(w, fv, props); err != nil { return err } - } else if err := writeAny(w, fv, props); err != nil { + } else if err := tm.writeAny(w, fv, props); err != nil { return err } @@ -389,7 +476,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { pv.Elem().Set(sv) } if pv.Type().Implements(extensionRangeType) { - if err := writeExtensions(w, pv); err != nil { + if err := tm.writeExtensions(w, pv); err != nil { return err } } @@ -419,20 +506,45 @@ func writeRaw(w *textWriter, b []byte) error { } // writeAny writes an arbitrary field. -func writeAny(w *textWriter, v reflect.Value, props *Properties) error { +func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) - if props != nil && len(props.CustomType) > 0 { - custom, ok := v.Interface().(Marshaler) - if ok { - data, err := custom.Marshal() + if props != nil { + if len(props.CustomType) > 0 { + custom, ok := v.Interface().(Marshaler) + if ok { + data, err := custom.Marshal() + if err != nil { + return err + } + if err := writeString(w, string(data)); err != nil { + return err + } + return nil + } + } else if props.StdTime { + t, ok := v.Interface().(time.Time) + if !ok { + return fmt.Errorf("stdtime is not time.Time, but %T", v.Interface()) + } + tproto, err := timestampProto(t) if err != nil { return err } - if err := writeString(w, string(data)); err != nil { - return err + props.StdTime = false + err = tm.writeAny(w, reflect.ValueOf(tproto), props) + props.StdTime = true + return err + } else if props.StdDuration { + d, ok := v.Interface().(time.Duration) + if !ok { + return fmt.Errorf("stdtime is not time.Duration, but %T", v.Interface()) } - return nil + dproto := durationProto(d) + props.StdDuration = false + err := tm.writeAny(w, reflect.ValueOf(dproto), props) + props.StdDuration = true + return err } } @@ -482,15 +594,15 @@ func writeAny(w *textWriter, v reflect.Value, props *Properties) error { } } w.indent() - if tm, ok := v.Interface().(encoding.TextMarshaler); ok { - text, err := tm.MarshalText() + if etm, ok := v.Interface().(encoding.TextMarshaler); ok { + text, err := etm.MarshalText() if err != nil { return err } if _, err = w.Write(text); err != nil { return err } - } else if err := writeStruct(w, v); err != nil { + } else if err := tm.writeStruct(w, v); err != nil { return err } w.unindent() @@ -634,7 +746,7 @@ func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // writeExtensions writes all the extensions in pv. // pv is assumed to be a pointer to a protocol message struct that is extendable. -func writeExtensions(w *textWriter, pv reflect.Value) error { +func (tm *TextMarshaler) writeExtensions(w *textWriter, pv reflect.Value) error { emap := extensionMaps[pv.Type().Elem()] e := pv.Interface().(Message) @@ -689,13 +801,13 @@ func writeExtensions(w *textWriter, pv reflect.Value) error { // Repeated extensions will appear as a slice. if !desc.repeated() { - if err := writeExtension(w, desc.Name, pb); err != nil { + if err := tm.writeExtension(w, desc.Name, pb); err != nil { return err } } else { v := reflect.ValueOf(pb) for i := 0; i < v.Len(); i++ { - if err := writeExtension(w, desc.Name, v.Index(i).Interface()); err != nil { + if err := tm.writeExtension(w, desc.Name, v.Index(i).Interface()); err != nil { return err } } @@ -704,7 +816,7 @@ func writeExtensions(w *textWriter, pv reflect.Value) error { return nil } -func writeExtension(w *textWriter, name string, pb interface{}) error { +func (tm *TextMarshaler) writeExtension(w *textWriter, name string, pb interface{}) error { if _, err := fmt.Fprintf(w, "[%s]:", name); err != nil { return err } @@ -713,7 +825,7 @@ func writeExtension(w *textWriter, name string, pb interface{}) error { return err } } - if err := writeAny(w, reflect.ValueOf(pb), nil); err != nil { + if err := tm.writeAny(w, reflect.ValueOf(pb), nil); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -740,12 +852,13 @@ func (w *textWriter) writeIndent() { // TextMarshaler is a configurable text format marshaler. type TextMarshaler struct { - Compact bool // use compact text format (one line). + Compact bool // use compact text format (one line). + ExpandAny bool // expand google.protobuf.Any messages of known types } // Marshal writes a given protocol buffer in text format. // The only errors returned are from w. -func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { +func (tm *TextMarshaler) Marshal(w io.Writer, pb Message) error { val := reflect.ValueOf(pb) if pb == nil || val.IsNil() { w.Write([]byte("")) @@ -760,11 +873,11 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { aw := &textWriter{ w: ww, complete: true, - compact: m.Compact, + compact: tm.Compact, } - if tm, ok := pb.(encoding.TextMarshaler); ok { - text, err := tm.MarshalText() + if etm, ok := pb.(encoding.TextMarshaler); ok { + text, err := etm.MarshalText() if err != nil { return err } @@ -778,7 +891,7 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { } // Dereference the received pointer so we don't have outer < and >. v := reflect.Indirect(val) - if err := writeStruct(aw, v); err != nil { + if err := tm.writeStruct(aw, v); err != nil { return err } if bw != nil { @@ -788,9 +901,9 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { } // Text is the same as Marshal, but returns the string directly. -func (m *TextMarshaler) Text(pb Message) string { +func (tm *TextMarshaler) Text(pb Message) string { var buf bytes.Buffer - m.Marshal(&buf, pb) + tm.Marshal(&buf, pb) return buf.String() } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/text_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/text_gogo.go index 5892674197fc..1d6c6aa0e41b 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/text_gogo.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/text_gogo.go @@ -33,10 +33,10 @@ import ( "reflect" ) -func writeEnum(w *textWriter, v reflect.Value, props *Properties) error { +func (tm *TextMarshaler) writeEnum(w *textWriter, v reflect.Value, props *Properties) error { m, ok := enumStringMaps[props.Enum] if !ok { - if err := writeAny(w, v, props); err != nil { + if err := tm.writeAny(w, v, props); err != nil { return err } } @@ -48,7 +48,7 @@ func writeEnum(w *textWriter, v reflect.Value, props *Properties) error { } s, ok := m[key] if !ok { - if err := writeAny(w, v, props); err != nil { + if err := tm.writeAny(w, v, props); err != nil { return err } } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/text_parser.go b/cmd/vendor/github.com/gogo/protobuf/proto/text_parser.go index bcd732c3c489..9db12e96018e 100644 --- a/cmd/vendor/github.com/gogo/protobuf/proto/text_parser.go +++ b/cmd/vendor/github.com/gogo/protobuf/proto/text_parser.go @@ -46,9 +46,13 @@ import ( "reflect" "strconv" "strings" + "time" "unicode/utf8" ) +// Error string emitted when deserializing Any and fields are already set +const anyRepeatedlyUnpacked = "Any message unpacked multiple times, or %q already set" + type ParseError struct { Message string Line int // 1-based line number @@ -168,7 +172,7 @@ func (p *textParser) advance() { p.cur.offset, p.cur.line = p.offset, p.line p.cur.unquoted = "" switch p.s[0] { - case '<', '>', '{', '}', ':', '[', ']', ';', ',': + case '<', '>', '{', '}', ':', '[', ']', ';', ',', '/': // Single symbol p.cur.value, p.s = p.s[0:1], p.s[1:len(p.s)] case '"', '\'': @@ -456,7 +460,10 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { fieldSet := make(map[string]bool) // A struct is a sequence of "name: value", terminated by one of // '>' or '}', or the end of the input. A name may also be - // "[extension]". + // "[extension]" or "[type/url]". + // + // The whole struct can also be an expanded Any message, like: + // [type/url] < ... struct contents ... > for { tok := p.next() if tok.err != nil { @@ -466,33 +473,74 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { break } if tok.value == "[" { - // Looks like an extension. + // Looks like an extension or an Any. // // TODO: Check whether we need to handle // namespace rooted names (e.g. ".something.Foo"). - tok = p.next() - if tok.err != nil { - return tok.err + extName, err := p.consumeExtName() + if err != nil { + return err + } + + if s := strings.LastIndex(extName, "/"); s >= 0 { + // If it contains a slash, it's an Any type URL. + messageName := extName[s+1:] + mt := MessageType(messageName) + if mt == nil { + return p.errorf("unrecognized message %q in google.protobuf.Any", messageName) + } + tok = p.next() + if tok.err != nil { + return tok.err + } + // consume an optional colon + if tok.value == ":" { + tok = p.next() + if tok.err != nil { + return tok.err + } + } + var terminator string + switch tok.value { + case "<": + terminator = ">" + case "{": + terminator = "}" + default: + return p.errorf("expected '{' or '<', found %q", tok.value) + } + v := reflect.New(mt.Elem()) + if pe := p.readStruct(v.Elem(), terminator); pe != nil { + return pe + } + b, err := Marshal(v.Interface().(Message)) + if err != nil { + return p.errorf("failed to marshal message of type %q: %v", messageName, err) + } + if fieldSet["type_url"] { + return p.errorf(anyRepeatedlyUnpacked, "type_url") + } + if fieldSet["value"] { + return p.errorf(anyRepeatedlyUnpacked, "value") + } + sv.FieldByName("TypeUrl").SetString(extName) + sv.FieldByName("Value").SetBytes(b) + fieldSet["type_url"] = true + fieldSet["value"] = true + continue } + var desc *ExtensionDesc // This could be faster, but it's functional. // TODO: Do something smarter than a linear scan. for _, d := range RegisteredExtensions(reflect.New(st).Interface().(Message)) { - if d.Name == tok.value { + if d.Name == extName { desc = d break } } if desc == nil { - return p.errorf("unrecognized extension %q", tok.value) - } - // Check the extension terminator. - tok = p.next() - if tok.err != nil { - return tok.err - } - if tok.value != "]" { - return p.errorf("unrecognized extension terminator %q", tok.value) + return p.errorf("unrecognized extension %q", extName) } props := &Properties{} @@ -550,7 +598,11 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { props = oop.Prop nv := reflect.New(oop.Type.Elem()) dst = nv.Elem().Field(0) - sv.Field(oop.Field).Set(nv) + field := sv.Field(oop.Field) + if !field.IsNil() { + return p.errorf("field '%s' would overwrite already parsed oneof '%s'", name, sv.Type().Field(oop.Field).Name) + } + field.Set(nv) } if !dst.IsValid() { return p.errorf("unknown field name %q in %v", name, st) @@ -657,6 +709,35 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { return reqFieldErr } +// consumeExtName consumes extension name or expanded Any type URL and the +// following ']'. It returns the name or URL consumed. +func (p *textParser) consumeExtName() (string, error) { + tok := p.next() + if tok.err != nil { + return "", tok.err + } + + // If extension name or type url is quoted, it's a single token. + if len(tok.value) > 2 && isQuote(tok.value[0]) && tok.value[len(tok.value)-1] == tok.value[0] { + name, err := unquoteC(tok.value[1:len(tok.value)-1], rune(tok.value[0])) + if err != nil { + return "", err + } + return name, p.consumeToken("]") + } + + // Consume everything up to "]" + var parts []string + for tok.value != "]" { + parts = append(parts, tok.value) + tok = p.next() + if tok.err != nil { + return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) + } + } + return strings.Join(parts, ""), nil +} + // consumeOptionalSeparator consumes an optional semicolon or comma. // It is used in readStruct to provide backward compatibility. func (p *textParser) consumeOptionalSeparator() error { @@ -717,6 +798,80 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error { } return nil } + if props.StdTime { + fv := v + p.back() + props.StdTime = false + tproto := ×tamp{} + err := p.readAny(reflect.ValueOf(tproto).Elem(), props) + props.StdTime = true + if err != nil { + return err + } + tim, err := timestampFromProto(tproto) + if err != nil { + return err + } + if props.Repeated { + t := reflect.TypeOf(v.Interface()) + if t.Kind() == reflect.Slice { + if t.Elem().Kind() == reflect.Ptr { + ts := fv.Interface().([]*time.Time) + ts = append(ts, &tim) + fv.Set(reflect.ValueOf(ts)) + return nil + } else { + ts := fv.Interface().([]time.Time) + ts = append(ts, tim) + fv.Set(reflect.ValueOf(ts)) + return nil + } + } + } + if reflect.TypeOf(v.Interface()).Kind() == reflect.Ptr { + v.Set(reflect.ValueOf(&tim)) + } else { + v.Set(reflect.Indirect(reflect.ValueOf(&tim))) + } + return nil + } + if props.StdDuration { + fv := v + p.back() + props.StdDuration = false + dproto := &duration{} + err := p.readAny(reflect.ValueOf(dproto).Elem(), props) + props.StdDuration = true + if err != nil { + return err + } + dur, err := durationFromProto(dproto) + if err != nil { + return err + } + if props.Repeated { + t := reflect.TypeOf(v.Interface()) + if t.Kind() == reflect.Slice { + if t.Elem().Kind() == reflect.Ptr { + ds := fv.Interface().([]*time.Duration) + ds = append(ds, &dur) + fv.Set(reflect.ValueOf(ds)) + return nil + } else { + ds := fv.Interface().([]time.Duration) + ds = append(ds, dur) + fv.Set(reflect.ValueOf(ds)) + return nil + } + } + } + if reflect.TypeOf(v.Interface()).Kind() == reflect.Ptr { + v.Set(reflect.ValueOf(&dur)) + } else { + v.Set(reflect.Indirect(reflect.ValueOf(&dur))) + } + return nil + } switch fv := v; fv.Kind() { case reflect.Slice: at := v.Type() @@ -759,12 +914,12 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error { fv.Set(reflect.Append(fv, reflect.New(at.Elem()).Elem())) return p.readAny(fv.Index(fv.Len()-1), props) case reflect.Bool: - // Either "true", "false", 1 or 0. + // true/1/t/True or false/f/0/False. switch tok.value { - case "true", "1": + case "true", "1", "t", "True": fv.SetBool(true) return nil - case "false", "0": + case "false", "0", "f", "False": fv.SetBool(false) return nil } diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/timestamp.go b/cmd/vendor/github.com/gogo/protobuf/proto/timestamp.go new file mode 100644 index 000000000000..9324f6542bcf --- /dev/null +++ b/cmd/vendor/github.com/gogo/protobuf/proto/timestamp.go @@ -0,0 +1,113 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +// This file implements operations on google.protobuf.Timestamp. + +import ( + "errors" + "fmt" + "time" +) + +const ( + // Seconds field of the earliest valid Timestamp. + // This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix(). + minValidSeconds = -62135596800 + // Seconds field just after the latest valid Timestamp. + // This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix(). + maxValidSeconds = 253402300800 +) + +// validateTimestamp determines whether a Timestamp is valid. +// A valid timestamp represents a time in the range +// [0001-01-01, 10000-01-01) and has a Nanos field +// in the range [0, 1e9). +// +// If the Timestamp is valid, validateTimestamp returns nil. +// Otherwise, it returns an error that describes +// the problem. +// +// Every valid Timestamp can be represented by a time.Time, but the converse is not true. +func validateTimestamp(ts *timestamp) error { + if ts == nil { + return errors.New("timestamp: nil Timestamp") + } + if ts.Seconds < minValidSeconds { + return fmt.Errorf("timestamp: %#v before 0001-01-01", ts) + } + if ts.Seconds >= maxValidSeconds { + return fmt.Errorf("timestamp: %#v after 10000-01-01", ts) + } + if ts.Nanos < 0 || ts.Nanos >= 1e9 { + return fmt.Errorf("timestamp: %#v: nanos not in range [0, 1e9)", ts) + } + return nil +} + +// TimestampFromProto converts a google.protobuf.Timestamp proto to a time.Time. +// It returns an error if the argument is invalid. +// +// Unlike most Go functions, if Timestamp returns an error, the first return value +// is not the zero time.Time. Instead, it is the value obtained from the +// time.Unix function when passed the contents of the Timestamp, in the UTC +// locale. This may or may not be a meaningful time; many invalid Timestamps +// do map to valid time.Times. +// +// A nil Timestamp returns an error. The first return value in that case is +// undefined. +func timestampFromProto(ts *timestamp) (time.Time, error) { + // Don't return the zero value on error, because corresponds to a valid + // timestamp. Instead return whatever time.Unix gives us. + var t time.Time + if ts == nil { + t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp + } else { + t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC() + } + return t, validateTimestamp(ts) +} + +// TimestampProto converts the time.Time to a google.protobuf.Timestamp proto. +// It returns an error if the resulting Timestamp is invalid. +func timestampProto(t time.Time) (*timestamp, error) { + seconds := t.Unix() + nanos := int32(t.Sub(time.Unix(seconds, 0))) + ts := ×tamp{ + Seconds: seconds, + Nanos: nanos, + } + if err := validateTimestamp(ts); err != nil { + return nil, err + } + return ts, nil +} diff --git a/cmd/vendor/github.com/gogo/protobuf/proto/timestamp_gogo.go b/cmd/vendor/github.com/gogo/protobuf/proto/timestamp_gogo.go new file mode 100644 index 000000000000..d42764743606 --- /dev/null +++ b/cmd/vendor/github.com/gogo/protobuf/proto/timestamp_gogo.go @@ -0,0 +1,229 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2016, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "reflect" + "time" +) + +var timeType = reflect.TypeOf((*time.Time)(nil)).Elem() + +type timestamp struct { + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` + Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"` +} + +func (m *timestamp) Reset() { *m = timestamp{} } +func (*timestamp) ProtoMessage() {} +func (*timestamp) String() string { return "timestamp" } + +func init() { + RegisterType((*timestamp)(nil), "gogo.protobuf.proto.timestamp") +} + +func (o *Buffer) decTimestamp() (time.Time, error) { + b, err := o.DecodeRawBytes(true) + if err != nil { + return time.Time{}, err + } + tproto := ×tamp{} + if err := Unmarshal(b, tproto); err != nil { + return time.Time{}, err + } + return timestampFromProto(tproto) +} + +func (o *Buffer) dec_time(p *Properties, base structPointer) error { + t, err := o.decTimestamp() + if err != nil { + return err + } + setPtrCustomType(base, p.field, &t) + return nil +} + +func (o *Buffer) dec_ref_time(p *Properties, base structPointer) error { + t, err := o.decTimestamp() + if err != nil { + return err + } + setCustomType(base, p.field, &t) + return nil +} + +func (o *Buffer) dec_slice_time(p *Properties, base structPointer) error { + t, err := o.decTimestamp() + if err != nil { + return err + } + newBas := appendStructPointer(base, p.field, reflect.SliceOf(reflect.PtrTo(timeType))) + var zero field + setPtrCustomType(newBas, zero, &t) + return nil +} + +func (o *Buffer) dec_slice_ref_time(p *Properties, base structPointer) error { + t, err := o.decTimestamp() + if err != nil { + return err + } + newBas := appendStructPointer(base, p.field, reflect.SliceOf(timeType)) + var zero field + setCustomType(newBas, zero, &t) + return nil +} + +func size_time(p *Properties, base structPointer) (n int) { + structp := structPointer_GetStructPointer(base, p.field) + if structPointer_IsNil(structp) { + return 0 + } + tim := structPointer_Interface(structp, timeType).(*time.Time) + t, err := timestampProto(*tim) + if err != nil { + return 0 + } + size := Size(t) + return size + sizeVarint(uint64(size)) + len(p.tagcode) +} + +func (o *Buffer) enc_time(p *Properties, base structPointer) error { + structp := structPointer_GetStructPointer(base, p.field) + if structPointer_IsNil(structp) { + return ErrNil + } + tim := structPointer_Interface(structp, timeType).(*time.Time) + t, err := timestampProto(*tim) + if err != nil { + return err + } + data, err := Marshal(t) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + return nil +} + +func size_ref_time(p *Properties, base structPointer) (n int) { + tim := structPointer_InterfaceAt(base, p.field, timeType).(*time.Time) + t, err := timestampProto(*tim) + if err != nil { + return 0 + } + size := Size(t) + return size + sizeVarint(uint64(size)) + len(p.tagcode) +} + +func (o *Buffer) enc_ref_time(p *Properties, base structPointer) error { + tim := structPointer_InterfaceAt(base, p.field, timeType).(*time.Time) + t, err := timestampProto(*tim) + if err != nil { + return err + } + data, err := Marshal(t) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + return nil +} + +func size_slice_time(p *Properties, base structPointer) (n int) { + ptims := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(reflect.PtrTo(timeType))).(*[]*time.Time) + tims := *ptims + for i := 0; i < len(tims); i++ { + if tims[i] == nil { + return 0 + } + tproto, err := timestampProto(*tims[i]) + if err != nil { + return 0 + } + size := Size(tproto) + n += len(p.tagcode) + size + sizeVarint(uint64(size)) + } + return n +} + +func (o *Buffer) enc_slice_time(p *Properties, base structPointer) error { + ptims := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(reflect.PtrTo(timeType))).(*[]*time.Time) + tims := *ptims + for i := 0; i < len(tims); i++ { + if tims[i] == nil { + return errRepeatedHasNil + } + tproto, err := timestampProto(*tims[i]) + if err != nil { + return err + } + data, err := Marshal(tproto) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + } + return nil +} + +func size_slice_ref_time(p *Properties, base structPointer) (n int) { + ptims := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(timeType)).(*[]time.Time) + tims := *ptims + for i := 0; i < len(tims); i++ { + tproto, err := timestampProto(tims[i]) + if err != nil { + return 0 + } + size := Size(tproto) + n += len(p.tagcode) + size + sizeVarint(uint64(size)) + } + return n +} + +func (o *Buffer) enc_slice_ref_time(p *Properties, base structPointer) error { + ptims := structPointer_InterfaceAt(base, p.field, reflect.SliceOf(timeType)).(*[]time.Time) + tims := *ptims + for i := 0; i < len(tims); i++ { + tproto, err := timestampProto(tims[i]) + if err != nil { + return err + } + data, err := Marshal(tproto) + if err != nil { + return err + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(data) + } + return nil +} diff --git a/cmd/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go b/cmd/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go index 1fc8ae8d7038..c7a45d6f0966 100644 --- a/cmd/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go +++ b/cmd/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go @@ -44,6 +44,7 @@ import ( "errors" "fmt" "io" + "math" "reflect" "sort" "strconv" @@ -51,6 +52,8 @@ import ( "time" "github.com/golang/protobuf/proto" + + stpb "github.com/golang/protobuf/ptypes/struct" ) // Marshaler is a configurable object for converting between @@ -72,6 +75,22 @@ type Marshaler struct { OrigName bool } +// JSONPBMarshaler is implemented by protobuf messages that customize the +// way they are marshaled to JSON. Messages that implement this should +// also implement JSONPBUnmarshaler so that the custom format can be +// parsed. +type JSONPBMarshaler interface { + MarshalJSONPB(*Marshaler) ([]byte, error) +} + +// JSONPBUnmarshaler is implemented by protobuf messages that customize +// the way they are unmarshaled from JSON. Messages that implement this +// should also implement JSONPBMarshaler so that the custom format can be +// produced. +type JSONPBUnmarshaler interface { + UnmarshalJSONPB(*Unmarshaler, []byte) error +} + // Marshal marshals a protocol buffer into JSON. func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error { writer := &errWriter{writer: out} @@ -89,6 +108,12 @@ func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) { type int32Slice []int32 +var nonFinite = map[string]float64{ + `"NaN"`: math.NaN(), + `"Infinity"`: math.Inf(1), + `"-Infinity"`: math.Inf(-1), +} + // For sorting extensions ids to ensure stable output. func (s int32Slice) Len() int { return len(s) } func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } @@ -100,6 +125,31 @@ type wkt interface { // marshalObject writes a struct to the Writer. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { + if jsm, ok := v.(JSONPBMarshaler); ok { + b, err := jsm.MarshalJSONPB(m) + if err != nil { + return err + } + if typeURL != "" { + // we are marshaling this object to an Any type + var js map[string]*json.RawMessage + if err = json.Unmarshal(b, &js); err != nil { + return fmt.Errorf("type %T produced invalid JSON: %v", v, err) + } + turl, err := json.Marshal(typeURL) + if err != nil { + return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err) + } + js["@type"] = (*json.RawMessage)(&turl) + if b, err = json.Marshal(js); err != nil { + return err + } + } + + out.write(string(b)) + return out.err + } + s := reflect.ValueOf(v).Elem() // Handle well-known types. @@ -126,8 +176,8 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeU out.write(x) out.write(`s"`) return out.err - case "Struct": - // Let marshalValue handle the `fields` map. + case "Struct", "ListValue": + // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice. // TODO: pass the correct Properties if needed. return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent) case "Timestamp": @@ -180,7 +230,7 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeU // IsNil will panic on most value kinds. switch value.Kind() { - case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + case reflect.Chan, reflect.Func, reflect.Interface: if value.IsNil() { continue } @@ -208,6 +258,10 @@ func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeU if value.Len() == 0 { continue } + case reflect.Map, reflect.Ptr, reflect.Slice: + if value.IsNil() { + continue + } } } @@ -371,10 +425,15 @@ func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v refle // marshalValue writes the value to the Writer. func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { - var err error v = reflect.Indirect(v) + // Handle nil pointer + if v.Kind() == reflect.Invalid { + out.write("null") + return out.err + } + // Handle repeated elements. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { out.write("[") @@ -404,9 +463,6 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle // Handle well-known types. // Most are handled up in marshalObject (because 99% are messages). - type wkt interface { - XXX_WellKnownType() string - } if wkt, ok := v.Interface().(wkt); ok { switch wkt.XXX_WellKnownType() { case "NullValue": @@ -494,6 +550,24 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle return out.err } + // Handle non-finite floats, e.g. NaN, Infinity and -Infinity. + if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { + f := v.Float() + var sval string + switch { + case math.IsInf(f, 1): + sval = `"Infinity"` + case math.IsInf(f, -1): + sval = `"-Infinity"` + case math.IsNaN(f): + sval = `"NaN"` + } + if sval != "" { + out.write(sval) + return out.err + } + } + // Default handling defers to the encoding/json library. b, err := json.Marshal(v.Interface()) if err != nil { @@ -569,12 +643,13 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe return u.unmarshalValue(target.Elem(), inputValue, prop) } - // Handle well-known types. - type wkt interface { - XXX_WellKnownType() string + if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok { + return jsu.UnmarshalJSONPB(u, []byte(inputValue)) } - if wkt, ok := target.Addr().Interface().(wkt); ok { - switch wkt.XXX_WellKnownType() { + + // Handle well-known types. + if w, ok := target.Addr().Interface().(wkt); ok { + switch w.XXX_WellKnownType() { case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": // "Wrappers use the same representation in JSON @@ -583,9 +658,72 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe // so we don't have to do any extra work. return u.unmarshalValue(target.Field(0), inputValue, prop) case "Any": - return fmt.Errorf("unmarshaling Any not supported yet") + // Use json.RawMessage pointer type instead of value to support pre-1.8 version. + // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see + // https://github.com/golang/go/issues/14493 + var jsonFields map[string]*json.RawMessage + if err := json.Unmarshal(inputValue, &jsonFields); err != nil { + return err + } + + val, ok := jsonFields["@type"] + if !ok || val == nil { + return errors.New("Any JSON doesn't have '@type'") + } + + var turl string + if err := json.Unmarshal([]byte(*val), &turl); err != nil { + return fmt.Errorf("can't unmarshal Any's '@type': %q", *val) + } + target.Field(0).SetString(turl) + + mname := turl + if slash := strings.LastIndex(mname, "/"); slash >= 0 { + mname = mname[slash+1:] + } + mt := proto.MessageType(mname) + if mt == nil { + return fmt.Errorf("unknown message type %q", mname) + } + + m := reflect.New(mt.Elem()).Interface().(proto.Message) + if _, ok := m.(wkt); ok { + val, ok := jsonFields["value"] + if !ok { + return errors.New("Any JSON doesn't have 'value'") + } + + if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil { + return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) + } + } else { + delete(jsonFields, "@type") + nestedProto, err := json.Marshal(jsonFields) + if err != nil { + return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err) + } + + if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil { + return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) + } + } + + b, err := proto.Marshal(m) + if err != nil { + return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err) + } + target.Field(1).SetBytes(b) + + return nil case "Duration": - unq, err := strconv.Unquote(string(inputValue)) + ivStr := string(inputValue) + if ivStr == "null" { + target.Field(0).SetInt(0) + target.Field(1).SetInt(0) + return nil + } + + unq, err := strconv.Unquote(ivStr) if err != nil { return err } @@ -600,7 +738,14 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe target.Field(1).SetInt(ns) return nil case "Timestamp": - unq, err := strconv.Unquote(string(inputValue)) + ivStr := string(inputValue) + if ivStr == "null" { + target.Field(0).SetInt(0) + target.Field(1).SetInt(0) + return nil + } + + unq, err := strconv.Unquote(ivStr) if err != nil { return err } @@ -611,6 +756,62 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe target.Field(0).SetInt(int64(t.Unix())) target.Field(1).SetInt(int64(t.Nanosecond())) return nil + case "Struct": + if string(inputValue) == "null" { + // Interpret a null struct as empty. + return nil + } + var m map[string]json.RawMessage + if err := json.Unmarshal(inputValue, &m); err != nil { + return fmt.Errorf("bad StructValue: %v", err) + } + target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{})) + for k, jv := range m { + pv := &stpb.Value{} + if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil { + return fmt.Errorf("bad value in StructValue for key %q: %v", k, err) + } + target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv)) + } + return nil + case "ListValue": + if string(inputValue) == "null" { + // Interpret a null ListValue as empty. + return nil + } + var s []json.RawMessage + if err := json.Unmarshal(inputValue, &s); err != nil { + return fmt.Errorf("bad ListValue: %v", err) + } + target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s), len(s)))) + for i, sv := range s { + if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil { + return err + } + } + return nil + case "Value": + ivStr := string(inputValue) + if ivStr == "null" { + target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{})) + } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil { + target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v})) + } else if v, err := strconv.Unquote(ivStr); err == nil { + target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v})) + } else if v, err := strconv.ParseBool(ivStr); err == nil { + target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v})) + } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil { + lv := &stpb.ListValue{} + target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv})) + return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop) + } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil { + sv := &stpb.Struct{} + target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv})) + return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop) + } else { + return fmt.Errorf("unrecognized type for Value %q", ivStr) + } + return nil } } @@ -694,6 +895,26 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe } } } + // Handle proto2 extensions. + if len(jsonFields) > 0 { + if ep, ok := target.Addr().Interface().(proto.Message); ok { + for _, ext := range proto.RegisteredExtensions(ep) { + name := fmt.Sprintf("[%s]", ext.Name) + raw, ok := jsonFields[name] + if !ok { + continue + } + delete(jsonFields, name) + nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem()) + if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil { + return err + } + if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil { + return err + } + } + } + } if !u.AllowUnknownFields && len(jsonFields) > 0 { // Pick any field to be the scapegoat. var f string @@ -766,6 +987,15 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe inputValue = inputValue[1 : len(inputValue)-1] } + // Non-finite numbers can be encoded as strings. + isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 + if isFloat { + if num, ok := nonFinite[string(inputValue)]; ok { + target.SetFloat(num) + return nil + } + } + // Use the encoding/json for parsing other value types. return json.Unmarshal(inputValue, target.Addr().Interface()) } diff --git a/cmd/vendor/github.com/golang/protobuf/proto/encode.go b/cmd/vendor/github.com/golang/protobuf/proto/encode.go index 68b9b30cfaed..2b30f84626ad 100644 --- a/cmd/vendor/github.com/golang/protobuf/proto/encode.go +++ b/cmd/vendor/github.com/golang/protobuf/proto/encode.go @@ -1075,10 +1075,17 @@ func (o *Buffer) enc_map(p *Properties, base structPointer) error { func (o *Buffer) enc_exts(p *Properties, base structPointer) error { exts := structPointer_Extensions(base, p.field) - if err := encodeExtensions(exts); err != nil { + + v, mu := exts.extensionsRead() + if v == nil { + return nil + } + + mu.Lock() + defer mu.Unlock() + if err := encodeExtensionsMap(v); err != nil { return err } - v, _ := exts.extensionsRead() return o.enc_map_body(v) } diff --git a/cmd/vendor/github.com/golang/protobuf/proto/extensions.go b/cmd/vendor/github.com/golang/protobuf/proto/extensions.go index 6b9b36374660..eaad21831263 100644 --- a/cmd/vendor/github.com/golang/protobuf/proto/extensions.go +++ b/cmd/vendor/github.com/golang/protobuf/proto/extensions.go @@ -154,6 +154,7 @@ type ExtensionDesc struct { Field int32 // field number Name string // fully-qualified name of extension, for text formatting Tag string // protobuf tag style + Filename string // name of the file in which the extension is defined } func (ed *ExtensionDesc) repeated() bool { diff --git a/cmd/vendor/github.com/golang/protobuf/proto/lib.go b/cmd/vendor/github.com/golang/protobuf/proto/lib.go index ac4ddbc07598..1c225504a013 100644 --- a/cmd/vendor/github.com/golang/protobuf/proto/lib.go +++ b/cmd/vendor/github.com/golang/protobuf/proto/lib.go @@ -73,7 +73,6 @@ for a protocol buffer variable v: When the .proto file specifies `syntax="proto3"`, there are some differences: - Non-repeated fields of non-message type are values instead of pointers. - - Getters are only generated for message and oneof fields. - Enum types do not get an Enum method. The simplest way to describe this is to see an example. diff --git a/cmd/vendor/github.com/golang/protobuf/ptypes/struct/struct.pb.go b/cmd/vendor/github.com/golang/protobuf/ptypes/struct/struct.pb.go new file mode 100644 index 000000000000..35a8ec59948f --- /dev/null +++ b/cmd/vendor/github.com/golang/protobuf/ptypes/struct/struct.pb.go @@ -0,0 +1,382 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: github.com/golang/protobuf/ptypes/struct/struct.proto + +/* +Package structpb is a generated protocol buffer package. + +It is generated from these files: + github.com/golang/protobuf/ptypes/struct/struct.proto + +It has these top-level messages: + Struct + Value + ListValue +*/ +package structpb + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +type NullValue int32 + +const ( + // Null value. + NullValue_NULL_VALUE NullValue = 0 +) + +var NullValue_name = map[int32]string{ + 0: "NULL_VALUE", +} +var NullValue_value = map[string]int32{ + "NULL_VALUE": 0, +} + +func (x NullValue) String() string { + return proto.EnumName(NullValue_name, int32(x)) +} +func (NullValue) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (NullValue) XXX_WellKnownType() string { return "NullValue" } + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +type Struct struct { + // Unordered map of dynamically typed values. + Fields map[string]*Value `protobuf:"bytes,1,rep,name=fields" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *Struct) Reset() { *m = Struct{} } +func (m *Struct) String() string { return proto.CompactTextString(m) } +func (*Struct) ProtoMessage() {} +func (*Struct) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (*Struct) XXX_WellKnownType() string { return "Struct" } + +func (m *Struct) GetFields() map[string]*Value { + if m != nil { + return m.Fields + } + return nil +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of that +// variants, absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +type Value struct { + // The kind of value. + // + // Types that are valid to be assigned to Kind: + // *Value_NullValue + // *Value_NumberValue + // *Value_StringValue + // *Value_BoolValue + // *Value_StructValue + // *Value_ListValue + Kind isValue_Kind `protobuf_oneof:"kind"` +} + +func (m *Value) Reset() { *m = Value{} } +func (m *Value) String() string { return proto.CompactTextString(m) } +func (*Value) ProtoMessage() {} +func (*Value) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*Value) XXX_WellKnownType() string { return "Value" } + +type isValue_Kind interface { + isValue_Kind() +} + +type Value_NullValue struct { + NullValue NullValue `protobuf:"varint,1,opt,name=null_value,json=nullValue,enum=google.protobuf.NullValue,oneof"` +} +type Value_NumberValue struct { + NumberValue float64 `protobuf:"fixed64,2,opt,name=number_value,json=numberValue,oneof"` +} +type Value_StringValue struct { + StringValue string `protobuf:"bytes,3,opt,name=string_value,json=stringValue,oneof"` +} +type Value_BoolValue struct { + BoolValue bool `protobuf:"varint,4,opt,name=bool_value,json=boolValue,oneof"` +} +type Value_StructValue struct { + StructValue *Struct `protobuf:"bytes,5,opt,name=struct_value,json=structValue,oneof"` +} +type Value_ListValue struct { + ListValue *ListValue `protobuf:"bytes,6,opt,name=list_value,json=listValue,oneof"` +} + +func (*Value_NullValue) isValue_Kind() {} +func (*Value_NumberValue) isValue_Kind() {} +func (*Value_StringValue) isValue_Kind() {} +func (*Value_BoolValue) isValue_Kind() {} +func (*Value_StructValue) isValue_Kind() {} +func (*Value_ListValue) isValue_Kind() {} + +func (m *Value) GetKind() isValue_Kind { + if m != nil { + return m.Kind + } + return nil +} + +func (m *Value) GetNullValue() NullValue { + if x, ok := m.GetKind().(*Value_NullValue); ok { + return x.NullValue + } + return NullValue_NULL_VALUE +} + +func (m *Value) GetNumberValue() float64 { + if x, ok := m.GetKind().(*Value_NumberValue); ok { + return x.NumberValue + } + return 0 +} + +func (m *Value) GetStringValue() string { + if x, ok := m.GetKind().(*Value_StringValue); ok { + return x.StringValue + } + return "" +} + +func (m *Value) GetBoolValue() bool { + if x, ok := m.GetKind().(*Value_BoolValue); ok { + return x.BoolValue + } + return false +} + +func (m *Value) GetStructValue() *Struct { + if x, ok := m.GetKind().(*Value_StructValue); ok { + return x.StructValue + } + return nil +} + +func (m *Value) GetListValue() *ListValue { + if x, ok := m.GetKind().(*Value_ListValue); ok { + return x.ListValue + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Value) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Value_OneofMarshaler, _Value_OneofUnmarshaler, _Value_OneofSizer, []interface{}{ + (*Value_NullValue)(nil), + (*Value_NumberValue)(nil), + (*Value_StringValue)(nil), + (*Value_BoolValue)(nil), + (*Value_StructValue)(nil), + (*Value_ListValue)(nil), + } +} + +func _Value_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Value) + // kind + switch x := m.Kind.(type) { + case *Value_NullValue: + b.EncodeVarint(1<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.NullValue)) + case *Value_NumberValue: + b.EncodeVarint(2<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.NumberValue)) + case *Value_StringValue: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeStringBytes(x.StringValue) + case *Value_BoolValue: + t := uint64(0) + if x.BoolValue { + t = 1 + } + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *Value_StructValue: + b.EncodeVarint(5<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.StructValue); err != nil { + return err + } + case *Value_ListValue: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ListValue); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Value.Kind has unexpected type %T", x) + } + return nil +} + +func _Value_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Value) + switch tag { + case 1: // kind.null_value + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Kind = &Value_NullValue{NullValue(x)} + return true, err + case 2: // kind.number_value + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.Kind = &Value_NumberValue{math.Float64frombits(x)} + return true, err + case 3: // kind.string_value + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Kind = &Value_StringValue{x} + return true, err + case 4: // kind.bool_value + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Kind = &Value_BoolValue{x != 0} + return true, err + case 5: // kind.struct_value + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Struct) + err := b.DecodeMessage(msg) + m.Kind = &Value_StructValue{msg} + return true, err + case 6: // kind.list_value + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ListValue) + err := b.DecodeMessage(msg) + m.Kind = &Value_ListValue{msg} + return true, err + default: + return false, nil + } +} + +func _Value_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Value) + // kind + switch x := m.Kind.(type) { + case *Value_NullValue: + n += proto.SizeVarint(1<<3 | proto.WireVarint) + n += proto.SizeVarint(uint64(x.NullValue)) + case *Value_NumberValue: + n += proto.SizeVarint(2<<3 | proto.WireFixed64) + n += 8 + case *Value_StringValue: + n += proto.SizeVarint(3<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(len(x.StringValue))) + n += len(x.StringValue) + case *Value_BoolValue: + n += proto.SizeVarint(4<<3 | proto.WireVarint) + n += 1 + case *Value_StructValue: + s := proto.Size(x.StructValue) + n += proto.SizeVarint(5<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case *Value_ListValue: + s := proto.Size(x.ListValue) + n += proto.SizeVarint(6<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +type ListValue struct { + // Repeated field of dynamically typed values. + Values []*Value `protobuf:"bytes,1,rep,name=values" json:"values,omitempty"` +} + +func (m *ListValue) Reset() { *m = ListValue{} } +func (m *ListValue) String() string { return proto.CompactTextString(m) } +func (*ListValue) ProtoMessage() {} +func (*ListValue) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (*ListValue) XXX_WellKnownType() string { return "ListValue" } + +func (m *ListValue) GetValues() []*Value { + if m != nil { + return m.Values + } + return nil +} + +func init() { + proto.RegisterType((*Struct)(nil), "google.protobuf.Struct") + proto.RegisterType((*Value)(nil), "google.protobuf.Value") + proto.RegisterType((*ListValue)(nil), "google.protobuf.ListValue") + proto.RegisterEnum("google.protobuf.NullValue", NullValue_name, NullValue_value) +} + +func init() { + proto.RegisterFile("github.com/golang/protobuf/ptypes/struct/struct.proto", fileDescriptor0) +} + +var fileDescriptor0 = []byte{ + // 417 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x41, 0x8b, 0xd3, 0x40, + 0x14, 0x80, 0x3b, 0xc9, 0x36, 0x98, 0x17, 0x59, 0x97, 0x11, 0xb4, 0xac, 0xa0, 0xa1, 0x7b, 0x09, + 0x22, 0x09, 0x56, 0x04, 0x31, 0x5e, 0x0c, 0xac, 0xbb, 0x60, 0x58, 0x62, 0x74, 0x57, 0xf0, 0x52, + 0x9a, 0x34, 0x8d, 0xa1, 0xd3, 0x99, 0x90, 0xcc, 0x28, 0x3d, 0xfa, 0x2f, 0x3c, 0x7b, 0xf4, 0xe8, + 0xaf, 0xf3, 0x28, 0x33, 0x93, 0x44, 0x69, 0x29, 0x78, 0x9a, 0xbe, 0x37, 0xdf, 0xfb, 0xe6, 0xbd, + 0xd7, 0xc0, 0xf3, 0xb2, 0xe2, 0x9f, 0x45, 0xe6, 0xe7, 0x6c, 0x13, 0x94, 0x8c, 0x2c, 0x68, 0x19, + 0xd4, 0x0d, 0xe3, 0x2c, 0x13, 0xab, 0xa0, 0xe6, 0xdb, 0xba, 0x68, 0x83, 0x96, 0x37, 0x22, 0xe7, + 0xdd, 0xe1, 0xab, 0x5b, 0x7c, 0xa7, 0x64, 0xac, 0x24, 0x85, 0xdf, 0xb3, 0xd3, 0xef, 0x08, 0xac, + 0xf7, 0x8a, 0xc0, 0x21, 0x58, 0xab, 0xaa, 0x20, 0xcb, 0x76, 0x82, 0x5c, 0xd3, 0x73, 0x66, 0x67, + 0xfe, 0x0e, 0xec, 0x6b, 0xd0, 0x7f, 0xa3, 0xa8, 0x73, 0xca, 0x9b, 0x6d, 0xda, 0x95, 0x9c, 0xbe, + 0x03, 0xe7, 0x9f, 0x34, 0x3e, 0x01, 0x73, 0x5d, 0x6c, 0x27, 0xc8, 0x45, 0x9e, 0x9d, 0xca, 0x9f, + 0xf8, 0x09, 0x8c, 0xbf, 0x2c, 0x88, 0x28, 0x26, 0x86, 0x8b, 0x3c, 0x67, 0x76, 0x6f, 0x4f, 0x7e, + 0x23, 0x6f, 0x53, 0x0d, 0xbd, 0x34, 0x5e, 0xa0, 0xe9, 0x2f, 0x03, 0xc6, 0x2a, 0x89, 0x43, 0x00, + 0x2a, 0x08, 0x99, 0x6b, 0x81, 0x94, 0x1e, 0xcf, 0x4e, 0xf7, 0x04, 0x57, 0x82, 0x10, 0xc5, 0x5f, + 0x8e, 0x52, 0x9b, 0xf6, 0x01, 0x3e, 0x83, 0xdb, 0x54, 0x6c, 0xb2, 0xa2, 0x99, 0xff, 0x7d, 0x1f, + 0x5d, 0x8e, 0x52, 0x47, 0x67, 0x07, 0xa8, 0xe5, 0x4d, 0x45, 0xcb, 0x0e, 0x32, 0x65, 0xe3, 0x12, + 0xd2, 0x59, 0x0d, 0x3d, 0x02, 0xc8, 0x18, 0xeb, 0xdb, 0x38, 0x72, 0x91, 0x77, 0x4b, 0x3e, 0x25, + 0x73, 0x1a, 0x78, 0xa5, 0x2c, 0x22, 0xe7, 0x1d, 0x32, 0x56, 0xa3, 0xde, 0x3f, 0xb0, 0xc7, 0x4e, + 0x2f, 0x72, 0x3e, 0x4c, 0x49, 0xaa, 0xb6, 0xaf, 0xb5, 0x54, 0xed, 0xfe, 0x94, 0x71, 0xd5, 0xf2, + 0x61, 0x4a, 0xd2, 0x07, 0x91, 0x05, 0x47, 0xeb, 0x8a, 0x2e, 0xa7, 0x21, 0xd8, 0x03, 0x81, 0x7d, + 0xb0, 0x94, 0xac, 0xff, 0x47, 0x0f, 0x2d, 0xbd, 0xa3, 0x1e, 0x3f, 0x00, 0x7b, 0x58, 0x22, 0x3e, + 0x06, 0xb8, 0xba, 0x8e, 0xe3, 0xf9, 0xcd, 0xeb, 0xf8, 0xfa, 0xfc, 0x64, 0x14, 0x7d, 0x43, 0x70, + 0x37, 0x67, 0x9b, 0x5d, 0x45, 0xe4, 0xe8, 0x69, 0x12, 0x19, 0x27, 0xe8, 0xd3, 0xd3, 0xff, 0xfd, + 0x30, 0x43, 0x7d, 0xd4, 0xd9, 0x6f, 0x84, 0x7e, 0x18, 0xe6, 0x45, 0x12, 0xfd, 0x34, 0x1e, 0x5e, + 0x68, 0x79, 0xd2, 0xf7, 0xf7, 0xb1, 0x20, 0xe4, 0x2d, 0x65, 0x5f, 0xe9, 0x07, 0x59, 0x99, 0x59, + 0x4a, 0xf5, 0xec, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9b, 0x6e, 0x5d, 0x3c, 0xfe, 0x02, 0x00, + 0x00, +} diff --git a/cmd/vendor/github.com/russross/blackfriday/inline.go b/cmd/vendor/github.com/russross/blackfriday/inline.go index c1f74752f679..4483b8f19fd2 100644 --- a/cmd/vendor/github.com/russross/blackfriday/inline.go +++ b/cmd/vendor/github.com/russross/blackfriday/inline.go @@ -488,6 +488,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int { } p.notes = append(p.notes, ref) + p.notesRecord[string(ref.link)] = struct{}{} link = ref.link title = ref.title @@ -498,9 +499,10 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int { return 0 } - if t == linkDeferredFootnote { + if t == linkDeferredFootnote && !p.isFootnote(lr) { lr.noteId = len(p.notes) + 1 p.notes = append(p.notes, lr) + p.notesRecord[string(lr.link)] = struct{}{} } // keep link and title from reference diff --git a/cmd/vendor/github.com/russross/blackfriday/markdown.go b/cmd/vendor/github.com/russross/blackfriday/markdown.go index 6d842d3b29c4..1722a738929b 100644 --- a/cmd/vendor/github.com/russross/blackfriday/markdown.go +++ b/cmd/vendor/github.com/russross/blackfriday/markdown.go @@ -218,7 +218,8 @@ type parser struct { // Footnotes need to be ordered as well as available to quickly check for // presence. If a ref is also a footnote, it's stored both in refs and here // in notes. Slice is nil if footnotes not enabled. - notes []*reference + notes []*reference + notesRecord map[string]struct{} } func (p *parser) getRef(refid string) (ref *reference, found bool) { @@ -241,6 +242,11 @@ func (p *parser) getRef(refid string) (ref *reference, found bool) { return ref, found } +func (p *parser) isFootnote(ref *reference) bool { + _, ok := p.notesRecord[string(ref.link)] + return ok +} + // // // Public interface @@ -376,6 +382,7 @@ func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte { if extensions&EXTENSION_FOOTNOTES != 0 { p.notes = make([]*reference, 0) + p.notesRecord = make(map[string]struct{}) } first := firstPass(p, input) diff --git a/contrib/raftexample/raft.go b/contrib/raftexample/raft.go index 84c8ffda35c9..b212dcb7c00d 100644 --- a/contrib/raftexample/raft.go +++ b/contrib/raftexample/raft.go @@ -107,9 +107,9 @@ func newRaftNode(id int, peers []string, join bool, getSnapshot func() ([]byte, } func (rc *raftNode) saveSnap(snap raftpb.Snapshot) error { - if err := rc.snapshotter.SaveSnap(snap); err != nil { - return err - } + // must save the snapshot index to the WAL before saving the + // snapshot to maintain the invariant that we only Open the + // wal at previously-saved snapshot indexes. walSnap := walpb.Snapshot{ Index: snap.Metadata.Index, Term: snap.Metadata.Term, @@ -117,6 +117,9 @@ func (rc *raftNode) saveSnap(snap raftpb.Snapshot) error { if err := rc.wal.SaveSnapshot(walSnap); err != nil { return err } + if err := rc.snapshotter.SaveSnap(snap); err != nil { + return err + } return rc.wal.ReleaseLockTo(snap.Metadata.Index) } diff --git a/e2e/ctl_v2_test.go b/e2e/ctl_v2_test.go index 8f877c25694c..c1635887dbe5 100644 --- a/e2e/ctl_v2_test.go +++ b/e2e/ctl_v2_test.go @@ -321,17 +321,31 @@ func TestCtlV2ClusterHealth(t *testing.T) { } }() - // has quorum + // all members available if err := etcdctlClusterHealth(epc, "cluster is healthy"); err != nil { t.Fatalf("cluster-health expected to be healthy (%v)", err) } - // cut quorum + // missing members, has quorum epc.procs[0].Stop() + + for i := 0; i < 3; i++ { + err := etcdctlClusterHealth(epc, "cluster is degraded") + if err == nil { + break + } else if i == 2 { + t.Fatalf("cluster-health expected to be degraded (%v)", err) + } + // possibly no leader yet; retry + time.Sleep(time.Second) + } + + // no quorum epc.procs[1].Stop() - if err := etcdctlClusterHealth(epc, "cluster is unhealthy"); err != nil { - t.Fatalf("cluster-health expected to be unhealthy (%v)", err) + if err := etcdctlClusterHealth(epc, "cluster is unavailable"); err != nil { + t.Fatalf("cluster-health expected to be unavailable (%v)", err) } + epc.procs[0], epc.procs[1] = nil, nil } diff --git a/etcdctl/ctlv2/command/cluster_health.go b/etcdctl/ctlv2/command/cluster_health.go index 95101785dc78..d1429649aede 100644 --- a/etcdctl/ctlv2/command/cluster_health.go +++ b/etcdctl/ctlv2/command/cluster_health.go @@ -70,7 +70,7 @@ func handleClusterHealth(c *cli.Context) error { } for { - health := false + healthyMembers := 0 for _, m := range ms { if len(m.ClientURLs) == 0 { fmt.Printf("member %s is unreachable: no available published client urls\n", m.ID) @@ -105,8 +105,8 @@ func handleClusterHealth(c *cli.Context) error { checked = true if result.Health == "true" || nresult.Health { - health = true fmt.Printf("member %s is healthy: got healthy result from %s\n", m.ID, url) + healthyMembers++ } else { fmt.Printf("member %s is unhealthy: got unhealthy result from %s\n", m.ID, url) } @@ -116,19 +116,20 @@ func handleClusterHealth(c *cli.Context) error { fmt.Printf("member %s is unreachable: %v are all unreachable\n", m.ID, m.ClientURLs) } } - if health { + switch healthyMembers { + case len(ms): fmt.Println("cluster is healthy") - } else { - fmt.Println("cluster is unhealthy") + case 0: + fmt.Println("cluster is unavailable") + default: + fmt.Println("cluster is degraded") } if !forever { - if health { + if healthyMembers == len(ms) { os.Exit(ExitSuccess) - return nil } os.Exit(ExitClusterNotHealthy) - return nil } fmt.Printf("\nnext check after 10 second...\n\n") diff --git a/etcdserver/api/v3election/v3electionpb/v3election.pb.go b/etcdserver/api/v3election/v3electionpb/v3election.pb.go index 2600756e300b..6ce0fd4709c6 100644 --- a/etcdserver/api/v3election/v3electionpb/v3election.pb.go +++ b/etcdserver/api/v3election/v3electionpb/v3election.pb.go @@ -67,6 +67,27 @@ func (m *CampaignRequest) String() string { return proto.CompactTextS func (*CampaignRequest) ProtoMessage() {} func (*CampaignRequest) Descriptor() ([]byte, []int) { return fileDescriptorV3Election, []int{0} } +func (m *CampaignRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *CampaignRequest) GetLease() int64 { + if m != nil { + return m.Lease + } + return 0 +} + +func (m *CampaignRequest) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + type CampaignResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // leader describes the resources used for holding leadereship of the election. @@ -111,6 +132,34 @@ func (m *LeaderKey) String() string { return proto.CompactTextString( func (*LeaderKey) ProtoMessage() {} func (*LeaderKey) Descriptor() ([]byte, []int) { return fileDescriptorV3Election, []int{2} } +func (m *LeaderKey) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *LeaderKey) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *LeaderKey) GetRev() int64 { + if m != nil { + return m.Rev + } + return 0 +} + +func (m *LeaderKey) GetLease() int64 { + if m != nil { + return m.Lease + } + return 0 +} + type LeaderRequest struct { // name is the election identifier for the leadership information. Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -121,6 +170,13 @@ func (m *LeaderRequest) String() string { return proto.CompactTextStr func (*LeaderRequest) ProtoMessage() {} func (*LeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptorV3Election, []int{3} } +func (m *LeaderRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + type LeaderResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // kv is the key-value pair representing the latest leader update. @@ -198,6 +254,13 @@ func (m *ProclaimRequest) GetLeader() *LeaderKey { return nil } +func (m *ProclaimRequest) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + type ProclaimResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } diff --git a/etcdserver/api/v3lock/v3lockpb/v3lock.pb.go b/etcdserver/api/v3lock/v3lockpb/v3lock.pb.go index 44bde286b2f4..20bd357525de 100644 --- a/etcdserver/api/v3lock/v3lockpb/v3lock.pb.go +++ b/etcdserver/api/v3lock/v3lockpb/v3lock.pb.go @@ -59,6 +59,20 @@ func (m *LockRequest) String() string { return proto.CompactTextStrin func (*LockRequest) ProtoMessage() {} func (*LockRequest) Descriptor() ([]byte, []int) { return fileDescriptorV3Lock, []int{0} } +func (m *LockRequest) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *LockRequest) GetLease() int64 { + if m != nil { + return m.Lease + } + return 0 +} + type LockResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // key is a key that will exist on etcd for the duration that the Lock caller @@ -79,6 +93,13 @@ func (m *LockResponse) GetHeader() *etcdserverpb.ResponseHeader { return nil } +func (m *LockResponse) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + type UnlockRequest struct { // key is the lock ownership key granted by Lock. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -89,6 +110,13 @@ func (m *UnlockRequest) String() string { return proto.CompactTextStr func (*UnlockRequest) ProtoMessage() {} func (*UnlockRequest) Descriptor() ([]byte, []int) { return fileDescriptorV3Lock, []int{2} } +func (m *UnlockRequest) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + type UnlockResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } diff --git a/etcdserver/apply.go b/etcdserver/apply.go index 9d520767720b..949f36ca7758 100644 --- a/etcdserver/apply.go +++ b/etcdserver/apply.go @@ -319,33 +319,36 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang } func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { - ok := true - for _, c := range rt.Compare { - if _, ok = a.applyCompare(c); !ok { - break - } - } - - var reqs []*pb.RequestOp - if ok { - reqs = rt.Success - } else { - reqs = rt.Failure - } + isWrite := !isTxnReadonly(rt) + txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read()) - if err := a.checkRequestPut(reqs); err != nil { - return nil, err + reqs, ok := a.compareToOps(txn, rt) + if isWrite { + if err := a.checkRequestPut(txn, reqs); err != nil { + txn.End() + return nil, err + } } - if err := a.checkRequestRange(reqs); err != nil { + if err := checkRequestRange(txn, reqs); err != nil { + txn.End() return nil, err } resps := make([]*pb.ResponseOp, len(reqs)) - - // When executing the operations of txn, etcd must hold the txn lock so - // readers do not see any intermediate results. - // TODO: use Read txn if only Ranges - txn := a.s.KV().Write() + txnResp := &pb.TxnResponse{ + Responses: resps, + Succeeded: ok, + Header: &pb.ResponseHeader{}, + } + + // When executing mutable txn ops, etcd must hold the txn lock so + // readers do not see any intermediate results. Since writes are + // serialized on the raft loop, the revision in the read view will + // be the revision of the write txn. + if isWrite { + txn.End() + txn = a.s.KV().Write() + } for i := range reqs { resps[i] = a.applyUnion(txn, reqs[i]) } @@ -355,23 +358,25 @@ func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { } txn.End() - txnResp := &pb.TxnResponse{} - txnResp.Header = &pb.ResponseHeader{} txnResp.Header.Revision = rev - txnResp.Responses = resps - txnResp.Succeeded = ok return txnResp, nil } -// applyCompare applies the compare request. -// It returns the revision at which the comparison happens. If the comparison -// succeeds, the it returns true. Otherwise it returns false. -func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) { - rr, err := a.s.KV().Range(c.Key, nil, mvcc.RangeOptions{}) - rev := rr.Rev +func (a *applierV3backend) compareToOps(rv mvcc.ReadView, rt *pb.TxnRequest) ([]*pb.RequestOp, bool) { + for _, c := range rt.Compare { + if !applyCompare(rv, c) { + return rt.Failure, false + } + } + return rt.Success, true +} +// applyCompare applies the compare request. +// If the comparison succeeds, it returns true. Otherwise, returns false. +func applyCompare(rv mvcc.ReadView, c *pb.Compare) bool { + rr, err := rv.Range(c.Key, nil, mvcc.RangeOptions{}) if err != nil { - return rev, false + return false } var ckv mvccpb.KeyValue if len(rr.KVs) != 0 { @@ -383,7 +388,7 @@ func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) { // We can treat non-existence as the empty set explicitly, such that // even a key with a value of length 0 bytes is still a real key // that was written that way - return rev, false + return false } } @@ -415,23 +420,15 @@ func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) { switch c.Result { case pb.Compare_EQUAL: - if result != 0 { - return rev, false - } + return result == 0 case pb.Compare_NOT_EQUAL: - if result == 0 { - return rev, false - } + return result != 0 case pb.Compare_GREATER: - if result != 1 { - return rev, false - } + return result > 0 case pb.Compare_LESS: - if result != -1 { - return rev, false - } + return result < 0 } - return rev, true + return true } func (a *applierV3backend) applyUnion(txn mvcc.TxnWrite, union *pb.RequestOp) *pb.ResponseOp { @@ -771,7 +768,7 @@ func (s *kvSortByValue) Less(i, j int) bool { return bytes.Compare(s.kvs[i].Value, s.kvs[j].Value) < 0 } -func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error { +func (a *applierV3backend) checkRequestPut(rv mvcc.ReadView, reqs []*pb.RequestOp) error { for _, requ := range reqs { tv, ok := requ.Request.(*pb.RequestOp_RequestPut) if !ok { @@ -783,7 +780,7 @@ func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error { } if preq.IgnoreValue || preq.IgnoreLease { // expects previous key-value, error if not exist - rr, err := a.s.KV().Range(preq.Key, nil, mvcc.RangeOptions{}) + rr, err := rv.Range(preq.Key, nil, mvcc.RangeOptions{}) if err != nil { return err } @@ -801,7 +798,7 @@ func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error { return nil } -func (a *applierV3backend) checkRequestRange(reqs []*pb.RequestOp) error { +func checkRequestRange(rv mvcc.ReadView, reqs []*pb.RequestOp) error { for _, requ := range reqs { tv, ok := requ.Request.(*pb.RequestOp_RequestRange) if !ok { @@ -812,10 +809,10 @@ func (a *applierV3backend) checkRequestRange(reqs []*pb.RequestOp) error { continue } - if greq.Revision > a.s.KV().Rev() { + if greq.Revision > rv.Rev() { return mvcc.ErrFutureRev } - if greq.Revision < a.s.KV().FirstRev() { + if greq.Revision < rv.FirstRev() { return mvcc.ErrCompacted } } diff --git a/etcdserver/config.go b/etcdserver/config.go index a2713c9e61a9..7ab77d986c3b 100644 --- a/etcdserver/config.go +++ b/etcdserver/config.go @@ -117,11 +117,41 @@ func (c *ServerConfig) advertiseMatchesCluster() error { sort.Strings(apurls) ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() - if !netutil.URLStringsEqual(ctx, apurls, urls.StringSlice()) { - umap := map[string]types.URLs{c.Name: c.PeerURLs} - return fmt.Errorf("--initial-cluster must include %s given --initial-advertise-peer-urls=%s", types.URLsMap(umap).String(), strings.Join(apurls, ",")) + if netutil.URLStringsEqual(ctx, apurls, urls.StringSlice()) { + return nil } - return nil + + initMap, apMap := make(map[string]struct{}), make(map[string]struct{}) + for _, url := range c.PeerURLs { + apMap[url.String()] = struct{}{} + } + for _, url := range c.InitialPeerURLsMap[c.Name] { + initMap[url.String()] = struct{}{} + } + + missing := []string{} + for url := range initMap { + if _, ok := apMap[url]; !ok { + missing = append(missing, url) + } + } + if len(missing) > 0 { + for i := range missing { + missing[i] = c.Name + "=" + missing[i] + } + mstr := strings.Join(missing, ",") + apStr := strings.Join(apurls, ",") + return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s ", mstr, apStr) + } + + for url := range apMap { + if _, ok := initMap[url]; !ok { + missing = append(missing, url) + } + } + mstr := strings.Join(missing, ",") + umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs}) + return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String()) } func (c *ServerConfig) MemberDir() string { return filepath.Join(c.DataDir, "member") } diff --git a/etcdserver/config_test.go b/etcdserver/config_test.go index bf0cd7f75d21..e574ab10dc04 100644 --- a/etcdserver/config_test.go +++ b/etcdserver/config_test.go @@ -107,6 +107,14 @@ func TestConfigVerifyLocalMember(t *testing.T) { true, }, + { + // Advertised peer URLs must match those in cluster-state + "node1=http://localhost:12345", + []string{"http://localhost:2380", "http://localhost:12345"}, + true, + + true, + }, { // Advertised peer URLs must match those in cluster-state "node1=http://localhost:2380", diff --git a/etcdserver/etcdserverpb/rpc.pb.go b/etcdserver/etcdserverpb/rpc.pb.go index 018a3652be7a..073d4a39468f 100644 --- a/etcdserver/etcdserverpb/rpc.pb.go +++ b/etcdserver/etcdserverpb/rpc.pb.go @@ -223,6 +223,34 @@ func (m *ResponseHeader) String() string { return proto.CompactTextSt func (*ResponseHeader) ProtoMessage() {} func (*ResponseHeader) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } +func (m *ResponseHeader) GetClusterId() uint64 { + if m != nil { + return m.ClusterId + } + return 0 +} + +func (m *ResponseHeader) GetMemberId() uint64 { + if m != nil { + return m.MemberId + } + return 0 +} + +func (m *ResponseHeader) GetRevision() int64 { + if m != nil { + return m.Revision + } + return 0 +} + +func (m *ResponseHeader) GetRaftTerm() uint64 { + if m != nil { + return m.RaftTerm + } + return 0 +} + type RangeRequest struct { // key is the first key for the range. If range_end is not given, the request only looks up key. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -273,6 +301,97 @@ func (m *RangeRequest) String() string { return proto.CompactTextStri func (*RangeRequest) ProtoMessage() {} func (*RangeRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } +func (m *RangeRequest) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *RangeRequest) GetRangeEnd() []byte { + if m != nil { + return m.RangeEnd + } + return nil +} + +func (m *RangeRequest) GetLimit() int64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *RangeRequest) GetRevision() int64 { + if m != nil { + return m.Revision + } + return 0 +} + +func (m *RangeRequest) GetSortOrder() RangeRequest_SortOrder { + if m != nil { + return m.SortOrder + } + return RangeRequest_NONE +} + +func (m *RangeRequest) GetSortTarget() RangeRequest_SortTarget { + if m != nil { + return m.SortTarget + } + return RangeRequest_KEY +} + +func (m *RangeRequest) GetSerializable() bool { + if m != nil { + return m.Serializable + } + return false +} + +func (m *RangeRequest) GetKeysOnly() bool { + if m != nil { + return m.KeysOnly + } + return false +} + +func (m *RangeRequest) GetCountOnly() bool { + if m != nil { + return m.CountOnly + } + return false +} + +func (m *RangeRequest) GetMinModRevision() int64 { + if m != nil { + return m.MinModRevision + } + return 0 +} + +func (m *RangeRequest) GetMaxModRevision() int64 { + if m != nil { + return m.MaxModRevision + } + return 0 +} + +func (m *RangeRequest) GetMinCreateRevision() int64 { + if m != nil { + return m.MinCreateRevision + } + return 0 +} + +func (m *RangeRequest) GetMaxCreateRevision() int64 { + if m != nil { + return m.MaxCreateRevision + } + return 0 +} + type RangeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // kvs is the list of key-value pairs matched by the range request. @@ -303,6 +422,20 @@ func (m *RangeResponse) GetKvs() []*mvccpb.KeyValue { return nil } +func (m *RangeResponse) GetMore() bool { + if m != nil { + return m.More + } + return false +} + +func (m *RangeResponse) GetCount() int64 { + if m != nil { + return m.Count + } + return 0 +} + type PutRequest struct { // key is the key, in bytes, to put into the key-value store. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -327,6 +460,48 @@ func (m *PutRequest) String() string { return proto.CompactTextString func (*PutRequest) ProtoMessage() {} func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } +func (m *PutRequest) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *PutRequest) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *PutRequest) GetLease() int64 { + if m != nil { + return m.Lease + } + return 0 +} + +func (m *PutRequest) GetPrevKv() bool { + if m != nil { + return m.PrevKv + } + return false +} + +func (m *PutRequest) GetIgnoreValue() bool { + if m != nil { + return m.IgnoreValue + } + return false +} + +func (m *PutRequest) GetIgnoreLease() bool { + if m != nil { + return m.IgnoreLease + } + return false +} + type PutResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // if prev_kv is set in the request, the previous key-value pair will be returned. @@ -371,6 +546,27 @@ func (m *DeleteRangeRequest) String() string { return proto.CompactTe func (*DeleteRangeRequest) ProtoMessage() {} func (*DeleteRangeRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{5} } +func (m *DeleteRangeRequest) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *DeleteRangeRequest) GetRangeEnd() []byte { + if m != nil { + return m.RangeEnd + } + return nil +} + +func (m *DeleteRangeRequest) GetPrevKv() bool { + if m != nil { + return m.PrevKv + } + return false +} + type DeleteRangeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // deleted is the number of keys deleted by the delete range request. @@ -391,6 +587,13 @@ func (m *DeleteRangeResponse) GetHeader() *ResponseHeader { return nil } +func (m *DeleteRangeResponse) GetDeleted() int64 { + if m != nil { + return m.Deleted + } + return 0 +} + func (m *DeleteRangeResponse) GetPrevKvs() []*mvccpb.KeyValue { if m != nil { return m.PrevKvs @@ -761,6 +964,27 @@ func (m *Compare) GetTargetUnion() isCompare_TargetUnion { return nil } +func (m *Compare) GetResult() Compare_CompareResult { + if m != nil { + return m.Result + } + return Compare_EQUAL +} + +func (m *Compare) GetTarget() Compare_CompareTarget { + if m != nil { + return m.Target + } + return Compare_VERSION +} + +func (m *Compare) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + func (m *Compare) GetVersion() int64 { if x, ok := m.GetTargetUnion().(*Compare_Version); ok { return x.Version @@ -957,6 +1181,13 @@ func (m *TxnResponse) GetHeader() *ResponseHeader { return nil } +func (m *TxnResponse) GetSucceeded() bool { + if m != nil { + return m.Succeeded + } + return false +} + func (m *TxnResponse) GetResponses() []*ResponseOp { if m != nil { return m.Responses @@ -980,6 +1211,20 @@ func (m *CompactionRequest) String() string { return proto.CompactTex func (*CompactionRequest) ProtoMessage() {} func (*CompactionRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{12} } +func (m *CompactionRequest) GetRevision() int64 { + if m != nil { + return m.Revision + } + return 0 +} + +func (m *CompactionRequest) GetPhysical() bool { + if m != nil { + return m.Physical + } + return false +} + type CompactionResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -1022,6 +1267,13 @@ func (m *HashResponse) GetHeader() *ResponseHeader { return nil } +func (m *HashResponse) GetHash() uint32 { + if m != nil { + return m.Hash + } + return 0 +} + type SnapshotRequest struct { } @@ -1052,6 +1304,20 @@ func (m *SnapshotResponse) GetHeader() *ResponseHeader { return nil } +func (m *SnapshotResponse) GetRemainingBytes() uint64 { + if m != nil { + return m.RemainingBytes + } + return 0 +} + +func (m *SnapshotResponse) GetBlob() []byte { + if m != nil { + return m.Blob + } + return nil +} + type WatchRequest struct { // request_union is a request to either create a new watcher or cancel an existing watcher. // @@ -1205,6 +1471,48 @@ func (m *WatchCreateRequest) String() string { return proto.CompactTe func (*WatchCreateRequest) ProtoMessage() {} func (*WatchCreateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{19} } +func (m *WatchCreateRequest) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *WatchCreateRequest) GetRangeEnd() []byte { + if m != nil { + return m.RangeEnd + } + return nil +} + +func (m *WatchCreateRequest) GetStartRevision() int64 { + if m != nil { + return m.StartRevision + } + return 0 +} + +func (m *WatchCreateRequest) GetProgressNotify() bool { + if m != nil { + return m.ProgressNotify + } + return false +} + +func (m *WatchCreateRequest) GetFilters() []WatchCreateRequest_FilterType { + if m != nil { + return m.Filters + } + return nil +} + +func (m *WatchCreateRequest) GetPrevKv() bool { + if m != nil { + return m.PrevKv + } + return false +} + type WatchCancelRequest struct { // watch_id is the watcher id to cancel so that no more events are transmitted. WatchId int64 `protobuf:"varint,1,opt,name=watch_id,json=watchId,proto3" json:"watch_id,omitempty"` @@ -1215,6 +1523,13 @@ func (m *WatchCancelRequest) String() string { return proto.CompactTe func (*WatchCancelRequest) ProtoMessage() {} func (*WatchCancelRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{20} } +func (m *WatchCancelRequest) GetWatchId() int64 { + if m != nil { + return m.WatchId + } + return 0 +} + type WatchResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // watch_id is the ID of the watcher that corresponds to the response. @@ -1253,6 +1568,41 @@ func (m *WatchResponse) GetHeader() *ResponseHeader { return nil } +func (m *WatchResponse) GetWatchId() int64 { + if m != nil { + return m.WatchId + } + return 0 +} + +func (m *WatchResponse) GetCreated() bool { + if m != nil { + return m.Created + } + return false +} + +func (m *WatchResponse) GetCanceled() bool { + if m != nil { + return m.Canceled + } + return false +} + +func (m *WatchResponse) GetCompactRevision() int64 { + if m != nil { + return m.CompactRevision + } + return 0 +} + +func (m *WatchResponse) GetCancelReason() string { + if m != nil { + return m.CancelReason + } + return "" +} + func (m *WatchResponse) GetEvents() []*mvccpb.Event { if m != nil { return m.Events @@ -1272,6 +1622,20 @@ func (m *LeaseGrantRequest) String() string { return proto.CompactTex func (*LeaseGrantRequest) ProtoMessage() {} func (*LeaseGrantRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{22} } +func (m *LeaseGrantRequest) GetTTL() int64 { + if m != nil { + return m.TTL + } + return 0 +} + +func (m *LeaseGrantRequest) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + type LeaseGrantResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // ID is the lease ID for the granted lease. @@ -1293,6 +1657,27 @@ func (m *LeaseGrantResponse) GetHeader() *ResponseHeader { return nil } +func (m *LeaseGrantResponse) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *LeaseGrantResponse) GetTTL() int64 { + if m != nil { + return m.TTL + } + return 0 +} + +func (m *LeaseGrantResponse) GetError() string { + if m != nil { + return m.Error + } + return "" +} + type LeaseRevokeRequest struct { // ID is the lease ID to revoke. When the ID is revoked, all associated keys will be deleted. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` @@ -1303,6 +1688,13 @@ func (m *LeaseRevokeRequest) String() string { return proto.CompactTe func (*LeaseRevokeRequest) ProtoMessage() {} func (*LeaseRevokeRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{24} } +func (m *LeaseRevokeRequest) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + type LeaseRevokeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -1329,6 +1721,13 @@ func (m *LeaseKeepAliveRequest) String() string { return proto.Compac func (*LeaseKeepAliveRequest) ProtoMessage() {} func (*LeaseKeepAliveRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{26} } +func (m *LeaseKeepAliveRequest) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + type LeaseKeepAliveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // ID is the lease ID from the keep alive request. @@ -1349,6 +1748,20 @@ func (m *LeaseKeepAliveResponse) GetHeader() *ResponseHeader { return nil } +func (m *LeaseKeepAliveResponse) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *LeaseKeepAliveResponse) GetTTL() int64 { + if m != nil { + return m.TTL + } + return 0 +} + type LeaseTimeToLiveRequest struct { // ID is the lease ID for the lease. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` @@ -1361,6 +1774,20 @@ func (m *LeaseTimeToLiveRequest) String() string { return proto.Compa func (*LeaseTimeToLiveRequest) ProtoMessage() {} func (*LeaseTimeToLiveRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{28} } +func (m *LeaseTimeToLiveRequest) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *LeaseTimeToLiveRequest) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type LeaseTimeToLiveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // ID is the lease ID from the keep alive request. @@ -1385,6 +1812,34 @@ func (m *LeaseTimeToLiveResponse) GetHeader() *ResponseHeader { return nil } +func (m *LeaseTimeToLiveResponse) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *LeaseTimeToLiveResponse) GetTTL() int64 { + if m != nil { + return m.TTL + } + return 0 +} + +func (m *LeaseTimeToLiveResponse) GetGrantedTTL() int64 { + if m != nil { + return m.GrantedTTL + } + return 0 +} + +func (m *LeaseTimeToLiveResponse) GetKeys() [][]byte { + if m != nil { + return m.Keys + } + return nil +} + type Member struct { // ID is the member ID for this member. ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` @@ -1401,6 +1856,34 @@ func (m *Member) String() string { return proto.CompactTextString(m) func (*Member) ProtoMessage() {} func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{30} } +func (m *Member) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *Member) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Member) GetPeerURLs() []string { + if m != nil { + return m.PeerURLs + } + return nil +} + +func (m *Member) GetClientURLs() []string { + if m != nil { + return m.ClientURLs + } + return nil +} + type MemberAddRequest struct { // peerURLs is the list of URLs the added member will use to communicate with the cluster. PeerURLs []string `protobuf:"bytes,1,rep,name=peerURLs" json:"peerURLs,omitempty"` @@ -1411,6 +1894,13 @@ func (m *MemberAddRequest) String() string { return proto.CompactText func (*MemberAddRequest) ProtoMessage() {} func (*MemberAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{31} } +func (m *MemberAddRequest) GetPeerURLs() []string { + if m != nil { + return m.PeerURLs + } + return nil +} + type MemberAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // member is the member information for the added member. @@ -1455,6 +1945,13 @@ func (m *MemberRemoveRequest) String() string { return proto.CompactT func (*MemberRemoveRequest) ProtoMessage() {} func (*MemberRemoveRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{33} } +func (m *MemberRemoveRequest) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + type MemberRemoveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // members is a list of all members after removing the member. @@ -1492,6 +1989,20 @@ func (m *MemberUpdateRequest) String() string { return proto.CompactT func (*MemberUpdateRequest) ProtoMessage() {} func (*MemberUpdateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{35} } +func (m *MemberUpdateRequest) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *MemberUpdateRequest) GetPeerURLs() []string { + if m != nil { + return m.PeerURLs + } + return nil +} + type MemberUpdateResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // members is a list of all members after updating the member. @@ -1591,6 +2102,27 @@ func (m *AlarmRequest) String() string { return proto.CompactTextStri func (*AlarmRequest) ProtoMessage() {} func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{41} } +func (m *AlarmRequest) GetAction() AlarmRequest_AlarmAction { + if m != nil { + return m.Action + } + return AlarmRequest_GET +} + +func (m *AlarmRequest) GetMemberID() uint64 { + if m != nil { + return m.MemberID + } + return 0 +} + +func (m *AlarmRequest) GetAlarm() AlarmType { + if m != nil { + return m.Alarm + } + return AlarmType_NONE +} + type AlarmMember struct { // memberID is the ID of the member associated with the raised alarm. MemberID uint64 `protobuf:"varint,1,opt,name=memberID,proto3" json:"memberID,omitempty"` @@ -1603,6 +2135,20 @@ func (m *AlarmMember) String() string { return proto.CompactTextStrin func (*AlarmMember) ProtoMessage() {} func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{42} } +func (m *AlarmMember) GetMemberID() uint64 { + if m != nil { + return m.MemberID + } + return 0 +} + +func (m *AlarmMember) GetAlarm() AlarmType { + if m != nil { + return m.Alarm + } + return AlarmType_NONE +} + type AlarmResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // alarms is a list of alarms associated with the alarm request. @@ -1662,6 +2208,41 @@ func (m *StatusResponse) GetHeader() *ResponseHeader { return nil } +func (m *StatusResponse) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +func (m *StatusResponse) GetDbSize() int64 { + if m != nil { + return m.DbSize + } + return 0 +} + +func (m *StatusResponse) GetLeader() uint64 { + if m != nil { + return m.Leader + } + return 0 +} + +func (m *StatusResponse) GetRaftIndex() uint64 { + if m != nil { + return m.RaftIndex + } + return 0 +} + +func (m *StatusResponse) GetRaftTerm() uint64 { + if m != nil { + return m.RaftTerm + } + return 0 +} + type AuthEnableRequest struct { } @@ -1688,6 +2269,20 @@ func (m *AuthenticateRequest) String() string { return proto.CompactT func (*AuthenticateRequest) ProtoMessage() {} func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{48} } +func (m *AuthenticateRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *AuthenticateRequest) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + type AuthUserAddRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` @@ -1698,6 +2293,20 @@ func (m *AuthUserAddRequest) String() string { return proto.CompactTe func (*AuthUserAddRequest) ProtoMessage() {} func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{49} } +func (m *AuthUserAddRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *AuthUserAddRequest) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + type AuthUserGetRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } @@ -1707,6 +2316,13 @@ func (m *AuthUserGetRequest) String() string { return proto.CompactTe func (*AuthUserGetRequest) ProtoMessage() {} func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{50} } +func (m *AuthUserGetRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type AuthUserDeleteRequest struct { // name is the name of the user to delete. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1717,6 +2333,13 @@ func (m *AuthUserDeleteRequest) String() string { return proto.Compac func (*AuthUserDeleteRequest) ProtoMessage() {} func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{51} } +func (m *AuthUserDeleteRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type AuthUserChangePasswordRequest struct { // name is the name of the user whose password is being changed. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1731,6 +2354,20 @@ func (*AuthUserChangePasswordRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{52} } +func (m *AuthUserChangePasswordRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *AuthUserChangePasswordRequest) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + type AuthUserGrantRoleRequest struct { // user is the name of the user which should be granted a given role. User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` @@ -1743,6 +2380,20 @@ func (m *AuthUserGrantRoleRequest) String() string { return proto.Com func (*AuthUserGrantRoleRequest) ProtoMessage() {} func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{53} } +func (m *AuthUserGrantRoleRequest) GetUser() string { + if m != nil { + return m.User + } + return "" +} + +func (m *AuthUserGrantRoleRequest) GetRole() string { + if m != nil { + return m.Role + } + return "" +} + type AuthUserRevokeRoleRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"` @@ -1753,6 +2404,20 @@ func (m *AuthUserRevokeRoleRequest) String() string { return proto.Co func (*AuthUserRevokeRoleRequest) ProtoMessage() {} func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{54} } +func (m *AuthUserRevokeRoleRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *AuthUserRevokeRoleRequest) GetRole() string { + if m != nil { + return m.Role + } + return "" +} + type AuthRoleAddRequest struct { // name is the name of the role to add to the authentication system. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1763,6 +2428,13 @@ func (m *AuthRoleAddRequest) String() string { return proto.CompactTe func (*AuthRoleAddRequest) ProtoMessage() {} func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{55} } +func (m *AuthRoleAddRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type AuthRoleGetRequest struct { Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"` } @@ -1772,6 +2444,13 @@ func (m *AuthRoleGetRequest) String() string { return proto.CompactTe func (*AuthRoleGetRequest) ProtoMessage() {} func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{56} } +func (m *AuthRoleGetRequest) GetRole() string { + if m != nil { + return m.Role + } + return "" +} + type AuthUserListRequest struct { } @@ -1797,6 +2476,13 @@ func (m *AuthRoleDeleteRequest) String() string { return proto.Compac func (*AuthRoleDeleteRequest) ProtoMessage() {} func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{59} } +func (m *AuthRoleDeleteRequest) GetRole() string { + if m != nil { + return m.Role + } + return "" +} + type AuthRoleGrantPermissionRequest struct { // name is the name of the role which will be granted the permission. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1811,6 +2497,13 @@ func (*AuthRoleGrantPermissionRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{60} } +func (m *AuthRoleGrantPermissionRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *AuthRoleGrantPermissionRequest) GetPerm() *authpb.Permission { if m != nil { return m.Perm @@ -1831,6 +2524,27 @@ func (*AuthRoleRevokePermissionRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{61} } +func (m *AuthRoleRevokePermissionRequest) GetRole() string { + if m != nil { + return m.Role + } + return "" +} + +func (m *AuthRoleRevokePermissionRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *AuthRoleRevokePermissionRequest) GetRangeEnd() string { + if m != nil { + return m.RangeEnd + } + return "" +} + type AuthEnableResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -1881,6 +2595,13 @@ func (m *AuthenticateResponse) GetHeader() *ResponseHeader { return nil } +func (m *AuthenticateResponse) GetToken() string { + if m != nil { + return m.Token + } + return "" +} + type AuthUserAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -1914,6 +2635,13 @@ func (m *AuthUserGetResponse) GetHeader() *ResponseHeader { return nil } +func (m *AuthUserGetResponse) GetRoles() []string { + if m != nil { + return m.Roles + } + return nil +} + type AuthUserDeleteResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -2037,6 +2765,13 @@ func (m *AuthRoleListResponse) GetHeader() *ResponseHeader { return nil } +func (m *AuthRoleListResponse) GetRoles() []string { + if m != nil { + return m.Roles + } + return nil +} + type AuthUserListResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` Users []string `protobuf:"bytes,2,rep,name=users" json:"users,omitempty"` @@ -2054,6 +2789,13 @@ func (m *AuthUserListResponse) GetHeader() *ResponseHeader { return nil } +func (m *AuthUserListResponse) GetUsers() []string { + if m != nil { + return m.Users + } + return nil +} + type AuthRoleDeleteResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` } @@ -10511,7 +11253,24 @@ func (m *WatchCreateRequest) Unmarshal(dAtA []byte) error { } m.ProgressNotify = bool(v != 0) case 5: - if wireType == 2 { + if wireType == 0 { + var v WatchCreateRequest_FilterType + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (WatchCreateRequest_FilterType(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Filters = append(m.Filters, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -10552,23 +11311,6 @@ func (m *WatchCreateRequest) Unmarshal(dAtA []byte) error { } m.Filters = append(m.Filters, v) } - } else if wireType == 0 { - var v WatchCreateRequest_FilterType - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (WatchCreateRequest_FilterType(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Filters = append(m.Filters, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Filters", wireType) } diff --git a/glide.lock b/glide.lock index 1d866cf7c56f..50172a864a2a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: cee1f2629857e9c2384ad89ff6014db09498c9af53771e5144ad3a4b510ff00e -updated: 2017-05-30T10:29:08.22609283-07:00 +hash: 936853fc7427741a35ce6114725318c5642f604214520d0e7d119c8f3ee9ab0a +updated: 2017-06-12T14:15:04.520854967-07:00 imports: - name: github.com/beorn7/perks version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 @@ -27,7 +27,7 @@ imports: - capnslog - dlopen - name: github.com/cpuguy83/go-md2man - version: bcc0a711c5e6bbe72c7cb13d81c7109b45267fd2 + version: 23709d0847197db6021a51fdb193e66e9222d4e7 subpackages: - md2man - name: github.com/dgrijalva/jwt-go @@ -37,7 +37,7 @@ imports: - name: github.com/ghodss/yaml version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7 - name: github.com/gogo/protobuf - version: 909568be09de550ed094403c2bf8a261b5bb730a + version: 100ba4e885062801d56799d78530b73b178a78f3 subpackages: - proto - name: github.com/golang/groupcache @@ -45,10 +45,11 @@ imports: subpackages: - lru - name: github.com/golang/protobuf - version: 4bd1920723d7b7c925de087aa32e2187708897f7 + version: 5a0f697c9ed9d68fef0116532c6e05cfeae00e55 subpackages: - jsonpb - proto + - ptypes/struct - name: github.com/google/btree version: 925471ac9e2131377a91e1595defec898166fe49 - name: github.com/grpc-ecosystem/go-grpc-prometheus @@ -94,7 +95,7 @@ imports: subpackages: - xfs - name: github.com/russross/blackfriday - version: 0ba0f2b6ed7c475a92e4df8641825cb7a11d1fa3 + version: 067529f716f4c3f5e37c8c95ddd59df1007290ae - name: github.com/spf13/cobra version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b - name: github.com/spf13/pflag @@ -127,7 +128,7 @@ imports: subpackages: - unix - name: golang.org/x/text - version: 4ee4af566555f5fbe026368b75596286a312663a + version: 210eee5cf7323015d097341bcf7166130d001cd8 subpackages: - secure/bidirule - transform diff --git a/glide.yaml b/glide.yaml index 90bcfeddc1ca..94e242e3c166 100644 --- a/glide.yaml +++ b/glide.yaml @@ -25,7 +25,7 @@ import: - package: github.com/ghodss/yaml version: v1.0.0 - package: github.com/gogo/protobuf - version: v0.3 + version: v0.4 subpackages: - proto - package: github.com/golang/groupcache @@ -33,7 +33,7 @@ import: subpackages: - lru - package: github.com/golang/protobuf - version: 4bd1920723d7b7c925de087aa32e2187708897f7 + version: 5a0f697c9ed9d68fef0116532c6e05cfeae00e55 subpackages: - jsonpb - proto diff --git a/integration/cluster_proxy.go b/integration/cluster_proxy.go index 3916553be867..613b61b9a4b0 100644 --- a/integration/cluster_proxy.go +++ b/integration/cluster_proxy.go @@ -21,7 +21,6 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/namespace" - pb "github.com/coreos/etcd/etcdserver/etcdserverpb" "github.com/coreos/etcd/proxy/grpcproxy" "github.com/coreos/etcd/proxy/grpcproxy/adapter" ) @@ -58,6 +57,7 @@ func toGRPC(c *clientv3.Client) grpcAPI { lp, lpch := grpcproxy.NewLeaseProxy(c) mp := grpcproxy.NewMaintenanceProxy(c) clp, _ := grpcproxy.NewClusterProxy(c, "", "") // without registering proxy URLs + authp := grpcproxy.NewAuthProxy(c) lockp := grpcproxy.NewLockProxy(c) electp := grpcproxy.NewElectionProxy(c) @@ -67,7 +67,7 @@ func toGRPC(c *clientv3.Client) grpcAPI { adapter.LeaseServerToLeaseClient(lp), adapter.WatchServerToWatchClient(wp), adapter.MaintenanceServerToMaintenanceClient(mp), - pb.NewAuthClient(c.ActiveConnection()), + adapter.AuthServerToAuthClient(authp), adapter.LockServerToLockClient(lockp), adapter.ElectionServerToElectionClient(electp), } diff --git a/integration/v3_grpc_test.go b/integration/v3_grpc_test.go index 8eaf4ec372fd..da04d76e63c7 100644 --- a/integration/v3_grpc_test.go +++ b/integration/v3_grpc_test.go @@ -328,6 +328,58 @@ func TestV3TxnRevision(t *testing.T) { } } +// Testv3TxnCmpHeaderRev tests that the txn header revision is set as expected +// when compared to the Succeeded field in the txn response. +func TestV3TxnCmpHeaderRev(t *testing.T) { + defer testutil.AfterTest(t) + clus := NewClusterV3(t, &ClusterConfig{Size: 1}) + defer clus.Terminate(t) + + kvc := toGRPC(clus.RandClient()).KV + + for i := 0; i < 10; i++ { + // Concurrently put a key with a txn comparing on it. + revc := make(chan int64, 1) + go func() { + defer close(revc) + pr := &pb.PutRequest{Key: []byte("k"), Value: []byte("v")} + presp, err := kvc.Put(context.TODO(), pr) + if err != nil { + t.Fatal(err) + } + revc <- presp.Header.Revision + }() + + // The read-only txn uses the optimized readindex server path. + txnget := &pb.RequestOp{Request: &pb.RequestOp_RequestRange{ + RequestRange: &pb.RangeRequest{Key: []byte("k")}}} + txn := &pb.TxnRequest{Success: []*pb.RequestOp{txnget}} + // i = 0 /\ Succeeded => put followed txn + cmp := &pb.Compare{ + Result: pb.Compare_EQUAL, + Target: pb.Compare_VERSION, + Key: []byte("k"), + TargetUnion: &pb.Compare_Version{Version: int64(i)}, + } + txn.Compare = append(txn.Compare, cmp) + + tresp, err := kvc.Txn(context.TODO(), txn) + if err != nil { + t.Fatal(err) + } + + prev := <-revc + // put followed txn; should eval to false + if prev > tresp.Header.Revision && !tresp.Succeeded { + t.Errorf("#%d: got else but put rev %d followed txn rev (%+v)", i, prev, tresp) + } + // txn follows put; should eval to true + if tresp.Header.Revision >= prev && tresp.Succeeded { + t.Errorf("#%d: got then but put rev %d preceded txn (%+v)", i, prev, tresp) + } + } +} + // TestV3PutIgnoreValue ensures that writes with ignore_value overwrites with previous key-value pair. func TestV3PutIgnoreValue(t *testing.T) { defer testutil.AfterTest(t) diff --git a/lease/lessor.go b/lease/lessor.go index 5120d1cfcdd3..6584a6fb488b 100644 --- a/lease/lessor.go +++ b/lease/lessor.go @@ -31,6 +31,10 @@ import ( const ( // NoLease is a special LeaseID representing the absence of a lease. NoLease = LeaseID(0) + + // maximum number of leases to revoke per iteration + // TODO: make this configurable? + leaseRevokeRate = 1000 ) var ( @@ -422,6 +426,10 @@ func (le *lessor) runLoop() { le.mu.Unlock() if len(ls) != 0 { + // rate limit + if len(ls) > leaseRevokeRate/2 { + ls = ls[:leaseRevokeRate/2] + } select { case <-le.stopC: return diff --git a/mvcc/kv.go b/mvcc/kv.go index e13cd647948e..6636347aa431 100644 --- a/mvcc/kv.go +++ b/mvcc/kv.go @@ -93,7 +93,9 @@ func (trw *txnReadWrite) DeleteRange(key, end []byte) (n, rev int64) { panic("un func (trw *txnReadWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) { panic("unexpected Put") } -func (trw *txnReadWrite) Changes() []mvccpb.KeyValue { panic("unexpected Changes") } +func (trw *txnReadWrite) Changes() []mvccpb.KeyValue { return nil } + +func NewReadOnlyTxnWrite(txn TxnRead) TxnWrite { return &txnReadWrite{txn} } type KV interface { ReadView diff --git a/proxy/grpcproxy/adapter/auth_client_adapter.go b/proxy/grpcproxy/adapter/auth_client_adapter.go new file mode 100644 index 000000000000..7f38a9ac76eb --- /dev/null +++ b/proxy/grpcproxy/adapter/auth_client_adapter.go @@ -0,0 +1,91 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package adapter + +import ( + pb "github.com/coreos/etcd/etcdserver/etcdserverpb" + "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +type as2ac struct{ as pb.AuthServer } + +func AuthServerToAuthClient(as pb.AuthServer) pb.AuthClient { + return &as2ac{as} +} + +func (s *as2ac) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (*pb.AuthEnableResponse, error) { + return s.as.AuthEnable(ctx, in) +} + +func (s *as2ac) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (*pb.AuthDisableResponse, error) { + return s.as.AuthDisable(ctx, in) +} + +func (s *as2ac) Authenticate(ctx context.Context, in *pb.AuthenticateRequest, opts ...grpc.CallOption) (*pb.AuthenticateResponse, error) { + return s.as.Authenticate(ctx, in) +} + +func (s *as2ac) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (*pb.AuthRoleAddResponse, error) { + return s.as.RoleAdd(ctx, in) +} + +func (s *as2ac) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (*pb.AuthRoleDeleteResponse, error) { + return s.as.RoleDelete(ctx, in) +} + +func (s *as2ac) RoleGet(ctx context.Context, in *pb.AuthRoleGetRequest, opts ...grpc.CallOption) (*pb.AuthRoleGetResponse, error) { + return s.as.RoleGet(ctx, in) +} + +func (s *as2ac) RoleList(ctx context.Context, in *pb.AuthRoleListRequest, opts ...grpc.CallOption) (*pb.AuthRoleListResponse, error) { + return s.as.RoleList(ctx, in) +} + +func (s *as2ac) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (*pb.AuthRoleRevokePermissionResponse, error) { + return s.as.RoleRevokePermission(ctx, in) +} + +func (s *as2ac) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (*pb.AuthRoleGrantPermissionResponse, error) { + return s.as.RoleGrantPermission(ctx, in) +} + +func (s *as2ac) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (*pb.AuthUserDeleteResponse, error) { + return s.as.UserDelete(ctx, in) +} + +func (s *as2ac) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (*pb.AuthUserAddResponse, error) { + return s.as.UserAdd(ctx, in) +} + +func (s *as2ac) UserGet(ctx context.Context, in *pb.AuthUserGetRequest, opts ...grpc.CallOption) (*pb.AuthUserGetResponse, error) { + return s.as.UserGet(ctx, in) +} + +func (s *as2ac) UserList(ctx context.Context, in *pb.AuthUserListRequest, opts ...grpc.CallOption) (*pb.AuthUserListResponse, error) { + return s.as.UserList(ctx, in) +} + +func (s *as2ac) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*pb.AuthUserGrantRoleResponse, error) { + return s.as.UserGrantRole(ctx, in) +} + +func (s *as2ac) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*pb.AuthUserRevokeRoleResponse, error) { + return s.as.UserRevokeRole(ctx, in) +} + +func (s *as2ac) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (*pb.AuthUserChangePasswordResponse, error) { + return s.as.UserChangePassword(ctx, in) +} diff --git a/raft/raftpb/raft.pb.go b/raft/raftpb/raft.pb.go index 3c45eef003c2..4c6e79d58a0c 100644 --- a/raft/raftpb/raft.pb.go +++ b/raft/raftpb/raft.pb.go @@ -1558,25 +1558,67 @@ func (m *ConfState) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) - } - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRaft + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + m.Nodes = append(m.Nodes, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + packedLen + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Nodes = append(m.Nodes, v) } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) } - m.Nodes = append(m.Nodes, v) default: iNdEx = preIndex skippy, err := skipRaft(dAtA[iNdEx:]) diff --git a/rafthttp/http.go b/rafthttp/http.go index 471028a61561..55df26e9b756 100644 --- a/rafthttp/http.go +++ b/rafthttp/http.go @@ -183,7 +183,8 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } dec := &messageDecoder{r: r.Body} - m, err := dec.decode() + // let snapshots be very large since they can exceed 512MB for large installations + m, err := dec.decodeLimit(uint64(1 << 63)) if err != nil { msg := fmt.Sprintf("failed to decode raft message (%v)", err) plog.Errorf(msg) diff --git a/rafthttp/msg_codec.go b/rafthttp/msg_codec.go index bf1f6bc003fd..ef59bc8883f6 100644 --- a/rafthttp/msg_codec.go +++ b/rafthttp/msg_codec.go @@ -48,12 +48,16 @@ var ( ) func (dec *messageDecoder) decode() (raftpb.Message, error) { + return dec.decodeLimit(readBytesLimit) +} + +func (dec *messageDecoder) decodeLimit(numBytes uint64) (raftpb.Message, error) { var m raftpb.Message var l uint64 if err := binary.Read(dec.r, binary.BigEndian, &l); err != nil { return m, err } - if l > readBytesLimit { + if l > numBytes { return m, ErrExceedSizeLimit } buf := make([]byte, int(l)) diff --git a/scripts/build-aci b/scripts/build-aci index b7efb2bccb16..7a4cff93b021 100755 --- a/scripts/build-aci +++ b/scripts/build-aci @@ -36,6 +36,7 @@ fi acbuild --debug begin TMPHOSTS="$(mktemp)" +ACI_ARCH="$(go2aci ${GOARCH})" acbuildEnd() { rm "$TMPHOSTS" @@ -71,7 +72,7 @@ acbuild --debug port add peer tcp 2380 acbuild --debug copy "$TMPHOSTS" /etc/hosts -acbuild --debug label add arch "$(go2aci ${GOARCH})" +acbuild --debug label add arch "${ACI_ARCH}" # mkdir default data-dir mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd @@ -80,4 +81,4 @@ mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd ln -s ./usr/local/bin/etcd .acbuild/currentaci/rootfs/etcd ln -s ./usr/local/bin/etcdctl .acbuild/currentaci/rootfs/etcdctl -acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${GOARCH}.aci +acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${ACI_ARCH}.aci diff --git a/scripts/genproto.sh b/scripts/genproto.sh index 4bfe9cdd6240..07ca14067f58 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -10,15 +10,15 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then exit 255 fi -# for now, be conservative about what version of protoc we expect -if ! [[ $(protoc --version) =~ "3.2.0" ]]; then - echo "could not find protoc 3.2.0, is it installed + in PATH?" +if ! [[ $(protoc --version) =~ "3.3.0" ]]; then + echo "could not find protoc 3.3.0, is it installed + in PATH?" exit 255 fi # directories containing protos to be built DIRS="./wal/walpb ./etcdserver/etcdserverpb ./snap/snappb ./raft/raftpb ./mvcc/mvccpb ./lease/leasepb ./auth/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb" + # exact version of packages to build GOGO_PROTO_SHA="8d70fb3182befc465c4a1eac8ad4d38ff49778e2" GRPC_GATEWAY_SHA="84398b94e188ee336f307779b57b3aa91af7063c" @@ -65,6 +65,7 @@ for dir in ${DIRS}; do sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go sed -i.bak -E 's/import fmt \"fmt\"//g' *.pb.go sed -i.bak -E 's/import _ \"github\.com\/coreos\/google\/api\"//g' *.pb.go + sed -i.bak -E 's/import _ \"google\.golang\.org\/genproto\/googleapis\/api\/annotations\"//g' *.pb.go rm -f *.bak goimports -w *.pb.go popd diff --git a/tools/benchmark/cmd/watch.go b/tools/benchmark/cmd/watch.go index b73e4f20eeb5..5b2f57fc9721 100644 --- a/tools/benchmark/cmd/watch.go +++ b/tools/benchmark/cmd/watch.go @@ -15,6 +15,7 @@ package cmd import ( + "context" "encoding/binary" "fmt" "math/rand" @@ -22,11 +23,11 @@ import ( "sync/atomic" "time" - v3 "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/report" "github.com/spf13/cobra" - "golang.org/x/net/context" + "golang.org/x/time/rate" "gopkg.in/cheggaaa/pb.v1" ) @@ -50,9 +51,9 @@ Each key is watched by (--total/--watched-key-total) watchers. } var ( - watchTotalStreams int - watchTotal int - watchedKeyTotal int + watchStreams int + watchWatchesPerStream int + watchedKeyTotal int watchPutRate int watchPutTotal int @@ -60,23 +61,27 @@ var ( watchKeySize int watchKeySpaceSize int watchSeqKeys bool +) - eventsTotal int +type watchedKeys struct { + watched []string + numWatchers map[string]int - nrWatchCompleted int32 - nrRecvCompleted int32 - watchCompletedNotifier chan struct{} - recvCompletedNotifier chan struct{} -) + watches []clientv3.WatchChan + + // ctx to control all watches + ctx context.Context + cancel context.CancelFunc +} func init() { RootCmd.AddCommand(watchCmd) - watchCmd.Flags().IntVar(&watchTotalStreams, "watchers", 10000, "Total number of watchers") - watchCmd.Flags().IntVar(&watchTotal, "total", 100000, "Total number of watch requests") - watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 10000, "Total number of keys to be watched") + watchCmd.Flags().IntVar(&watchStreams, "streams", 10, "Total watch streams") + watchCmd.Flags().IntVar(&watchWatchesPerStream, "watch-per-stream", 100, "Total watchers per stream") + watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 1, "Total number of keys to be watched") - watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 100, "Number of keys to put per second") - watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 10000, "Number of put requests") + watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 0, "Number of keys to put per second") + watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 1000, "Number of put requests") watchCmd.Flags().IntVar(&watchKeySize, "key-size", 32, "Key size of watch request") watchCmd.Flags().IntVar(&watchKeySpaceSize, "key-space-size", 1, "Maximum possible keys") @@ -88,124 +93,155 @@ func watchFunc(cmd *cobra.Command, args []string) { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", watchKeySpaceSize) os.Exit(1) } - - watched := make([]string, watchedKeyTotal) - numWatchers := make(map[string]int) - for i := range watched { - k := make([]byte, watchKeySize) - if watchSeqKeys { - binary.PutVarint(k, int64(i%watchKeySpaceSize)) - } else { - binary.PutVarint(k, int64(rand.Intn(watchKeySpaceSize))) - } - watched[i] = string(k) + grpcConns := int(totalClients) + if totalClients > totalConns { + grpcConns = int(totalConns) + } + wantedConns := 1 + (watchStreams / 100) + if grpcConns < wantedConns { + fmt.Fprintf(os.Stderr, "warning: grpc limits 100 streams per client connection, have %d but need %d\n", grpcConns, wantedConns) } - - requests := make(chan string, totalClients) - clients := mustCreateClients(totalClients, totalConns) + wk := newWatchedKeys() + benchMakeWatches(clients, wk) + benchPutWatches(clients, wk) +} - streams := make([]v3.Watcher, watchTotalStreams) +func benchMakeWatches(clients []*clientv3.Client, wk *watchedKeys) { + streams := make([]clientv3.Watcher, watchStreams) for i := range streams { - streams[i] = v3.NewWatcher(clients[i%len(clients)]) + streams[i] = clientv3.NewWatcher(clients[i%len(clients)]) } - // watching phase - bar = pb.New(watchTotal) + keyc := make(chan string, watchStreams) + bar = pb.New(watchStreams * watchWatchesPerStream) bar.Format("Bom !") bar.Start() - atomic.StoreInt32(&nrWatchCompleted, int32(0)) - watchCompletedNotifier = make(chan struct{}) - - r := report.NewReportRate("%4.4f") - for i := range streams { - go doWatch(streams[i], requests, r.Results()) + r := newReport() + rch := r.Results() + + wg.Add(len(streams) + 1) + wc := make(chan []clientv3.WatchChan, len(streams)) + for _, s := range streams { + go func(s clientv3.Watcher) { + defer wg.Done() + var ws []clientv3.WatchChan + for i := 0; i < watchWatchesPerStream; i++ { + k := <-keyc + st := time.Now() + wch := s.Watch(wk.ctx, k) + rch <- report.Result{Start: st, End: time.Now()} + ws = append(ws, wch) + bar.Increment() + } + wc <- ws + }(s) } - go func() { - for i := 0; i < watchTotal; i++ { - key := watched[i%len(watched)] - requests <- key - numWatchers[key]++ + defer func() { + close(keyc) + wg.Done() + }() + for i := 0; i < watchStreams*watchWatchesPerStream; i++ { + key := wk.watched[i%len(wk.watched)] + keyc <- key + wk.numWatchers[key]++ } - close(requests) }() rc := r.Run() - <-watchCompletedNotifier + wg.Wait() bar.Finish() close(r.Results()) fmt.Printf("Watch creation summary:\n%s", <-rc) - // put phase - eventsTotal = 0 + for i := 0; i < len(streams); i++ { + wk.watches = append(wk.watches, (<-wc)...) + } +} + +func newWatchedKeys() *watchedKeys { + watched := make([]string, watchedKeyTotal) + for i := range watched { + k := make([]byte, watchKeySize) + if watchSeqKeys { + binary.PutVarint(k, int64(i%watchKeySpaceSize)) + } else { + binary.PutVarint(k, int64(rand.Intn(watchKeySpaceSize))) + } + watched[i] = string(k) + } + ctx, cancel := context.WithCancel(context.TODO()) + return &watchedKeys{ + watched: watched, + numWatchers: make(map[string]int), + ctx: ctx, + cancel: cancel, + } +} + +func benchPutWatches(clients []*clientv3.Client, wk *watchedKeys) { + eventsTotal := 0 for i := 0; i < watchPutTotal; i++ { - eventsTotal += numWatchers[watched[i%len(watched)]] + eventsTotal += wk.numWatchers[wk.watched[i%len(wk.watched)]] } bar = pb.New(eventsTotal) bar.Format("Bom !") bar.Start() - atomic.StoreInt32(&nrRecvCompleted, 0) - recvCompletedNotifier = make(chan struct{}) - putreqc := make(chan v3.Op) + r := newReport() - r = report.NewReportRate("%4.4f") - for i := 0; i < watchPutTotal; i++ { - go func(c *v3.Client) { - for op := range putreqc { - if _, err := c.Do(context.TODO(), op); err != nil { - fmt.Fprintf(os.Stderr, "failed to Put for watch benchmark: %v\n", err) - os.Exit(1) - } - } - }(clients[i%len(clients)]) + wg.Add(len(wk.watches)) + nrRxed := int32(eventsTotal) + for _, w := range wk.watches { + go func(wc clientv3.WatchChan) { + defer wg.Done() + recvWatchChan(wc, r.Results(), &nrRxed) + wk.cancel() + }(w) } + putreqc := make(chan clientv3.Op, len(clients)) go func() { + defer close(putreqc) for i := 0; i < watchPutTotal; i++ { - putreqc <- v3.OpPut(watched[i%(len(watched))], "data") - // TODO: use a real rate-limiter instead of sleep. - time.Sleep(time.Second / time.Duration(watchPutRate)) + putreqc <- clientv3.OpPut(wk.watched[i%(len(wk.watched))], "data") } - close(putreqc) }() - rc = r.Run() - <-recvCompletedNotifier + limit := rate.NewLimiter(rate.Limit(watchPutRate), 1) + for _, cc := range clients { + go func(c *clientv3.Client) { + for op := range putreqc { + if err := limit.Wait(context.TODO()); err != nil { + panic(err) + } + if _, err := c.Do(context.TODO(), op); err != nil { + panic(err) + } + } + }(cc) + } + + rc := r.Run() + wg.Wait() bar.Finish() close(r.Results()) fmt.Printf("Watch events received summary:\n%s", <-rc) -} -func doWatch(stream v3.Watcher, requests <-chan string, results chan<- report.Result) { - for r := range requests { - st := time.Now() - wch := stream.Watch(context.TODO(), r) - results <- report.Result{Start: st, End: time.Now()} - bar.Increment() - go recvWatchChan(wch, results) - } - atomic.AddInt32(&nrWatchCompleted, 1) - if atomic.LoadInt32(&nrWatchCompleted) == int32(watchTotalStreams) { - watchCompletedNotifier <- struct{}{} - } } -func recvWatchChan(wch v3.WatchChan, results chan<- report.Result) { +func recvWatchChan(wch clientv3.WatchChan, results chan<- report.Result, nrRxed *int32) { for r := range wch { st := time.Now() for range r.Events { results <- report.Result{Start: st, End: time.Now()} bar.Increment() - atomic.AddInt32(&nrRecvCompleted, 1) - } - - if atomic.LoadInt32(&nrRecvCompleted) == int32(eventsTotal) { - recvCompletedNotifier <- struct{}{} - break + if atomic.AddInt32(nrRxed, -1) <= 0 { + return + } } } }