runtime: support bootstrap runtime configuration. (#6759)

As a first step towards TDS, add support for proto/JSON representations of the runtime virtual file
system and plumb this into bootstrap.

"numerator" and "denominator" are now reserved keywords and can't be used in runtime keys; this
simplifies parsing significantly. It is not expected that these are widely used, due to the
difficulty of ensuring atomicity if representing fractions, but this is technically a runtime API
breaking change.

Risk level: Medium (due to the reserved word changes).
Testing: Unit and server tests added.

Part of #6708

Signed-off-by: Harvey Tuch <htuch@google.com>

api/envoy/config/bootstrap/v2/bootstrap.proto

@@ -23,6 +23,7 @@ import "envoy/config/metrics/v2/stats.proto";
 import "envoy/config/overload/v2alpha/overload.proto";
 
 import "google/protobuf/duration.proto";
+import "google/protobuf/struct.proto";
 
 import "validate/validate.proto";
 import "gogoproto/gogo.proto";
@@ -205,8 +206,8 @@ message Runtime {
   // symbolic link. An atomic link swap is used when a new tree should be
   // switched to. This parameter specifies the path to the symbolic link. Envoy
   // will watch the location for changes and reload the file system tree when
-  // they happen.
-  string symlink_root = 1 [(validate.rules).string.min_bytes = 1];
+  // they happen. If this parameter is not set, there will be no disk based runtime.
+  string symlink_root = 1;
 
   // Specifies the subdirectory to load within the root directory. This is
   // useful if multiple systems share the same delivery mechanism. Envoy
@@ -220,4 +221,10 @@ message Runtime {
   // Sometimes it is useful to have a per service cluster directory for runtime
   // configuration. See below for exactly how the override directory is used.
   string override_subdirectory = 3;
+
+  // Static base runtime. This will be :ref:`overridden
+  // <config_runtime_layering>` by other runtime layers, e.g.
+  // disk or admin. This follows the :ref:`runtime protobuf JSON representation
+  // encoding <config_runtime_proto_json>`.
+  google.protobuf.Struct base = 4;
 }

api/test/validate/pgv_test.cc

@@ -54,7 +54,7 @@ template <class Proto> struct TestCase {
 // from data plane API.
 int main(int argc, char* argv[]) {
   envoy::config::bootstrap::v2::Bootstrap invalid_bootstrap;
-  invalid_bootstrap.mutable_runtime();
+  invalid_bootstrap.mutable_static_resources()->add_clusters();
   // This is a baseline test of the validation features we care about. It's
   // probably not worth adding in every filter and field that we want to valid
   // in the API upfront, but as regressions occur, this is the place to add the

docs/root/configuration/runtime.rst

@@ -3,29 +3,67 @@
 Runtime
 =======
 
-The :ref:`runtime configuration <arch_overview_runtime>` specifies the location of the local file
-system tree that contains re-loadable configuration elements. Values can be viewed at the
-:ref:`/runtime admin endpoint <operations_admin_interface_runtime>`. Values can be modified and
-added at the :ref:`/runtime_modify admin endpoint <operations_admin_interface_runtime_modify>`. If
-runtime is not configured, an empty provider is used which has the effect of using all defaults
-built into the code, except for any values added via `/runtime_modify`.
+The :ref:`runtime configuration <arch_overview_runtime>` specifies a virtual file system tree that
+contains re-loadable configuration elements. This virtual file system can be realized via a series
+of local file system, static bootstrap configuration and admin console derived overlays.
 
-.. attention::
+* :ref:`v2 API reference <envoy_api_msg_config.bootstrap.v2.Runtime>`
 
-  Use the :ref:`/runtime_modify<operations_admin_interface_runtime_modify>` endpoint with care.
-  Changes are effectively immediately. It is **critical** that the admin interface is :ref:`properly
-  secured <operations_admin_interface_security>`.
+.. _config_virtual_filesystem:
 
+Virtual file system
+-------------------
 
-* :ref:`v2 API reference <envoy_api_msg_config.bootstrap.v2.Runtime>`
+.. _config_runtime_layering:
+
+Layering
+++++++++
+
+The runtime can be viewed as virtual file system consisting of multiple layers:
+
+1. :ref:`Static bootstrap configuration <config_runtime_bootstrap>`
+2. :ref:`Local disk file system <config_runtime_local_disk>`
+3. :ref:`Local disk file system *override_subdirectory* <config_runtime_local_disk>`
+4. :ref:`Admin console overrides <config_runtime_admin>`
+
+with values in higher layers overriding corresponding values in lower layers.
+
+.. _config_runtime_file_system:
 
 File system layout
-------------------
+++++++++++++++++++
 
 Various sections of the configuration guide describe the runtime settings that are available.
 For example, :ref:`here <config_cluster_manager_cluster_runtime>` are the runtime settings for
 upstream clusters.
 
+Each '.' in a runtime key indicates a new directory in the hierarchy,
+The terminal portion of a path is the file. The contents of the file constitute the runtime value.
+When reading numeric values from a file, spaces and new lines will be ignored.
+
+*numerator* or *denominator* are reserved keywords and may not appear in any directory.
+
+.. _config_runtime_bootstrap:
+
+Static bootstrap
+++++++++++++++++
+
+A static base runtime may be specified in the :ref:`bootstrap configuration
+<envoy_api_field_config.bootstrap.v2.Runtime.base>` via a :ref:`protobuf JSON representation
+<config_runtime_proto_json>`.
+
+.. _config_runtime_local_disk:
+
+Local disk file system
+++++++++++++++++++++++
+
+When the :ref:`runtime virtual file system <config_runtime_file_system>` is realized on a local
+disk, it is rooted at *symlink_root* +
+*subdirectory*. For example, the *health_check.min_interval* key would have the following full
+file system path (using the symbolic link):
+
+``/srv/runtime/current/envoy/health_check/min_interval``
+
 Assume that the folder ``/srv/runtime/v1`` points to the actual file system path where global
 runtime configurations are stored. The following would be a typical configuration setting for
 runtime:
@@ -36,15 +74,6 @@ runtime:
 
 Where ``/srv/runtime/current`` is a symbolic link to ``/srv/runtime/v1``.
 
-Each '.' in a runtime key indicates a new directory in the hierarchy, rooted at *symlink_root* +
-*subdirectory*. For example, the *health_check.min_interval* key would have the following full
-file system path (using the symbolic link):
-
-``/srv/runtime/current/envoy/health_check/min_interval``
-
-The terminal portion of a path is the file. The contents of the file constitute the runtime value.
-When reading numeric values from a file, spaces and new lines will be ignored.
-
 The *override_subdirectory* is used along with the :option:`--service-cluster` CLI option. Assume
 that :option:`--service-cluster` has been set to ``my-cluster``. Envoy will first look for the
 *health_check.min_interval* key in the following full file system path:
@@ -54,20 +83,10 @@ that :option:`--service-cluster` has been set to ``my-cluster``. Envoy will firs
 If found, the value will override any value found in the primary lookup path. This allows the user
 to customize the runtime values for individual clusters on top of global defaults.
 
-.. _config_runtime_comments:
-
-Comments
---------
-
-Lines starting with ``#`` as the first character are treated as comments.
-
-Comments can be used to provide context on an existing value. Comments are also useful in an
-otherwise empty file to keep a placeholder for deployment in a time of need.
-
 .. _config_runtime_symbolic_link_swap:
 
 Updating runtime values via symbolic link swap
-----------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 There are two steps to update any runtime value. First, create a hard copy of the entire runtime
 tree and update the desired runtime values. Second, atomically swap the symbolic link root from the
@@ -79,6 +98,55 @@ old tree to the new runtime tree, using the equivalent of the following command:
 
 It's beyond the scope of this document how the file system data is deployed, garbage collected, etc.
 
+.. _config_runtime_admin:
+
+Admin console
++++++++++++++
+
+Values can be viewed at the
+:ref:`/runtime admin endpoint <operations_admin_interface_runtime>`. Values can be modified and
+added at the :ref:`/runtime_modify admin endpoint <operations_admin_interface_runtime_modify>`. If
+runtime is not configured, an empty provider is used which has the effect of using all defaults
+built into the code, except for any values added via `/runtime_modify`.
+
+.. attention::
+
+  Use the :ref:`/runtime_modify<operations_admin_interface_runtime_modify>` endpoint with care.
+  Changes are effectively immediately. It is **critical** that the admin interface is :ref:`properly
+  secured <operations_admin_interface_security>`.
+
+.. _config_runtime_proto_json:
+
+Protobuf and JSON representation
+--------------------------------
+
+The runtime :ref:`file system <config_runtime_file_system>` can be represented inside a proto3
+message as a `google.protobuf.Struct
+<https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Struct>`_
+modeling a JSON object with the following rules:
+
+* Dot separators map to tree edges.
+* Scalar leaves (integer, strings, booleans) are represented with their respective JSON type.
+* :ref:`FractionalPercent <envoy_api_msg_type.FractionalPercent>` is represented with via its
+  `canonical JSON encoding <https://developers.google.com/protocol-buffers/docs/proto3#json>`_.
+
+An example representation of a setting for the *health_check.min_interval* key in YAML is:
+
+.. code-block:: yaml
+
+  health_check:
+    min_interval: 5
+
+.. _config_runtime_comments:
+
+Comments
+--------
+
+Lines starting with ``#`` as the first character are treated as comments.
+
+Comments can be used to provide context on an existing value. Comments are also useful in an
+otherwise empty file to keep a placeholder for deployment in a time of need.
+
 Using runtime overrides for deprecated features
 -----------------------------------------------
 

docs/root/intro/version_history.rst

@@ -25,6 +25,8 @@ Version history
 * router: per try timeouts will no longer start before the downstream request has been received
   in full by the router. This ensures that the per try timeout does not account for slow
   downstreams and that will not start before the global timeout.
+* runtime: added support for statically :ref:`specifying the runtime in the bootstrap configuration
+  <envoy_api_field_config.bootstrap.v2.Runtime.base>`.
 * upstream: added :ref:`upstream_cx_pool_overflow <config_cluster_manager_cluster_stats>` for the connection pool circuit breaker.
 * upstream: an EDS management server can now force removal of a host that is still passing active
   health checking by first marking the host as failed via EDS health check and subsequently removing

include/envoy/server/configuration.h

@@ -20,9 +20,9 @@ namespace Configuration {
 /**
  * Configuration for local disk runtime support.
  */
-class Runtime {
+class DiskRuntime {
 public:
-  virtual ~Runtime() {}
+  virtual ~DiskRuntime() {}
 
   /**
    * @return const std::string& the root symlink to watch for swapping.
@@ -136,9 +136,15 @@ class Initial {
   virtual absl::optional<std::string> flagsPath() PURE;
 
   /**
-   * @return Runtime* the local disk runtime configuration or nullptr if there is no configuration.
+   * @return const ProtobufWkt::Struct& base runtime snapshot.
    */
-  virtual Runtime* runtime() PURE;
+  virtual const ProtobufWkt::Struct& baseRuntime() PURE;
+
+  /**
+   * @return DiskRuntime* the local disk runtime configuration or nullptr if there is no
+   * configuration.
+   */
+  virtual DiskRuntime* diskRuntime() PURE;
 };
 
 } // namespace Configuration

source/common/runtime/runtime_impl.cc

@@ -293,6 +293,12 @@ SnapshotImpl::Entry SnapshotImpl::createEntry(const std::string& value) {
   return entry;
 }
 
+SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value) {
+  // This isn't the smartest way to do it; we're round-tripping via YAML, this should be optimized
+  // if runtime parsing becomes performance sensitive.
+  return createEntry(MessageUtil::getYamlStringFromMessage(value, false, false));
+}
+
 bool SnapshotImpl::parseEntryBooleanValue(Entry& entry) {
   absl::string_view stripped = entry.raw_string_value_;
   stripped = absl::StripAsciiWhitespace(stripped);
@@ -401,19 +407,58 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix
   }
 }
 
-LoaderImpl::LoaderImpl(RandomGenerator& generator, Stats::Store& store,
-                       ThreadLocal::SlotAllocator& tls)
-    : LoaderImpl(DoNotLoadSnapshot{}, generator, store, tls) {
-  loadNewSnapshot();
+ProtoLayer::ProtoLayer(const ProtobufWkt::Struct& proto) : OverrideLayerImpl{"base"} {
+  for (const auto& f : proto.fields()) {
+    walkProtoValue(f.second, f.first);
+  }
 }
 
-LoaderImpl::LoaderImpl(DoNotLoadSnapshot /* unused */, RandomGenerator& generator,
+void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string& prefix) {
+  switch (v.kind_case()) {
+  case ProtobufWkt::Value::KIND_NOT_SET:
+  case ProtobufWkt::Value::kListValue:
+  case ProtobufWkt::Value::kNullValue:
+    throw EnvoyException(fmt::format("Invalid runtime entry value for {}", prefix));
+    break;
+  case ProtobufWkt::Value::kStringValue:
+    values_.emplace(prefix, SnapshotImpl::createEntry(v.string_value()));
+    break;
+  case ProtobufWkt::Value::kNumberValue:
+  case ProtobufWkt::Value::kBoolValue:
+    values_.emplace(prefix, SnapshotImpl::createEntry(v));
+    break;
+  case ProtobufWkt::Value::kStructValue: {
+    const ProtobufWkt::Struct& s = v.struct_value();
+    if (s.fields().empty() || s.fields().find("numerator") != s.fields().end() ||
+        s.fields().find("denominator") != s.fields().end()) {
+      values_.emplace(prefix, SnapshotImpl::createEntry(v));
+      break;
+    }
+    for (const auto& f : s.fields()) {
+      walkProtoValue(f.second, prefix + "." + f.first);
+    }
+    break;
+  }
+  default:
+    NOT_REACHED_GCOVR_EXCL_LINE;
+  }
+}
+
+LoaderImpl::LoaderImpl(const ProtobufWkt::Struct& base, RandomGenerator& generator,
                        Stats::Store& store, ThreadLocal::SlotAllocator& tls)
-    : generator_(generator), stats_(generateStats(store)), admin_layer_(stats_),
+    : LoaderImpl(DoNotLoadSnapshot{}, base, generator, store, tls) {
+  loadNewSnapshot();
+}
+
+LoaderImpl::LoaderImpl(DoNotLoadSnapshot /* unused */, const ProtobufWkt::Struct& base,
+                       RandomGenerator& generator, Stats::Store& store,
+                       ThreadLocal::SlotAllocator& tls)
+    : generator_(generator), stats_(generateStats(store)), admin_layer_(stats_), base_(base),
       tls_(tls.allocateSlot()) {}
 
 std::unique_ptr<SnapshotImpl> LoaderImpl::createNewSnapshot() {
   std::vector<Snapshot::OverrideLayerConstPtr> layers;
+  layers.emplace_back(std::make_unique<const ProtoLayer>(base_));
   layers.emplace_back(std::make_unique<const AdminLayer>(admin_layer_));
   return std::make_unique<SnapshotImpl>(generator_, stats_, std::move(layers));
 }
@@ -432,13 +477,11 @@ void LoaderImpl::mergeValues(const std::unordered_map<std::string, std::string>&
   loadNewSnapshot();
 }
 
-DiskBackedLoaderImpl::DiskBackedLoaderImpl(Event::Dispatcher& dispatcher,
-                                           ThreadLocal::SlotAllocator& tls,
-                                           const std::string& root_symlink_path,
-                                           const std::string& subdir,
-                                           const std::string& override_dir, Stats::Store& store,
-                                           RandomGenerator& generator, Api::Api& api)
-    : LoaderImpl(DoNotLoadSnapshot{}, generator, store, tls),
+DiskBackedLoaderImpl::DiskBackedLoaderImpl(
+    Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, const ProtobufWkt::Struct& base,
+    const std::string& root_symlink_path, const std::string& subdir,
+    const std::string& override_dir, Stats::Store& store, RandomGenerator& generator, Api::Api& api)
+    : LoaderImpl(DoNotLoadSnapshot{}, base, generator, store, tls),
       watcher_(dispatcher.createFilesystemWatcher()), root_path_(root_symlink_path + "/" + subdir),
       override_path_(root_symlink_path + "/" + override_dir), api_(api) {
   watcher_->addWatch(root_symlink_path, Filesystem::Watcher::Events::MovedTo,
@@ -456,6 +499,7 @@ RuntimeStats LoaderImpl::generateStats(Stats::Store& store) {
 
 std::unique_ptr<SnapshotImpl> DiskBackedLoaderImpl::createNewSnapshot() {
   std::vector<Snapshot::OverrideLayerConstPtr> layers;
+  layers.emplace_back(std::make_unique<const ProtoLayer>(base_));
   try {
     layers.push_back(std::make_unique<DiskLayer>("root", root_path_, api_));
     if (api_.fileSystem().directoryExists(override_path_)) {