Add SSL support in API server

* Add a static yamlgen/deploy/cert.yaml file to set up cert-manager
* Update models and build.rs to accommodate certs mounting and deploying
  and adjusting server's port number
* Update apiserver's server and client to use ssl
* Expose 8443 as apiserver port in dockerfile
* Bump SDK to 0.26.0 for openssl dh algorithm and stdio feature

Cargo.toml

@@ -5,5 +5,5 @@ members = [
     "controller",
     "models",
     "yamlgen",
-    "integ"
+    "integ",
 ]

Dockerfile

@@ -45,4 +45,4 @@ COPY --from=build \
     /licenses/bottlerocket-sdk/
 
 # Expose apiserver port
-EXPOSE 8080
+EXPOSE 8443

Makefile

@@ -29,7 +29,7 @@ DESTDIR = .
 # tarball.
 DISTFILE = $(DESTDIR:/=)/$(subst /,_,$(IMAGE_NAME)).tar.gz
 
-BOTTLEROCKET_SDK_VERSION = v0.25.1
+BOTTLEROCKET_SDK_VERSION = v0.26.0
 BOTTLEROCKET_SDK_ARCH = $(UNAME_ARCH)
 
 BUILDER_IMAGE = public.ecr.aws/bottlerocket/bottlerocket-sdk-$(BOTTLEROCKET_SDK_ARCH):$(BOTTLEROCKET_SDK_VERSION)

apiserver/Cargo.toml

@@ -15,8 +15,9 @@ server = []
 models = { path = "../models", version = "0.1.0" }
 
 # tracing-actix-web version must align with actix-web version
-actix-web = { version = "4.0.0-beta.9", default-features = false }
+actix-web = { version = "4.0.0-beta.9", features = ["openssl"], default-features = false }
 actix-web-opentelemetry = { version = "0.11.0-beta.5", features = ["metrics"] }
+openssl = { version = "0.10" }
 opentelemetry = { version = "0.16", features = ["rt-tokio-current-thread"]}
 opentelemetry-prometheus = "0.9"
 tracing = "0.1"

apiserver/src/api/mod.rs

@@ -14,7 +14,8 @@ use crate::{
     telemetry,
 };
 use models::constants::{
-    AGENT, APISERVER_HEALTH_CHECK_ROUTE, APISERVER_SERVICE_NAME, LABEL_COMPONENT, NAMESPACE,
+    AGENT, APISERVER_HEALTH_CHECK_ROUTE, APISERVER_SERVICE_NAME, CERT_KEY_NAME, LABEL_COMPONENT,
+    NAMESPACE, PRIVATE_KEY_NAME, PUBLIC_KEY_NAME, TLS_KEY_MOUNT_PATH,
 };
 use models::node::{BottlerocketShadowClient, BottlerocketShadowSelector};
 
@@ -32,6 +33,7 @@ use kube::{
     runtime::{reflector, utils::try_flatten_touched, watcher::watcher},
     ResourceExt,
 };
+use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod, SslMode, SslOptions};
 use opentelemetry::global::meter;
 use snafu::{OptionExt, ResultExt};
 use std::env;
@@ -146,6 +148,18 @@ pub async fn run_server<T: 'static + BottlerocketShadowClient>(
 
     event!(Level::DEBUG, ?server_addr, "Server addr localhost.");
 
+    let mut builder = SslAcceptor::mozilla_modern_v5(SslMethod::tls()).context(error::SSLError)?;
+
+    builder
+        .set_certificate_chain_file(format!("{}/{}", TLS_KEY_MOUNT_PATH, PUBLIC_KEY_NAME))
+        .context(error::SSLError)?;
+    builder
+        .set_private_key_file(
+            format!("{}/{}", TLS_KEY_MOUNT_PATH, PRIVATE_KEY_NAME),
+            SslFiletype::PEM,
+        )
+        .context(error::SSLError)?;
+
     let server = HttpServer::new(move || {
         App::new()
             .wrap(
@@ -177,7 +191,7 @@ pub async fn run_server<T: 'static + BottlerocketShadowClient>(
                 web::get().to(ping::health_check),
             )
     })
-    .bind(server_addr)
+    .bind_openssl(server_addr, builder)
     .context(error::HttpServerError)?
     .run();
 

apiserver/src/client/error.rs

@@ -67,4 +67,7 @@ pub enum ClientError {
         source
     ))]
     IOError { source: Box<dyn std::error::Error> },
+
+    #[snafu(display("Failed to create https client due to {}", source))]
+    CreateClientError { source: reqwest::Error },
 }

apiserver/src/client/webclient.rs

@@ -8,21 +8,24 @@ use crate::{
     UncordonBottlerocketShadowRequest, UpdateBottlerocketShadowRequest,
 };
 use models::{
-    constants::{APISERVER_SERVICE_NAME, APISERVER_SERVICE_PORT, NAMESPACE},
+    constants::{
+        APISERVER_SERVICE_NAME, APISERVER_SERVICE_PORT, CERT_KEY_NAME, NAMESPACE, PUBLIC_KEY_NAME,
+        TLS_KEY_MOUNT_PATH,
+    },
     node::{BottlerocketShadow, BottlerocketShadowSelector, BottlerocketShadowStatus},
 };
 
 use async_trait::async_trait;
 use snafu::ResultExt;
+use std::fs;
+use std::io::Read;
 use tokio::time::Duration;
 use tokio_retry::{
     strategy::{jitter, ExponentialBackoff},
     Retry,
 };
 use tracing::instrument;
 
-use std::fs;
-
 // The web client uses exponential backoff.
 // These values configure how long to delay between tries.
 const RETRY_BASE_DELAY: Duration = Duration::from_millis(100);
@@ -74,7 +77,7 @@ impl K8SAPIServerClient {
 
     /// Protocol scheme for communicating with the server.
     pub fn scheme() -> String {
-        "http".to_string()
+        "https".to_string()
     }
 
     /// Returns the domain on which the server can be reached.
@@ -95,6 +98,28 @@ impl K8SAPIServerClient {
             .header(HEADER_BRUPOP_NODE_NAME, &node_selector.node_name)
             .header(HEADER_BRUPOP_K8S_AUTH_TOKEN, &self.auth_token()?))
     }
+
+    /// Returns the https client configured to use self-signed certificate
+    fn https_client() -> Result<reqwest::Client> {
+        let mut buf = Vec::new();
+
+        let public_key_path = format!("{}/{}", TLS_KEY_MOUNT_PATH, CERT_KEY_NAME);
+        std::fs::File::open(public_key_path)
+            .map_err(|err| Box::new(err) as Box<dyn std::error::Error>)
+            .context(error::IOError)?
+            .read_to_end(&mut buf)
+            .map_err(|err| Box::new(err) as Box<dyn std::error::Error>)
+            .context(error::IOError)?;
+
+        let cert = reqwest::Certificate::from_pem(&buf).context(error::CreateClientError)?;
+
+        let client = reqwest::Client::builder()
+            .add_root_certificate(cert)
+            .connection_verbose(true)
+            .build()
+            .context(error::CreateClientError)?;
+        Ok(client)
+    }
 }
 
 #[async_trait]
@@ -105,10 +130,10 @@ impl APIServerClient for K8SAPIServerClient {
         req: CreateBottlerocketShadowRequest,
     ) -> Result<BottlerocketShadow> {
         Retry::spawn(retry_strategy(), || async {
-            let http_client = reqwest::Client::new();
+            let https_client = Self::https_client()?;
 
             let request_builder = self.add_common_request_headers(
-                http_client.post(format!(
+                https_client.post(format!(
                     "{}://{}{}",
                     Self::scheme(),
                     Self::server_domain(),
@@ -158,10 +183,9 @@ impl APIServerClient for K8SAPIServerClient {
         req: UpdateBottlerocketShadowRequest,
     ) -> Result<BottlerocketShadowStatus> {
         Retry::spawn(retry_strategy(), || async {
-            let http_client = reqwest::Client::new();
-
+            let https_client = Self::https_client()?;
             let request_builder = self.add_common_request_headers(
-                http_client.put(format!(
+                https_client.put(format!(
                     "{}://{}{}",
                     Self::scheme(),
                     Self::server_domain(),
@@ -212,10 +236,9 @@ impl APIServerClient for K8SAPIServerClient {
         req: CordonAndDrainBottlerocketShadowRequest,
     ) -> Result<()> {
         Retry::spawn(retry_strategy(), || async {
-            let http_client = reqwest::Client::new();
-
+            let https_client = Self::https_client()?;
             let request_builder = self.add_common_request_headers(
-                http_client.post(format!(
+                https_client.post(format!(
                     "{}://{}{}",
                     Self::scheme(),
                     Self::server_domain(),
@@ -254,10 +277,9 @@ impl APIServerClient for K8SAPIServerClient {
     #[instrument]
     async fn uncordon_node(&self, req: UncordonBottlerocketShadowRequest) -> Result<()> {
         Retry::spawn(retry_strategy(), || async {
-            let http_client = reqwest::Client::new();
-
+            let https_client = Self::https_client()?;
             let request_builder = self.add_common_request_headers(
-                http_client.post(format!(
+                https_client.post(format!(
                     "{}://{}{}",
                     Self::scheme(),
                     Self::server_domain(),

apiserver/src/error.rs

@@ -41,6 +41,9 @@ pub enum Error {
 
     #[snafu(display("Failed to drain Node: '{}'", source))]
     BottlerocketShadowDrain { source: BottlerocketShadowError },
+
+    #[snafu(display("Failed to set up SslAcceptorBuilder : {:?}", source))]
+    SSLError { source: openssl::error::ErrorStack },
 }
 
 impl ResponseError for Error {}

apiserver/src/main.rs

@@ -1,6 +1,7 @@
 use apiserver::api::{self, APIServerSettings};
 use apiserver::error::{self, Result};
 use apiserver::telemetry::init_telemetry;
+use models::constants::APISERVER_INTERNAL_PORT;
 use models::node::K8SBottlerocketShadowClient;
 use tracing::{event, Level};
 
@@ -40,7 +41,7 @@ async fn run_server() -> Result<()> {
 
     let settings = APIServerSettings {
         node_client: K8SBottlerocketShadowClient::new(k8s_client.clone()),
-        server_port: 8080,
+        server_port: APISERVER_INTERNAL_PORT as u16,
     };
 
     api::run_server(settings, k8s_client, Some(prometheus_exporter)).await

models/src/agent.rs

@@ -1,14 +1,15 @@
 use crate::constants::{
     AGENT, AGENT_NAME, APISERVER_SERVICE_NAME, APP_COMPONENT, APP_MANAGED_BY, APP_PART_OF, BRUPOP,
-    BRUPOP_INTERFACE_VERSION, LABEL_BRUPOP_INTERFACE_NAME, LABEL_COMPONENT, NAMESPACE,
+    BRUPOP_INTERFACE_VERSION, LABEL_BRUPOP_INTERFACE_NAME, LABEL_COMPONENT, NAMESPACE, SECRET_NAME,
+    TLS_KEY_MOUNT_PATH,
 };
 use k8s_openapi::api::apps::v1::{DaemonSet, DaemonSetSpec};
 use k8s_openapi::api::core::v1::{
     Affinity, Container, EnvVar, EnvVarSource, HostPathVolumeSource, LocalObjectReference,
     NodeAffinity, NodeSelector, NodeSelectorRequirement, NodeSelectorTerm, ObjectFieldSelector,
     PodSpec, PodTemplateSpec, ProjectedVolumeSource, ResourceRequirements, SELinuxOptions,
-    SecurityContext, ServiceAccount, ServiceAccountTokenProjection, Volume, VolumeMount,
-    VolumeProjection,
+    SecretVolumeSource, SecurityContext, ServiceAccount, ServiceAccountTokenProjection, Volume,
+    VolumeMount, VolumeProjection,
 };
 use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
@@ -206,6 +207,11 @@ pub fn agent_daemonset(agent_image: String, image_pull_secret: Option<String>) -
                                 mount_path: TOKEN_PROJECTION_MOUNT_PATH.to_string(),
                                 ..Default::default()
                             },
+                            VolumeMount {
+                                name: "bottlerocket-tls-keys".to_string(),
+                                mount_path: TLS_KEY_MOUNT_PATH.to_string(),
+                                ..Default::default()
+                            },
                         ]),
                         security_context: Some(SecurityContext {
                             se_linux_options: Some(SELinuxOptions {
@@ -252,6 +258,15 @@ pub fn agent_daemonset(agent_image: String, image_pull_secret: Option<String>) -
                             }),
                             ..Default::default()
                         },
+                        Volume {
+                            name: "bottlerocket-tls-keys".to_string(),
+                            secret: Some(SecretVolumeSource {
+                                secret_name: Some(SECRET_NAME.to_string()),
+                                optional: Some(false),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
                     ]),
                     ..Default::default()
                 }),

models/src/apiserver.rs

@@ -1,7 +1,7 @@
 use crate::constants::{
     APISERVER, APISERVER_HEALTH_CHECK_ROUTE, APISERVER_INTERNAL_PORT, APISERVER_MAX_UNAVAILABLE,
     APISERVER_SERVICE_NAME, APISERVER_SERVICE_PORT, APP_COMPONENT, APP_MANAGED_BY, APP_PART_OF,
-    BRUPOP, BRUPOP_DOMAIN_LIKE_NAME, LABEL_COMPONENT, NAMESPACE,
+    BRUPOP, BRUPOP_DOMAIN_LIKE_NAME, LABEL_COMPONENT, NAMESPACE, SECRET_NAME, TLS_KEY_MOUNT_PATH,
 };
 use crate::node::{K8S_NODE_PLURAL, K8S_NODE_STATUS};
 use k8s_openapi::api::apps::v1::{
@@ -10,7 +10,7 @@ use k8s_openapi::api::apps::v1::{
 use k8s_openapi::api::core::v1::{
     Affinity, Container, ContainerPort, HTTPGetAction, LocalObjectReference, NodeAffinity,
     NodeSelector, NodeSelectorRequirement, NodeSelectorTerm, PodSpec, PodTemplateSpec, Probe,
-    Service, ServiceAccount, ServicePort, ServiceSpec,
+    SecretVolumeSource, Service, ServiceAccount, ServicePort, ServiceSpec, Volume, VolumeMount,
 };
 use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
@@ -246,6 +246,7 @@ pub fn apiserver_deployment(
                             http_get: Some(HTTPGetAction {
                                 path: Some(APISERVER_HEALTH_CHECK_ROUTE.to_string()),
                                 port: IntOrString::Int(APISERVER_INTERNAL_PORT),
+                                scheme: Some("HTTPS".to_string()),
                                 ..Default::default()
                             }),
                             initial_delay_seconds: Some(5),
@@ -255,13 +256,28 @@ pub fn apiserver_deployment(
                             http_get: Some(HTTPGetAction {
                                 path: Some(APISERVER_HEALTH_CHECK_ROUTE.to_string()),
                                 port: IntOrString::Int(APISERVER_INTERNAL_PORT),
+                                scheme: Some("HTTPS".to_string()),
                                 ..Default::default()
                             }),
                             initial_delay_seconds: Some(5),
                             ..Default::default()
                         }),
+                        volume_mounts: Some(vec![VolumeMount {
+                            name: "bottlerocket-tls-keys".to_string(),
+                            mount_path: TLS_KEY_MOUNT_PATH.to_string(),
+                            ..Default::default()
+                        }]),
                         ..Default::default()
                     }],
+                    volumes: Some(vec![Volume {
+                        name: "bottlerocket-tls-keys".to_string(),
+                        secret: Some(SecretVolumeSource {
+                            secret_name: Some(SECRET_NAME.to_string()),
+                            optional: Some(false),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    }]),
                     image_pull_secrets,
                     service_account_name: Some(BRUPOP_APISERVER_SERVICE_ACCOUNT.to_string()),
                     ..Default::default()

models/src/constants.rs

@@ -18,6 +18,13 @@ pub const BRUPOP_DOMAIN_LIKE_NAME: &str = brupop_domain!();
 pub const LABEL_BRUPOP_INTERFACE_NAME: &str = "bottlerocket.aws/updater-interface-version";
 pub const BRUPOP_INTERFACE_VERSION: &str = "2.0.0";
 
+// In name space secret name for SSL communication in API server.
+pub const SECRET_NAME: &str = "brupop-tls";
+pub const CERT_KEY_NAME: &str = "ca.crt";
+pub const PUBLIC_KEY_NAME: &str = "tls.crt";
+pub const PRIVATE_KEY_NAME: &str = "tls.key";
+pub const TLS_KEY_MOUNT_PATH: &str = "/etc/brupop-tls-keys";
+
 // Label keys
 pub const LABEL_COMPONENT: &str = brupop_domain!("component");
 
@@ -31,8 +38,8 @@ pub const APP_CREATED_BY: &str = "app.kubernetes.io/created-by";
 
 // apiserver constants
 pub const APISERVER: &str = "apiserver";
-pub const APISERVER_INTERNAL_PORT: i32 = 8080; // The internal port on which the the apiservice is hosted.
-pub const APISERVER_SERVICE_PORT: i32 = 80; // The k8s service port hosting the apiserver.
+pub const APISERVER_INTERNAL_PORT: i32 = 8443; // The internal port on which the the apiservice is hosted.
+pub const APISERVER_SERVICE_PORT: i32 = 443; // The k8s service port hosting the apiserver.
 pub const APISERVER_MAX_UNAVAILABLE: &str = "33%"; // The maximum number of unavailable nodes for the apiserver deployment.
 pub const APISERVER_HEALTH_CHECK_ROUTE: &str = "/ping"; // Route used for apiserver k8s liveness and readiness checks.
 pub const APISERVER_SERVICE_NAME: &str = "brupop-apiserver"; // The name for the `svc` fronting the apiserver.