cloud-api-adaptor: initialization of secure-cooms

Use peer-pods-cm to initialize adaptor's secure-comms
Use agent-protocol-forwarder.service to initialize forwarder's secure-comms

Signed-off-by: David Hadas <david.hadas@gmail.com>

src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go

@@ -9,6 +9,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
 	daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
@@ -22,7 +23,6 @@ import (
 const programName = "agent-protocol-forwarder"
 
 type Config struct {
-	secureComms         bool
 	tlsConfig           *tlsutil.TLSConfig
 	daemonConfig        daemon.Config
 	configPath          string
@@ -47,12 +47,13 @@ func load(path string, obj interface{}) error {
 }
 
 func (cfg *Config) Setup() (cmd.Starter, error) {
-
 	var (
-		showVersion bool
-		disableTLS  bool
-		secureComms bool
-		tlsConfig   tlsutil.TLSConfig
+		showVersion          bool
+		disableTLS           bool
+		secureComms          bool
+		secureCommsInbounds  string
+		secureCommsOutbounds string
+		tlsConfig            tlsutil.TLSConfig
 	)
 
 	cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) {
@@ -67,30 +68,29 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 		flags.StringVar(&tlsConfig.KeyFile, "cert-key", "", "cert key")
 		flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
 		flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
-		flags.BoolVar(&secureComms, "secure-comms", true, "Use SSH to secure communication between cluster and peer pods")
+		flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
+		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
 	})
 
-	if secureComms {
-		cfg.secureComms = true
-		cfg.listenAddr = "127.0.0.1:" + daemon.DefaultListenPort
-		ppssh.InitSshServer([]string{"B:KBS:8080"}, []string{"K:KATAAPI:" + daemon.DefaultListenPort}, ppssh.GetSecret(getKey))
-	} else {
-		if !disableTLS {
-			cfg.tlsConfig = &tlsConfig
-		}
-	}
-
 	cmd.ShowVersion(programName)
 
 	if showVersion {
 		cmd.Exit(0)
 	}
 
-	for path, obj := range map[string]interface{}{
-		cfg.configPath: &cfg.daemonConfig,
-	} {
-		if err := load(path, obj); err != nil {
-			return nil, err
+	if err := load(cfg.configPath, &cfg.daemonConfig); err != nil {
+		return nil, err
+	}
+
+	if secureComms {
+		cfg.listenAddr = "127.0.0.1:15150"
+		inbounds := append([]string{"K:KATAAPI:15150"}, strings.Split(secureCommsInbounds, ",")...)
+		outbounds := append([]string{"B:KBS:8080"}, strings.Split(secureCommsOutbounds, ",")...)
+		ppssh.InitSshServer(inbounds, outbounds, ppssh.GetSecret(getKey))
+	} else {
+		if !disableTLS {
+			cfg.tlsConfig = &tlsConfig
 		}
 	}
 

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -8,7 +8,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"log"
 	"os"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
@@ -50,7 +49,6 @@ func printHelp(out io.Writer) {
 }
 
 func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
-	log.Printf("ARGS: %v", os.Args)
 
 	if len(os.Args) < 2 {
 		printHelp(os.Stderr)
@@ -85,9 +83,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 	}
 
 	var (
-		disableTLS  bool
-		tlsConfig   tlsutil.TLSConfig
-		secureComms bool
+		disableTLS           bool
+		tlsConfig            tlsutil.TLSConfig
+		secureComms          bool
+		secureCommsInbounds  string
+		secureCommsOutbounds string
 	)
 
 	cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
@@ -108,7 +108,9 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&tlsConfig.KeyFile, "cert-key", "", "cert key")
 		flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
 		flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
-		flags.BoolVar(&secureComms, "secure-comms", true, "Use SSH to secure communication between cluster and peer pods")
+		flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
+		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
 		flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")
 
 		flags.StringVar(&cfg.networkConfig.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider")
@@ -127,6 +129,10 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 
 	if secureComms {
 		cfg.serverConfig.SecureComms = true
+		cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
+		cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
+
+		cfg.serverConfig.AAKBCParams = "cc_kbc::http://127.0.0.1:8080"
 	} else {
 		if !disableTLS {
 			cfg.serverConfig.TLSConfig = &tlsConfig

src/cloud-api-adaptor/docs/SecureComms

@@ -0,0 +1,39 @@
+# Secure Comms
+
+Here we describe how to setup the Secure Comms feature of the Adaptor and Forwarder
+
+## Setup
+
+### Deploy CAA
+Use any of the option for installing CAA depending on the cloud driver used.
+
+### Deploy KBS-Operator
+Deploy KBS-Operator by following instructions at [https://github.com/confidential-containers/kbs-operator?tab=readme-ov-file#getting-started](KBS Operator Getting Started). 
+
+Copy the kbs-client secret from the `kbs-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:
+
+```sh
+kubectl get secret kbs-client -n kbs-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
+```
+
+### Build podvm to enforce Secure-Comms
+
+Change the `podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
+```sh
+ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags.
+
+### Activate Secure-Comms in peer-pods-cm
+
+Add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace:
+```sh
+apiVersion: v1
+data:
+    ...
+    SECURE_COMMS: "true"
+    ...
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` flags. 

src/cloud-api-adaptor/entrypoint.sh

@@ -19,6 +19,9 @@ optionals+=""
 [[ "${AA_KBC_PARAMS}" ]] && optionals+="-aa-kbc-params ${AA_KBC_PARAMS} "
 [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} "
 [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify "
+[[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
+[[ "${SECURE_COMMS_INBOUNDS}" == "true" ]] && optionals+="-secure-comms-inbounds "
+[[ "${SECURE_COMMS_OUTBOUNDS}" == "true" ]] && optionals+="-secure-comms-outbounds "
 
 test_vars() {
     for i in "$@"; do

src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml

@@ -22,6 +22,7 @@ configMapGenerator:
   - LIBVIRT_NET="default" # set
   - LIBVIRT_POOL="default" # set
   - DISABLECVM="true" # set as false to enable confidential VM
+  - SECURE_COMMS="false" # set as true to enable Secure Comms
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
   #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/containerd/containerd/pkg/cri/annotations"
@@ -29,12 +30,11 @@ import (
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh"
-	//"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh"
 )
 
 const (
-	Version = "0.0.0"
-	KBS_URL = "http://kbs-service.kbs-operator-system:8080"
+	Version     = "0.0.0"
+	KBS_ADDRESS = "kbs-service.kbs-operator-system:8080"
 )
 
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
@@ -78,12 +78,14 @@ func (s *cloudService) removeSandbox(id sandboxID) error {
 }
 
 func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode,
-	secureComms bool, podsDir, daemonPort, aaKBCParams string) Service {
+	secureComms bool, secureCommsInbounds, secureCommsOutbounds, podsDir, daemonPort, aaKBCParams string) Service {
 	var err error
 	var sshClient *wnssh.SshClient
 
 	if secureComms {
-		sshClient, err = wnssh.InitSshClient([]string{"K:KATAAPI:0"}, []string{"B:KBS:kbs-service.kbs-operator-system:8080"}, KBS_URL)
+		inbounds := append([]string{"K:KATAAPI:0"}, strings.Split(secureCommsInbounds, ",")...)
+		outbounds := append([]string{"B:KBS:" + KBS_ADDRESS}, strings.Split(secureCommsOutbounds, ",")...)
+		sshClient, err = wnssh.InitSshClient(inbounds, outbounds, KBS_ADDRESS)
 		if err != nil {
 			log.Fatalf("InitSshClient failed %v", err)
 		}

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go

@@ -114,7 +114,7 @@ func TestCloudService(t *testing.T) {
 		podsDir: dir,
 	}
 
-	s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, dir, forwarder.DefaultListenPort, "")
+	s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", dir, forwarder.DefaultListenPort, "")
 
 	assert.NotNil(t, s)
 

src/cloud-api-adaptor/pkg/adaptor/server.go

@@ -42,6 +42,8 @@ type ServerConfig struct {
 	AAKBCParams             string
 	EnableCloudConfigVerify bool
 	SecureComms             bool
+	SecureCommsInbounds     string
+	SecureCommsOutbounds    string
 }
 
 type Server interface {
@@ -67,7 +69,8 @@ func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetw
 	logger.Printf("server config: %#v", cfg)
 
 	agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.CriSocketPath, cfg.TLSConfig, cfg.ProxyTimeout)
-	cloudService := cloud.NewService(provider, agentFactory, workerNode, cfg.SecureComms, cfg.PodsDir, cfg.ForwarderPort, cfg.AAKBCParams)
+	cloudService := cloud.NewService(provider, agentFactory, workerNode,
+		cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.PodsDir, cfg.ForwarderPort, cfg.AAKBCParams)
 	vmInfoService := vminfo.NewService(cloudService)
 
 	return &server{

src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshproxy"
@@ -120,6 +121,10 @@ func InitSshServer(inbound_strings, outbounds_strings []string, getSecret GetSec
 	var wg sync.WaitGroup
 
 	for _, tag := range inbound_strings {
+		tag = strings.TrimSpace(tag)
+		if tag == "" {
+			continue
+		}
 		port, _, name, phase, err := sshproxy.ParseTag(tag)
 		if err != nil {
 			logger.Fatalf("failed to parse inbound tag %s: %v", tag, err)

src/cloud-api-adaptor/pkg/securecomms/sshutil/sshutil.go

@@ -11,7 +11,7 @@ const SSHPORT = "2222"
 const PpSecureCommsVersion = "v0.2"
 const KBS = "KBS"
 
-var Logger = log.New(log.Writer(), "[adaptor/scomms] ", log.LstdFlags|log.Lmsgprefix)
+var Logger = log.New(log.Writer(), "[secure-comms] ", log.LstdFlags|log.Lmsgprefix)
 
 // RsaPrivateKeyPEM return a PEM for the RSA Private Key
 func RsaPrivateKeyPEM(pKey *rsa.PrivateKey) []byte {

src/cloud-api-adaptor/pkg/securecomms/wnssh/kbsclient.go

@@ -16,9 +16,9 @@ type KbsClient struct {
 	url       string
 }
 
-func InitKbsClient(url string) *KbsClient {
+func InitKbsClient(address string) *KbsClient {
 	return &KbsClient{
-		url: url,
+		url: "http://" + address,
 	}
 }
 

src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go

@@ -55,7 +55,7 @@ func PpSecretName(sid string) string {
 // Structure of an inbound tag: "<MyPort>:<InboundName>:<Phase>"
 // Structure of an outbound tag: "<DesPort>:<DesHost>:<outboundName>:<Phase>"
 // Phasde may be "A" (Attestation), "K" (Kubernetes), or "B" (Both)
-func InitSshClient(inbound_strings, outbound_strings []string, kbsUrl string) (*SshClient, error) {
+func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string) (*SshClient, error) {
 	logger.Printf("Using PP Secure Comms: InitSshClient version %s", sshutil.PpSecureCommsVersion)
 
 	err := kubemgr.InitKubeMgr()
@@ -86,7 +86,7 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsUrl string) (*
 		return nil, fmt.Errorf("failed to read KBS Client Secret: %w", err)
 	}
 
-	kc := InitKbsClient(kbsUrl)
+	kc := InitKbsClient(kbsAddress)
 	err = kc.SetPemSecret(kbscPrivateKey)
 	if err != nil {
 		return nil, fmt.Errorf("KbsClient - %v", err)