Adding a remove-clusters command

app/kubemci/cmd/cmd.go

@@ -40,6 +40,7 @@ func NewCommand(in io.Reader, out, err io.Writer) *cobra.Command {
 		NewCmdGetStatus(out, err),
 		NewCmdGetVersion(out, err),
 		newCmdList(out, err),
+		newCmdRemoveClusters(out, err),
 	)
 	rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
 	return rootCmd

app/kubemci/cmd/delete.go

@@ -18,9 +18,11 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/hashicorp/go-multierror"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/spf13/cobra"
 	"k8s.io/api/extensions/v1beta1"
+	// gcp is needed for GKE cluster auth to work.
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 
 	"github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/cloudinterface"
 	gcplb "github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/loadbalancer"

app/kubemci/cmd/remove_clusters.go

@@ -0,0 +1,160 @@
+// Copyright 2018 Google, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/spf13/cobra"
+	"k8s.io/api/extensions/v1beta1"
+	// gcp is needed for GKE cluster auth to work.
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+
+	"github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/cloudinterface"
+	gcplb "github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/loadbalancer"
+	gcputils "github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/utils"
+	"github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/ingress"
+	"github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/kubeutils"
+)
+
+var (
+	removeClustersShortDesc = "Remove an existing multicluster ingress from some clusters."
+	removeClustersLongDesc  = `Remove an existing multicluster ingress from some clusters.
+
+	Takes a load balancer name and a list of clusters and removes the existing multicluster ingress from those clusters.
+	If the clusters have already been deleted, you can run "kubemci create --force" with the updated cluster list to update the
+	load balancer to be restricted to those clusters. That will however not delete the ingress from old clusters (which is fine
+	if the clusters have been deleted already).
+`
+)
+
+type removeClustersOptions struct {
+	// Name of the YAML file containing ingress spec.
+	// Required.
+	IngressFilename string
+	// Path to kubeconfig file.
+	// Required.
+	KubeconfigFilename string
+	// Names of the contexts to use from the kubeconfig file.
+	KubeContexts []string
+	// Name of the load balancer.
+	// Required.
+	LBName string
+	// Name of the GCP project in which the load balancer is configured.
+	GCPProject string
+	// Overwrite values when they differ from what's requested. If
+	// the resource does not exist, or is already the correct
+	// value, then 'force' is a no-op.
+	ForceUpdate bool
+	// Name of the namespace for the ingress when none is provided (mismatch of option with spec causes an error).
+	Namespace string
+}
+
+func newCmdRemoveClusters(out, err io.Writer) *cobra.Command {
+	var options removeClustersOptions
+	cmd := &cobra.Command{
+		Use:   "remove-clusters",
+		Short: removeClustersShortDesc,
+		Long:  removeClustersLongDesc,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := validateRemoveClustersArgs(&options, args); err != nil {
+				fmt.Println(err)
+				return
+			}
+			if err := runRemoveClusters(&options, args); err != nil {
+				fmt.Println("Error removing clusters:", err)
+				return
+			}
+		},
+	}
+	addRemoveClustersFlags(cmd, &options)
+	return cmd
+}
+
+func addRemoveClustersFlags(cmd *cobra.Command, options *removeClustersOptions) error {
+	cmd.Flags().StringVarP(&options.IngressFilename, "ingress", "i", options.IngressFilename, "[required] filename containing ingress spec")
+	cmd.Flags().StringVarP(&options.KubeconfigFilename, "kubeconfig", "k", options.KubeconfigFilename, "[required] path to kubeconfig file")
+	cmd.Flags().StringSliceVar(&options.KubeContexts, "kubecontexts", options.KubeContexts, "[optional] contexts in the kubeconfig file to remove the ingress from")
+	cmd.Flags().StringVarP(&options.GCPProject, "gcp-project", "", options.GCPProject, "[required] name of the gcp project")
+	cmd.Flags().BoolVarP(&options.ForceUpdate, "force", "f", options.ForceUpdate, "[optional] overwrite existing settings if they are different")
+	cmd.Flags().StringVarP(&options.Namespace, "namespace", "n", options.Namespace, "[optional] namespace for the ingress only if left unspecified by ingress spec")
+	return nil
+}
+
+func validateRemoveClustersArgs(options *removeClustersOptions, args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("unexpected args: %v. Expected one arg as name of load balancer.", args)
+	}
+	// Verify that the required options are not missing.
+	if options.IngressFilename == "" {
+		return fmt.Errorf("unexpected missing argument ingress.")
+	}
+	if options.GCPProject == "" {
+		project, err := gcputils.GetProjectFromGCloud()
+		if project == "" || err != nil {
+			return fmt.Errorf("unexpected cannot determine GCP project. Either set --gcp-project flag, or set a default project with gcloud such that gcloud config get-value project returns that")
+		}
+		options.GCPProject = project
+	}
+	if options.KubeconfigFilename == "" {
+		return fmt.Errorf("unexpected missing argument kubeconfig.")
+	}
+	return nil
+}
+
+// runRemoveClusters removes the given load balancer from the given list of clusters.
+func runRemoveClusters(options *removeClustersOptions, args []string) error {
+	options.LBName = args[0]
+
+	// Unmarshal the YAML into ingress struct.
+	var ing v1beta1.Ingress
+	if err := ingress.UnmarshallAndApplyDefaults(options.IngressFilename, options.Namespace, &ing); err != nil {
+		return fmt.Errorf("error in unmarshalling the yaml file %s, err: %s", options.IngressFilename, err)
+	}
+	cloudInterface, err := cloudinterface.NewGCECloudInterface(options.GCPProject)
+	if err != nil {
+		err := fmt.Errorf("error in creating cloud interface: %s", err)
+		fmt.Println(err)
+		return err
+	}
+	// Get clients for all clusters
+	clients, err := kubeutils.GetClients(options.KubeconfigFilename, options.KubeContexts)
+	if err != nil {
+		return err
+	}
+
+	lbs, err := gcplb.NewLoadBalancerSyncer(options.LBName, clients, cloudInterface, options.GCPProject)
+	if err != nil {
+		return err
+	}
+	if delErr := lbs.RemoveFromClusters(&ing, clients, options.ForceUpdate); delErr != nil {
+		err = multierror.Append(err, delErr)
+	}
+
+	// Delete ingress resource from clusters
+	is := ingress.NewIngressSyncer()
+	if is == nil {
+		err = multierror.Append(err, fmt.Errorf("unexpected ingress syncer is nil"))
+		// No point in proceeding.
+		return err
+	}
+	isErr := is.DeleteIngress(&ing, clients)
+	if isErr != nil {
+		err = multierror.Append(err, isErr)
+	}
+	return err
+}

app/kubemci/pkg/gcp/backendservice/backendservicesyncer.go

@@ -89,6 +89,38 @@ func (b *BackendServiceSyncer) DeleteBackendServices(ports []ingressbe.ServicePo
 	return nil
 }
 
+// See interface for comment.
+func (b *BackendServiceSyncer) RemoveFromClusters(ports []ingressbe.ServicePort, removeIGLinks []string) error {
+	fmt.Println("Removing backend services from clusters")
+	var err error
+	for _, p := range ports {
+		if beErr := b.removeFromClusters(p, removeIGLinks); beErr != nil {
+			beErr = fmt.Errorf("Error %s in removing backend service for port %v", beErr, p)
+			fmt.Printf("Error in removing backend service for port %v: %v. Continuing.\n", p, beErr)
+			// Try removing backend services for all ports and return all errors at once.
+			err = multierror.Append(err, beErr)
+			continue
+		}
+	}
+	return err
+}
+
+func (b *BackendServiceSyncer) removeFromClusters(port ingressbe.ServicePort, removeIGLinks []string) error {
+	name := b.namer.BeServiceName(port.Port)
+
+	existingBE, err := b.bsp.GetGlobalBackendService(name)
+	if err != nil {
+		err := fmt.Errorf("error in fetching existing backend service %s: %s", name, err)
+		fmt.Println(err)
+		return err
+	}
+	// Remove clusters from the backend service.
+	desiredBE := b.desiredBackendServiceWithoutClusters(existingBE, removeIGLinks)
+	glog.V(5).Infof("Existing backend service: %v\n, desired: %v\n", *existingBE, *desiredBE)
+	_, err = b.updateBackendService(desiredBE)
+	return err
+}
+
 func (b *BackendServiceSyncer) deleteBackendService(port ingressbe.ServicePort) error {
 	name := b.namer.BeServiceName(port.Port)
 	glog.V(2).Infof("Deleting backend service %s", name)
@@ -238,6 +270,25 @@ func (b *BackendServiceSyncer) desiredBackendService(lbName string, port ingress
 	}
 }
 
+// desiredBackendServiceWithoutClusters returns the desired backend service after removing the given instance groups from the given existing backend service.
+func (b *BackendServiceSyncer) desiredBackendServiceWithoutClusters(existingBE *compute.BackendService, removeIGLinks []string) *compute.BackendService {
+	// Compute the backends to be removed.
+	removeBackends := desiredBackends(removeIGLinks)
+	removeBackendsMap := map[string]bool{}
+	for _, v := range removeBackends {
+		removeBackendsMap[v.Group] = true
+	}
+	var newBackends []*compute.Backend
+	for _, v := range existingBE.Backends {
+		if !removeBackendsMap[v.Group] {
+			newBackends = append(newBackends, v)
+		}
+	}
+	desiredBE := existingBE
+	desiredBE.Backends = newBackends
+	return desiredBE
+}
+
 func desiredBackends(igLinks []string) []*compute.Backend {
 	// Sort the slice so we get determistic results.
 	sort.Strings(igLinks)

app/kubemci/pkg/gcp/backendservice/backendservicesyncer_test.go

@@ -166,3 +166,63 @@ func TestDeleteBackendService(t *testing.T) {
 		t.Errorf("unexpected nil error, expected NotFound")
 	}
 }
+
+func TestRemoveFromClusters(t *testing.T) {
+	lbName := "lb-name"
+	port := int64(32211)
+	portName := "portName"
+	ig1Link := "ig1Link"
+	ig2Link := "ig2Link"
+	hcLink := "hcLink"
+	kubeSvcName := "ingress-svc"
+	// Should create the backend service as expected.
+	bsp := ingressbe.NewFakeBackendServices(func(op int, be *compute.BackendService) error { return nil })
+	namer := utilsnamer.NewNamer("mci1", lbName)
+	beName := namer.BeServiceName(port)
+	bss := NewBackendServiceSyncer(namer, bsp)
+	ports := []ingressbe.ServicePort{
+		{
+			Port:     port,
+			Protocol: "HTTP",
+			SvcName:  types.NamespacedName{Name: kubeSvcName},
+		},
+	}
+	// Create the backend service for 2 clusters.
+	if _, err := bss.EnsureBackendService(lbName, ports, healthcheck.HealthChecksMap{
+		port: &compute.HealthCheck{
+			SelfLink: hcLink,
+		},
+	}, NamedPortsMap{
+		port: &compute.NamedPort{
+			Port: port,
+			Name: portName,
+		},
+	}, []string{ig1Link, ig2Link}, false /*forceUpdate*/); err != nil {
+		t.Fatalf("expected no error in ensuring backend service, actual: %v", err)
+	}
+	be, err := bsp.GetGlobalBackendService(beName)
+	if err != nil {
+		t.Fatalf("expected nil error, actual: %v", err)
+	}
+	// Verify that the backend has 2 backends, one for each cluster.
+	if len(be.Backends) != 2 {
+		t.Fatalf("expected the backend service to have 2 backends for igs [%s %s], actual: %v", ig1Link, ig2Link, be.Backends)
+	}
+
+	// Verify that the backend for the second cluster is removed after running RemoveFromClusters.
+	if err := bss.RemoveFromClusters(ports, []string{ig2Link}); err != nil {
+		t.Fatalf("unexpected error in removing from clusters: %s", err)
+	}
+	be, err = bsp.GetGlobalBackendService(beName)
+	if err != nil {
+		t.Fatalf("expected nil error, actual: %v", err)
+	}
+	if len(be.Backends) != 1 || be.Backends[0].Group != ig1Link {
+		t.Fatalf("expected the backend service to have 1 backend for ig: %s, actual: %v", ig1Link, be.Backends)
+	}
+
+	// Cleanup
+	if err := bss.DeleteBackendServices(ports); err != nil {
+		t.Fatalf("unexpected error in deleting backend services: %s", err)
+	}
+}

app/kubemci/pkg/gcp/backendservice/fake_backendservicesyncer.go

@@ -61,3 +61,28 @@ func (h *FakeBackendServiceSyncer) DeleteBackendServices(ports []ingressbe.Servi
 	h.EnsuredBackendServices = nil
 	return nil
 }
+
+func (h *FakeBackendServiceSyncer) RemoveFromClusters(ports []ingressbe.ServicePort, removeIGLinks []string) error {
+	// Convert array to maps for easier lookups.
+	affectedPorts := make(map[int64]bool, len(ports))
+	for _, v := range ports {
+		affectedPorts[v.Port] = true
+	}
+	removeLinks := make(map[string]bool, len(removeIGLinks))
+	for _, v := range removeIGLinks {
+		removeLinks[v] = true
+	}
+	for _, v := range h.EnsuredBackendServices {
+		if _, has := affectedPorts[v.Port.Port]; !has {
+			continue
+		}
+		newIGLinks := []string{}
+		for _, ig := range v.IGLinks {
+			if _, has := removeLinks[ig]; !has {
+				newIGLinks = append(newIGLinks, ig)
+			}
+		}
+		v.IGLinks = newIGLinks
+	}
+	return nil
+}

app/kubemci/pkg/gcp/backendservice/interfaces.go

@@ -37,4 +37,6 @@ type BackendServiceSyncerInterface interface {
 	EnsureBackendService(lbName string, ports []ingressbe.ServicePort, hcMap healthcheck.HealthChecksMap, npMap NamedPortsMap, igLinks []string, forceUpdate bool) (BackendServicesMap, error)
 	// DeleteBackendServices deletes all backend services that would have been created by EnsureBackendService.
 	DeleteBackendServices(ports []ingressbe.ServicePort) error
+	// RemoveFromClusters removes the clusters corresponding to the given removeIGLinks from the existing backend services corresponding to the given ports.
+	RemoveFromClusters(ports []ingressbe.ServicePort, removeIGLinks []string) error
 }

app/kubemci/pkg/gcp/firewallrule/fake_firewallrulesyncer.go

@@ -50,3 +50,19 @@ func (h *FakeFirewallRuleSyncer) DeleteFirewallRules() error {
 	h.EnsuredFirewallRules = nil
 	return nil
 }
+
+func (h *FakeFirewallRuleSyncer) RemoveFromClusters(lbName string, removeIGLinks map[string][]string) error {
+	for _, v := range h.EnsuredFirewallRules {
+		if v.LBName != lbName {
+			continue
+		}
+		newIGLinks := map[string][]string{}
+		for clusterName, igValues := range v.IGLinks {
+			if _, has := removeIGLinks[clusterName]; !has {
+				newIGLinks[clusterName] = igValues
+			}
+		}
+		v.IGLinks = newIGLinks
+	}
+	return nil
+}