Refactor controller initialization (part 2)

pkg/authorization/authorizer/subjects_test.go

@@ -49,6 +49,9 @@ func TestSubjects(t *testing.T) {
 			"system:serviceaccount:openshift-infra:build-controller",
 			"system:serviceaccount:openshift-infra:deployer-controller",
 			"system:serviceaccount:openshift-infra:template-instance-controller",
+			"system:serviceaccount:openshift-infra:template-instance-controller",
+			"system:serviceaccount:openshift-infra:build-pod-controller",
+			"system:serviceaccount:openshift-infra:build-controller",
 		),
 		expectedGroups: sets.NewString("RootUsers", "system:cluster-admins", "system:cluster-readers", "system:masters", "system:nodes"),
 	}

pkg/cmd/server/bootstrappolicy/controller_policy.go

@@ -51,10 +51,33 @@ func init() {
 		Rules: []rbac.PolicyRule{
 			rbac.NewRule("get", "list", "watch", "update", "delete").Groups(buildGroup, legacyBuildGroup).Resources("builds").RuleOrDie(),
 			rbac.NewRule("get").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
-			rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
+			rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/optimizeddocker", "builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
 			rbac.NewRule("get").Groups(imageGroup, legacyImageGroup).Resources("imagestreams").RuleOrDie(),
 			rbac.NewRule("get", "list", "create", "delete").Groups(kapiGroup).Resources("pods").RuleOrDie(),
-			rbac.NewRule("get").Groups(kapiGroup).Resources("namespaces").RuleOrDie(),
+			eventsRule(),
+		},
+	})
+
+	// build-pod-controller
+	addControllerRole(rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraBuildPodControllerServiceAccountName},
+		Rules: []rbac.PolicyRule{
+			rbac.NewRule("create", "get", "list", "watch", "update", "delete").Groups(buildGroup, legacyBuildGroup).Resources("builds").RuleOrDie(),
+			rbac.NewRule("get", "list", "create", "delete").Groups(kapiGroup).Resources("pods").RuleOrDie(),
+			rbac.NewRule("get").Groups(kapiGroup).Resources("secrets").RuleOrDie(),
+			rbac.NewRule("get").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
+			// Needed for strategyrestriction admission
+			rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/optimizeddocker", "builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
+			eventsRule(),
+		},
+	})
+
+	// build-config-change-controller
+	addControllerRole(rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraBuildConfigChangeControllerServiceAccountName},
+		Rules: []rbac.PolicyRule{
+			rbac.NewRule("get", "list", "watch").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
+			rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs/instantiate").RuleOrDie(),
 			eventsRule(),
 		},
 	})
@@ -103,8 +126,39 @@ func init() {
 		},
 	})
 
+	// template-instance-controller
 	controllerRoleBindings = append(controllerRoleBindings,
 		rbac.NewClusterBinding(EditRoleName).SAs(DefaultOpenShiftInfraNamespace, InfraTemplateInstanceControllerServiceAccountName).BindingOrDie())
+
+	// origin-namespace-controller
+	addControllerRole(rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraOriginNamespaceServiceAccountName},
+		Rules: []rbac.PolicyRule{
+			rbac.NewRule("get", "list", "watch").Groups(kapiGroup).Resources("namespaces").RuleOrDie(),
+			rbac.NewRule("update").Groups(kapiGroup).Resources("namespaces/finalize", "namespaces/status").RuleOrDie(),
+			eventsRule(),
+		},
+	})
+
+	// serviceaccount-controller
+	addControllerRole(rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraServiceAccountControllerServiceAccountName},
+		Rules: []rbac.PolicyRule{
+			rbac.NewRule("get", "list", "watch", "create", "update", "patch", "delete").Groups(kapiGroup).Resources("serviceaccounts").RuleOrDie(),
+			eventsRule(),
+		},
+	})
+
+	// serviceaccount-pull-secrets-controller
+	addControllerRole(rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraServiceAccountPullSecretsControllerServiceAccountName},
+		Rules: []rbac.PolicyRule{
+			rbac.NewRule("get", "list", "watch", "create", "update").Groups(kapiGroup).Resources("serviceaccounts").RuleOrDie(),
+			rbac.NewRule("get", "list", "watch", "create", "update", "patch", "delete").Groups(kapiGroup).Resources("secrets").RuleOrDie(),
+			rbac.NewRule("get", "list", "watch").Groups(kapiGroup).Resources("services").RuleOrDie(),
+			eventsRule(),
+		},
+	})
 }
 
 // ControllerRoles returns the cluster roles used by controllers

pkg/cmd/server/bootstrappolicy/infra_sa_policy.go

@@ -23,12 +23,18 @@ import (
 )
 
 const (
-	InfraBuildControllerServiceAccountName             = "build-controller"
-	InfraImageTriggerControllerServiceAccountName      = "imagetrigger-controller"
-	ImageTriggerControllerRoleName                     = "system:imagetrigger-controller"
-	InfraDeploymentConfigControllerServiceAccountName  = "deploymentconfig-controller"
-	InfraDeploymentTriggerControllerServiceAccountName = "deployment-trigger-controller"
-	InfraDeployerControllerServiceAccountName          = "deployer-controller"
+	InfraBuildControllerServiceAccountName                     = "build-controller"
+	InfraImageTriggerControllerServiceAccountName              = "imagetrigger-controller"
+	ImageTriggerControllerRoleName                             = "system:imagetrigger-controller"
+	InfraDeploymentConfigControllerServiceAccountName          = "deploymentconfig-controller"
+	InfraDeploymentTriggerControllerServiceAccountName         = "deployment-trigger-controller"
+	InfraDeployerControllerServiceAccountName                  = "deployer-controller"
+	InfraOriginNamespaceServiceAccountName                     = "origin-namespace-controller"
+	InfraServiceAccountControllerServiceAccountName            = "serviceaccount-controller"
+	InfraServiceAccountPullSecretsControllerServiceAccountName = "serviceaccount-pull-secrets-controller"
+	InfraServiceAccountTokensControllerServiceAccountName      = "serviceaccount-tokens-controller"
+	InfraBuildPodControllerServiceAccountName                  = "build-pod-controller"
+	InfraBuildConfigChangeControllerServiceAccountName         = "build-config-change-controller"
 
 	InfraPersistentVolumeBinderControllerServiceAccountName = "pv-binder-controller"
 	PersistentVolumeBinderControllerRoleName                = "system:pv-binder-controller"

pkg/cmd/server/origin/controller.go

@@ -1,15 +1,90 @@
 package origin
 
 import (
+	"fmt"
+	"io/ioutil"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/util/cert"
 	kapi "k8s.io/kubernetes/pkg/api"
+	kubecontroller "k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/serviceaccount"
 
+	"github.com/golang/glog"
+	"github.com/openshift/origin/pkg/cmd/server/crypto"
 	"github.com/openshift/origin/pkg/cmd/server/origin/controller"
 )
 
+// NewOpenShiftControllerPreStartInitializers returns list of initializers for controllers
+// that needed to be run before any other controller is started.
+// Typically this has to done for the serviceaccount-tokens controller as it provides
+// tokens to other controllers.
+func (c *MasterConfig) NewOpenShiftControllerPreStartInitializers() (map[string]controller.InitFunc, error) {
+	ret := map[string]controller.InitFunc{}
+
+	saTokens := controller.ServiceAccountTokensControllerOptions{
+		RootClientBuilder: kubecontroller.SimpleControllerClientBuilder{
+			ClientConfig: &c.PrivilegedLoopbackClientConfig,
+		},
+	}
+
+	if len(c.Options.ServiceAccountConfig.PrivateKeyFile) == 0 {
+		glog.Infof("Skipped starting Service Account Token Manager, no private key specified")
+		return nil, nil
+	}
+
+	var err error
+
+	saTokens.PrivateKey, err = serviceaccount.ReadPrivateKey(c.Options.ServiceAccountConfig.PrivateKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading signing key for Service Account Token Manager: %v", err)
+	}
+
+	if len(c.Options.ServiceAccountConfig.MasterCA) > 0 {
+		saTokens.RootCA, err = ioutil.ReadFile(c.Options.ServiceAccountConfig.MasterCA)
+		if err != nil {
+			return nil, fmt.Errorf("error reading master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
+		}
+		if _, err := cert.ParseCertsPEM(saTokens.RootCA); err != nil {
+			return nil, fmt.Errorf("error parsing master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
+		}
+	}
+
+	if c.Options.ControllerConfig.ServiceServingCert.Signer != nil && len(c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile) > 0 {
+		certFile := c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile
+		serviceServingCA, err := ioutil.ReadFile(certFile)
+		if err != nil {
+			return nil, fmt.Errorf("error reading ca file for Service Serving Certificate Signer: %s: %v", certFile, err)
+		}
+		if _, err := crypto.CertsFromPEM(serviceServingCA); err != nil {
+			return nil, fmt.Errorf("error parsing ca file for Service Serving Certificate Signer: %s: %v", certFile, err)
+		}
+
+		// if we have a rootCA bundle add that too.  The rootCA will be used when hitting the default master service, since those are signed
+		// using a different CA by default.  The rootCA's key is more closely guarded than ours and if it is compromised, that power could
+		// be used to change the trusted signers for every pod anyway, so we're already effectively trusting it.
+		if len(saTokens.RootCA) > 0 {
+			saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, saTokens.RootCA...)
+			saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, []byte("\n")...)
+		}
+		saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, serviceServingCA...)
+	}
+	ret["serviceaccount-tokens"] = saTokens.RunController
+
+	return ret, nil
+}
+
 func (c *MasterConfig) NewOpenshiftControllerInitializers() (map[string]controller.InitFunc, error) {
 	ret := map[string]controller.InitFunc{}
 
+	serviceAccount := controller.ServiceAccountControllerOptions{
+		ManagedNames: c.Options.ServiceAccountConfig.ManagedNames,
+	}
+	ret["serviceaccount"] = serviceAccount.RunController
+
+	ret["serviceaccount-pull-secrets"] = controller.RunServiceAccountPullSecretsController
+	ret["origin-namespace"] = controller.RunOriginNamespaceController
+
 	// initialize build controller
 	storageVersion := c.Options.EtcdStorageConfig.OpenShiftStorageVersion
 	groupVersion := schema.GroupVersion{Group: "", Version: storageVersion}
@@ -23,6 +98,8 @@ func (c *MasterConfig) NewOpenshiftControllerInitializers() (map[string]controll
 		Codec: codec,
 	}
 	ret["build"] = buildControllerConfig.RunController
+	ret["build-pod"] = controller.RunBuildPodController
+	ret["build-config-change"] = controller.RunBuildConfigChangeController
 
 	// initialize apps.openshift.io controllers
 	vars, err := c.GetOpenShiftClientEnvVars()

pkg/cmd/server/origin/controller/build.go

@@ -8,6 +8,7 @@ import (
 	builddefaults "github.com/openshift/origin/pkg/build/admission/defaults"
 	buildoverrides "github.com/openshift/origin/pkg/build/admission/overrides"
 	buildclient "github.com/openshift/origin/pkg/build/client"
+	buildpodcontroller "github.com/openshift/origin/pkg/build/controller/buildpod"
 	buildcontrollerfactory "github.com/openshift/origin/pkg/build/controller/factory"
 	buildstrategy "github.com/openshift/origin/pkg/build/controller/strategy"
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
@@ -82,3 +83,30 @@ func (c *BuildControllerConfig) RunController(ctx ControllerContext) (bool, erro
 	deleteController.Run()
 	return true, nil
 }
+
+func RunBuildPodController(ctx ControllerContext) (bool, error) {
+	go buildpodcontroller.NewBuildPodController(
+		ctx.DeprecatedOpenshiftInformers.Builds().Informer(),
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Pods(),
+		ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
+		ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
+		ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
+	).Run(5, ctx.Stop)
+	return true, nil
+}
+
+func RunBuildConfigChangeController(ctx ControllerContext) (bool, error) {
+	clientName := bootstrappolicy.InfraBuildConfigChangeControllerServiceAccountName
+	bcInstantiator := buildclient.NewOSClientBuildConfigInstantiatorClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName))
+	factory := buildcontrollerfactory.BuildConfigControllerFactory{
+		Client:                  ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName),
+		KubeClient:              ctx.ClientBuilder.KubeInternalClientOrDie(clientName),
+		ExternalKubeClient:      ctx.ClientBuilder.ClientOrDie(clientName),
+		BuildConfigInstantiator: bcInstantiator,
+		BuildLister:             buildclient.NewOSClientBuildClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
+		BuildConfigGetter:       buildclient.NewOSClientBuildConfigClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
+		BuildDeleter:            buildclient.NewBuildDeleter(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
+	}
+	go factory.Create().Run()
+	return true, nil
+}

pkg/cmd/server/origin/controller/project.go

@@ -0,0 +1,15 @@
+package controller
+
+import (
+	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
+	projectcontroller "github.com/openshift/origin/pkg/project/controller"
+)
+
+func RunOriginNamespaceController(ctx ControllerContext) (bool, error) {
+	controller := projectcontroller.NewProjectFinalizerController(
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Namespaces(),
+		ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraOriginNamespaceServiceAccountName),
+	)
+	go controller.Run(ctx.Stop, 5)
+	return true, nil
+}

pkg/cmd/server/origin/controller/serviceaccount.go

@@ -0,0 +1,103 @@
+package controller
+
+import (
+	"github.com/golang/glog"
+
+	kapiv1 "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/controller"
+	sacontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
+	"k8s.io/kubernetes/pkg/serviceaccount"
+
+	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
+	serviceaccountcontrollers "github.com/openshift/origin/pkg/serviceaccounts/controllers"
+)
+
+type ServiceAccountControllerOptions struct {
+	ManagedNames []string
+}
+
+func (c *ServiceAccountControllerOptions) RunController(ctx ControllerContext) (bool, error) {
+	if len(c.ManagedNames) == 0 {
+		glog.Infof("Skipped starting Service Account Manager, no managed names specified")
+		return false, nil
+	}
+
+	options := sacontroller.DefaultServiceAccountsControllerOptions()
+	options.ServiceAccounts = []kapiv1.ServiceAccount{}
+
+	for _, saName := range c.ManagedNames {
+		sa := kapiv1.ServiceAccount{}
+		sa.Name = saName
+
+		options.ServiceAccounts = append(options.ServiceAccounts, sa)
+	}
+
+	go sacontroller.NewServiceAccountsController(
+		ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().ServiceAccounts(),
+		ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().Namespaces(),
+		ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountControllerServiceAccountName),
+		options).Run(3, ctx.Stop)
+
+	return true, nil
+}
+
+type ServiceAccountTokensControllerOptions struct {
+	RootCA           []byte
+	ServiceServingCA []byte
+	PrivateKey       interface{}
+
+	RootClientBuilder controller.SimpleControllerClientBuilder
+}
+
+func (c *ServiceAccountTokensControllerOptions) RunController(ctx ControllerContext) (bool, error) {
+	go sacontroller.NewTokensController(
+		ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().ServiceAccounts(),
+		ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().Secrets(),
+		c.RootClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountTokensControllerServiceAccountName),
+		sacontroller.TokensControllerOptions{
+			TokenGenerator:   serviceaccount.JWTTokenGenerator(c.PrivateKey),
+			RootCA:           c.RootCA,
+			ServiceServingCA: c.ServiceServingCA,
+		},
+	).Run(int(ctx.KubeControllerContext.Options.ConcurrentSATokenSyncs), ctx.Stop)
+	return true, nil
+}
+
+func RunServiceAccountPullSecretsController(ctx ControllerContext) (bool, error) {
+	kc := ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraServiceAccountPullSecretsControllerServiceAccountName)
+
+	go serviceaccountcontrollers.NewDockercfgDeletedController(
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Secrets(),
+		kc,
+		serviceaccountcontrollers.DockercfgDeletedControllerOptions{},
+	).Run(ctx.Stop)
+
+	go serviceaccountcontrollers.NewDockercfgTokenDeletedController(
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Secrets(),
+		kc,
+		serviceaccountcontrollers.DockercfgTokenDeletedControllerOptions{},
+	).Run(ctx.Stop)
+
+	dockerURLsInitialized := make(chan struct{})
+	dockercfgController := serviceaccountcontrollers.NewDockercfgController(
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().ServiceAccounts(),
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Secrets(),
+		kc,
+		serviceaccountcontrollers.DockercfgControllerOptions{DockerURLsInitialized: dockerURLsInitialized},
+	)
+	go dockercfgController.Run(5, ctx.Stop)
+
+	dockerRegistryControllerOptions := serviceaccountcontrollers.DockerRegistryServiceControllerOptions{
+		RegistryNamespace:     "default",
+		RegistryServiceName:   "docker-registry",
+		DockercfgController:   dockercfgController,
+		DockerURLsInitialized: dockerURLsInitialized,
+	}
+	go serviceaccountcontrollers.NewDockerRegistryServiceController(
+		ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Secrets(),
+		kc,
+		dockerRegistryControllerOptions,
+	).Run(10, ctx.Stop)
+
+	return true, nil
+}