Skip to content

Commit

Permalink
migrate serviceaccount and rest of build controllers to new controlle…
Browse files Browse the repository at this point in the history
…r initialization
  • Loading branch information
mfojtik committed May 25, 2017
1 parent 5ef63e6 commit 047fb4d
Show file tree
Hide file tree
Showing 12 changed files with 623 additions and 205 deletions.
3 changes: 3 additions & 0 deletions pkg/authorization/authorizer/subjects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ func TestSubjects(t *testing.T) {
"system:serviceaccount:openshift-infra:build-controller",
"system:serviceaccount:openshift-infra:deployer-controller",
"system:serviceaccount:openshift-infra:template-instance-controller",
"system:serviceaccount:openshift-infra:template-instance-controller",
"system:serviceaccount:openshift-infra:build-pod-controller",
"system:serviceaccount:openshift-infra:build-controller",
),
expectedGroups: sets.NewString("RootUsers", "system:cluster-admins", "system:cluster-readers", "system:masters", "system:nodes"),
}
Expand Down
58 changes: 56 additions & 2 deletions pkg/cmd/server/bootstrappolicy/controller_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,33 @@ func init() {
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch", "update", "delete").Groups(buildGroup, legacyBuildGroup).Resources("builds").RuleOrDie(),
rbac.NewRule("get").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/optimizeddocker", "builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
rbac.NewRule("get").Groups(imageGroup, legacyImageGroup).Resources("imagestreams").RuleOrDie(),
rbac.NewRule("get", "list", "create", "delete").Groups(kapiGroup).Resources("pods").RuleOrDie(),
rbac.NewRule("get").Groups(kapiGroup).Resources("namespaces").RuleOrDie(),
eventsRule(),
},
})

// build-pod-controller
addControllerRole(rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraBuildPodControllerServiceAccountName},
Rules: []rbac.PolicyRule{
rbac.NewRule("create", "get", "list", "watch", "update", "delete").Groups(buildGroup, legacyBuildGroup).Resources("builds").RuleOrDie(),
rbac.NewRule("get", "list", "create", "delete").Groups(kapiGroup).Resources("pods").RuleOrDie(),
rbac.NewRule("get").Groups(kapiGroup).Resources("secrets").RuleOrDie(),
rbac.NewRule("get").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
// Needed for strategyrestriction admission
rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/optimizeddocker", "builds/docker", "builds/source", "builds/custom", "builds/jenkinspipeline").RuleOrDie(),
eventsRule(),
},
})

// build-config-change-controller
addControllerRole(rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraBuildConfigChangeControllerServiceAccountName},
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs").RuleOrDie(),
rbac.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs/instantiate").RuleOrDie(),
eventsRule(),
},
})
Expand Down Expand Up @@ -103,8 +126,39 @@ func init() {
},
})

// template-instance-controller
controllerRoleBindings = append(controllerRoleBindings,
rbac.NewClusterBinding(EditRoleName).SAs(DefaultOpenShiftInfraNamespace, InfraTemplateInstanceControllerServiceAccountName).BindingOrDie())

// origin-namespace-controller
addControllerRole(rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraOriginNamespaceServiceAccountName},
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch").Groups(kapiGroup).Resources("namespaces").RuleOrDie(),
rbac.NewRule("update").Groups(kapiGroup).Resources("namespaces/finalize", "namespaces/status").RuleOrDie(),
eventsRule(),
},
})

// serviceaccount-controller
addControllerRole(rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraServiceAccountControllerServiceAccountName},
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch", "create", "update", "patch", "delete").Groups(kapiGroup).Resources("serviceaccounts").RuleOrDie(),
eventsRule(),
},
})

// serviceaccount-pull-secrets-controller
addControllerRole(rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + InfraServiceAccountPullSecretsControllerServiceAccountName},
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch", "create", "update").Groups(kapiGroup).Resources("serviceaccounts").RuleOrDie(),
rbac.NewRule("get", "list", "watch", "create", "update", "patch", "delete").Groups(kapiGroup).Resources("secrets").RuleOrDie(),
rbac.NewRule("get", "list", "watch").Groups(kapiGroup).Resources("services").RuleOrDie(),
eventsRule(),
},
})
}

// ControllerRoles returns the cluster roles used by controllers
Expand Down
18 changes: 12 additions & 6 deletions pkg/cmd/server/bootstrappolicy/infra_sa_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,18 @@ import (
)

const (
InfraBuildControllerServiceAccountName = "build-controller"
InfraImageTriggerControllerServiceAccountName = "imagetrigger-controller"
ImageTriggerControllerRoleName = "system:imagetrigger-controller"
InfraDeploymentConfigControllerServiceAccountName = "deploymentconfig-controller"
InfraDeploymentTriggerControllerServiceAccountName = "deployment-trigger-controller"
InfraDeployerControllerServiceAccountName = "deployer-controller"
InfraBuildControllerServiceAccountName = "build-controller"
InfraImageTriggerControllerServiceAccountName = "imagetrigger-controller"
ImageTriggerControllerRoleName = "system:imagetrigger-controller"
InfraDeploymentConfigControllerServiceAccountName = "deploymentconfig-controller"
InfraDeploymentTriggerControllerServiceAccountName = "deployment-trigger-controller"
InfraDeployerControllerServiceAccountName = "deployer-controller"
InfraOriginNamespaceServiceAccountName = "origin-namespace-controller"
InfraServiceAccountControllerServiceAccountName = "serviceaccount-controller"
InfraServiceAccountPullSecretsControllerServiceAccountName = "serviceaccount-pull-secrets-controller"
InfraServiceAccountTokensControllerServiceAccountName = "serviceaccount-tokens-controller"
InfraBuildPodControllerServiceAccountName = "build-pod-controller"
InfraBuildConfigChangeControllerServiceAccountName = "build-config-change-controller"

InfraPersistentVolumeBinderControllerServiceAccountName = "pv-binder-controller"
PersistentVolumeBinderControllerRoleName = "system:pv-binder-controller"
Expand Down
77 changes: 77 additions & 0 deletions pkg/cmd/server/origin/controller.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,90 @@
package origin

import (
"fmt"
"io/ioutil"

"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/util/cert"
kapi "k8s.io/kubernetes/pkg/api"
kubecontroller "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/serviceaccount"

"github.com/golang/glog"
"github.com/openshift/origin/pkg/cmd/server/crypto"
"github.com/openshift/origin/pkg/cmd/server/origin/controller"
)

// NewOpenShiftControllerPreStartInitializers returns list of initializers for controllers
// that needed to be run before any other controller is started.
// Typically this has to done for the serviceaccount-tokens controller as it provides
// tokens to other controllers.
func (c *MasterConfig) NewOpenShiftControllerPreStartInitializers() (map[string]controller.InitFunc, error) {
ret := map[string]controller.InitFunc{}

saTokens := controller.ServiceAccountTokensControllerOptions{
RootClientBuilder: kubecontroller.SimpleControllerClientBuilder{
ClientConfig: &c.PrivilegedLoopbackClientConfig,
},
}

if len(c.Options.ServiceAccountConfig.PrivateKeyFile) == 0 {
glog.Infof("Skipped starting Service Account Token Manager, no private key specified")
return nil, nil
}

var err error

saTokens.PrivateKey, err = serviceaccount.ReadPrivateKey(c.Options.ServiceAccountConfig.PrivateKeyFile)
if err != nil {
return nil, fmt.Errorf("error reading signing key for Service Account Token Manager: %v", err)
}

if len(c.Options.ServiceAccountConfig.MasterCA) > 0 {
saTokens.RootCA, err = ioutil.ReadFile(c.Options.ServiceAccountConfig.MasterCA)
if err != nil {
return nil, fmt.Errorf("error reading master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
if _, err := cert.ParseCertsPEM(saTokens.RootCA); err != nil {
return nil, fmt.Errorf("error parsing master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
}

if c.Options.ControllerConfig.ServiceServingCert.Signer != nil && len(c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile) > 0 {
certFile := c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile
serviceServingCA, err := ioutil.ReadFile(certFile)
if err != nil {
return nil, fmt.Errorf("error reading ca file for Service Serving Certificate Signer: %s: %v", certFile, err)
}
if _, err := crypto.CertsFromPEM(serviceServingCA); err != nil {
return nil, fmt.Errorf("error parsing ca file for Service Serving Certificate Signer: %s: %v", certFile, err)
}

// if we have a rootCA bundle add that too. The rootCA will be used when hitting the default master service, since those are signed
// using a different CA by default. The rootCA's key is more closely guarded than ours and if it is compromised, that power could
// be used to change the trusted signers for every pod anyway, so we're already effectively trusting it.
if len(saTokens.RootCA) > 0 {
saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, saTokens.RootCA...)
saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, []byte("\n")...)
}
saTokens.ServiceServingCA = append(saTokens.ServiceServingCA, serviceServingCA...)
}
ret["serviceaccount-tokens"] = saTokens.RunController

return ret, nil
}

func (c *MasterConfig) NewOpenshiftControllerInitializers() (map[string]controller.InitFunc, error) {
ret := map[string]controller.InitFunc{}

serviceAccount := controller.ServiceAccountControllerOptions{
ManagedNames: c.Options.ServiceAccountConfig.ManagedNames,
}
ret["serviceaccount"] = serviceAccount.RunController

ret["serviceaccount-pull-secrets"] = controller.RunServiceAccountPullSecretsController
ret["origin-namespace"] = controller.RunOriginNamespaceController

// initialize build controller
storageVersion := c.Options.EtcdStorageConfig.OpenShiftStorageVersion
groupVersion := schema.GroupVersion{Group: "", Version: storageVersion}
Expand All @@ -23,6 +98,8 @@ func (c *MasterConfig) NewOpenshiftControllerInitializers() (map[string]controll
Codec: codec,
}
ret["build"] = buildControllerConfig.RunController
ret["build-pod"] = controller.RunBuildPodController
ret["build-config-change"] = controller.RunBuildConfigChangeController

// initialize apps.openshift.io controllers
vars, err := c.GetOpenShiftClientEnvVars()
Expand Down
28 changes: 28 additions & 0 deletions pkg/cmd/server/origin/controller/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
builddefaults "github.com/openshift/origin/pkg/build/admission/defaults"
buildoverrides "github.com/openshift/origin/pkg/build/admission/overrides"
buildclient "github.com/openshift/origin/pkg/build/client"
buildpodcontroller "github.com/openshift/origin/pkg/build/controller/buildpod"
buildcontrollerfactory "github.com/openshift/origin/pkg/build/controller/factory"
buildstrategy "github.com/openshift/origin/pkg/build/controller/strategy"
configapi "github.com/openshift/origin/pkg/cmd/server/api"
Expand Down Expand Up @@ -82,3 +83,30 @@ func (c *BuildControllerConfig) RunController(ctx ControllerContext) (bool, erro
deleteController.Run()
return true, nil
}

func RunBuildPodController(ctx ControllerContext) (bool, error) {
go buildpodcontroller.NewBuildPodController(
ctx.DeprecatedOpenshiftInformers.Builds().Informer(),
ctx.DeprecatedOpenshiftInformers.InternalKubernetesInformers().Core().InternalVersion().Pods(),
ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraBuildPodControllerServiceAccountName),
).Run(5, ctx.Stop)
return true, nil
}

func RunBuildConfigChangeController(ctx ControllerContext) (bool, error) {
clientName := bootstrappolicy.InfraBuildConfigChangeControllerServiceAccountName
bcInstantiator := buildclient.NewOSClientBuildConfigInstantiatorClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName))
factory := buildcontrollerfactory.BuildConfigControllerFactory{
Client: ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName),
KubeClient: ctx.ClientBuilder.KubeInternalClientOrDie(clientName),
ExternalKubeClient: ctx.ClientBuilder.ClientOrDie(clientName),
BuildConfigInstantiator: bcInstantiator,
BuildLister: buildclient.NewOSClientBuildClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
BuildConfigGetter: buildclient.NewOSClientBuildConfigClient(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
BuildDeleter: buildclient.NewBuildDeleter(ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(clientName)),
}
go factory.Create().Run()
return true, nil
}
13 changes: 13 additions & 0 deletions pkg/cmd/server/origin/controller/project.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package controller

import (
"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
projectcontroller "github.com/openshift/origin/pkg/project/controller"
)

func RunOriginNamespaceController(ctx ControllerContext) (bool, error) {
factory := projectcontroller.NamespaceControllerFactory{}
factory.KubeClient = ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraOriginNamespaceServiceAccountName)
go factory.Create().Run()
return true, nil
}
85 changes: 85 additions & 0 deletions pkg/cmd/server/origin/controller/serviceaccount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package controller

import (
"github.com/golang/glog"

kapiv1 "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/controller"
sacontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/serviceaccount"

"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
serviceaccountcontrollers "github.com/openshift/origin/pkg/serviceaccounts/controllers"
)

type ServiceAccountControllerOptions struct {
ManagedNames []string
}

func (c *ServiceAccountControllerOptions) RunController(ctx ControllerContext) (bool, error) {
if len(c.ManagedNames) == 0 {
glog.Infof("Skipped starting Service Account Manager, no managed names specified")
return false, nil
}

options := sacontroller.DefaultServiceAccountsControllerOptions()
options.ServiceAccounts = []kapiv1.ServiceAccount{}

for _, saName := range c.ManagedNames {
sa := kapiv1.ServiceAccount{}
sa.Name = saName

options.ServiceAccounts = append(options.ServiceAccounts, sa)
}

go sacontroller.NewServiceAccountsController(
ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().ServiceAccounts(),
ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().Namespaces(),
ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountControllerServiceAccountName),
options).Run(3, ctx.Stop)

return true, nil
}

type ServiceAccountTokensControllerOptions struct {
RootCA []byte
ServiceServingCA []byte
PrivateKey interface{}

RootClientBuilder controller.SimpleControllerClientBuilder
}

func (c *ServiceAccountTokensControllerOptions) RunController(ctx ControllerContext) (bool, error) {
go sacontroller.NewTokensController(
ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().ServiceAccounts(),
ctx.DeprecatedOpenshiftInformers.KubernetesInformers().Core().V1().Secrets(),
c.RootClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountTokensControllerServiceAccountName),
sacontroller.TokensControllerOptions{
TokenGenerator: serviceaccount.JWTTokenGenerator(c.PrivateKey),
RootCA: c.RootCA,
ServiceServingCA: c.ServiceServingCA,
},
).Run(int(ctx.KubeControllerContext.Options.ConcurrentSATokenSyncs), ctx.Stop)
return true, nil
}

func RunServiceAccountPullSecretsController(ctx ControllerContext) (bool, error) {
kc := ctx.ClientBuilder.KubeInternalClientOrDie(bootstrappolicy.InfraServiceAccountPullSecretsControllerServiceAccountName)

serviceaccountcontrollers.NewDockercfgDeletedController(kc, serviceaccountcontrollers.DockercfgDeletedControllerOptions{}).Run()
serviceaccountcontrollers.NewDockercfgTokenDeletedController(kc, serviceaccountcontrollers.DockercfgTokenDeletedControllerOptions{}).Run()

dockerURLsIntialized := make(chan struct{})
dockercfgController := serviceaccountcontrollers.NewDockercfgController(kc, serviceaccountcontrollers.DockercfgControllerOptions{DockerURLsIntialized: dockerURLsIntialized})
go dockercfgController.Run(5, ctx.Stop)

dockerRegistryControllerOptions := serviceaccountcontrollers.DockerRegistryServiceControllerOptions{
RegistryNamespace: "default",
RegistryServiceName: "docker-registry",
DockercfgController: dockercfgController,
DockerURLsIntialized: dockerURLsIntialized,
}
go serviceaccountcontrollers.NewDockerRegistryServiceController(kc, dockerRegistryControllerOptions).Run(10, ctx.Stop)

return true, nil
}
Loading

0 comments on commit 047fb4d

Please sign in to comment.