From f09dabc6f5ff034e331adc786ece351a5471aff4 Mon Sep 17 00:00:00 2001 From: Charlie Drage Date: Mon, 15 May 2017 16:35:46 -0400 Subject: [PATCH] Add v3 support of Docker Compose This does a major refactor on the compose.go functions as well as brings in a new era of v3 support to Kompose. Similar to how we utilize libcompose, we utilize docker/cli's "stack deploy" code which has a built-in v3 parser. We convert the parsed structure to our own and then convert it to Kubernetes/OpenShift artifacts. --- pkg/app/app.go | 1 + pkg/loader/compose/compose.go | 304 ++++++++++++++++++++++++++++++++-- 2 files changed, 289 insertions(+), 16 deletions(-) diff --git a/pkg/app/app.go b/pkg/app/app.go index f01fd5eec7..27f541c7a2 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -210,6 +210,7 @@ func Convert(opt kobject.ConvertOptions) { komposeObject := kobject.KomposeObject{ ServiceConfigs: make(map[string]kobject.ServiceConfig), } + // Possible place to parse based on version number?? komposeObject, err = l.LoadFile(opt.InputFiles) if err != nil { log.Fatalf(err.Error()) diff --git a/pkg/loader/compose/compose.go b/pkg/loader/compose/compose.go index 542f5c315c..cb47b72b13 100644 --- a/pkg/loader/compose/compose.go +++ b/pkg/loader/compose/compose.go @@ -18,6 +18,7 @@ package compose import ( "fmt" + "io/ioutil" "net" "os" "path/filepath" @@ -28,10 +29,13 @@ import ( "k8s.io/kubernetes/pkg/api" log "github.com/Sirupsen/logrus" + "github.com/docker/cli/cli/compose/loader" + "github.com/docker/cli/cli/compose/types" "github.com/docker/libcompose/config" "github.com/docker/libcompose/lookup" "github.com/docker/libcompose/project" "github.com/fatih/structs" + "github.com/ghodss/yaml" "github.com/kubernetes-incubator/kompose/pkg/kobject" "github.com/pkg/errors" ) @@ -178,6 +182,26 @@ func loadEnvVars(envars []string) []kobject.EnvVar { return envs } +// Convert Docker Compose v3 ports to kobject.Ports +func loadV3Ports(ports []types.ServicePortConfig) ([]kobject.Ports, error) { + komposePorts := []kobject.Ports{} + + for _, port := range ports { + + // Convert to a kobject struct with ports + // TODO convert "IP" somehow + komposePorts = append(komposePorts, kobject.Ports{ + HostPort: int32(port.Published), + ContainerPort: int32(port.Target), + HostIP: "", + Protocol: api.Protocol(port.Protocol), + }) + + } + + return komposePorts, nil +} + // Load ports from compose file func loadPorts(composePorts []string) ([]kobject.Ports, error) { ports := []kobject.Ports{} @@ -273,12 +297,123 @@ func loadPorts(composePorts []string) ([]kobject.Ports, error) { return ports, nil } -// LoadFile loads compose file into KomposeObject +// LoadFile loads a compose file into KomposeObject func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) { + + // What will finally be returned komposeObject := kobject.KomposeObject{ ServiceConfigs: make(map[string]kobject.ServiceConfig), LoadedFrom: "compose", } + + // Load the json / yaml file in order to get the version value + version, err := getVersionFromFile(files[0]) + if err != nil { + return kobject.KomposeObject{}, errors.Wrap(err, "Unable to load yaml/json file for version parsing") + } + log.Debugf("Docker Compose version: %s", version) + + // Convert based on version + if version == "1" || version == "1.0" || + version == "2" || version == "2.0" { + komposeObject, err := parseV1V2ComposeWithLibcompose(files) + if err != nil { + return komposeObject, err + } + return komposeObject, nil + } else if version == "3" || version == "3.0" { + komposeObject, err := parseV3WithDockerCLI(files) + if err != nil { + return komposeObject, err + } + return komposeObject, nil + } else { + log.Fatalf("Version %s of Docker Compose is not supported. Please use Version 1 or 2.", version) + } + + return komposeObject, nil +} + +// getComposeFileDir returns compose file directory +func getComposeFileDir(inputFiles []string) (string, error) { + // Lets assume all the docker-compose files are in the same directory + inputFile := inputFiles[0] + if strings.Index(inputFile, "/") != 0 { + workDir, err := os.Getwd() + if err != nil { + return "", err + } + inputFile = filepath.Join(workDir, inputFile) + } + return filepath.Dir(inputFile), nil +} + +// The purpose of this is not to deploy, but to be able to parse +// v3 of Docker Compose into a suitable format. In this case, whatever is returned +// by docker/cli's ServiceConfig +func parseV3WithDockerCLI(files []string) (kobject.KomposeObject, error) { + + // STEP 1 + // In order to get V3 parsing to work, we have to go through some preliminary steps + // for us to hack up github.com/docker/cli + + // Gather the working directory + workingDir, err := getComposeFileDir(files) + if err != nil { + return kobject.KomposeObject{}, err + } + + // Load and then parse the YAML first! ( + loadedFile, err := ioutil.ReadFile(files[0]) + if err != nil { + return kobject.KomposeObject{}, err + } + + // Parse the Compose File + parsedComposeFile, err := loader.ParseYAML(loadedFile) + if err != nil { + return kobject.KomposeObject{}, err + } + + // Config file + configFile := types.ConfigFile{ + Filename: files[0], + Config: parsedComposeFile, + } + + // Config details + // TODO: Load environment variables + configDetails := types.ConfigDetails{ + WorkingDir: workingDir, + ConfigFiles: []types.ConfigFile{configFile}, + Environment: nil, + } + + // STEP 2 + // Finally, we load it in order to retrieve the parsed output configuration! + // This will output a Config (https://github.com/docker/cli/blob/9e413798bf1904e9d62451f2128b8ebc012b18d7/cli/compose/types/types.go#L65) + // Which is similar to our version of ServiceConfig + // returns Config + config, err := loader.Load(configDetails) + if err != nil { + return kobject.KomposeObject{}, err + } + + // STEP 3 + // Finally, we convert the object from docker/cli's ServiceConfig to our appropriate one + komposeObject, err := dockerComposeToKomposeMapping(config) + if err != nil { + return kobject.KomposeObject{}, err + } + + return komposeObject, nil +} + +// Parse Docker Compose with libcompose (only supports v1 and v2). Eventually we will +// switch to using only libcompose once v3 is supported. +func parseV1V2ComposeWithLibcompose(files []string) (kobject.KomposeObject, error) { + + // Gather the appropriate context for parsing context := &project.Context{} context.ComposeFiles = files @@ -301,18 +436,150 @@ func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) { } } - // load compose file into composeObject + // Load the context and let's start parsing composeObject := project.NewProject(context, nil, nil) err := composeObject.Parse() if err != nil { return kobject.KomposeObject{}, errors.Wrap(err, "composeObject.Parse() failed, Failed to load compose file") } + // Check for all unsupported keys + // TODO: Try to get to 100%, if there's something unsupported, we need to explain as to WHY. noSupKeys := checkUnsupportedKey(composeObject) for _, keyName := range noSupKeys { log.Warningf("Unsupported %s key - ignoring", keyName) } + // Map the parsed struct to a struct we understand (kobject) + komposeObject, err := libComposeToKomposeMapping(composeObject) + if err != nil { + return kobject.KomposeObject{}, err + } + + return komposeObject, nil +} + +func handleServiceType(ServiceType string) (string, error) { + switch strings.ToLower(ServiceType) { + case "", "clusterip": + return string(api.ServiceTypeClusterIP), nil + case "nodeport": + return string(api.ServiceTypeNodePort), nil + case "loadbalancer": + return string(api.ServiceTypeLoadBalancer), nil + default: + return "", errors.New("Unknown value " + ServiceType + " , supported values are 'NodePort, ClusterIP or LoadBalancer'") + } +} + +func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.KomposeObject, error) { + + // Step 1. Initialize what's going to be returned + komposeObject := kobject.KomposeObject{ + ServiceConfigs: make(map[string]kobject.ServiceConfig), + LoadedFrom: "compose", + } + + // Step 2. Parse through the object and conver it to kobject.KomposeObject! + // Here we "clean up" the service configuration so we return something that includes + // all relevant information as well as avoid the unsupported keys as well. + for _, composeServiceConfig := range composeObject.Services { + + // Standard import + // No need to modify before importation + name := composeServiceConfig.Name + serviceConfig := kobject.ServiceConfig{} + serviceConfig.Image = composeServiceConfig.Image + serviceConfig.WorkingDir = composeServiceConfig.WorkingDir + serviceConfig.Annotations = map[string]string(composeServiceConfig.Labels) + serviceConfig.CPUQuota = int64(composeServiceConfig.CPUQuota) + serviceConfig.CapAdd = composeServiceConfig.CapAdd + serviceConfig.CapDrop = composeServiceConfig.CapDrop + serviceConfig.Expose = composeServiceConfig.Expose + serviceConfig.Privileged = composeServiceConfig.Privileged + serviceConfig.Restart = composeServiceConfig.Restart + serviceConfig.User = composeServiceConfig.User + serviceConfig.VolumesFrom = composeServiceConfig.VolumesFrom + serviceConfig.Stdin = composeServiceConfig.StdinOpen + serviceConfig.Tty = composeServiceConfig.Tty + serviceConfig.MemLimit = composeServiceConfig.MemLimit + serviceConfig.TmpFs = composeServiceConfig.Tmpfs + serviceConfig.ContainerName = composeServiceConfig.ContainerName + serviceConfig.Command = composeServiceConfig.Entrypoint + serviceConfig.Args = composeServiceConfig.Command + + // Build support? + // Not supported with dockerCompose? Odd.. Will have to implement manually? + // serviceConfig.Build = composeServiceConfig.Build.Context + // serviceConfig.Dockerfile = composeServiceConfig.Build.Dockerfile + + // Gather the environment values + // DockerCompose uses map[string]*string while we use []string + // So let's convert that using this hack + var envKeys []string + for key, _ := range composeServiceConfig.Environment { + envKeys = append(envKeys, key) + } + envs := loadEnvVars(envKeys) + serviceConfig.Environment = envs + + // Parse the ports + // v3 uses a new format called "long syntax" starting in 3.2 + // https://docs.docker.com/compose/compose-file/#ports + ports, err := loadV3Ports(composeServiceConfig.Ports) + if err != nil { + return kobject.KomposeObject{}, errors.Wrap(err, "loadPorts failed. "+name+" failed to load ports from compose file") + } + serviceConfig.Port = ports + + // Parse the volumes + if composeServiceConfig.Volumes != nil { + for _, volume := range composeServiceConfig.Volumes.Volumes { + v := normalizeServiceNames(volume.String()) + serviceConfig.Volumes = append(serviceConfig.Volumes, v) + } + } + + // Label handler + // Labels used to influence conversion of kompose will be handled + // from here for docker-compose. Each loader will have such handler. + for key, value := range composeServiceConfig.Labels { + switch key { + case "kompose.service.type": + serviceType, err := handleServiceType(value) + if err != nil { + return kobject.KomposeObject{}, errors.Wrap(err, "handleServiceType failed") + } + + serviceConfig.ServiceType = serviceType + case "kompose.service.expose": + serviceConfig.ExposeService = strings.ToLower(value) + } + } + + // Log if the name will been changed + if normalizeServiceNames(name) != name { + log.Infof("Service name in docker-compose has been changed from %q to %q", name, normalizeServiceNames(name)) + } + + // Final step, add to the array! + komposeObject.ServiceConfigs[normalizeServiceNames(name)] = serviceConfig + } + + return komposeObject, nil +} + +// Uses libcompose's APIProject type and converts it to a Kompose object for us to understand +func libComposeToKomposeMapping(composeObject *project.Project) (kobject.KomposeObject, error) { + + // Initialize what's going to be returned + komposeObject := kobject.KomposeObject{ + ServiceConfigs: make(map[string]kobject.ServiceConfig), + LoadedFrom: "compose", + } + + // Here we "clean up" the service configuration so we return something that includes + // all relevant information as well as avoid the unsupported keys as well. for name, composeServiceConfig := range composeObject.ServiceConfigs.All() { serviceConfig := kobject.ServiceConfig{} serviceConfig.Image = composeServiceConfig.Image @@ -377,23 +644,28 @@ func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) { log.Infof("Service name in docker-compose has been changed from %q to %q", name, normalizeServiceNames(name)) } } - return komposeObject, nil } -func handleServiceType(ServiceType string) (string, error) { - switch strings.ToLower(ServiceType) { - case "", "clusterip": - return string(api.ServiceTypeClusterIP), nil - case "nodeport": - return string(api.ServiceTypeNodePort), nil - case "loadbalancer": - return string(api.ServiceTypeLoadBalancer), nil - default: - return "", errors.New("Unknown value " + ServiceType + " , supported values are 'NodePort, ClusterIP or LoadBalancer'") - } -} - func normalizeServiceNames(svcName string) string { return strings.Replace(svcName, "_", "-", -1) } + +func getVersionFromFile(file string) (string, error) { + type ComposeVersion struct { + Version string `json:"version"` // This affects YAML as well + } + var version ComposeVersion + + loadedFile, err := ioutil.ReadFile(file) + if err != nil { + return "", err + } + + err = yaml.Unmarshal(loadedFile, &version) + if err != nil { + return "", err + } + + return version.Version, nil +}