Skip to content

Commit

Permalink
Add v3 support
Browse files Browse the repository at this point in the history 
This does a major refactor on the compose.go functions as well as brings
in a new era of v3 support to Kompose.

Similar to how we utilize libcompose, we utilize docker/cli's "stack
deploy" code which has a built-in v3 parser. We convert the parsed
structure to our own and then convert it to Kubernetes/OpenShift
artifacts.
  • Loading branch information
@cdrage
cdrage committed Jun 5, 2017
1 parent 9197799 commit 42178ab
Show file tree
Hide file tree
Showing 11 changed files with 2,330 additions and 6 deletions.
317 changes: 311 additions & 6 deletions pkg/loader/compose/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,25 @@ package compose

import (
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"

libcomposeyaml "github.com/docker/libcompose/yaml"
"k8s.io/kubernetes/pkg/api"

log "github.com/Sirupsen/logrus"
"github.com/docker/cli/cli/compose/loader"
"github.com/docker/cli/cli/compose/types"
"github.com/docker/libcompose/config"
"github.com/docker/libcompose/lookup"
"github.com/docker/libcompose/project"
"github.com/fatih/structs"
"github.com/ghodss/yaml"
"github.com/kubernetes-incubator/kompose/pkg/kobject"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -177,6 +182,50 @@ func loadEnvVars(envars []string) []kobject.EnvVar {
return envs
}

// Convert the Docker Compose v3 volumes to []string (the old way)
// TODO: Check to see if it's a "bind" or "volume". Ignore for now.
// See: https://docs.docker.com/compose/compose-file/#long-syntax-2
func loadV3Volumes(volumes []types.ServiceVolumeConfig) []string {
var volArray []string
for _, vol := range volumes {

// There will *always* be Source when parsing
v := vol.Source

if vol.Target != "" {
v = v + ":" + vol.Target
}

if vol.ReadOnly {
v = v + ":ro"
}

volArray = append(volArray, v)
}
return volArray
}

// Convert Docker Compose v3 ports to kobject.Ports
func loadV3Ports(ports []types.ServicePortConfig) []kobject.Ports {
komposePorts := []kobject.Ports{}

for _, port := range ports {

// Convert to a kobject struct with ports
// NOTE: V3 doesn't use IP (they utilize Swarm instead for host-networking).
// Thus, IP is blank.
komposePorts = append(komposePorts, kobject.Ports{
HostPort: int32(port.Published),
ContainerPort: int32(port.Target),
HostIP: "",
Protocol: api.Protocol(strings.ToUpper(string(port.Protocol))),
})

}

return komposePorts
}

// Load ports from compose file
func loadPorts(composePorts []string) ([]kobject.Ports, error) {
ports := []kobject.Ports{}
Expand Down Expand Up @@ -272,12 +321,131 @@ func loadPorts(composePorts []string) ([]kobject.Ports, error) {
return ports, nil
}

// LoadFile loads compose file into KomposeObject
// LoadFile loads a compose file into KomposeObject
func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
komposeObject := kobject.KomposeObject{
ServiceConfigs: make(map[string]kobject.ServiceConfig),
LoadedFrom: "compose",

// Load the json / yaml file in order to get the version value
var version string

for _, file := range files {
composeVersion, err := getVersionFromFile(file)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "Unable to load yaml/json file for version parsing")
}

// Check that the previous file loaded matches.
if len(files) > 0 && version != "" && version != composeVersion {
return kobject.KomposeObject{}, errors.New("All files passed must be the same version")
}
version = composeVersion
}

log.Debugf("Docker Compose version: %s", version)

// Convert based on version
switch version {
// Use libcompose for 1 or 2
// If blank, it's assumed it's 1 or 2
case "", "1", "1.0", "2", "2.0":
komposeObject, err := parseV1V2ComposeWithLibcompose(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
// Use docker/cli for 3
case "3", "3.0":
komposeObject, err := parseV3WithDockerCLI(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
default:
return kobject.KomposeObject{}, fmt.Errorf("Version %s of Docker Compose is not supported. Please use version 1, 2 or 3", version)
}

}

// getComposeFileDir returns compose file directory
// Assume all the docker-compose files are in the same directory
// TODO: fix (check if file exists)
func getComposeFileDir(inputFiles []string) (string, error) {
inputFile := inputFiles[0]
if strings.Index(inputFile, "/") != 0 {
workDir, err := os.Getwd()
if err != nil {
return "", errors.Wrap(err, "Unable to retrieve compose file directory")
}
inputFile = filepath.Join(workDir, inputFile)
}
return filepath.Dir(inputFile), nil
}

// The purpose of this is not to deploy, but to be able to parse
// v3 of Docker Compose into a suitable format. In this case, whatever is returned
// by docker/cli's ServiceConfig
func parseV3WithDockerCLI(files []string) (kobject.KomposeObject, error) {

// In order to get V3 parsing to work, we have to go through some preliminary steps
// for us to hack up github.com/docker/cli in order to correctly convert to a kobject.KomposeObject

// Gather the working directory
workingDir, err := getComposeFileDir(files)
if err != nil {
return kobject.KomposeObject{}, err
}

// Load and then parse the YAML first!
loadedFile, err := ioutil.ReadFile(files[0])
if err != nil {
return kobject.KomposeObject{}, err
}

// Parse the Compose File
parsedComposeFile, err := loader.ParseYAML(loadedFile)
if err != nil {
return kobject.KomposeObject{}, err
}

// Config file
configFile := types.ConfigFile{
Filename: files[0],
Config: parsedComposeFile,
}

// Config details
// Environment is nil as docker/cli loads the appropriate environmental values itself
configDetails := types.ConfigDetails{
WorkingDir: workingDir,
ConfigFiles: []types.ConfigFile{configFile},
Environment: nil,
}

// Actual config
// We load it in order to retrieve the parsed output configuration!
// This will output a github.com/docker/cli ServiceConfig
// Which is similar to our version of ServiceConfig
config, err := loader.Load(configDetails)
if err != nil {
return kobject.KomposeObject{}, err
}

// TODO: Check all "unsupported" keys and output details
// Specifically, keys such as "volumes_from" are not supported in V3.

// Finally, we convert the object from docker/cli's ServiceConfig to our appropriate one
komposeObject, err := dockerComposeToKomposeMapping(config)
if err != nil {
return kobject.KomposeObject{}, err
}

return komposeObject, nil
}

// Parse Docker Compose with libcompose (only supports v1 and v2). Eventually we will
// switch to using only libcompose once v3 is supported.
func parseV1V2ComposeWithLibcompose(files []string) (kobject.KomposeObject, error) {

// Gather the appropriate context for parsing
context := &project.Context{}
context.ComposeFiles = files

Expand All @@ -300,7 +468,7 @@ func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
}
}

// load compose file into composeObject
// Load the context and let's start parsing
composeObject := project.NewProject(context, nil, nil)
err := composeObject.Parse()
if err != nil {
Expand All @@ -312,6 +480,125 @@ func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
log.Warningf("Unsupported %s key - ignoring", keyName)
}

// Map the parsed struct to a struct we understand (kobject)
komposeObject, err := libComposeToKomposeMapping(composeObject)
if err != nil {
return kobject.KomposeObject{}, err
}

return komposeObject, nil
}

func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.KomposeObject, error) {

// Step 1. Initialize what's going to be returned
komposeObject := kobject.KomposeObject{
ServiceConfigs: make(map[string]kobject.ServiceConfig),
LoadedFrom: "compose",
}

// Step 2. Parse through the object and conver it to kobject.KomposeObject!
// Here we "clean up" the service configuration so we return something that includes
// all relevant information as well as avoid the unsupported keys as well.
for _, composeServiceConfig := range composeObject.Services {

// Standard import
// No need to modify before importation
name := composeServiceConfig.Name
serviceConfig := kobject.ServiceConfig{}
serviceConfig.Image = composeServiceConfig.Image
serviceConfig.WorkingDir = composeServiceConfig.WorkingDir
serviceConfig.Annotations = map[string]string(composeServiceConfig.Labels)
serviceConfig.CapAdd = composeServiceConfig.CapAdd
serviceConfig.CapDrop = composeServiceConfig.CapDrop
serviceConfig.Expose = composeServiceConfig.Expose
serviceConfig.Privileged = composeServiceConfig.Privileged
serviceConfig.Restart = composeServiceConfig.Restart
serviceConfig.User = composeServiceConfig.User
serviceConfig.Stdin = composeServiceConfig.StdinOpen
serviceConfig.Tty = composeServiceConfig.Tty
serviceConfig.TmpFs = composeServiceConfig.Tmpfs
serviceConfig.ContainerName = composeServiceConfig.ContainerName
serviceConfig.Command = composeServiceConfig.Entrypoint
serviceConfig.Args = composeServiceConfig.Command

// This is a bit messy since we use yaml.MemStringorInt
// TODO: Refactor yaml.MemStringorInt in kobject.go to int64
// Since Deploy.Resources.Limits does not initialize, we must check type Resources before continuing
if (composeServiceConfig.Deploy.Resources != types.Resources{}) {
serviceConfig.MemLimit = libcomposeyaml.MemStringorInt(composeServiceConfig.Deploy.Resources.Limits.MemoryBytes)
}

// POOF. volumes_From is gone in v3. docker/cli will error out of volumes_from is added in v3
// serviceConfig.VolumesFrom = composeServiceConfig.VolumesFrom

// TODO
// Build is not yet supported, see:
// https://github.com/docker/cli/blob/master/cli/compose/types/types.go#L9
// We will have to *manually* add this / parse.
// serviceConfig.Build = composeServiceConfig.Build.Context
// serviceConfig.Dockerfile = composeServiceConfig.Build.Dockerfile

// Gather the environment values
// DockerCompose uses map[string]*string while we use []string
// So let's convert that using this hack
var envKeys []string
for key := range composeServiceConfig.Environment {
envKeys = append(envKeys, key)
}
envs := loadEnvVars(envKeys)
serviceConfig.Environment = envs

// Parse the ports
// v3 uses a new format called "long syntax" starting in 3.2
// https://docs.docker.com/compose/compose-file/#ports
serviceConfig.Port = loadV3Ports(composeServiceConfig.Ports)

// Parse the volumes
// Again, in v3, we use the "long syntax" for volumes in terms of parsing
// https://docs.docker.com/compose/compose-file/#long-syntax-2
serviceConfig.Volumes = loadV3Volumes(composeServiceConfig.Volumes)

// Label handler
// Labels used to influence conversion of kompose will be handled
// from here for docker-compose. Each loader will have such handler.
for key, value := range composeServiceConfig.Labels {
switch key {
case "kompose.service.type":
serviceType, err := handleServiceType(value)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "handleServiceType failed")
}

serviceConfig.ServiceType = serviceType
case "kompose.service.expose":
serviceConfig.ExposeService = strings.ToLower(value)
}
}

// Log if the name will been changed
if normalizeServiceNames(name) != name {
log.Infof("Service name in docker-compose has been changed from %q to %q", name, normalizeServiceNames(name))
}

// Final step, add to the array!
komposeObject.ServiceConfigs[normalizeServiceNames(name)] = serviceConfig
}

return komposeObject, nil
}

// Uses libcompose's APIProject type and converts it to a Kompose object for us to understand
func libComposeToKomposeMapping(composeObject *project.Project) (kobject.KomposeObject, error) {

// Initialize what's going to be returned
komposeObject := kobject.KomposeObject{
ServiceConfigs: make(map[string]kobject.ServiceConfig),
LoadedFrom: "compose",
}

// Here we "clean up" the service configuration so we return something that includes
// all relevant information as well as avoid the unsupported keys as well.
for name, composeServiceConfig := range composeObject.ServiceConfigs.All() {
serviceConfig := kobject.ServiceConfig{}
serviceConfig.Image = composeServiceConfig.Image
Expand Down Expand Up @@ -388,7 +675,6 @@ func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
log.Infof("Service name in docker-compose has been changed from %q to %q", name, normalizeServiceNames(name))
}
}

return komposeObject, nil
}

Expand All @@ -408,3 +694,22 @@ func handleServiceType(ServiceType string) (string, error) {
func normalizeServiceNames(svcName string) string {
return strings.Replace(svcName, "_", "-", -1)
}

func getVersionFromFile(file string) (string, error) {
type ComposeVersion struct {
Version string `json:"version"` // This affects YAML as well
}
var version ComposeVersion

loadedFile, err := ioutil.ReadFile(file)
if err != nil {
return "", err
}

err = yaml.Unmarshal(loadedFile, &version)
if err != nil {
return "", err
}

return version.Version, nil
}
Loading

0 comments on commit 42178ab

Please sign in to comment.