diff --git a/Makefile b/Makefile index 0b2176a..9ff7905 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,8 @@ binary: dependencies test: dependencies # Run tests. TODO: more tests - GOPATH="${GOPATH}" go test -tags "${BUILDTAGS}" "${PKGNAME}/model" + #GOPATH="${GOPATH}" go test -tags "${BUILDTAGS}" "${PKGNAME}/model" + GOPATH="${GOPATH}" go test -tags "${BUILDTAGS}" "${PKGNAME}/model/compose" format: # Format the go code diff --git a/README.md b/README.md index dc2465d..a127915 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ cntnr uses [runc/libcontainer](https://github.com/opencontainers/runc/blob/v1.0. - [runrootless](https://github.com/AkihiroSuda/runrootless) - [singularity](http://singularity.lbl.gov/) - [orca-build](https://github.com/cyphar/orca-build) +- [rkt-compose](https://github.com/mgoltzsche/rkt-compose) ## Roadmap diff --git a/cmd/compose.go b/cmd/compose.go index de6b42c..be4d9b5 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -16,6 +16,7 @@ package cmd import ( "github.com/mgoltzsche/cntnr/model" + "github.com/mgoltzsche/cntnr/model/compose" "github.com/spf13/cobra" ) @@ -42,7 +43,7 @@ func runComposeRun(cmd *cobra.Command, args []string) error { return usageError("No compose file argument provided") } - project, err := model.LoadProject(args[0], loggers.Warn) + project, err := compose.Load(args[0], ".", compose.GetEnv(), loggers.Warn) if err != nil { return err } diff --git a/cmd/serviceflags.go b/cmd/serviceflags.go index ac9ba2d..de03f0e 100644 --- a/cmd/serviceflags.go +++ b/cmd/serviceflags.go @@ -79,7 +79,8 @@ func (c *bundleFlags) InitRunFlags(f *pflag.FlagSet) { func (c *bundleFlags) curr() *model.Service { if c.app == nil { - c.app = model.NewService("") + s := model.NewService("") + c.app = &s } return c.app } @@ -300,7 +301,7 @@ type cVolumeMount bundleFlags func (c *cVolumeMount) Set(s string) (err error) { v := model.VolumeMount{} - if err = model.ParseVolumeMount(s, &v); err != nil { + if err = model.ParseBindMount(s, &v); err != nil { return } v.Source, err = filepath.Abs(v.Source) diff --git a/model/compose/dctransform.go b/model/compose/dctransform.go new file mode 100644 index 0000000..36de0c5 --- /dev/null +++ b/model/compose/dctransform.go @@ -0,0 +1,242 @@ +package compose + +import ( + "io/ioutil" + "os" + "strings" + + "github.com/docker/cli/cli/compose/loader" + "github.com/docker/cli/cli/compose/types" + "github.com/mgoltzsche/cntnr/log" + "github.com/mgoltzsche/cntnr/model" + exterrors "github.com/mgoltzsche/cntnr/pkg/errors" + "github.com/mgoltzsche/cntnr/pkg/sliceutils" + "github.com/pkg/errors" +) + +// +// Currently only Docker Compose 3 schema is supported. +// Let's hope soon the older schema versions are supported as well +// when this is merged: https://github.com/docker/cli/pull/573 +// and containers/image updated their github.com/docker/docker dependency +// + +// TODO: use project +func Load(file, cwd string, env map[string]string, warn log.Logger) (r *model.CompoundServices, err error) { + defer exterrors.Wrapd(&err, "load docker compose file") + b, err := ioutil.ReadFile(file) + if err != nil { + return + } + dcyml, err := loader.ParseYAML(b) + if err != nil { + return + } + cfg, err := loader.Load(types.ConfigDetails{ + WorkingDir: cwd, + ConfigFiles: []types.ConfigFile{types.ConfigFile{file, dcyml}}, + Environment: env, + }) + if err != nil { + return + } + return transform(cfg, cwd, warn) +} + +func GetEnv() map[string]string { + r := map[string]string{} + for _, entry := range os.Environ() { + s := strings.SplitN(entry, "=", 2) + if len(s) == 2 { + r[s[0]] = s[1] + } else { + r[s[0]] = "" + } + } + return r +} + +func transform(cfg *types.Config, cwd string, warn log.Logger) (r *model.CompoundServices, err error) { + services, err := toServices(cfg.Services) + if err != nil { + return + } + r = &model.CompoundServices{ + Dir: cwd, + Volumes: toVolumes(cfg.Volumes, warn), + Services: services, + // TODO: map networks, secrets + } + return +} + +func toVolumes(vols map[string]types.VolumeConfig, warn log.Logger) map[string]model.Volume { + r := map[string]model.Volume{} + for name, vol := range vols { + v := model.Volume{} + if vol.External.External { + v.External = vol.Name + if vol.External.Name != "" { + v.External = vol.External.Name + } + } else { + warn.Printf("adding unsupported volume %v as temporary volume", vol) + } + r[name] = v + } + return r +} + +func toServices(services []types.ServiceConfig) (r map[string]model.Service, err error) { + r = map[string]model.Service{} + for _, service := range services { + if r[service.Name], err = toService(service); err != nil { + return + } + } + return +} + +func toService(s types.ServiceConfig) (r model.Service, err error) { + r = model.NewService(s.Name) + r.Build = toBuild(s.Build) + r.CapAdd = s.CapAdd + r.CapDrop = s.CapDrop + // s.CgroupParent + r.Command = []string(s.Command) + // TODO: + // DependsOn + // CredentialSpec + // Deploy + // Devices + r.Dns = []string(s.DNS) + r.DnsSearch = []string(s.DNSSearch) + r.Domainname = s.DomainName + r.Entrypoint = []string(s.Entrypoint) + r.Environment = toStringMap(s.Environment) + // EnvFile + r.Expose = []string(s.Expose) + // ExternalLinks + if r.ExtraHosts, err = toExtraHosts(s.ExtraHosts); err != nil { + return + } + r.Hostname = s.ContainerName + // Healthcheck + r.Image = s.Image + // Ipc + // Labels + // Links + // Logging + // MacAddress + // NetworkMode + // Pid + if r.Ports, err = toPorts(s.Ports); err != nil { + return + } + // Privileged + r.ReadOnly = s.ReadOnly + // Restart + // Secrets + // SecurityOpt + r.StdinOpen = s.StdinOpen + r.StopGracePeriod = s.StopGracePeriod + r.StopSignal = s.StopSignal + // Tmpfs + r.Tty = s.Tty + // Ulimits + r.User = toUser(s.User) + r.Volumes = toVolumeMounts(s.Volumes) + r.Cwd = s.WorkingDir + // Isolation + return +} + +func toBuild(s types.BuildConfig) (r *model.ImageBuild) { + if s.Context != "" || s.Dockerfile != "" { + r = &model.ImageBuild{ + Context: s.Context, + Dockerfile: s.Dockerfile, + Args: toStringMap(s.Args), + } + } + return +} + +func toStringMap(m types.MappingWithEquals) map[string]string { + r := map[string]string{} + for k, v := range (map[string]*string)(m) { + if v == nil { + r[k] = "" + } else { + r[k] = *v + } + } + return r +} + +func toExtraHosts(hl types.HostsList) ([]model.ExtraHost, error) { + l := []string(hl) + r := make([]model.ExtraHost, 0, len(l)) + for _, h := range hl { + he := strings.SplitN(h, ":", 2) + if len(he) != 2 { + return nil, errors.Errorf("invalid extra_hosts entry: expected format host:ip but was %q", h) + } + r = append(r, model.ExtraHost{ + Name: he[0], + Ip: he[1], + }) + } + return r, nil +} + +func toPorts(ports []types.ServicePortConfig) (r []model.PortBinding, err error) { + r = make([]model.PortBinding, 0, len(ports)) + for _, p := range ports { + if p.Target > 65535 { + return nil, errors.Errorf("invalid target port %d exceeded range", p.Target) + } + if p.Published > 65535 { + return nil, errors.Errorf("invalid published port %d exceeded range", p.Published) + } + r = append(r, model.PortBinding{ + Target: uint16(p.Target), + Published: uint16(p.Published), + Protocol: p.Protocol, + }) + // TODO: checkout p.Mode + } + return +} + +func toUser(s string) (u *model.User) { + ug := strings.SplitN(s, ":", 2) + if len(ug) == 2 { + u = &model.User{ug[0], ug[1]} + } else { + u = &model.User{ug[0], ug[0]} + } + return +} + +func toVolumeMounts(vols []types.ServiceVolumeConfig) []model.VolumeMount { + r := []model.VolumeMount{} + for _, vol := range vols { + var opts []string + if vol.Bind != nil && vol.Bind.Propagation != "" { + opts = strings.Split(vol.Bind.Propagation, ":") + } + if vol.ReadOnly { + sliceutils.AddToSet(&opts, "ro") + } + // TODO: resolve entry of type 'volume' + r = append(r, model.VolumeMount{ + Type: vol.Type, // 'volume', 'bind' or 'tmpfs' + Source: vol.Source, + Target: vol.Target, + Options: opts, + // TODO: Tmpfs, Consistency + }) + } + return r +} diff --git a/model/compose/dctransform_test.go b/model/compose/dctransform_test.go new file mode 100644 index 0000000..74217e5 --- /dev/null +++ b/model/compose/dctransform_test.go @@ -0,0 +1,27 @@ +package compose + +import ( + "fmt" + "io/ioutil" + "testing" + + "github.com/mgoltzsche/cntnr/log" + "github.com/mgoltzsche/cntnr/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoad(t *testing.T) { + dcFile := "../../vendor/github.com/docker/cli/cli/compose/loader/full-example.yml" + b, err := ioutil.ReadFile("full-example.json") + require.NoError(t, err) + expected, err := model.FromJSON(b) + require.NoError(t, err) + env := map[string]string{} + env["HOME"] = "/home/user" + actual, err := Load(dcFile, "../../vendor/github.com/docker/cli/cli/compose/loader", env, log.NewNopLogger()) + require.NoError(t, err) + fmt.Println(actual.JSON()) + assert.Equal(t, expected.Services, actual.Services) + assert.Equal(t, expected.Volumes, actual.Volumes) +} diff --git a/model/compose/full-example.json b/model/compose/full-example.json new file mode 100644 index 0000000..3000aad --- /dev/null +++ b/model/compose/full-example.json @@ -0,0 +1,241 @@ +{ + "services": { + "foo": { + "image": "redis", + "build": { + "context": "./dir", + "dockerfile": "Dockerfile", + "args": { + "foo": "bar" + } + }, + "entrypoint": [ + "/code/entrypoint.sh", + "-p", + "3000" + ], + "command": [ + "bundle", + "exec", + "thin", + "-p", + "3000" + ], + "working_dir": "/code", + "environment": { + "BAR": "bar_from_env_file_2", + "BAZ": "baz_from_service_def", + "FOO": "foo_from_env_file", + "QUX": "" + }, + "user": { + "uid": "someone", + "gid": "someone" + }, + "cap_add": [ + "ALL" + ], + "cap_drop": [ + "NET_ADMIN", + "SYS_ADMIN" + ], + "seccomp": "default", + "hostname": "my-web-container", + "domainname": "foo.com", + "dns": [ + "8.8.8.8", + "9.9.9.9" + ], + "dns_search": [ + "dc1.example.com", + "dc2.example.com" + ], + "extra_hosts": [ + { + "name": "somehost", + "ip": "162.242.195.82" + }, + { + "name": "otherhost", + "ip": "50.31.209.229" + } + ], + "ports": [ + { + "target": 3000, + "protocol": "tcp" + }, + { + "target": 3000, + "protocol": "tcp" + }, + { + "target": 3001, + "protocol": "tcp" + }, + { + "target": 3002, + "protocol": "tcp" + }, + { + "target": 3003, + "protocol": "tcp" + }, + { + "target": 3004, + "protocol": "tcp" + }, + { + "target": 3005, + "protocol": "tcp" + }, + { + "target": 8000, + "published": 8000, + "protocol": "tcp" + }, + { + "target": 8080, + "published": 9090, + "protocol": "tcp" + }, + { + "target": 8081, + "published": 9091, + "protocol": "tcp" + }, + { + "target": 22, + "published": 49100, + "protocol": "tcp" + }, + { + "target": 8001, + "published": 8001, + "protocol": "tcp" + }, + { + "target": 5000, + "published": 5000, + "protocol": "tcp" + }, + { + "target": 5001, + "published": 5001, + "protocol": "tcp" + }, + { + "target": 5002, + "published": 5002, + "protocol": "tcp" + }, + { + "target": 5003, + "published": 5003, + "protocol": "tcp" + }, + { + "target": 5004, + "published": 5004, + "protocol": "tcp" + }, + { + "target": 5005, + "published": 5005, + "protocol": "tcp" + }, + { + "target": 5006, + "published": 5006, + "protocol": "tcp" + }, + { + "target": 5007, + "published": 5007, + "protocol": "tcp" + }, + { + "target": 5008, + "published": 5008, + "protocol": "tcp" + }, + { + "target": 5009, + "published": 5009, + "protocol": "tcp" + }, + { + "target": 5010, + "published": 5010, + "protocol": "tcp" + } + ], + "stdin_open": true, + "tty": true, + "read_only": true, + "expose": [ + "3000", + "8000" + ], + "volumes": [ + { + "type": "volume", + "target": "/var/lib/mysql" + }, + { + "type": "bind", + "source": "/opt/data", + "target": "/var/lib/mysql" + }, + { + "type": "bind", + "source": "../../vendor/github.com/docker/cli/cli/compose/loader", + "target": "/code" + }, + { + "type": "bind", + "source": "../../vendor/github.com/docker/cli/cli/compose/loader/static", + "target": "/var/www/html" + }, + { + "type": "bind", + "source": "/home/user/configs", + "target": "/etc/configs/", + "options": [ + "ro" + ] + }, + { + "type": "volume", + "source": "datavolume", + "target": "/var/lib/mysql" + }, + { + "type": "bind", + "source": "../../vendor/github.com/docker/cli/cli/compose/loader/opt", + "target": "/opt" + }, + { + "type": "tmpfs", + "target": "/opt" + } + ], + "stop_signal": "SIGUSR1", + "stop_grace_period": 20000000000 + } + }, + "volumes": { + "another-volume": {}, + "external-volume": { + "external": "external-volume" + }, + "external-volume3": { + "external": "this-is-volume3" + }, + "other-external-volume": { + "external": "my-cool-volume" + }, + "other-volume": {}, + "some-volume": {} + } +} \ No newline at end of file diff --git a/model/transform.go b/model/ocitransform.go similarity index 100% rename from model/transform.go rename to model/ocitransform.go diff --git a/model/reader.go b/model/reader.go index ceddf1b..56a5bc2 100644 --- a/model/reader.go +++ b/model/reader.go @@ -19,12 +19,12 @@ import ( "gopkg.in/yaml.v2" ) -func LoadProject(file string, warn log.Logger) (r *Project, err error) { +func LoadProject(file string, warn log.Logger) (r *CompoundServices, err error) { file, err = filepath.Abs(file) if err != nil { return } - r = &Project{Dir: filepath.Dir(file)} + r = &CompoundServices{Dir: filepath.Dir(file)} env, err := readEnvironment() if err != nil { return @@ -34,7 +34,7 @@ func LoadProject(file string, warn log.Logger) (r *Project, err error) { return } -func loadFromJSON(file string, r *Project) error { +func loadFromJSON(file string, r *CompoundServices) error { b, err := ioutil.ReadFile(filepath.FromSlash(file)) if err != nil { return err @@ -42,7 +42,7 @@ func loadFromJSON(file string, r *Project) error { return json.Unmarshal(b, r) } -func loadFromComposeYAML(file string, sub Substitution, r *Project) error { +func loadFromComposeYAML(file string, sub Substitution, r *CompoundServices) error { c, err := readComposeYAML(file) if err != nil { return err @@ -61,7 +61,7 @@ func readComposeYAML(file string) (*dockerCompose, error) { return dc, err } -func convertCompose(c *dockerCompose, sub Substitution, r *Project) error { +func convertCompose(c *dockerCompose, sub Substitution, r *CompoundServices) error { if c.Services == nil || len(c.Services) == 0 { return errors.New("no services defined in: " + c.Dir) } @@ -70,7 +70,7 @@ func convertCompose(c *dockerCompose, sub Substitution, r *Project) error { for k, v := range c.Services { s := NewService(k) envFileEnv := map[string]string{} - err := convertComposeService(c, v, sub, r, s, envFileEnv) + err := convertComposeService(c, v, sub, r, &s, envFileEnv) if err != nil { return err } @@ -82,7 +82,7 @@ func convertCompose(c *dockerCompose, sub Substitution, r *Project) error { } } - r.Services[k] = *s + r.Services[k] = s } return nil } @@ -117,7 +117,7 @@ func toVolumes(c *dockerCompose, sub Substitution, rp *map[string]Volume, path s return nil } -func convertComposeService(c *dockerCompose, s *dcService, sub Substitution, p *Project, d *Service, envFileEnv map[string]string) (err error) { +func convertComposeService(c *dockerCompose, s *dcService, sub Substitution, p *CompoundServices, d *Service, envFileEnv map[string]string) (err error) { l := "service." + d.Name // Extend service (convert recursively) @@ -332,7 +332,7 @@ func toVolumeMounts(dcVols []interface{}, sub Substitution, baseFile, destBaseFi switch t := e.(type) { case string: - if err = ParseVolumeMount(sub(e.(string)), &v); err != nil { + if err = ParseBindMount(sub(e.(string)), &v); err != nil { return errors.Wrap(err, path) } case map[interface{}]interface{}: @@ -372,8 +372,9 @@ func toVolumeMounts(dcVols []interface{}, sub Substitution, baseFile, destBaseFi return nil } -func ParseVolumeMount(expr string, r *VolumeMount) (err error) { +func ParseBindMount(expr string, r *VolumeMount) (err error) { r.Options = []string{} + r.Type = "bind" s := strings.Split(expr, ":") switch len(s) { case 0: @@ -523,19 +524,19 @@ func toStringMap(v interface{}, sub Substitution, r map[string]string, path stri } } -func toDuration(v, defaultVal string, sub Substitution, p string) (time.Duration, error) { +func toDuration(v, defaultVal string, sub Substitution, p string) (*time.Duration, error) { v = sub(v) if v == "" { v = defaultVal } if v == "" { - return 0, nil + return nil, nil } d, e := time.ParseDuration(v) if e != nil { - return 0, errors.Errorf("%s: duration expected but found %q", p, v) + return nil, errors.Errorf("%s: duration expected but found %q", p, v) } - return d, nil + return &d, nil } func toBool(v interface{}, sub Substitution, path string) (bool, error) { diff --git a/model/volumes.go b/model/resolve.go similarity index 100% rename from model/volumes.go rename to model/resolve.go diff --git a/model/types.go b/model/types.go index 4a76c49..5e9a7f2 100644 --- a/model/types.go +++ b/model/types.go @@ -5,18 +5,28 @@ import ( "strconv" "strings" "time" + + "github.com/pkg/errors" ) -type Project struct { +func FromJSON(b []byte) (r CompoundServices, err error) { + if err = json.Unmarshal(b, &r); err != nil { + err = errors.Wrap(err, "unmarshal CompoundServices") + } + for k, s := range r.Services { + // TODO: better use slice instead of map for services to avoid copying service struct in such cases + s.Name = k + r.Services[k] = s + } + return +} + +type CompoundServices struct { Dir string `json:"-"` Services map[string]Service `json:"services"` Volumes map[string]Volume `json:"volumes,omitempty"` } -func NewProject() *Project { - return &Project{"", map[string]Service{}, map[string]Volume{}} -} - type Service struct { Name string `json:"-"` Bundle string `json:"bundle,omitempty"` @@ -43,9 +53,9 @@ type Service struct { Expose []string `json:"expose,omitempty"` Volumes []VolumeMount `json:"volumes,omitempty"` // TODO: handle check - HealthCheck *Check `json:"healthcheck,omitempty"` - StopSignal string `json:"stop_signal,omitempty"` - StopGracePeriod time.Duration `json:"stop_grace_period"` + HealthCheck *Check `json:"healthcheck,omitempty"` + StopSignal string `json:"stop_signal,omitempty"` + StopGracePeriod *time.Duration `json:"stop_grace_period,omitempty"` // TODO: uid/gid mapping: spec.AddLinuxUIDMapping(hostid, containerid, size), ... AddLinuxGIDMapping } @@ -131,17 +141,17 @@ type Volume struct { type Check struct { Command []string `json:"cmd"` //Http string `json:"http"` - Interval time.Duration `json:"interval"` - Timeout time.Duration `json:"timeout"` - Retries uint `json:"retries,omitempty"` - Disable bool `json:"disable,omitempty"` + Interval *time.Duration `json:"interval"` + Timeout *time.Duration `json:"timeout"` + Retries uint `json:"retries,omitempty"` + Disable bool `json:"disable,omitempty"` } -func NewService(name string) *Service { - return &Service{Name: name, Seccomp: "default", StopGracePeriod: time.Duration(10000000000)} +func NewService(name string) Service { + return Service{Name: name, Seccomp: "default"} } -func (c *Project) JSON() string { +func (c *CompoundServices) JSON() string { return toJSON(c) } diff --git a/vendor.conf b/vendor.conf index e91d738..3efa953 100644 --- a/vendor.conf +++ b/vendor.conf @@ -64,6 +64,11 @@ github.com/golang/protobuf 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 #golang.org/x/sys 0e0164865330d5cf1c00247be08330bf96e2f87c https://github.com/golang/sys +# Docker Compose (Cannot use newer version due to conflict with older dependencies from containers/image) +github.com/docker/cli 11dfa23a5d491ed0b63c6ced36746012c97d4370 +github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715 + + # Console github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e golang.org/x/sys 7ddbeae9ae08c6a06a59597f0c9edbc5ff2444ce https://github.com/golang/sys