From b71370e41b4cca2902fb4b82418252915462c378 Mon Sep 17 00:00:00 2001 From: lsautia Date: Thu, 5 Aug 2021 09:45:07 -0700 Subject: [PATCH 1/5] add map merge functionality --- docs/templating-language.md | 25 ++++++++++++++++++++ go.mod | 2 +- go.sum | 9 ++------ template/funcs.go | 14 +++++++++++ template/template.go | 2 ++ template/template_test.go | 46 +++++++++++++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/docs/templating-language.md b/docs/templating-language.md index dfcf00200..10e163345 100644 --- a/docs/templating-language.md +++ b/docs/templating-language.md @@ -54,6 +54,8 @@ provides the following functions: - [in](#in) - [loop](#loop) - [join](#join) + - [mergeMap](#mergemap) + - [mergeMapWithOverride](#mergemapwithoverride) - [trimSpace](#trimspace) - [parseBool](#parsebool) - [parseFloat](#parsefloat) @@ -1223,6 +1225,29 @@ Takes the given list of strings as a pipe and joins them on the provided string: {{ $items | join "," }} ``` +### `mergeMap` + +Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will not be overridden by piped map + +```golang +{{ $base := tree "base" | explode }} +{{ $overrides := tree "overrides" | explode | mergeMap $base}} +{{ with $overrides }} +{{ .a.b.c }}{{ end }} +``` + +### `mergeMapWithOverride` + +Takes the result from a [`tree`](#tree) or [`ls`](#ls) call and converts it into a deeply-nested +map for parsing/traversing. + +```golang +{{ $base := tree "base" | explode }} +{{ $overrides := tree "overrides" | explode | mergeMap $base}} +{{ with $overrides }} +{{ .a.b.c }}{{ end }} +``` + ### `trimSpace` Takes the provided input and trims all whitespace, tabs and newlines: diff --git a/go.mod b/go.mod index 101c47848..e21a98d14 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( github.com/hashicorp/logutils v1.0.0 github.com/hashicorp/serf v0.9.4 // indirect github.com/hashicorp/vault/api v1.0.5-0.20190730042357-746c0b111519 + github.com/imdario/mergo v0.3.12 github.com/mattn/go-colorable v0.1.7 // indirect - github.com/mattn/go-shellwords v1.0.12 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/hashstructure v1.0.0 github.com/mitchellh/mapstructure v1.3.3 diff --git a/go.sum b/go.sum index 0d67b84b1..3091d4888 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eikenb/go-shellwords v1.0.12-0.20210603231637-cbc54604f068 h1:y0DBnMgUmMeM+y1h7R2Hv0IZNiAcK5FAjNbRbbJNe3A= -github.com/eikenb/go-shellwords v1.0.12-0.20210603231637-cbc54604f068/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -124,6 +122,8 @@ github.com/hashicorp/vault/sdk v0.1.14-0.20190730042320-0dc007d98cc8 h1:fLUoZ8cI github.com/hashicorp/vault/sdk v0.1.14-0.20190730042320-0dc007d98cc8/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -146,10 +146,6 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= @@ -248,7 +244,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/template/funcs.go b/template/funcs.go index 66990b1c2..bfccb6544 100644 --- a/template/funcs.go +++ b/template/funcs.go @@ -26,6 +26,7 @@ import ( dep "github.com/hashicorp/consul-template/dependency" "github.com/hashicorp/consul/api" socktmpl "github.com/hashicorp/go-sockaddr/template" + "github.com/imdario/mergo" "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) @@ -739,6 +740,19 @@ func containsSomeFunc(retTrue, invert bool) func([]interface{}, interface{}) (bo } } +// mergeMap is used to merge two maps +func mergeMap(dstMap map[string]interface{}, srcMap map[string]interface{}, args ...func(*mergo.Config)) (map[string]interface{}, error) { + if err := mergo.Map(&dstMap, srcMap, args...); err != nil { + return nil, err + } + return dstMap, nil +} + +// mergeMapWithOverride is used to merge two maps with dstMap overriding vaules in srcMap +func mergeMapWithOverride(dstMap map[string]interface{}, srcMap map[string]interface{}) (map[string]interface{}, error) { + return mergeMap(dstMap, srcMap, mergo.WithOverride) +} + // explode is used to expand a list of keypairs into a deeply-nested hash. func explode(pairs []*dep.KeyPair) (map[string]interface{}, error) { m := make(map[string]interface{}) diff --git a/template/template.go b/template/template.go index 3dbf10114..379620db8 100644 --- a/template/template.go +++ b/template/template.go @@ -264,6 +264,8 @@ func funcMap(i *funcMapInput) template.FuncMap { "executeTemplate": executeTemplateFunc(i.t), "explode": explode, "explodeMap": explodeMap, + "mergeMap": mergeMap, + "mergeMapWithOverride": mergeMapWithOverride, "in": in, "indent": indent, "loop": loop, diff --git a/template/template_test.go b/template/template_test.go index a810535be..72aae972e 100644 --- a/template/template_test.go +++ b/template/template_test.go @@ -1191,6 +1191,52 @@ func TestTemplate_Execute(t *testing.T) { "bar", false, }, + { + "helper_mergeMap", + &NewTemplateInput{ + Contents: `{{ $base := "{\"voo\":{\"bar\":\"v\"}}" | parseJSON}}{{ $role := tree "list" | explode | mergeMap $base}}{{ range $k, $v := $role }}{{ $k }}{{ $v }}{{ end }}`, + }, + &ExecuteInput{ + Brain: func() *Brain { + b := NewBrain() + d, err := dep.NewKVListQuery("list") + if err != nil { + t.Fatal(err) + } + b.Remember(d, []*dep.KeyPair{ + &dep.KeyPair{Key: "", Value: ""}, + &dep.KeyPair{Key: "foo/bar", Value: "a"}, + &dep.KeyPair{Key: "zip/zap", Value: "b"}, + }) + return b + }(), + }, + "foomap[bar:a]voomap[bar:v]zipmap[zap:b]", + false, + }, + { + "helper_mergeMapWithOverride", + &NewTemplateInput{ + Contents: `{{ $base := "{\"zip\":{\"zap\":\"t\"},\"voo\":{\"bar\":\"v\"}}" | parseJSON}}{{ $role := tree "list" | explode | mergeMapWithOverride $base}}{{ range $k, $v := $role }}{{ $k }}{{ $v }}{{ end }}`, + }, + &ExecuteInput{ + Brain: func() *Brain { + b := NewBrain() + d, err := dep.NewKVListQuery("list") + if err != nil { + t.Fatal(err) + } + b.Remember(d, []*dep.KeyPair{ + &dep.KeyPair{Key: "", Value: ""}, + &dep.KeyPair{Key: "foo/bar", Value: "a"}, + &dep.KeyPair{Key: "zip/zap", Value: "b"}, + }) + return b + }(), + }, + "foomap[bar:a]voomap[bar:v]zipmap[zap:b]", + false, + }, { "helper_explode", &NewTemplateInput{ From 2adfa87663098282e4285b160fad6402dc197535 Mon Sep 17 00:00:00 2001 From: lsautia Date: Thu, 5 Aug 2021 10:38:04 -0700 Subject: [PATCH 2/5] fix readme for mergeMap --- docs/templating-language.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/templating-language.md b/docs/templating-language.md index 10e163345..4718bd89a 100644 --- a/docs/templating-language.md +++ b/docs/templating-language.md @@ -1227,7 +1227,7 @@ Takes the given list of strings as a pipe and joins them on the provided string: ### `mergeMap` -Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will not be overridden by piped map +Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will not be overridden by piped map. ```golang {{ $base := tree "base" | explode }} @@ -1238,8 +1238,7 @@ Takes the result from [`explode`](#explode) and an exploded argument then merges ### `mergeMapWithOverride` -Takes the result from a [`tree`](#tree) or [`ls`](#ls) call and converts it into a deeply-nested -map for parsing/traversing. +Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will be overridden by piped map. ```golang {{ $base := tree "base" | explode }} From 162c28f679d33cbd30304c120b26283194d7f544 Mon Sep 17 00:00:00 2001 From: Lee Sautia Date: Fri, 6 Aug 2021 16:18:09 -0700 Subject: [PATCH 3/5] Update docs/templating-language.md Co-authored-by: lornasong --- docs/templating-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templating-language.md b/docs/templating-language.md index 4718bd89a..3b1e34ca4 100644 --- a/docs/templating-language.md +++ b/docs/templating-language.md @@ -1227,7 +1227,7 @@ Takes the given list of strings as a pipe and joins them on the provided string: ### `mergeMap` -Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will not be overridden by piped map. +Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source will not be overridden by piped map. ```golang {{ $base := tree "base" | explode }} From e015ba9e617ddfad66d5d7ad05689508d0a28c1b Mon Sep 17 00:00:00 2001 From: Lee Sautia Date: Fri, 6 Aug 2021 16:18:29 -0700 Subject: [PATCH 4/5] Update docs/templating-language.md Co-authored-by: lornasong --- docs/templating-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templating-language.md b/docs/templating-language.md index 3b1e34ca4..ab18da9a3 100644 --- a/docs/templating-language.md +++ b/docs/templating-language.md @@ -1238,7 +1238,7 @@ Takes the result from [`explode`](#explode) and an exploded argument then merges ### `mergeMapWithOverride` -Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source is will be overridden by piped map. +Takes the result from [`explode`](#explode) and an exploded argument then merges it both maps. The argument's source will be overridden by piped map. ```golang {{ $base := tree "base" | explode }} From dbb0b1672089ff2a512f198bc2070a2c152be7ad Mon Sep 17 00:00:00 2001 From: lsautia Date: Fri, 6 Aug 2021 16:20:04 -0700 Subject: [PATCH 5/5] Fix documentation --- docs/templating-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templating-language.md b/docs/templating-language.md index ab18da9a3..4ad80fd8d 100644 --- a/docs/templating-language.md +++ b/docs/templating-language.md @@ -1242,7 +1242,7 @@ Takes the result from [`explode`](#explode) and an exploded argument then merges ```golang {{ $base := tree "base" | explode }} -{{ $overrides := tree "overrides" | explode | mergeMap $base}} +{{ $overrides := tree "overrides" | explode | mergeMapWithOverride $base}} {{ with $overrides }} {{ .a.b.c }}{{ end }} ```