Beats dashboard should use custom index patterns when setup.dashboards.index is set

dev-tools/mage/check.go

@@ -36,6 +36,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
+	"github.com/elastic/beats/v7/libbeat/dashboards"
 	"github.com/elastic/beats/v7/libbeat/processors/dissect"
 )
 
@@ -260,6 +261,13 @@ func checkDashboardForErrors(file string, d []byte) bool {
 		fmt.Println("  ", err)
 	}
 
+	replaced := dashboards.ReplaceIndexInDashboardObject("my-test-index-*", d)
+	if bytes.Contains(replaced, []byte(BeatName+"-*")) {
+		hasErrors = true
+		fmt.Printf(">> Cannot modify all index pattern references in dashboard - %s\n", file)
+		fmt.Println(string(replaced))
+	}
+
 	return hasErrors
 }
 

libbeat/dashboards/decode.go

@@ -30,11 +30,13 @@ import (
 
 var (
 	responseToDecode = []string{
-		"attributes.uiStateJSON",
-		"attributes.visState",
+		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+		"attributes.layerListJSON",
+		"attributes.mapStateJSON",
 		"attributes.optionsJSON",
 		"attributes.panelsJSON",
-		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+		"attributes.uiStateJSON",
+		"attributes.visState",
 	}
 )
 

libbeat/dashboards/modify_json.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"regexp"
 
 	"github.com/pkg/errors"
 
@@ -46,11 +47,6 @@ type JSONObject struct {
 	Attributes JSONObjectAttribute `json:"attributes"`
 }
 
-// JSONFormat contains a list of JSON object
-type JSONFormat struct {
-	Objects []JSONObject `json:"objects"`
-}
-
 // ReplaceIndexInIndexPattern replaces an index in a dashboard content body
 func ReplaceIndexInIndexPattern(index string, content common.MapStr) (err error) {
 	if index == "" {
@@ -128,43 +124,62 @@ func ReplaceIndexInSavedObject(logger *logp.Logger, index string, kibanaSavedObj
 		}
 		kibanaSavedObject["searchSourceJSON"] = searchSourceJSON
 	}
-	if visStateJSON, ok := kibanaSavedObject["visState"].(string); ok {
-		visStateJSON = ReplaceIndexInVisState(logger, index, visStateJSON)
-		kibanaSavedObject["visState"] = visStateJSON
+	if visState, ok := kibanaSavedObject["visState"].(map[string]interface{}); ok {
+		kibanaSavedObject["visState"] = ReplaceIndexInVisState(logger, index, visState)
 	}
 
 	return kibanaSavedObject
 }
 
-// ReplaceIndexInVisState replaces index appearing in visState params objects
-func ReplaceIndexInVisState(logger *logp.Logger, index string, visStateJSON string) string {
-
-	var visState map[string]interface{}
-	err := json.Unmarshal([]byte(visStateJSON), &visState)
-	if err != nil {
-		logger.Errorf("Fail to unmarshal visState: %v", err)
-		return visStateJSON
-	}
+var timeLionIdxRegexp = regexp.MustCompile(`index=\".*beat-\*\"`)
 
+// ReplaceIndexInVisState replaces index appearing in visState params objects
+func ReplaceIndexInVisState(logger *logp.Logger, index string, visState map[string]interface{}) map[string]interface{} {
 	params, ok := visState["params"].(map[string]interface{})
 	if !ok {
-		return visStateJSON
+		return visState
 	}
 
 	// Don't set it if it was not set before
-	if pattern, ok := params["index_pattern"].(string); !ok || len(pattern) == 0 {
-		return visStateJSON
+	if pattern, ok := params["index_pattern"].(string); ok && len(pattern) != 0 {
+		params["index_pattern"] = index
+	}
+
+	if s, ok := params["series"].([]interface{}); ok {
+		for i, ser := range s {
+			if series, ok := ser.(map[string]interface{}); ok {
+				if _, ok := series["series_index_pattern"]; !ok {
+					continue
+				}
+				series["series_index_pattern"] = index
+				s[i] = series
+			}
+		}
+		params["series"] = s
 	}
 
-	params["index_pattern"] = index
+	if annotations, ok := params["annotations"].([]interface{}); ok {
+		for i, ann := range annotations {
+			annotation, ok := ann.(map[string]interface{})
+			if !ok {
+				continue
+			}
+			if _, ok = annotation["index_pattern"]; !ok {
+				continue
+			}
+			annotation["index_pattern"] = index
+			annotations[i] = annotation
+		}
+		params["annotations"] = annotations
+	}
 
-	d, err := json.Marshal(visState)
-	if err != nil {
-		logger.Errorf("Fail to marshal visState: %v", err)
-		return visStateJSON
+	if expr, ok := params["expression"].(string); ok {
+		params["expression"] = timeLionIdxRegexp.ReplaceAllString(expr, `index="`+index+`"`)
 	}
 
-	return string(d)
+	visState["params"] = replaceIndexInParamControls(logger, index, params)
+
+	return visState
 }
 
 // ReplaceIndexInDashboardObject replaces references to the index pattern in dashboard objects
@@ -195,10 +210,28 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 		attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(logger, index, kibanaSavedObject)
 	}
 
-	if visState, ok := attributes["visState"].(string); ok {
+	if visState, ok := attributes["visState"].(map[string]interface{}); ok {
 		attributes["visState"] = ReplaceIndexInVisState(logger, index, visState)
 	}
 
+	if layerListJSON, ok := attributes["layerListJSON"].([]interface{}); ok {
+		attributes["layerListJSON"] = replaceIndexInLayerListJSON(logger, index, layerListJSON)
+	}
+
+	if mapStateJSON, ok := attributes["mapStateJSON"].(map[string]interface{}); ok {
+		attributes["mapStateJSON"] = replaceIndexInMapStateJSON(logger, index, mapStateJSON)
+	}
+
+	if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok {
+		attributes["panelsJSON"] = replaceIndexInPanelsJSON(logger, index, panelsJSON)
+	}
+
+	objectMap["attributes"] = attributes
+
+	if references, ok := objectMap["references"].([]interface{}); ok {
+		objectMap["references"] = replaceIndexInReferences(index, references)
+	}
+
 	b, err := json.Marshal(objectMap)
 	if err != nil {
 		logger.Error("Error marshaling modified dashboard: %+v", err)
@@ -208,6 +241,140 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 	return b
 }
 
+func replaceIndexInLayerListJSON(logger *logp.Logger, index string, layerListJSON []interface{}) []interface{} {
+	for i, layerListElem := range layerListJSON {
+		elem, ok := layerListElem.(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		if joins, ok := elem["joins"].([]interface{}); ok {
+			for j, join := range joins {
+				if pos, ok := join.(map[string]interface{}); ok {
+					for key, val := range pos {
+						if joinElems, ok := val.(map[string]interface{}); ok {
+							if _, ok := joinElems["indexPatternTitle"]; ok {
+								joinElems["indexPatternTitle"] = index
+								pos[key] = joinElems
+							}
+						}
+					}
+					joins[j] = pos
+				}
+			}
+			elem["joins"] = joins
+		}
+		layerListJSON[i] = elem
+	}
+	return layerListJSON
+}
+
+func replaceIndexInMapStateJSON(logger *logp.Logger, index string, mapState map[string]interface{}) map[string]interface{} {
+	if filters, ok := mapState["filters"].([]interface{}); ok {
+		for i, f := range filters {
+			if filter, ok := f.(map[string]interface{}); ok {
+				if meta, ok := filter["meta"].(map[string]interface{}); ok {
+					if _, ok := meta["index"]; !ok {
+						continue
+					}
+					meta["index"] = index
+					filter["meta"] = meta
+				}
+				filters[i] = filter
+			}
+		}
+		mapState["filters"] = filters
+	}
+
+	return mapState
+}
+
+func replaceIndexInPanelsJSON(logger *logp.Logger, index string, panelsJSON []interface{}) []interface{} {
+	for i, p := range panelsJSON {
+		if panel, ok := p.(map[string]interface{}); ok {
+			config, ok := panel["embeddableConfig"].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			if configAttr, ok := config["attributes"].(map[string]interface{}); ok {
+				if references, ok := configAttr["references"].([]interface{}); ok {
+					configAttr["references"] = replaceIndexInReferences(index, references)
+				}
+				if layerListJSON, ok := configAttr["layerListJSON"].(string); ok {
+					configAttr["layerListJSON"] = replaceIndexInLaterListStr(logger, index, layerListJSON)
+				}
+				config["attributes"] = configAttr
+			}
+
+			if savedVis, ok := config["savedVis"].(map[string]interface{}); ok {
+				if params, ok := savedVis["params"].(map[string]interface{}); ok {
+					savedVis["params"] = replaceIndexInParamControls(logger, index, params)
+				}
+				config["savedVis"] = savedVis
+			}
+
+			panel["embeddableConfig"] = config
+			panelsJSON[i] = panel
+		}
+	}
+	return panelsJSON
+}
+
+func replaceIndexInParamControls(logger *logp.Logger, index string, params map[string]interface{}) map[string]interface{} {
+	if contolsList, ok := params["controls"].([]interface{}); ok {
+		for i, ctrl := range contolsList {
+			if control, ok := ctrl.(map[string]interface{}); ok {
+				if _, ok := control["indexPattern"]; ok {
+					control["indexPattern"] = index
+					contolsList[i] = control
+				}
+			}
+		}
+		params["controls"] = contolsList
+	}
+	return params
+}
+
+func replaceIndexInLaterListStr(logger *logp.Logger, index string, layerListJSON string) string {
+	var layerList []map[string]interface{}
+	err := json.Unmarshal([]byte(layerListJSON), &layerList)
+	if err != nil {
+		logger.Error("Failed to unmarshal json: %+v", err)
+		return layerListJSON
+	}
+	for i, elem := range layerList {
+		if descriptor, ok := elem["sourceDescriptor"].(map[string]interface{}); ok {
+			if _, ok := descriptor["indexPatternId"]; ok {
+				descriptor["indexPatternId"] = index
+			}
+			elem["sourceDescriptor"] = descriptor
+			layerList[i] = elem
+		}
+	}
+
+	b, err := json.Marshal(layerList)
+	if err != nil {
+		logger.Error("Failed to marshal to json: %+v", err)
+		return layerListJSON
+	}
+	return string(b)
+
+}
+
+func replaceIndexInReferences(index string, references []interface{}) []interface{} {
+	for i, ref := range references {
+		if reference, ok := ref.(map[string]interface{}); ok {
+			if refType, ok := reference["type"].(string); ok {
+				if refType == "index-pattern" {
+					reference["id"] = index
+				}
+			}
+			references[i] = reference
+		}
+	}
+	return references
+}
+
 func EncodeJSONObjects(content []byte) []byte {
 	logger := logp.NewLogger("dashboards")
 

libbeat/dashboards/modify_json_test.go

@@ -77,9 +77,34 @@ func TestReplaceIndexInDashboardObject(t *testing.T) {
 			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`),
 		},
 		{
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`),
+			[]byte(`{"attributes":{"layerListJSON":[{"joins":[{"leftField":"iso2","right":{"indexPatternTitle":"filebeat-*"}}]}]}}`),
 			"otherindex-*",
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`),
+			[]byte(`{"attributes":{"layerListJSON":[{"joins":[{"leftField":"iso2","right":{"indexPatternTitle":"otherindex-*"}}]}]}}`),
+		},
+		{
+			[]byte(`{"attributes":{"panelsJSON":[{"embeddableConfig":{"attributes":{"references":[{"id":"filebeat-*","type":"index-pattern"}]}}}]}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"panelsJSON":[{"embeddableConfig":{"attributes":{"references":[{"id":"otherindex-*","type":"index-pattern"}]}}}]}}`),
+		},
+		{
+			[]byte(`{"attributes":{},"references":[{"id":"auditbeat-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{},"references":[{"id":"otherindex-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]}`),
+		},
+		{
+			[]byte(`{"attributes":{"visState":{"params":{"index_pattern":"winlogbeat-*"}}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"visState":{"params":{"index_pattern":"otherindex-*"}}}}`),
+		},
+		{
+			[]byte(`{"attributes":{"visState":{"params":{"series":[{"series_index_pattern":"filebeat-*"}]}}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"visState":{"params":{"series":[{"series_index_pattern":"otherindex-*"}]}}}}`),
+		},
+		{
+			[]byte(`{"attributes":{"mapStateJSON":{"filters":[{"meta":{"index":"filebeat-*"}}]}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"mapStateJSON":{"filters":[{"meta":{"index":"otherindex-*"}}]}}}`),
 		},
 	}