Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use time.Duration directly in GetStartTimeEndTime function #12033

Merged
merged 2 commits into from
May 7, 2019
Merged

Use time.Duration directly in GetStartTimeEndTime function #12033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
038bc50
Remove convertPeriodToDuration and use duration directly in GetStartT…
kaiyan-sheng May 2, 2019
1f99ad7
Pass period in time.Duration type
kaiyan-sheng May 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 6 additions & 21 deletions x-pack/metricbeat/module/aws/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,9 @@ type Config struct {
// MetricSet is the base metricset for all aws metricsets
type MetricSet struct {
mb.BaseMetricSet
RegionsList []string
DurationString string
PeriodInSec int
AwsConfig *awssdk.Config
RegionsList []string
Period time.Duration
AwsConfig *awssdk.Config
}

// ModuleName is the name of this module.
Expand Down Expand Up @@ -77,16 +76,10 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {

awsConfig.Region = config.DefaultRegion

durationString, periodSec := convertPeriodToDuration(config.Period)
if err != nil {
return nil, err
}

metricSet := MetricSet{
BaseMetricSet: base,
DurationString: durationString,
PeriodInSec: periodSec,
AwsConfig: &awsConfig,
BaseMetricSet: base,
Period: config.Period,
AwsConfig: &awsConfig,
}

// Construct MetricSet with a full regions list
Expand Down Expand Up @@ -120,14 +113,6 @@ func getRegions(svc ec2iface.EC2API) (completeRegionsList []string, err error) {
return
}

func convertPeriodToDuration(period time.Duration) (string, int) {
// Set starttime double the default frequency earlier than the endtime in order to make sure
// GetMetricDataRequest gets the latest data point for each metric.
duration := "-" + (period * 2).String()
numberPeriod := int(period.Seconds())
return duration, numberPeriod
}

// StringInSlice checks if a string is already exists in list
func StringInSlice(str string, list []string) bool {
for _, v := range list {
Expand Down
36 changes: 0 additions & 36 deletions x-pack/metricbeat/module/aws/aws_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package aws
import (
"fmt"
"testing"
"time"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand Down Expand Up @@ -48,38 +47,3 @@ func TestGetRegions(t *testing.T) {
assert.Equal(t, 1, len(regionsList))
assert.Equal(t, regionName, regionsList[0])
}

func TestConvertPeriodToDuration(t *testing.T) {
cases := []struct {
period time.Duration
expectedDuration string
expectedPeriodNumber int
}{
{
period: time.Duration(300) * time.Second,
expectedDuration: "-10m0s",
expectedPeriodNumber: 300,
},
{
period: time.Duration(10) * time.Minute,
expectedDuration: "-20m0s",
expectedPeriodNumber: 600,
},
{
period: time.Duration(30) * time.Second,
expectedDuration: "-1m0s",
expectedPeriodNumber: 30,
},
{
period: time.Duration(60) * time.Second,
expectedDuration: "-2m0s",
expectedPeriodNumber: 60,
},
}

for _, c := range cases {
duration, periodSec := convertPeriodToDuration(c.period)
assert.Equal(t, c.expectedDuration, duration)
assert.Equal(t, c.expectedPeriodNumber, periodSec)
}
}
kaiyan-sheng marked this conversation as resolved.
Show resolved Hide resolved
20 changes: 9 additions & 11 deletions x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,18 +91,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "error GetStartTimeEndTime")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

// Get listMetrics and namespacesTotal from configuration
listMetrics, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs)
for _, regionName := range m.MetricSet.RegionsList {
awsConfig := m.MetricSet.AwsConfig.Copy()
awsConfig.Region = regionName
svcCloudwatch := cloudwatch.New(awsConfig)
err := createEvents(svcCloudwatch, listMetrics, regionName, m.PeriodInSec, startTime, endTime, report)
err := createEvents(svcCloudwatch, listMetrics, regionName, m.Period, startTime, endTime, report)
if err != nil {
return errors.Wrap(err, "createEvents failed")
}
Expand All @@ -124,7 +121,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
continue
}

err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.PeriodInSec, startTime, endTime, report)
err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.Period, startTime, endTime, report)
if err != nil {
return errors.Wrap(err, "createEvents failed for region "+regionName)
}
Expand All @@ -150,7 +147,7 @@ func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []st
return listMetrics, namespacesTotal
}

func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
for i, listMetric := range listMetricsOutput {
metricDataQuery := createMetricDataQuery(listMetric, i, period)
Expand Down Expand Up @@ -181,15 +178,16 @@ func constructLabel(metric cloudwatch.Metric) string {
return label
}

func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, index int, period time.Duration) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
id := "cw" + strconv.Itoa(index)
label := constructLabel(metric)
periodInSec := int64(period.Seconds())

metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSec,
Stat: &statistic,
Metric: &metric,
},
Expand Down Expand Up @@ -267,7 +265,7 @@ func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloud
return listMetricsOutput
}

func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error {
func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period time.Duration, startTime time.Time, endTime time.Time, report mb.ReporterV2) error {
identifiers := getIdentifiers(listMetricsTotal)
// Initialize events map per region, which stores one event per identifierValue
events := map[string]mb.Event{}
Expand All @@ -280,7 +278,7 @@ func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwat
var eventsNoIdentifier []mb.Event

// Construct metricDataQueries
metricDataQueries := constructMetricQueries(listMetricsTotal, int64(period))
metricDataQueries := constructMetricQueries(listMetricsTotal, period)
if len(metricDataQueries) == 0 {
return nil
}
Expand Down
22 changes: 10 additions & 12 deletions x-pack/metricbeat/module/aws/ec2/ec2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand Down Expand Up @@ -51,8 +52,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
}

// Check if period is set to be multiple of 60s or 300s
remainder300 := metricSet.PeriodInSec % 300
remainder60 := metricSet.PeriodInSec % 60
remainder300 := int(metricSet.Period.Seconds()) % 300
remainder60 := int(metricSet.Period.Seconds()) % 60
if remainder300 != 0 || remainder60 != 0 {
err := errors.New("period needs to be set to 60s (or a multiple of 60s) if detailed monitoring is " +
"enabled for EC2 instances or set to 300s (or a multiple of 300s) if EC2 instances has basic monitoring. " +
Expand All @@ -70,10 +71,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "Error ParseDuration")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

for _, regionName := range m.MetricSet.RegionsList {
awsConfig := m.MetricSet.AwsConfig.Copy()
Expand Down Expand Up @@ -102,7 +100,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {

var metricDataQueriesTotal []cloudwatch.MetricDataQuery
for _, instanceID := range instanceIDs {
metricDataQueriesTotal = append(metricDataQueriesTotal, constructMetricQueries(listMetricsOutput, instanceID, m.PeriodInSec)...)
metricDataQueriesTotal = append(metricDataQueriesTotal, constructMetricQueries(listMetricsOutput, instanceID, m.Period)...)
}

var metricDataOutput []cloudwatch.MetricDataResult
Expand Down Expand Up @@ -139,11 +137,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
return nil
}

func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, periodInSec int) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
metricDataQueryEmpty := cloudwatch.MetricDataQuery{}
for i, listMetric := range listMetricsOutput {
metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, periodInSec)
metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, period)
if metricDataQuery == metricDataQueryEmpty {
continue
}
Expand Down Expand Up @@ -260,9 +258,9 @@ func getInstancesPerRegion(svc ec2iface.EC2API) (instanceIDs []string, instances
return
}

func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index int, periodInSec int) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index int, period time.Duration) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
period := int64(periodInSec)
periodInSeconds := int64(period.Seconds())
id := metricsetName + strconv.Itoa(index)
metricDims := metric.Dimensions

Expand All @@ -273,7 +271,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index in
metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSeconds,
Stat: &statistic,
Metric: &metric,
},
Expand Down
2 changes: 1 addition & 1 deletion x-pack/metricbeat/module/aws/ec2/ec2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func TestConstructMetricQueries(t *testing.T) {
}

listMetricsOutput := []cloudwatch.Metric{listMetric}
metricDataQuery := constructMetricQueries(listMetricsOutput, instanceID, 300)
metricDataQuery := constructMetricQueries(listMetricsOutput, instanceID, 5*time.Minute)
assert.Equal(t, 1, len(metricDataQuery))
assert.Equal(t, "i-123 CPUUtilization", *metricDataQuery[0].Label)
assert.Equal(t, "Average", *metricDataQuery[0].MetricStat.Stat)
Expand Down
20 changes: 9 additions & 11 deletions x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/pkg/errors"
Expand Down Expand Up @@ -55,7 +56,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
}

// Check if period is set to be multiple of 86400s
remainder := metricSet.PeriodInSec % 86400
remainder := int(metricSet.Period.Seconds()) % 86400
if remainder != 0 {
err := errors.New("period needs to be set to 86400s (or a multiple of 86400s). " +
"To avoid data missing or extra costs, please make sure period is set correctly " +
Expand All @@ -74,10 +75,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
namespace := "AWS/S3"
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "Error ParseDuration")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

// GetMetricData for AWS S3 from Cloudwatch
for _, regionName := range m.MetricSet.RegionsList {
Expand All @@ -96,7 +94,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
continue
}

metricDataQueries := constructMetricQueries(listMetricsOutputs, m.PeriodInSec)
metricDataQueries := constructMetricQueries(listMetricsOutputs, m.Period)
// Use metricDataQueries to make GetMetricData API calls
metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime)
if err != nil {
Expand Down Expand Up @@ -142,7 +140,7 @@ func getBucketNames(listMetricsOutputs []cloudwatch.Metric) (bucketNames []strin
return
}

func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
metricDataQueryEmpty := cloudwatch.MetricDataQuery{}
metricNames := []string{"NumberOfObjects", "BucketSizeBytes"}
Expand All @@ -151,7 +149,7 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec
continue
}

metricDataQuery := createMetricDataQuery(listMetric, periodInSec, i)
metricDataQuery := createMetricDataQuery(listMetric, period, i)
if metricDataQuery == metricDataQueryEmpty {
continue
}
Expand All @@ -160,9 +158,9 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec
return metricDataQueries
}

func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, period time.Duration, index int) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
period := int64(periodInSec)
periodInSec := int64(period.Seconds())
id := "s3d" + strconv.Itoa(index)
metricDims := metric.Dimensions
bucketName := ""
Expand All @@ -180,7 +178,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int)
metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSec,
Stat: &statistic,
Metric: &metric,
},
Expand Down
Loading