Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Metricbeat] Change Account ID to Project ID in gcp.billing module #26412

Merged
merged 5 commits into from
Jun 25, 2021
Merged

[Metricbeat] Change Account ID to Project ID in gcp.billing module #26412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b88364f
#26357: Change Account ID to Project ID
legoguy1000 Jun 22, 2021
23b51d7
Update changelog
legoguy1000 Jun 22, 2021
5f2f9bf
query additional items
legoguy1000 Jun 23, 2021
b62ff80
fix typo
legoguy1000 Jun 24, 2021
2352c7f
fix Big Query per comments
legoguy1000 Jun 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
- Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]

*Packetbeat*

Expand Down
25 changes: 15 additions & 10 deletions x-pack/metricbeat/module/gcp/billing/billing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client,
SELECT
invoice.month,
project.id,
project.name,
billing_account_id,
cost_type,
(SUM(CAST(cost * 1000000 AS int64))
+ SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000
Expand All @@ -217,8 +219,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client,
WHERE project.id IS NOT NULL
AND invoice.month = '%s'
AND cost_type = '%s'
GROUP BY 1, 2, 3
ORDER BY 1 ASC, 2 ASC, 3 ASC;`, tableMeta.tableFullID, month, costType)
GROUP BY 1, 2, 3, 4, 5
ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, tableMeta.tableFullID, month, costType)

q := client.Query(query)
m.logger.Debug("bigquery query = ", query)
Expand Down Expand Up @@ -261,26 +263,29 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client,
return events, err
}

if len(row) == 4 {
if len(row) == 6 {
events = append(events, createEvents(row, m.config.ProjectID))
}
}
return events, nil
}

func createEvents(rowItems []bigquery.Value, accountID string) mb.Event {
func createEvents(rowItems []bigquery.Value, projectID string) mb.Event {
event := mb.Event{}
event.MetricSetFields = common.MapStr{
"invoice_month": rowItems[0],
"project_id": rowItems[1],
"cost_type": rowItems[2],
"total": rowItems[3],
"invoice_month": rowItems[0],
"project_id": rowItems[1],
"project_name": rowItems[2],
"billing_account_id": rowItems[3],
"cost_type": rowItems[4],
"total": rowItems[5],
}

event.RootFields = common.MapStr{
"cloud.provider": "gcp",
"cloud.account.id": accountID,
"cloud.account.name": accountID,
"cloud.project.id": projectID,
"cloud.project.name": rowItems[2],
"cloud.account.id": rowItems[3],
}

// create eventID for each current_date + invoice_month + project_id + cost_type
Expand Down