Add config option to select a different azure cloud env in the azure-eventhub input and azure module

CHANGELOG.next.asciidoc

@@ -257,6 +257,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379]
 - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491]
 - Release Google Cloud module as GA. {pull}17511[17511]
+- Add config option to select a different azure cloud env in the azure-eventhub input and azure module. {issue}17649[17649] {pull}17659[17659]
 - Added new Checkpoint Syslog filebeat module. {pull}17682[17682]
 - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]

filebeat/docs/modules/azure.asciidoc

@@ -43,6 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
       connection_string: ""
       storage_account: ""
       storage_account_key: ""
+      resource_manager_endpoint: ""
 
    auditlogs:
      enabled: false
@@ -52,6 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
        connection_string: ""
        storage_account: ""
        storage_account_key: ""
+       resource_manager_endpoint: ""
 
    signinlogs:
      enabled: false
@@ -61,6 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
        connection_string: ""
        storage_account: ""
        storage_account_key: ""
+       resource_manager_endpoint: ""
 
 ```
 
@@ -90,6 +93,16 @@ The name of the storage account the state/offsets will be stored and updated.
 _string_
 The storage account key, this key will be used to authorize access to data in your storage account.
 
+`resource_manager_endpoint` ::
+_string_
+Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
+Ex:
+https://management.chinacloudapi.cn/ for azure ChinaCloud
+https://management.microsoftazure.de/ for azure GermanCloud
+https://management.azure.com/ for azure PublicCloud
+https://management.usgovcloudapi.net/ for azure USGovernmentCloud
+Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints.
+
 include::../include/what-happens.asciidoc[]
 
 include::../include/gs-link.asciidoc[]

x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc

@@ -28,6 +28,8 @@ Example configuration:
   storage_account: "azureeph"
   storage_account_key: "....."
   storage_account_container: ""
+  resource_manager_endpoint: ""
+
 ----
 
 ==== Configuration options
@@ -36,7 +38,7 @@ The `azure-eventhub` input supports the following configuration:
 
 ==== `eventhub`
 
-The name of the eventhub users would like to read from.
+The name of the eventhub users would like to read from, field required.
 
 ==== `consumer_group`
 
@@ -50,14 +52,23 @@ A Blob Storage account is required in order to store/retrieve/update the offset
 
 ==== `storage_account`
 
-The name of the storage account.
+The name of the storage account. Required.
 
 ==== `storage_account_key`
 
-The storage account key, this key will be used to authorize access to data in your storage account.
+The storage account key, this key will be used to authorize access to data in your storage account, option is required.
 
 ==== `storage_account_container`
 
 Optional, the name of the storage account container you would like to store the offset information in.
 
+==== `resource_manager_endpoint`
+
+Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
+Ex:
+https://management.chinacloudapi.cn/ for azure ChinaCloud
+https://management.microsoftazure.de/ for azure GermanCloud
+https://management.azure.com/ for azure PublicCloud
+https://management.usgovcloudapi.net/ for azure USGovernmentCloud
+Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints.
 

x-pack/filebeat/filebeat.reference.yml

@@ -306,15 +306,15 @@ filebeat.modules:
   activitylogs:
     enabled: true
     var:
-      # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
+      # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
       eventhub: "insights-operational-logs"
-      # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
+      # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
       consumer_group: "$Default"
       # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string
       connection_string: ""
-      # the name of the storage account the state/offsets will be stored and updated.
+      # the name of the storage account the state/offsets will be stored and updated
       storage_account: ""
-      #The storage account key, this key will be used to authorize access to data in your storage account.
+      # the storage account key, this key will be used to authorize access to data in your storage account
       storage_account_key: ""
 
   auditlogs:

x-pack/filebeat/input/azureeventhub/config.go

@@ -17,6 +17,8 @@ type azureInputConfig struct {
 	SAName      string `config:"storage_account"`
 	SAKey       string `config:"storage_account_key"`
 	SAContainer string `config:"storage_account_container"`
+	// by default the azure public environment is used, to override, users can provide a specific resource manager endpoint
+	OverrideEnvironment string `config:"resource_manager_endpoint"`
 }
 
 const ephContainerName = "filebeat"

x-pack/filebeat/input/azureeventhub/eph.go

@@ -16,14 +16,26 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 )
 
+// users can select from one of the already defined azure cloud envs
+var environments = map[string]azure.Environment{
+	azure.ChinaCloud.ResourceManagerEndpoint:        azure.ChinaCloud,
+	azure.GermanCloud.ResourceManagerEndpoint:       azure.GermanCloud,
+	azure.PublicCloud.ResourceManagerEndpoint:       azure.PublicCloud,
+	azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud,
+}
+
 // runWithEPH will consume ingested events using the Event Processor Host (EPH) https://github.com/Azure/azure-event-hubs-go#event-processor-host, https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host
 func (a *azureInput) runWithEPH() error {
 	// create a new Azure Storage Leaser / Checkpointer
 	cred, err := azblob.NewSharedKeyCredential(a.config.SAName, a.config.SAKey)
 	if err != nil {
 		return err
 	}
-	leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, azure.PublicCloud)
+	env, err := getAzureEnvironment(a.config.OverrideEnvironment)
+	if err != nil {
+		return err
+	}
+	leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, env)
 	if err != nil {
 		return err
 	}
@@ -74,3 +86,15 @@ func (a *azureInput) runWithEPH() error {
 	}
 	return nil
 }
+
+func getAzureEnvironment(overrideResManager string) (azure.Environment, error) {
+	// if no overrride is set then the azure public cloud is used
+	if overrideResManager == "" {
+		return azure.PublicCloud, nil
+	}
+	if env, ok := environments[overrideResManager]; ok {
+		return env, nil
+	}
+	// can retrieve hybrid env from the resource manager endpoint
+	return azure.EnvironmentFromURL(overrideResManager)
+}

x-pack/filebeat/input/azureeventhub/eph_test.go

@@ -7,6 +7,8 @@ package azureeventhub
 import (
 	"testing"
 
+	"github.com/Azure/go-autorest/autorest/azure"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -26,3 +28,17 @@ func TestRunWithEPH(t *testing.T) {
 	err := input.runWithEPH()
 	assert.Error(t, err, '7')
 }
+
+func TestGetAzureEnvironment(t *testing.T) {
+	resMan := ""
+	env, err := getAzureEnvironment(resMan)
+	assert.NoError(t, err)
+	assert.Equal(t, env, azure.PublicCloud)
+	resMan = "https://management.microsoftazure.de/"
+	env, err = getAzureEnvironment(resMan)
+	assert.NoError(t, err)
+	assert.Equal(t, env, azure.GermanCloud)
+	resMan = "http://management.invalidhybrid.com/"
+	env, err = getAzureEnvironment(resMan)
+	assert.Errorf(t, err, "invalid character 'F' looking for beginning of value")
+}

x-pack/filebeat/module/azure/_meta/config.yml

@@ -3,15 +3,15 @@
   activitylogs:
     enabled: true
     var:
-      # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
+      # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
       eventhub: "insights-operational-logs"
-      # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
+      # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
       consumer_group: "$Default"
       # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string
       connection_string: ""
-      # the name of the storage account the state/offsets will be stored and updated.
+      # the name of the storage account the state/offsets will be stored and updated
       storage_account: ""
-      #The storage account key, this key will be used to authorize access to data in your storage account.
+      # the storage account key, this key will be used to authorize access to data in your storage account
       storage_account_key: ""
 
   auditlogs:

x-pack/filebeat/module/azure/_meta/docs.asciidoc

@@ -38,6 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
       connection_string: ""
       storage_account: ""
       storage_account_key: ""
+      resource_manager_endpoint: ""
 
    auditlogs:
      enabled: false
@@ -47,6 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
        connection_string: ""
        storage_account: ""
        storage_account_key: ""
+       resource_manager_endpoint: ""
 
    signinlogs:
      enabled: false
@@ -56,6 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
        connection_string: ""
        storage_account: ""
        storage_account_key: ""
+       resource_manager_endpoint: ""
 
 ```
 
@@ -85,6 +88,16 @@ The name of the storage account the state/offsets will be stored and updated.
 _string_
 The storage account key, this key will be used to authorize access to data in your storage account.
 
+`resource_manager_endpoint` ::
+_string_
+Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
+Ex:
+https://management.chinacloudapi.cn/ for azure ChinaCloud
+https://management.microsoftazure.de/ for azure GermanCloud
+https://management.azure.com/ for azure PublicCloud
+https://management.usgovcloudapi.net/ for azure USGovernmentCloud
+Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints.
+
 include::../include/what-happens.asciidoc[]
 
 include::../include/gs-link.asciidoc[]

x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml

@@ -4,3 +4,4 @@ eventhub: {{ .eventhub }}
 consumer_group: {{ .consumer_group }}
 storage_account: {{ .storage_account }}
 storage_account_key: {{ .storage_account_key }}
+resource_manager_endpoint: {{ .resource_manager_endpoint }}

x-pack/filebeat/module/azure/activitylogs/manifest.yml

@@ -10,6 +10,7 @@ var:
   - name: connection_string
   - name: storage_account
   - name: storage_account_key
+  - name: resource_manager_endpoint
 
 ingest_pipeline:
   - ingest/pipeline.json

x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml

@@ -4,4 +4,5 @@ eventhub: {{ .eventhub }}
 consumer_group: {{ .consumer_group }}
 storage_account: {{ .storage_account }}
 storage_account_key: {{ .storage_account_key }}
+resource_manager_endpoint: {{ .resource_manager_endpoint }}
 

x-pack/filebeat/module/azure/auditlogs/manifest.yml

@@ -10,6 +10,7 @@ var:
   - name: connection_string
   - name: storage_account
   - name: storage_account_key
+  - name: resource_manager_endpoint
 
 ingest_pipeline:
   - ingest/pipeline.json

x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml

@@ -4,3 +4,4 @@ eventhub: {{ .eventhub }}
 consumer_group: {{ .consumer_group }}
 storage_account: {{ .storage_account }}
 storage_account_key: {{ .storage_account_key }}
+resource_manager_endpoint: {{ .resource_manager_endpoint }}

x-pack/filebeat/module/azure/signinlogs/manifest.yml

@@ -10,6 +10,7 @@ var:
   - name: connection_string
   - name: storage_account
   - name: storage_account_key
+  - name: resource_manager_endpoint
 
 ingest_pipeline:
   - ingest/pipeline.json

x-pack/filebeat/modules.d/azure.yml.disabled

@@ -6,15 +6,15 @@
   activitylogs:
     enabled: true
     var:
-      # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
+      # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub
       eventhub: "insights-operational-logs"
-      # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
+      # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module
       consumer_group: "$Default"
       # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string
       connection_string: ""
-      # the name of the storage account the state/offsets will be stored and updated.
+      # the name of the storage account the state/offsets will be stored and updated
       storage_account: ""
-      #The storage account key, this key will be used to authorize access to data in your storage account.
+      # the storage account key, this key will be used to authorize access to data in your storage account
       storage_account_key: ""
 
   auditlogs: