From da5c33ae9d14b074a9993a30649613a3b7ec9267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20REY?= Date: Mon, 22 Aug 2022 15:00:58 +0200 Subject: [PATCH] fix(datasource-elastic): allow support of elastic version > 7 In latest versions of Grafana, the elastic version is no more provided as an integer even if it is still supported and won't return any error. To ease configuration, the module will accept the version format as displayed by the WUI (ex: "7.10+") and will map the value to what the API expects ("7.10.0"). The added values are the one supported by Grafana 8 and 9 (greater than elastic 7.0 which was already supported). The module will keep on supporting the old format to avoid a breaking change with significant impact. Fixes: #263 --- changelogs/fragments/fix-263.yaml | 2 + plugins/modules/grafana_datasource.py | 50 ++++++++++---- .../grafana_datasource/tasks/elastic.yml | 67 ++++++++++++++++--- 3 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 changelogs/fragments/fix-263.yaml diff --git a/changelogs/fragments/fix-263.yaml b/changelogs/fragments/fix-263.yaml new file mode 100644 index 00000000..7ad03b9b --- /dev/null +++ b/changelogs/fragments/fix-263.yaml @@ -0,0 +1,2 @@ +bugfixes: + - Add support for more elasticsearch version as datasource (#263) diff --git a/plugins/modules/grafana_datasource.py b/plugins/modules/grafana_datasource.py index 96cf82e5..1ec36958 100644 --- a/plugins/modules/grafana_datasource.py +++ b/plugins/modules/grafana_datasource.py @@ -137,13 +137,16 @@ - Version 56 is for elasticsearch 5.6+ where you can specify the C(max_concurrent_shard_requests) option. choices: - - 2 - - 5 - - 56 - - 60 - - 70 - default: 5 - type: int + - "2" + - "5" + - "56" + - "60" + - "70" + - "7.7+" + - "7.10+" + - "8.0+" + default: "7.10+" + type: str max_concurrent_shard_requests: description: - Starting with elasticsearch 5.6, you can specify the max concurrent shard per @@ -490,6 +493,13 @@ from ansible_collections.community.grafana.plugins.module_utils import base +ES_VERSION_MAPPING = { + "7.7+": "7.7.0", + "7.10+": "7.10.0", + "8.0+": "8.0.0", +} + + def compare_datasources(new, current, compareSecureData=True): if 'uid' in current: del current['uid'] @@ -581,12 +591,22 @@ def get_datasource_payload(data): # datasource type related parameters if data['ds_type'] == 'elasticsearch': - json_data['esVersion'] = data['es_version'] + + json_data['maxConcurrentShardRequests'] = data['max_concurrent_shard_requests'] json_data['timeField'] = data['time_field'] if data.get('interval'): json_data['interval'] = data['interval'] - if data['es_version'] >= 56: - json_data['maxConcurrentShardRequests'] = data['max_concurrent_shard_requests'] + + # Handle changes in es_version format in Grafana < 8.x which used to + # be integers and is now semver format + try: + es_version = int(data['es_version']) + if es_version < 56: + json_data.pop('maxConcurrentShardRequests') + except ValueError: + # Retrieve the Semver format expected by API + es_version = ES_VERSION_MAPPING.get(data['es_version']) + json_data['esVersion'] = es_version if data['ds_type'] == 'elasticsearch' or data['ds_type'] == 'influxdb': if data.get('time_interval'): @@ -725,7 +745,9 @@ def setup_module_object(): tls_skip_verify=dict(type='bool', default=False), is_default=dict(default=False, type='bool'), org_id=dict(default=1, type='int'), - es_version=dict(type='int', default=5, choices=[2, 5, 56, 60, 70]), + es_version=dict(type='str', default="7.10+", choices=["2", "5", "56", "60", + "70", "7.7+", "7.10+", + "8.0+"]), max_concurrent_shard_requests=dict(type='int', default=256), time_field=dict(default='@timestamp', type='str'), time_interval=dict(type='str'), @@ -770,9 +792,9 @@ def setup_module_object(): ['ds_type', 'mysql', ['database']], ['ds_type', 'postgres', ['database', 'sslmode']], ['ds_type', 'cloudwatch', ['aws_auth_type', 'aws_default_region']], - ['es_version', 56, ['max_concurrent_shard_requests']], - ['es_version', 60, ['max_concurrent_shard_requests']], - ['es_version', 70, ['max_concurrent_shard_requests']] + ['es_version', "56", ['max_concurrent_shard_requests']], + ['es_version', "60", ['max_concurrent_shard_requests']], + ['es_version', "70", ['max_concurrent_shard_requests']] ], ) return module diff --git a/tests/integration/targets/grafana_datasource/tasks/elastic.yml b/tests/integration/targets/grafana_datasource/tasks/elastic.yml index dc00d571..7d3b6fc0 100644 --- a/tests/integration/targets/grafana_datasource/tasks/elastic.yml +++ b/tests/integration/targets/grafana_datasource/tasks/elastic.yml @@ -1,7 +1,8 @@ -- name: Create elasticsearch datasource +--- +- name: Create elasticsearch datasource with legacy elasticsearch format register: result grafana_datasource: - name: "datasource/elastic" + name: "datasource/elasticLegacy" grafana_url: "{{ grafana_url }}" grafana_user: "{{ grafana_username }}" grafana_password: "{{ grafana_password }}" @@ -35,6 +36,52 @@ - result.datasource.jsonData.timeField == '@timestamp' - not result.datasource.jsonData.tlsAuth - result.datasource.jsonData.tlsAuthWithCACert + - result.datasource.name == 'datasource/elasticLegacy' + - result.datasource.orgId == 1 + - ('password' not in result.datasource) or (result.datasource.password == '') + - result.datasource.type == 'elasticsearch' + - result.datasource.url == 'https://elastic.company.com:9200' + - result.datasource.user == '' + - not result.datasource.withCredentials + - "result.msg == 'Datasource datasource/elasticLegacy created'" + +- name: Create elasticsearch datasource with new elsaticsearch version format + register: result + grafana_datasource: + name: "datasource/elastic" + grafana_url: "{{ grafana_url }}" + grafana_user: "{{ grafana_username }}" + grafana_password: "{{ grafana_password }}" + org_id: '1' + ds_type: elasticsearch + ds_url: https://elastic.company.com:9200 + database: '[logstash_]YYYY.MM.DD' + basic_auth_user: grafana + basic_auth_password: '******' + time_field: '@timestamp' + time_interval: 1m + interval: Daily + es_version: "7.10+" + max_concurrent_shard_requests: 42 + tls_ca_cert: /etc/ssl/certs/ca.pem + +- debug: + var: result + +- assert: + that: + - result.changed + - result.datasource.basicAuth + - result.datasource.basicAuthUser == 'grafana' + - result.datasource.access == 'proxy' + - result.datasource.database == '[logstash_]YYYY.MM.DD' + - not result.datasource.isDefault + - result.datasource.jsonData.esVersion == "7.10.0" + - result.datasource.jsonData.interval == 'Daily' + - result.datasource.jsonData.maxConcurrentShardRequests == 42 + - result.datasource.jsonData.timeField == '@timestamp' + - not result.datasource.jsonData.tlsAuth + - result.datasource.jsonData.tlsAuthWithCACert - result.datasource.name == 'datasource/elastic' - result.datasource.orgId == 1 - ('password' not in result.datasource) or (result.datasource.password == '') @@ -60,7 +107,7 @@ time_field: '@timestamp' time_interval: 1m interval: Daily - es_version: 56 + es_version: "7.10+" max_concurrent_shard_requests: 42 tls_ca_cert: /etc/ssl/certs/ca.pem @@ -75,7 +122,7 @@ - result.datasource.access == 'proxy' - result.datasource.database == '[logstash_]YYYY.MM.DD' - not result.datasource.isDefault - - result.datasource.jsonData.esVersion == 56 + - result.datasource.jsonData.esVersion == '7.10.0' - result.datasource.jsonData.interval == 'Daily' - result.datasource.jsonData.maxConcurrentShardRequests == 42 - result.datasource.jsonData.timeField == '@timestamp' @@ -105,7 +152,7 @@ time_field: '@timestamp' time_interval: 1m interval: Daily - es_version: 56 + es_version: "7.10+" max_concurrent_shard_requests: 42 tls_ca_cert: /etc/ssl/certs/ca.pem @@ -120,7 +167,7 @@ - result.datasource.access == 'proxy' - result.datasource.database == '[logstash_]YYYY.MM.DD' - not result.datasource.isDefault - - result.datasource.jsonData.esVersion == 56 + - result.datasource.jsonData.esVersion == '7.10.0' - result.datasource.jsonData.interval == 'Daily' - result.datasource.jsonData.maxConcurrentShardRequests == 42 - result.datasource.jsonData.timeField == '@timestamp' @@ -150,7 +197,7 @@ time_field: '@timestamp' time_interval: 1m interval: Daily - es_version: 56 + es_version: "7.10+" max_concurrent_shard_requests: 42 tls_ca_cert: /etc/ssl/certs/ca.pem enforce_secure_data: false @@ -170,7 +217,7 @@ - result.datasource.access == 'proxy' - result.datasource.database == '[logstash_]YYYY.MM.DD' - not result.datasource.isDefault - - result.datasource.jsonData.esVersion == 56 + - result.datasource.jsonData.esVersion == '7.10.0' - result.datasource.jsonData.interval == 'Daily' - result.datasource.jsonData.maxConcurrentShardRequests == 42 - result.datasource.jsonData.timeField == '@timestamp' @@ -201,7 +248,7 @@ time_field: '@timestamp' time_interval: 1m interval: Daily - es_version: 56 + es_version: "7.10+" max_concurrent_shard_requests: 42 tls_ca_cert: /etc/ssl/certs/ca.pem enforce_secure_data: true @@ -221,7 +268,7 @@ - result.datasource.access == 'proxy' - result.datasource.database == '[logstash_]YYYY.MM.DD' - not result.datasource.isDefault - - result.datasource.jsonData.esVersion == 56 + - result.datasource.jsonData.esVersion == '7.10.0' - result.datasource.jsonData.interval == 'Daily' - result.datasource.jsonData.maxConcurrentShardRequests == 42 - result.datasource.jsonData.timeField == '@timestamp'