fix(datasource-elastic): allow support of elastic version > 7

In latest versions of Grafana, the elastic version is no more provided
as an integer even if it is still supported and won't return any error.
To ease configuration, the module will accept the version format as
displayed by the WUI (ex: "7.10+") and will map the value to what the API
expects ("7.10.0").
The added values are the one supported by Grafana 8 and 9 (greater than
elastic 7.0 which was already supported).

The module will keep on supporting the old format to avoid a breaking
change with significant impact.

Fixes: #263

changelogs/fragments/fix-263.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+  - Add support for more elasticsearch version as datasource (#263)

plugins/modules/grafana_datasource.py

@@ -137,13 +137,16 @@
     - Version 56 is for elasticsearch 5.6+ where you can specify the C(max_concurrent_shard_requests)
       option.
     choices:
-    - 2
-    - 5
-    - 56
-    - 60
-    - 70
-    default: 5
-    type: int
+    - "2"
+    - "5"
+    - "56"
+    - "60"
+    - "70"
+    - "7.7+"
+    - "7.10+"
+    - "8.0+"
+    default: "7.10+"
+    type: str
   max_concurrent_shard_requests:
     description:
     - Starting with elasticsearch 5.6, you can specify the max concurrent shard per
@@ -490,6 +493,13 @@
 from ansible_collections.community.grafana.plugins.module_utils import base
 
 
+ES_VERSION_MAPPING = {
+    "7.7+": "7.7.0",
+    "7.10+": "7.10.0",
+    "8.0+": "8.0.0",
+}
+
+
 def compare_datasources(new, current, compareSecureData=True):
     if 'uid' in current:
         del current['uid']
@@ -581,12 +591,22 @@ def get_datasource_payload(data):
 
     # datasource type related parameters
     if data['ds_type'] == 'elasticsearch':
-        json_data['esVersion'] = data['es_version']
+
+        json_data['maxConcurrentShardRequests'] = data['max_concurrent_shard_requests']
         json_data['timeField'] = data['time_field']
         if data.get('interval'):
             json_data['interval'] = data['interval']
-        if data['es_version'] >= 56:
-            json_data['maxConcurrentShardRequests'] = data['max_concurrent_shard_requests']
+
+        # Handle changes in es_version format in Grafana < 8.x which used to
+        # be integers and is now semver format
+        try:
+            es_version = int(data['es_version'])
+            if es_version < 56:
+                json_data.pop('maxConcurrentShardRequests')
+        except ValueError:
+            # Retrieve the Semver format expected by API
+            es_version = ES_VERSION_MAPPING.get(data['es_version'])
+        json_data['esVersion'] = es_version
 
     if data['ds_type'] == 'elasticsearch' or data['ds_type'] == 'influxdb':
         if data.get('time_interval'):
@@ -725,7 +745,9 @@ def setup_module_object():
         tls_skip_verify=dict(type='bool', default=False),
         is_default=dict(default=False, type='bool'),
         org_id=dict(default=1, type='int'),
-        es_version=dict(type='int', default=5, choices=[2, 5, 56, 60, 70]),
+        es_version=dict(type='str', default="7.10+", choices=["2", "5", "56", "60",
+                                                              "70", "7.7+", "7.10+",
+                                                              "8.0+"]),
         max_concurrent_shard_requests=dict(type='int', default=256),
         time_field=dict(default='@timestamp', type='str'),
         time_interval=dict(type='str'),
@@ -770,9 +792,9 @@ def setup_module_object():
             ['ds_type', 'mysql', ['database']],
             ['ds_type', 'postgres', ['database', 'sslmode']],
             ['ds_type', 'cloudwatch', ['aws_auth_type', 'aws_default_region']],
-            ['es_version', 56, ['max_concurrent_shard_requests']],
-            ['es_version', 60, ['max_concurrent_shard_requests']],
-            ['es_version', 70, ['max_concurrent_shard_requests']]
+            ['es_version', "56", ['max_concurrent_shard_requests']],
+            ['es_version', "60", ['max_concurrent_shard_requests']],
+            ['es_version', "70", ['max_concurrent_shard_requests']]
         ],
     )
     return module

tests/integration/targets/grafana_datasource/tasks/elastic.yml

@@ -1,7 +1,8 @@
-- name: Create elasticsearch datasource
+---
+- name: Create elasticsearch datasource with legacy elasticsearch format
   register: result
   grafana_datasource:
-    name: "datasource/elastic"
+    name: "datasource/elasticLegacy"
     grafana_url: "{{ grafana_url }}"
     grafana_user: "{{ grafana_username }}"
     grafana_password: "{{ grafana_password }}"
@@ -35,6 +36,52 @@
     - result.datasource.jsonData.timeField == '@timestamp'
     - not result.datasource.jsonData.tlsAuth
     - result.datasource.jsonData.tlsAuthWithCACert
+    - result.datasource.name == 'datasource/elasticLegacy'
+    - result.datasource.orgId == 1
+    - ('password' not in result.datasource) or (result.datasource.password == '')
+    - result.datasource.type == 'elasticsearch'
+    - result.datasource.url == 'https://elastic.company.com:9200'
+    - result.datasource.user == ''
+    - not result.datasource.withCredentials
+    - "result.msg == 'Datasource datasource/elasticLegacy created'"
+
+- name: Create elasticsearch datasource with new elsaticsearch version format
+  register: result
+  grafana_datasource:
+    name: "datasource/elastic"
+    grafana_url: "{{ grafana_url }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    org_id: '1'
+    ds_type: elasticsearch
+    ds_url: https://elastic.company.com:9200
+    database: '[logstash_]YYYY.MM.DD'
+    basic_auth_user: grafana
+    basic_auth_password: '******'
+    time_field: '@timestamp'
+    time_interval: 1m
+    interval: Daily
+    es_version: "7.10+"
+    max_concurrent_shard_requests: 42
+    tls_ca_cert: /etc/ssl/certs/ca.pem
+
+- debug:
+    var: result
+
+- assert:
+    that:
+    - result.changed
+    - result.datasource.basicAuth
+    - result.datasource.basicAuthUser == 'grafana'
+    - result.datasource.access == 'proxy'
+    - result.datasource.database == '[logstash_]YYYY.MM.DD'
+    - not result.datasource.isDefault
+    - result.datasource.jsonData.esVersion == "7.10.0"
+    - result.datasource.jsonData.interval == 'Daily'
+    - result.datasource.jsonData.maxConcurrentShardRequests == 42
+    - result.datasource.jsonData.timeField == '@timestamp'
+    - not result.datasource.jsonData.tlsAuth
+    - result.datasource.jsonData.tlsAuthWithCACert
     - result.datasource.name == 'datasource/elastic'
     - result.datasource.orgId == 1
     - ('password' not in result.datasource) or (result.datasource.password == '')
@@ -60,7 +107,7 @@
     time_field: '@timestamp'
     time_interval: 1m
     interval: Daily
-    es_version: 56
+    es_version: "7.10+"
     max_concurrent_shard_requests: 42
     tls_ca_cert: /etc/ssl/certs/ca.pem
 
@@ -75,7 +122,7 @@
     - result.datasource.access == 'proxy'
     - result.datasource.database == '[logstash_]YYYY.MM.DD'
     - not result.datasource.isDefault
-    - result.datasource.jsonData.esVersion == 56
+    - result.datasource.jsonData.esVersion == '7.10.0'
     - result.datasource.jsonData.interval == 'Daily'
     - result.datasource.jsonData.maxConcurrentShardRequests == 42
     - result.datasource.jsonData.timeField == '@timestamp'
@@ -105,7 +152,7 @@
     time_field: '@timestamp'
     time_interval: 1m
     interval: Daily
-    es_version: 56
+    es_version: "7.10+"
     max_concurrent_shard_requests: 42
     tls_ca_cert: /etc/ssl/certs/ca.pem
 
@@ -120,7 +167,7 @@
     - result.datasource.access == 'proxy'
     - result.datasource.database == '[logstash_]YYYY.MM.DD'
     - not result.datasource.isDefault
-    - result.datasource.jsonData.esVersion == 56
+    - result.datasource.jsonData.esVersion == '7.10.0'
     - result.datasource.jsonData.interval == 'Daily'
     - result.datasource.jsonData.maxConcurrentShardRequests == 42
     - result.datasource.jsonData.timeField == '@timestamp'
@@ -150,7 +197,7 @@
     time_field: '@timestamp'
     time_interval: 1m
     interval: Daily
-    es_version: 56
+    es_version: "7.10+"
     max_concurrent_shard_requests: 42
     tls_ca_cert: /etc/ssl/certs/ca.pem
     enforce_secure_data: false
@@ -170,7 +217,7 @@
     - result.datasource.access == 'proxy'
     - result.datasource.database == '[logstash_]YYYY.MM.DD'
     - not result.datasource.isDefault
-    - result.datasource.jsonData.esVersion == 56
+    - result.datasource.jsonData.esVersion == '7.10.0'
     - result.datasource.jsonData.interval == 'Daily'
     - result.datasource.jsonData.maxConcurrentShardRequests == 42
     - result.datasource.jsonData.timeField == '@timestamp'
@@ -201,7 +248,7 @@
     time_field: '@timestamp'
     time_interval: 1m
     interval: Daily
-    es_version: 56
+    es_version: "7.10+"
     max_concurrent_shard_requests: 42
     tls_ca_cert: /etc/ssl/certs/ca.pem
     enforce_secure_data: true
@@ -221,7 +268,7 @@
     - result.datasource.access == 'proxy'
     - result.datasource.database == '[logstash_]YYYY.MM.DD'
     - not result.datasource.isDefault
-    - result.datasource.jsonData.esVersion == 56
+    - result.datasource.jsonData.esVersion == '7.10.0'
     - result.datasource.jsonData.interval == 'Daily'
     - result.datasource.jsonData.maxConcurrentShardRequests == 42
     - result.datasource.jsonData.timeField == '@timestamp'