Add support for Elasticsearch ILM Polices

plugin/storage/es/esRollover.py

@@ -8,6 +8,7 @@
 import requests
 import ssl
 import sys
+import re
 from pathlib2 import Path
 from requests.auth import HTTPBasicAuth
 
@@ -57,17 +58,17 @@ def main():
     if str2bool(os.getenv('ARCHIVE', 'false')):
         write_alias = prefix + ARCHIVE_INDEX + '-write'
         read_alias = prefix + ARCHIVE_INDEX + '-read'
-        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span-archive', 'jaeger-span')
+        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span-archive', 'jaeger-span', prefix)
     else:
         write_alias = prefix + 'jaeger-span-write'
         read_alias = prefix + 'jaeger-span-read'
-        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span', 'jaeger-span')
+        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span', 'jaeger-span', prefix)
         write_alias = prefix + 'jaeger-service-write'
         read_alias = prefix + 'jaeger-service-read'
-        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-service', 'jaeger-service')
+        perform_action(action, client, write_alias, read_alias, prefix+'jaeger-service', 'jaeger-service', prefix)
 
 
-def perform_action(action, client, write_alias, read_alias, index_to_rollover, template_name):
+def perform_action(action, client, write_alias, read_alias, index_to_rollover, template_name, prefix):
     if action == 'init':
         shards = os.getenv('SHARDS', SHARDS)
         replicas = os.getenv('REPLICAS', REPLICAS)
@@ -76,7 +77,7 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
             mapping = Path('./mappings/'+template_name+'-7.json').read_text()
         else:
             mapping = Path('./mappings/'+template_name+'.json').read_text()
-        create_index_template(fix_mapping(mapping, shards, replicas), template_name)
+        create_index_template(fix_mapping(mapping, shards, replicas, prefix), prefix+template_name+"-with-ilm")
 
         index = index_to_rollover + '-000001'
         create_index(client, index)
@@ -118,9 +119,12 @@ def create_aliases(client, alias_name, archive_index_name):
     """
     ilo = curator.IndexList(client)
     ilo.filter_by_regex(kind='regex', value='^'+archive_index_name+'$')
-    alias = curator.Alias(client=client, name=alias_name)
     for index in ilo.working_list():
         print("Adding index {} to alias {}".format(index, alias_name))
+    if re.search(r'write', alias_name):
+        alias = curator.Alias(client=client, name=alias_name, extra_settings={'is_write_index': True})
+    else:
+        alias = curator.Alias(client=client, name=alias_name)
     alias.add(ilo)
     alias.do_action()
 
@@ -174,9 +178,10 @@ def str2bool(v):
     return v.lower() in ('true', '1')
 
 
-def fix_mapping(mapping, shards, replicas):
+def fix_mapping(mapping, shards, replicas, prefix):
     mapping = mapping.replace("${__NUMBER_OF_SHARDS__}", str(shards))
     mapping = mapping.replace("${__NUMBER_OF_REPLICAS__}", str(replicas))
+    mapping = mapping.replace("${__PREFIX__}", str(prefix))
     return mapping
 
 

plugin/storage/es/mappings/jaeger-service-7.json

@@ -1,10 +1,18 @@
 {
-  "index_patterns": "*jaeger-service-*",
+  "index_patterns": "${__PREFIX__}jaeger-service-*",
+  "order": 1,
+  "aliases": {
+    "${__PREFIX__}jaeger-service-read" : {}
+  },
   "settings":{
     "index.number_of_shards": ${__NUMBER_OF_SHARDS__},
     "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__},
     "index.mapping.nested_fields.limit":50,
-    "index.requests.cache.enable":true
+    "index.requests.cache.enable":true,
+    "lifecycle": {
+        "name": "jaeger",
+        "rollover_alias": "${__PREFIX__}jaeger-service-write"
+    }
   },
   "mappings":{
     "dynamic_templates":[

plugin/storage/es/mappings/jaeger-span-7.json

@@ -1,10 +1,18 @@
 {
-  "index_patterns": "*jaeger-span-*", 
+  "index_patterns": "${__PREFIX__}jaeger-span-*",
+  "order": 1,
+  "aliases": {
+    "${__PREFIX__}jaeger-span-read": {}
+  },
   "settings":{
     "index.number_of_shards": ${__NUMBER_OF_SHARDS__},
     "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__},
     "index.mapping.nested_fields.limit":50,
-    "index.requests.cache.enable":true
+    "index.requests.cache.enable":true,
+    "lifecycle": {
+      "name": "jaeger",
+      "rollover_alias": "${__PREFIX__}jaeger-span-write"
+    }
   },
   "mappings":{
     "dynamic_templates":[