diff --git a/esrally/metrics.py b/esrally/metrics.py
index f447c8789..3230e0704 100644
--- a/esrally/metrics.py
+++ b/esrally/metrics.py
@@ -62,13 +62,15 @@ def delete_by_query(self, index, body):
         return self.guarded(self._client.delete_by_query, index=index, body=body)
 
     def delete(self, index, id):
+        # ignore 404 status code (NotFoundError) when index does not exist
         return self.guarded(self._client.delete, index=index, id=id, ignore=404)
 
     def get_index(self, name):
         return self.guarded(self._client.indices.get, name=name)
 
-    def create_index(self, index, body=None):
-        return self.guarded(self._client.indices.create, index=index, body=body, ignore=400)
+    def create_index(self, index):
+        # ignore 400 status code (BadRequestError) when index already exists
+        return self.guarded(self._client.indices.create, index=index, ignore=400)
 
     def exists(self, index):
         return self.guarded(self._client.indices.exists, index=index)
@@ -94,6 +96,7 @@ def search(self, index, body):
     def guarded(self, target, *args, **kwargs):
         # pylint: disable=import-outside-toplevel
         import elasticsearch
+        import elasticsearch.helpers
         from elastic_transport import ApiError, TransportError
 
         max_execution_count = 10
@@ -1773,8 +1776,9 @@ def _at_midnight(race_timestamp):
             )
         else:
             if not self.client.exists(index="rally-annotations"):
-                body = self.index_template_provider.annotations_template()
-                self.client.create_index(index="rally-annotations", body=body)
+                # create or overwrite template on index creation
+                self.client.put_template("rally-annotations", self.index_template_provider.annotations_template())
+                self.client.create_index(index="rally-annotations")
             self.client.index(
                 index="rally-annotations",
                 id=annotation_id,
diff --git a/esrally/resources/annotation-template.json b/esrally/resources/annotation-template.json
index 540bc5891..60b760a1a 100644
--- a/esrally/resources/annotation-template.json
+++ b/esrally/resources/annotation-template.json
@@ -1,6 +1,8 @@
 {
+  "index_patterns": ["rally-annotations"],
   "settings": {
-    "number_of_shards": 1
+    "index":{
+    }
   },
   "mappings": {
     "dynamic_templates": [
diff --git a/tests/metrics_test.py b/tests/metrics_test.py
index 225caf4b6..87e371e31 100644
--- a/tests/metrics_test.py
+++ b/tests/metrics_test.py
@@ -2607,6 +2607,7 @@ def test_primary_and_replica_shard_count_specified_index_template_update(self):
         _index_template_provider = metrics.IndexTemplateProvider(self.cfg)
 
         templates = [
+            _index_template_provider.annotations_template(),
             _index_template_provider.metrics_template(),
             _index_template_provider.races_template(),
             _index_template_provider.results_template(),
@@ -2627,6 +2628,7 @@ def test_primary_shard_count_specified_index_template_update(self):
         _index_template_provider = metrics.IndexTemplateProvider(self.cfg)
 
         templates = [
+            _index_template_provider.annotations_template(),
             _index_template_provider.metrics_template(),
             _index_template_provider.races_template(),
             _index_template_provider.results_template(),
@@ -2649,6 +2651,7 @@ def test_replica_shard_count_specified_index_template_update(self):
         _index_template_provider = metrics.IndexTemplateProvider(self.cfg)
 
         templates = [
+            _index_template_provider.annotations_template(),
             _index_template_provider.metrics_template(),
             _index_template_provider.races_template(),
             _index_template_provider.results_template(),
@@ -2672,6 +2675,7 @@ def test_primary_shard_count_less_than_one(self):
         with pytest.raises(exceptions.SystemSetupError) as ctx:
             # pylint: disable=unused-variable
             templates = [
+                _index_template_provider.annotations_template(),
                 _index_template_provider.metrics_template(),
                 _index_template_provider.races_template(),
                 _index_template_provider.results_template(),
@@ -2693,6 +2697,7 @@ def test_primary_and_replica_shard_counts_passed_as_strings(self):
         _index_template_provider = metrics.IndexTemplateProvider(self.cfg)
 
         templates = [
+            _index_template_provider.annotations_template(),
             _index_template_provider.metrics_template(),
             _index_template_provider.races_template(),
             _index_template_provider.results_template(),