No es index smf fsd

README.md

@@ -35,10 +35,10 @@ You must supply a "wrapper", which provides these functions:
 * build the container image for your benchmark, with all the packages, python modules, etc. that are required to run it.
 * runs the benchmark and stores the benchmark-specific results to an elasticsearch server
 
-Your ripsaw benchmark will define several environment variables:
+Your ripsaw benchmark will define several environment variables relevant to Elasticsearch:
 * es - hostname of elasticsearch server
 * es_port - port number of elasticsearch server (default 9020)
-* es_index - prefix of index name for your results in elasticsearch (default "ripsaw")
+* es_index - OPTIONAL - default is "snafu-tool" - define the prefix of the ES index name
 
 It will then invoke your wrapper via the command:
 
@@ -84,6 +84,32 @@ server that is viewable with Kibana and Grafana!
 
 Look at some of the other benchmarks for examples of how this works.
 
+## how do I post results to Elasticsearch from my wrapper?
+
+Every snafu benchmark will use Elasticsearch index name of the form **orchestrator-benchmark-doctype**, consisting of the 3
+components:
+
+* orchestrator - software running the benchmark - usually "ripsaw" at this point
+* benchmark - typically the tool name, something like "iperf" or "fio"
+* doctype - type of documents being placed in this index.
+
+If you are using run_snafu.py, construct an elastic search document in the usual way, and then use the python "yield" statement (do not return!) a **document** and **doctype**, where **document** is a python dictionary representing an Elasticsearch document, and **doctype** is the end of the index name.  For example, any ripsaw benchmark will be defining an index name that begins with ripsaw, but your wrapper can create whatever indexes it wants with that prefix.  For example, to create an index named ripsaw-iperf-results, you just do something like this:
+
+- optionally, in roles/your-benchmark/defaults/main.yml, you can override the default if you need to:
+
+```
+es_index: ripsaw-iperf
+```
+
+- in your snafu wrapper, to post a document to Elasticsearch, you **MUST**:
+
+```
+    yield my_doc, 'results'
+```
+
+run_snafu.py concatenates the doctype with the es_index component associated with the benchmark to generate the
+full index name, and posts document **my__doc** to it.
+
 ## how do I integrate snafu wrapper into my ripsaw benchmark?
 
 You just replace the commands to run the workload in your ripsaw benchmark 
@@ -98,19 +124,17 @@ run_snafu.py for access to Elasticsearch:
       spec:
         containers:
           env:
-{% if es_server is defined %}
           - name: uuid
             value: "{{ uuid }}"
           - name: test_user
             value: "{{ test_user }}"
           - name: clustername
             value: "{{ clustername }}"
+{% if elasticsearch.server is defined %}
           - name: es
-            value: "{{ es_server }}"
+            value: "{{ elasticsearch.server }}"
           - name: es_port
-            value: "{{ es_port }}"
-          - name: es_index
-            value: "{{ es_index }}"
+            value: "{{ elasticsearch.port }}"
 {% endif %}
 ```
 

cluster_loader/trigger_cluster_loader.py

@@ -83,4 +83,4 @@ def emit_actions(self):
         output_template['sample'] = self.sample
         output_template['test_name'] = self.test_name
         output_template.update(cl_output_dict)
-        yield output_template, 'snafu-cl'
+        yield output_template, 'cl'

fio_wrapper/fio_analyzer.py

@@ -132,5 +132,5 @@ def emit_actions(self):
                 importdoc['ceph_benchmark_test']['test_data'] = tmp_doc
                 importdoc['cluster_name'] = self.cluster_name
                 #TODO add ID to document
-                index = "-analyzed-result"
+                index = "analyzed-result"
                 yield importdoc, index

fio_wrapper/trigger_fio.py

@@ -271,7 +271,7 @@ def emit_actions(self):
             self.fio_analyzer_obj.add_fio_result_documents(fio_result_documents, earliest_starttime)
 
             #from the returned normalized fio json document yield up for indexing
-            index = "-results"
+            index = "results"
             for document in fio_result_documents:
                 yield document, index
 
@@ -292,7 +292,7 @@ def emit_actions(self):
             fio_log_documents = self._log_payload(job_dir, self.user, self.uuid, self.sample, self.fio_jobs_dict, fio_version, fio_starttime, hosts, job)
 
             #if indexing is turned on yield back normalized data
-            index = "-log"
+            index = "log"
             for document in fio_log_documents:
                 yield document, index
             if self.histogram_process:
@@ -308,6 +308,6 @@ def emit_actions(self):
                 histogram_documents = self._histogram_payload(histogram_output_file, self.user, self.uuid, self.sample, self.fio_jobs_dict, fio_version, earliest_starttime, hosts, job)
                 #if indexing is turned on yield back normalized data
 
-                index = "-hist-log"
+                index = "hist-log"
                 for document in histogram_documents:
                     yield document, index

fs_drift_wrapper/trigger_fs_drift.py

@@ -75,7 +75,7 @@ def emit_actions(self):
                 thrd['uuid'] = self.uuid
                 thrd['user'] = self.user
                 thrd['params'] = params
-                yield thrd, '-results'
+                yield thrd, 'results'
 
         # process response time data
 
@@ -129,4 +129,4 @@ def emit_actions(self):
                     interval['90%'] = float(flds[8])
                     interval['95%'] = float(flds[9])
                     interval['99%'] = float(flds[10])
-                    yield interval, '-rsptimes'
+                    yield interval, 'rsptimes'

run_snafu.py

@@ -115,7 +115,7 @@ def process_generator(index_args, parser):
         for data_object in wrapper_object.run():
             for action, index in data_object.emit_actions():
 
-                es_index = index_args.prefix + index
+                es_index = index_args.prefix + '-' + index
                 es_valid_document = { "_index": es_index,
                                       "_op_type": "create",
                                       "_source": action,

smallfile_wrapper/trigger_smallfile.py

@@ -86,7 +86,7 @@ def emit_actions(self):
                     thrd['host'] = self.host
                     thrd['tid'] = tid
                     thrd['date'] = timestamp
-                    yield thrd, '-results'
+                    yield thrd, 'results'
 
             # process response time data
 
@@ -135,7 +135,7 @@ def emit_actions(self):
                             interval['90%'] = float(flds[8])
                             interval['95%'] = float(flds[9])
                             interval['99%'] = float(flds[10])
-                            yield interval, '-rsptimes'
+                            yield interval, 'rsptimes'
 
         # clean up anything created by smallfile so that the next sample will work
         # this is brutally inefficient, best way to clean up is to