Add field type for version strings

docs/reference/mapping/types/keyword.asciidoc

@@ -129,3 +129,5 @@ The following parameters are accepted by `keyword` fields:
 include::constant-keyword.asciidoc[]
 
 include::wildcard.asciidoc[]
+
+include::version.asciidoc[]

docs/reference/mapping/types/version.asciidoc

@@ -0,0 +1,127 @@
+[role="xpack"]
+[testenv="basic"]
+[discrete]
+[[version-field-type]]
+=== Version field type
+
+The `version` field type is a specialization of the `keyword` field for
+handling software version values and to support specialized precedence
+rules for them. Precedence is defined following the rules outlined by
+https://semver.org/[Semantic Versioning], which for example means that
+major, minor and patch version parts are sorted numerically (i.e. 
+"2.1.0" < "2.4.1" < "2.11.2") and pre-release versions are sorted before
+release versiond (i.e. "1.0.0-alpha" < "1.0.0").
+
+You index a `version` field as follows
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_version": {
+        "type": "version"
+      }
+    }
+  }
+}
+
+--------------------------------------------------
+
+The field offers the same search capabilities as a regular keyword field. It 
+can e.g. be searched for exact matches using `match` or `term` queries and
+supports prefix and wildcard searches. The main benefit is that `range` queries
+will honour Semver ordering, so a `range` query between "1.0.0" and "1.5.0"
+will include versions of "1.2.3" but not "1.11.2" for example. Note that this
+would be different when using a regular `keyword` field for indexing where ordering
+is alphabetical.
+
+Software versions are expected to follow the
+https://semver.org/[Semantic Versioning rules] schema and precedence rules with
+the notable exception that more or less than three main version identifiers are
+allowed (i.e. "1.2" or "1.2.3.4" qualify as valid versions while they wouldn't under
+strict Semver rules). Version strings that are not valid under the Semver definition
+(e.g. "1.2.alpha.4") can still be indexed and retrieved as exact matches, however they
+will all appear _after_ any valid version with regular alphabetical ordering. The empty
+String "" is considered invalid and sorted after all valid versions, but before other
+invalid ones.
+
+[discrete]
+[[version-params]]
+==== Parameters for version fields
+
+The following parameters are accepted by `version` fields:
+
+[horizontal]
+
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.
+
+[discrete]
+==== Limitations
+
+This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those
+type of queries work in this field, you should consider using a regular `keyword` field if
+you strongly rely on these kind of queries.
+
+==== Script support
+
+The `version` fields offers some specialized access to detailed information derived from
+valid version strings like the Major, Minor or Patch release number, whether the version value
+is valid according to Semver or if it is a pre-release version. This can be helpful when e.g.
+filtering for only released versions or running aggregations on parts of the version.
+The following query, for example, filters for released versions and groups them by Major version
+using a `terms` aggregation:
+
+[source,console]
+--------------------------------------------------
+POST my-index-000001/_search
+{
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "script": {
+            "script": {
+              "source": "doc['my_version'].isRelease() == true"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "aggs": {
+    "group_major": {
+      "terms": {
+        "script": { "source": "doc['my_version'].getMajor()"},
+        "order": {
+          "_key": "asc"
+        }
+      }
+    }
+  }
+}
+
+--------------------------------------------------
+// TEST[continued]
+
+Functions available on via doc values in scripting are:
+
+[horizontal]
+
+isValid()::
+    Returns `true` if the field contains a version thats legal according to the Semantic Versioning rules
+
+isRelease()::
+    Returns `true` if the field contains a valid release version, `false` if it is a pre-release version or invalid.
+
+getMajor()::
+    Returns an Integer value of the Major version if the version is valid, or null otherwise
+
+getMinor()::
+    Returns an Integer value of the Minor  version if the version is valid, or null otherwise.
+
+getPatch()::
+    Returns an Integer value of the Patch version if the version is valid, or null otherwise.

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;

server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java

@@ -33,9 +33,10 @@
 
 /** Base {@link MappedFieldType} implementation for a field that is indexed
  *  with the inverted index. */
-abstract class TermBasedFieldType extends SimpleMappedFieldType {
+public abstract class TermBasedFieldType extends SimpleMappedFieldType {
 
-    TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, Map<String, String> meta) {
+    public TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo,
+        Map<String, String> meta) {
         super(name, isSearchable, hasDocValues, textSearchInfo, meta);
     }
 

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java

@@ -101,7 +101,7 @@ public void collect(int doc, long bucket) throws IOException {
                     for (int i = 0; i < valuesCount; i++) {
                         BytesRef value = values.nextValue();
                         if (value.length > 0) {
-                            String valueStr = value.utf8ToString();
+                            String valueStr = (String) format.format(value);
                             int length = valueStr.length();
                             totalLength.increment(bucket, length);
 

x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml

@@ -0,0 +1,107 @@
+# Integration tests for the version field
+#
+---
+setup:
+
+  - skip:
+      features: headers
+      version: " - 7.99.99"
+      reason: "version field is added to 8.0 first"
+
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                properties:
+                  version:
+                    type: version
+
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "test_index", "_id" : "1" } }'
+            - '{"version": "1.1.0" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "2" } }'
+            - '{"version": "2.0.0-beta" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "3" } }'
+            - '{"version": "3.1.0" }'
+
+---
+"Store malformed":
+  - do:
+        indices.create:
+          index: test_malformed
+          body:
+              mappings:
+                properties:
+                  version:
+                    type: version
+
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "1" } }'
+            - '{"version": "1.1.0" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "2" } }'
+            - '{"version": "2.0.0-beta" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "3" } }'
+            - '{"version": "v3.1.0" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "4" } }'
+            - '{"version": "1.el6" }'
+
+  - do:
+        search:
+          index: test_malformed
+          body:
+            query: { "match" : { "version" : "1.el6" } }
+
+  - do:
+        search:
+          index: test_malformed
+          body:
+            query: { "match_all" : { } }
+            sort:
+              version: asc
+
+  - match: { hits.total.value: 4 }
+  - match: { hits.hits.0._source.version: "1.1.0" }
+  - match: { hits.hits.1._source.version: "2.0.0-beta" }
+  - match: { hits.hits.2._source.version: "1.el6" }
+  - match: { hits.hits.3._source.version: "v3.1.0" }
+
+---
+"Basic ranges":
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gt" : "1.1.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 2 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "1.1.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 3 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "2.0.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 1 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "2.0.0-alpha", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 2 }