Create an Ingest API

package.json

@@ -179,7 +179,8 @@
     "portscanner": "1.0.0",
     "simple-git": "1.8.0",
     "sinon": "1.17.2",
-    "source-map": "0.4.4"
+    "source-map": "0.4.4",
+    "supertest-as-promised": "2.0.2"
   },
   "engines": {
     "node": "0.12.9",

src/plugins/kibana/index.js

@@ -1,3 +1,5 @@
+const ingest = require('./server/routes/api/ingest');
+
 module.exports = function (kibana) {
   return new kibana.Plugin({
 
@@ -43,6 +45,10 @@ module.exports = function (kibana) {
           };
         }
       }
+    },
+
+    init: function (server, options) {
+      ingest(server);
     }
   });
 

src/plugins/kibana/server/lib/__tests__/case_conversion.js

@@ -0,0 +1,35 @@
+const expect = require('expect.js');
+const { keysToSnakeCaseShallow, keysToCamelCaseShallow } = require('../case_conversion');
+const _ = require('lodash');
+
+describe('keysToSnakeCaseShallow', function () {
+
+  it('should convert all of an object\'s keys to snake case', function () {
+    const result = keysToSnakeCaseShallow({
+      camelCase: 'camel_case',
+      'kebab-case': 'kebab_case',
+      snake_case: 'snake_case'
+    });
+
+    _.forEach(result, function (value, key) {
+      expect(key).to.be(value);
+    });
+  });
+
+});
+
+describe('keysToCamelCaseShallow', function () {
+
+  it('should convert all of an object\'s keys to camel case', function () {
+    const result = keysToCamelCaseShallow({
+      camelCase: 'camelCase',
+      'kebab-case': 'kebabCase',
+      snake_case: 'snakeCase'
+    });
+
+    _.forEach(result, function (value, key) {
+      expect(key).to.be(value);
+    });
+  });
+
+});

src/plugins/kibana/server/lib/__tests__/convert_pattern_and_template_name.js

@@ -0,0 +1,31 @@
+const {templateToPattern, patternToTemplate} = require('../convert_pattern_and_template_name');
+const expect = require('expect.js');
+
+describe('convertPatternAndTemplateName', function () {
+
+  describe('templateToPattern', function () {
+
+    it('should convert an index template\'s name to its matching index pattern\'s title', function () {
+      expect(templateToPattern('kibana-logstash-*')).to.be('logstash-*');
+    });
+
+    it('should throw an error if the template name isn\'t a valid kibana namespaced name', function () {
+      expect(templateToPattern).withArgs('logstash-*').to.throwException('not a valid kibana namespaced template name');
+      expect(templateToPattern).withArgs('').to.throwException(/not a valid kibana namespaced template name/);
+    });
+
+  });
+
+  describe('patternToTemplate', function () {
+
+    it('should convert an index pattern\'s title to its matching index template\'s name', function () {
+      expect(patternToTemplate('logstash-*')).to.be('kibana-logstash-*');
+    });
+
+    it('should throw an error if the pattern is empty', function () {
+      expect(patternToTemplate).withArgs('').to.throwException(/pattern must not be empty/);
+    });
+
+  });
+
+});

src/plugins/kibana/server/lib/__tests__/create_mappings_from_pattern_fields.js

@@ -0,0 +1,82 @@
+const createMappingsFromPatternFields = require('../create_mappings_from_pattern_fields');
+const expect = require('expect.js');
+const _ = require('lodash');
+
+let testFields;
+
+describe('createMappingsFromPatternFields', function () {
+
+  beforeEach(function () {
+    testFields = [
+      {
+        'name': 'ip',
+        'type': 'ip'
+      },
+      {
+        'name': 'agent',
+        'type': 'string'
+      },
+      {
+        'name': 'bytes',
+        'type': 'number'
+      }
+    ];
+  });
+
+  it('should throw an error if the argument is empty', function () {
+    expect(createMappingsFromPatternFields).to.throwException(/argument must not be empty/);
+  });
+
+  it('should not modify the original argument', function () {
+    const testFieldClone = _.cloneDeep(testFields);
+    const mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings.ip).to.not.be(testFields[0]);
+    expect(_.isEqual(testFields, testFieldClone)).to.be.ok();
+  });
+
+  it('should set the same default mapping for all non-strings', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    _.forEach(mappings, function (mapping) {
+      if (mapping.type !== 'string') {
+        expect(_.isEqual(mapping, {
+          type: mapping.type,
+          index: 'not_analyzed',
+          doc_values: true
+        })).to.be.ok();
+      }
+    });
+  });
+
+  it('should give strings a multi-field mapping', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    _.forEach(mappings, function (mapping) {
+      if (mapping.type === 'string') {
+        expect(mapping).to.have.property('fields');
+      }
+    });
+  });
+
+  it('should handle nested fields', function () {
+    testFields.push({name: 'geo.coordinates', type: 'geo_point'});
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings).to.have.property('geo');
+    expect(mappings.geo).to.have.property('properties');
+    expect(mappings.geo.properties).to.have.property('coordinates');
+    expect(_.isEqual(mappings.geo.properties.coordinates, {
+      type: 'geo_point',
+      index: 'not_analyzed',
+      doc_values: true
+    })).to.be.ok();
+  });
+
+  it('should map all number fields as an ES double', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings).to.have.property('bytes');
+    expect(mappings.bytes).to.have.property('type', 'double');
+  });
+});

src/plugins/kibana/server/lib/__tests__/handle_es_error.js

@@ -0,0 +1,41 @@
+var expect = require('expect.js');
+var Boom = require('boom');
+var esErrors = require('elasticsearch').errors;
+var handleESError = require('../handle_es_error');
+
+describe('handleESError', function () {
+
+  it('should transform elasticsearch errors into boom errors with the same status code', function () {
+    var conflict = handleESError(new esErrors.Conflict());
+    expect(conflict.isBoom).to.be(true);
+    expect(conflict.output.statusCode).to.be(409);
+
+    var forbidden = handleESError(new esErrors[403]);
+    expect(forbidden.isBoom).to.be(true);
+    expect(forbidden.output.statusCode).to.be(403);
+
+    var notFound = handleESError(new esErrors.NotFound());
+    expect(notFound.isBoom).to.be(true);
+    expect(notFound.output.statusCode).to.be(404);
+
+    var badRequest = handleESError(new esErrors.BadRequest());
+    expect(badRequest.isBoom).to.be(true);
+    expect(badRequest.output.statusCode).to.be(400);
+  });
+
+  it('should return an unknown error without transforming it', function () {
+    var unknown = new Error('mystery error');
+    expect(handleESError(unknown)).to.be(unknown);
+  });
+
+  it('should return a boom 503 server timeout error for ES connection errors', function () {
+    expect(handleESError(new esErrors.ConnectionFault()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.ServiceUnavailable()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.NoConnections()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.RequestTimeout()).output.statusCode).to.be(503);
+  });
+
+  it('should throw an error if called with a non-error argument', function () {
+    expect(handleESError).withArgs('notAnError').to.throwException();
+  });
+});

src/plugins/kibana/server/lib/__tests__/init_default_field_props.js

@@ -0,0 +1,74 @@
+const initDefaultFieldProps = require('../init_default_field_props');
+const expect = require('expect.js');
+const _ = require('lodash');
+let fields;
+
+const testData = [
+  {
+    'name': 'ip',
+    'type': 'ip'
+  }, {
+    'name': '@timestamp',
+    'type': 'date'
+  }, {
+    'name': 'agent',
+    'type': 'string'
+  }, {
+    'name': 'bytes',
+    'type': 'number'
+  },
+  {
+    'name': 'geo.coordinates',
+    'type': 'geo_point'
+  }
+];
+
+describe('initDefaultFieldProps', function () {
+
+  beforeEach(function () {
+    fields = _.cloneDeep(testData);
+  });
+
+  it('should throw an error if no argument is passed or the argument is not an array', function () {
+    expect(initDefaultFieldProps).to.throwException(/requires an array argument/);
+    expect(initDefaultFieldProps).withArgs({}).to.throwException(/requires an array argument/);
+  });
+
+  it('should set the same defaults for everything but strings', function () {
+    const results = initDefaultFieldProps(fields);
+    _.forEach(results, function (field) {
+      if (field.type !== 'string') {
+        expect(field).to.have.property('indexed', true);
+        expect(field).to.have.property('analyzed', false);
+        expect(field).to.have.property('doc_values', true);
+        expect(field).to.have.property('scripted', false);
+        expect(field).to.have.property('count', 0);
+      }
+    });
+  });
+
+  it('should make string fields analyzed', function () {
+    const results = initDefaultFieldProps(fields);
+    _.forEach(results, function (field) {
+      if (field.type === 'string' && !_.contains(field.name, 'raw')) {
+        expect(field).to.have.property('indexed', true);
+        expect(field).to.have.property('analyzed', true);
+        expect(field).to.have.property('doc_values', false);
+        expect(field).to.have.property('scripted', false);
+        expect(field).to.have.property('count', 0);
+      }
+    });
+  });
+
+  it('should create an extra raw non-analyzed field for strings', function () {
+    const results = initDefaultFieldProps(fields);
+    const rawField = _.find(results, function (field) {
+      return _.contains(field.name, 'raw');
+    });
+    expect(rawField).to.have.property('indexed', true);
+    expect(rawField).to.have.property('analyzed', false);
+    expect(rawField).to.have.property('doc_values', true);
+    expect(rawField).to.have.property('scripted', false);
+    expect(rawField).to.have.property('count', 0);
+  });
+});

src/plugins/kibana/server/lib/case_conversion.js

@@ -0,0 +1,15 @@
+const _ = require('lodash');
+
+module.exports = {
+  keysToSnakeCaseShallow: function (object) {
+    return _.mapKeys(object, (value, key) => {
+      return _.snakeCase(key);
+    });
+  },
+
+  keysToCamelCaseShallow: function (object) {
+    return _.mapKeys(object, (value, key) => {
+      return _.camelCase(key);
+    });
+  }
+};

src/plugins/kibana/server/lib/convert_pattern_and_template_name.js

@@ -0,0 +1,22 @@
+// To avoid index template naming collisions the index pattern creation API
+// namespaces template names by prepending 'kibana-' to the matching pattern's title.
+// e.g. a pattern with title `logstash-*` will have a matching template named `kibana-logstash-*`.
+// This module provides utility functions for easily converting between template and pattern names.
+
+module.exports = {
+  templateToPattern: (templateName) => {
+    if (templateName.indexOf('kibana-') === -1) {
+      throw new Error('not a valid kibana namespaced template name');
+    }
+
+    return templateName.slice(templateName.indexOf('-') + 1);
+  },
+
+  patternToTemplate: (patternName) => {
+    if (patternName === '') {
+      throw new Error('pattern must not be empty');
+    }
+
+    return `kibana-${patternName.toLowerCase()}`;
+  }
+};

src/plugins/kibana/server/lib/create_mappings_from_pattern_fields.js

@@ -0,0 +1,38 @@
+const _ = require('lodash');
+
+// Creates an ES field mapping from a single field object in a kibana index pattern
+module.exports = function createMappingsFromPatternFields(fields) {
+  if (_.isEmpty(fields)) {
+    throw new Error('argument must not be empty');
+  }
+
+  const mappings = {};
+
+  _.forEach(fields, function (field) {
+    let mapping;
+
+    if (field.type === 'string') {
+      mapping = {
+        type: 'string',
+        index: 'analyzed',
+        omit_norms: true,
+        fielddata: {format: 'disabled'},
+        fields: {
+          raw: {type: 'string', index: 'not_analyzed', doc_values: true, ignore_above: 256}
+        }
+      };
+    }
+    else {
+      const fieldType = field.type === 'number' ? 'double' : field.type;
+      mapping = {
+        type: fieldType,
+        index: 'not_analyzed',
+        doc_values: true
+      };
+    }
+
+    _.set(mappings, field.name.split('.').join('.properties.'), mapping);
+  });
+
+  return mappings;
+};

src/plugins/kibana/server/lib/handle_es_error.js

@@ -0,0 +1,26 @@
+const Boom = require('boom');
+const esErrors = require('elasticsearch').errors;
+const _ = require('lodash');
+
+module.exports = function handleESError(error) {
+  if (!(error instanceof Error)) {
+    throw new Error('Expected an instance of Error');
+  }
+
+  if (error instanceof esErrors.ConnectionFault ||
+    error instanceof esErrors.ServiceUnavailable ||
+    error instanceof esErrors.NoConnections ||
+    error instanceof esErrors.RequestTimeout) {
+    return Boom.serverTimeout(error);
+  } else if (error instanceof esErrors.Conflict || _.contains(error.message, 'index_template_already_exists')) {
+    return Boom.conflict(error);
+  } else if (error instanceof esErrors[403]) {
+    return Boom.forbidden(error);
+  } else if (error instanceof esErrors.NotFound) {
+    return Boom.notFound(error);
+  } else if (error instanceof esErrors.BadRequest || error instanceof TypeError) {
+    return Boom.badRequest(error);
+  } else {
+    return error;
+  }
+};