Add support for multiple paths per fileset (#3195)

We generally need more than one path per OS, because the logs location is not always the same. For example, depending on the linux distribution and how you installed it, MySQL can have it's error logs in a number of default "paths". The solution is to configure them all, which means that Filebeat might try to access unexisting folders. This also improves the python prototype to accept multiple modules and to accept namespaced parameters. E.g.: ./filebeat.py --modules=nginx,syslog -M nginx.access.paths=...
elastic · Dec 15, 2016 · cc3c257 · cc3c257
1 parent 2f207bd
commit cc3c257
Show file tree

Hide file tree

Showing 12 changed files with 101 additions and 59 deletions.
diff --git a/filebeat/filebeat.py b/filebeat/filebeat.py
@@ -13,27 +13,23 @@
 def main():
     parser = argparse.ArgumentParser(
         description="PROTOTYPE: start filebeat with a module configuration")
-    parser.add_argument("--module", default="",
+    parser.add_argument("--modules", default="",
                         help="From branch")
-    parser.add_argument("--nginx", action="store_true",
-                        help="Shortcut for --module nginx")
     parser.add_argument("--es", default="http://localhost:9200",
                         help="Elasticsearch URL")
-    parser.add_argument("-E", nargs="*", type=str, default=None,
+    parser.add_argument("-M", nargs="*", type=str, default=None,
                         help="Variables overrides. e.g. path=/test")
 
     args = parser.parse_args()
     print args
 
-    if args.nginx:
-        args.module = "nginx"
-
-    if args.module == "":
-        print("You need to specify a module")
+    modules = args.modules.split(",")
+    if len(modules) == 0:
+        print("You need to specify at least a module")
         sys.exit(1)
 
     load_dashboards(args)
-    load_datasets(args, args.module)
+    load_datasets(args, modules)
 
 
 def load_dashboards(args):
@@ -43,22 +39,23 @@ def load_dashboards(args):
     subprocess.Popen(cmd).wait()
 
 
-def load_datasets(args, module):
-    path = os.path.join("module", module)
-    if not os.path.isdir(path):
-        print("Module {} not found".format(module))
-        sys.exit(1)
-    print("Found module {} in {}".format(module, path))
+def load_datasets(args, modules):
+    prospectors = ""
+    for module in modules:
+        path = os.path.join("module", module)
+        if not os.path.isdir(path):
+            print("Module {} not found".format(module))
+            sys.exit(1)
+        print("Found module {} in {}".format(module, path))
 
-    filesets = [name for name in os.listdir(path) if
-                os.path.isfile(os.path.join(path, name, "manifest.yml"))]
+        filesets = [name for name in os.listdir(path) if
+                    os.path.isfile(os.path.join(path, name, "manifest.yml"))]
 
-    print("Found filesets: {}".format(filesets))
+        print("Found filesets: {}".format(filesets))
 
-    prospectors = ""
-    for fileset in filesets:
-        prospectors += load_fileset(args, module, fileset,
-                                    os.path.join(path, fileset))
+        for fileset in filesets:
+            prospectors += load_fileset(args, module, fileset,
+                                        os.path.join(path, fileset))
 
     run_filebeat(args, prospectors)
 
@@ -67,7 +64,7 @@ def load_datasets(args, module):
 
 def load_fileset(args, module, fileset, path):
     manifest = yaml.load(file(os.path.join(path, "manifest.yml"), "r"))
-    var = evaluate_vars(args, manifest["vars"])
+    var = evaluate_vars(args, manifest["vars"], module, fileset)
     var["beat"] = dict(module=module, fileset=fileset, path=path, args=args)
     print("Evaluated variables: {}".format(var))
 
@@ -77,25 +74,35 @@ def load_fileset(args, module, fileset, path):
     return var["beat"]["prospectors"]
 
 
-def evaluate_vars(args, var_in):
+def evaluate_vars(args, var_in, module, fileset):
     var = {
         "builtin": get_builtin_vars()
     }
     for name, vals in var_in.items():
         var[name] = vals["default"]
-
         if sys.platform == "darwin" and "os.darwin" in vals:
             var[name] = vals["os.darwin"]
         elif sys.platform == "windows" and "os.windows" in vals:
             var[name] = vals["os.windows"]
 
-        var[name] = Template(var[name]).render(var)
+        if isinstance(var[name], basestring):
+            var[name] = Template(var[name]).render(var)
+        elif isinstance(var[name], list):
+            # only supports array of strings atm
+            var[name] = [Template(x).render(var) for x in var[name]]
 
     # overrides
-    if args.E is not None:
-        for pair in args.E:
+    if args.M is not None:
+        for pair in args.M:
             key, val = pair.partition("=")[::2]
-            var[key] = val
+            if key.startswith("{}.{}.".format(module, fileset)):
+                key = key[len("{}.{}.".format(module, fileset)):]
+
+                # this is a hack in the prototype only, because
+                # here we don't know the type of each variable type.
+                if key == "paths":
+                    val = val.split(",")
+                var[key] = val
 
     return var
 

diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml
@@ -1,6 +1,9 @@
 - input_type: log
   paths:
-    - {{path}}
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
   fields:
     source_type: mysql-error
     pipeline_id: {{beat.pipeline_id}}
diff --git a/filebeat/module/mysql/error/manifest.yml b/filebeat/module/mysql/error/manifest.yml
@@ -1,10 +1,14 @@
 module_version: 1.0
 
 vars:
-  path:
-    default: /var/log/mysql/error.log
-    os.darwin: /usr/local/var/mysql/{{builtin.hostname}}.{{builtin.domain}}.err
-    os.windows: "c:/programdata/MySQL/MySQL Server*/error.log"
+  paths:
+    default:
+      - /var/log/mysql/error.log*
+      - /var/log/mysqld.log*
+    os.darwin:
+      - /usr/local/var/mysql/{{builtin.hostname}}.{{builtin.domain}}.err*
+    os.windows:
+      - "c:/programdata/MySQL/MySQL Server*/error.log*"
 
 ingest_pipeline: ingest/pipeline.json
 prospectors:

diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml
@@ -1,6 +1,9 @@
 - input_type: log
   paths:
-    - {{path}}
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
   multiline:
     pattern: "^# User@Host: "
     negate: true

diff --git a/filebeat/module/mysql/slowlog/manifest.yml b/filebeat/module/mysql/slowlog/manifest.yml
@@ -1,10 +1,14 @@
 module_version: 1.0
 
 vars:
-  path:
-    default: /var/log/mysql/mysql-slow.log
-    os.darwin: /usr/local/var/mysql/{{builtin.hostname}}-slow.log
-    os.windows: "c:/programdata/MySQL/MySQL Server*/mysql-slow.log"
+  paths:
+    default:
+      - /var/log/mysql/mysql-slow.log*
+      - /var/lib/mysql/{{builtin.hostname}}-slow.log
+    os.darwin:
+      - /usr/local/var/mysql/{{builtin.hostname}}-slow.log*
+    os.windows:
+      - "c:/programdata/MySQL/MySQL Server*/mysql-slow.log*"
 
 ingest_pipeline: ingest/pipeline.json
 prospectors:

diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml
@@ -1,6 +1,9 @@
 - input_type: log
   paths:
-  - {{path}}/access*.log
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
   fields:
     source_type: nginx-access
     pipeline_id: {{beat.pipeline_id}}
diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml
@@ -1,10 +1,13 @@
 module_version: 1.0
 
 vars:
-  path:
-    default: /var/log/nginx
-    os.darwin: /usr/local/var/log/nginx
-    os.windows: c:/programfiles/nginx/logs
+  paths:
+    default:
+      - /var/log/nginx/access.log*
+    os.darwin:
+      - /usr/local/var/log/nginx/access.log*
+    os.windows:
+      - c:/programfiles/nginx/logs/access.log*
   pipeline:
     # options: with_plugins, no_plugins, json_with_plugins, json_no_plugins
     default: with_plugins

diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml
@@ -1,6 +1,9 @@
 - input_type: log
   paths:
-  - {{path}}/error.log
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
   fields:
     source_type: nginx-error
     pipeline_id: {{beat.pipeline_id}}

diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json
@@ -6,11 +6,7 @@
       "patterns": [
         "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}"
       ],
-      "on_failure" : [{
-        "set" : {
-          "field" : "error",
-          "value" : "The error pattern didn't match on this event."
-        }
+      "ignore_missing": true
     }
   },{
     "remove":{
@@ -31,5 +27,11 @@
     "remove": {
       "field": "nginx.error.time"
     }
+  }],
+  "on_failure" : [{
+    "set" : {
+      "field" : "error",
+      "value" : "{{ _ingest.on_failure_message }}"
+    }
   }]
 }
diff --git a/filebeat/module/nginx/error/manifest.yml b/filebeat/module/nginx/error/manifest.yml
@@ -1,10 +1,13 @@
 module_version: 1.0
 
 vars:
-  path:
-    default: /var/log/nginx
-    os.darwin: /usr/local/var/log/nginx
-    os.windows: c:/programfiles/nginx/logs
+  paths:
+    default:
+      - /var/log/nginx/error.log*
+    os.darwin:
+      - /usr/local/var/log/nginx/error.log*
+    os.windows:
+      - c:/programfiles/nginx/logs/error.log*
 
 ingest_pipeline: ingest/pipeline.json
 prospectors:

diff --git a/filebeat/module/syslog/system/config/system.yml b/filebeat/module/syslog/system/config/system.yml
@@ -1,6 +1,9 @@
 - input_type: log
   paths:
-    - {{path}}
+  {%- for path in paths %}
+   - {{path}}
+  {%- endfor %}
+  exclude_files: [".gz$"]
   multiline:
     pattern: "^\\s"
     match: after

diff --git a/filebeat/module/syslog/system/manifest.yml b/filebeat/module/syslog/system/manifest.yml
@@ -1,9 +1,13 @@
 module_version: 1.0
 
 vars:
-  path:
-    default: /var/log/messages
-    os.darwin: /var/log/system.log
+  paths:
+    default:
+      - /var/log/messages*
+      - /var/log/syslog*
+    os.darwin:
+      - /var/log/system.log*
+    os.windows: []
 
 ingest_pipeline: ingest/pipeline.json
 prospectors: