[Filebeat] Add postgresql csv fileset

CHANGELOG.next.asciidoc

@@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest PostgreSQL CSV logs. {pull}23334[23334]
 - Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]

filebeat/docs/fields.asciidoc

@@ -105572,6 +105572,191 @@ Fields from PostgreSQL logs.
 
 
 
+[float]
+=== log
+
+Fields from the PostgreSQL CSV log files.
+
+
+
+*`postgresql.log.database`*::
++
+--
+Name of database.
+
+example: postgres
+
+--
+
+*`postgresql.log.client_addr`*::
++
+--
+Host where the connection originated from.
+
+example: 127.0.0.1
+
+--
+
+*`postgresql.log.client_port`*::
++
+--
+Port where the connection originated from.
+
+example: 59700
+
+--
+
+*`postgresql.log.session_id`*::
++
+--
+PostgreSQL session.
+
+example: 5ff1dd98.22
+
+--
+
+*`postgresql.log.session_line_num`*::
++
+--
+Line number inside a session. (session_id,session_line_num) identifies uniquely an event.
+
+
+type: long
+
+--
+
+*`postgresql.log.command_tag`*::
++
+--
+Type of session's current command. The complete list can be found at: src/include/tcop/cmdtaglist.h
+
+
+example: SELECT
+
+--
+
+*`postgresql.log.session_start_time`*::
++
+--
+Time when this session started.
+
+type: date
+
+--
+
+*`postgresql.log.virtual_transaction_id`*::
++
+--
+Backend local transaction id .
+
+--
+
+*`postgresql.log.transaction_id`*::
++
+--
+The id of current transaction.
+
+type: long
+
+--
+
+*`postgresql.log.error_severity`*::
++
+--
+Valid values are DEBUG5, DEBUG4, DEBUG3, DEBUG2, DEBUG1, INFO, NOTICE, WARNING, ERROR, LOG, FATAL, and PANIC.
+
+example: LOG
+
+--
+
+*`postgresql.log.sql_state_code`*::
++
+--
+State code returned by Postgres (if any). This code is not a number. See also https://www.postgresql.org/docs/current/errcodes-appendix.html
+
+
+--
+
+*`postgresql.log.message`*::
++
+--
+Main message of this event.
+
+--
+
+*`postgresql.log.detail`*::
++
+--
+--
+
+*`postgresql.log.hint`*::
++
+--
+--
+
+*`postgresql.log.internal_query`*::
++
+--
+internal query that led to the error (if any).
+
+--
+
+*`postgresql.log.internal_query_pos`*::
++
+--
+character count of the error position (if any).
+
+type: long
+
+--
+
+*`postgresql.log.context`*::
++
+--
+Error context.
+
+--
+
+*`postgresql.log.query`*::
++
+--
+User query that led to the error (if any and enabled by log_min_error_statement).
+
+--
+
+*`postgresql.log.query_pos`*::
++
+--
+character count of the error position (if any).
+
+type: long
+
+--
+
+*`postgresql.log.location`*::
++
+--
+Location of the error in the PostgreSQL source code (if log_error_verbosity is set to verbose).
+
+--
+
+*`postgresql.log.application_name`*::
++
+--
+Name of the application of this event. It is defined by the client.
+
+--
+
+*`postgresql.log.backend_type`*::
++
+--
+Type of backend of this event. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. In addition, background workers registered by extensions may have additional types.
+
+
+example: client backend
+
+--
+
 [float]
 === log
 

filebeat/module/postgresql/_meta/docs.asciidoc

@@ -13,8 +13,13 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-The +{modulename}+ module was tested with logs from versions 9.5 on Ubuntu, 9.6
-on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3.
+This module comes in two flavours: a parser of log files based on Linux distribution
+defaults, and a CSV log parser, that you need to enable in database configuration.
+
+The +{modulename}+ module using `.log` was tested with logs from versions 9.5 on Ubuntu,
+ 9.6 on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3.
+
+The +{modulename}+ module using `.csv` was tested using versions 11 and 13.
 
 include::../include/configuring-intro.asciidoc[]
 
@@ -68,4 +73,31 @@ image::./images/filebeat-postgresql-slowlog-overview.png[]
 
 :fileset_ex!:
 
+:fileset_ex: csv
+
+Since the PostgreSQL CSV log file is a well-defined format, 
+there is almost no configuration to be done in filebeat, just the filepath
+
+On the other hand, it's necessary to configure postgresql to emit `.csv` logs.
+The recommended parameters are:
+
+```
+logging_collector = 'on';
+log_destination = 'csvlog';
+log_statement = 'none';
+log_checkpoints = on;
+log_connections = on;
+log_disconnections = on;
+log_lock_waits = on;
+log_min_duration_statement = 0;
+```
+
+In busy servers, `log_min_duration_statement` can cause contention, so you can assign 
+a value greater than 0.
+
+Both `log_connections` and `log_disconnections` can cause a lot of events if you don't have
+persistent connections, so enable with care.
+
+:fileset_ex!:
+
 :modulename!:

filebeat/module/postgresql/csv/_meta/fields.yml

@@ -0,0 +1,71 @@
+- name: log
+  type: group
+  description: >
+    Fields from the PostgreSQL CSV log files.
+  fields:
+    - name: database
+      example: "postgres"
+      description: Name of database.
+    - name: client_addr
+      example: "127.0.0.1"
+      description: Host where the connection originated from.
+    - name: client_port
+      example: "59700"
+      description: Port where the connection originated from.
+    - name: session_id
+      description: PostgreSQL session.
+      example: "5ff1dd98.22"
+    - name: session_line_num
+      type: long
+      description: >
+        Line number inside a session.
+        (session_id,session_line_num) identifies uniquely an event.
+    - name: command_tag
+      example: "SELECT"
+      description: >
+        Type of session's current command.
+        The complete list can be found at: src/include/tcop/cmdtaglist.h
+    - name: session_start_time
+      type: date
+      description: Time when this session started.
+    - name: virtual_transaction_id
+      description: Backend local transaction id .
+    - name: transaction_id
+      type: long
+      description: The id of current transaction.
+    - name: error_severity
+      example: "LOG"
+      description: Valid values are DEBUG5, DEBUG4, DEBUG3, DEBUG2, DEBUG1, INFO, NOTICE, WARNING, ERROR, LOG, FATAL, and PANIC. 
+    - name: sql_state_code
+      description: >
+        State code returned by Postgres (if any).
+        This code is not a number.
+        See also https://www.postgresql.org/docs/current/errcodes-appendix.html
+    - name: message
+      description: Main message of this event.
+    - name: detail
+    - name: hint
+    - name: internal_query
+      description: internal query that led to the error (if any).
+    - name: internal_query_pos
+      type: long
+      description: character count of the error position (if any).
+    - name: context
+      description: Error context.
+    - name: query
+      description: User query that led to the error (if any and enabled by log_min_error_statement).
+    - name: query_pos
+      type: long
+      description: character count of the error position (if any).
+    - name: location
+      description: Location of the error in the PostgreSQL source code (if log_error_verbosity is set to verbose).
+    - name: application_name
+      description: Name of the application of this event. It is defined by the client.
+    - name: backend_type
+      example: "client backend"
+      description: > 
+        Type of backend of this event.
+        Possible types are autovacuum launcher, autovacuum worker, logical replication launcher,
+        logical replication worker, parallel worker, background writer, client backend, checkpointer,
+        startup, walreceiver, walsender and walwriter.
+        In addition, background workers registered by extensions may have additional types.

filebeat/module/postgresql/csv/config/csv.yml

@@ -0,0 +1,15 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+multiline:
+  pattern: '^\d{4}-\d{2}-\d{2} '
+  negate: true
+  match: after
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0
+