From 2b381b1fd4e098a83d82728302541de20fb71b55 Mon Sep 17 00:00:00 2001 From: StormLiangMS <89824293+StormLiangMS@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:52:14 +0800 Subject: [PATCH] Revert "revert [syslog] Add remote syslog configuration (cherry-pick to 202305) (#15897) (#16179)" (#16549) This reverts commit 164fa102c01d3c32ccf95ab94f8db03f53c7c900. --- files/image_config/rsyslog/rsyslog-config.sh | 5 +- files/image_config/rsyslog/rsyslog.conf.j2 | 72 ++++--- src/sonic-yang-models/doc/Configuration.md | 108 ++++++---- .../tests/files/sample_config_db.json | 13 +- .../tests/yang_model_tests/tests/syslog.json | 59 +++++- .../yang_model_tests/tests_config/syslog.json | 184 ++++++++++++++++-- .../yang-models/sonic-syslog.yang | 78 +++++++- 7 files changed, 437 insertions(+), 82 deletions(-) diff --git a/files/image_config/rsyslog/rsyslog-config.sh b/files/image_config/rsyslog/rsyslog-config.sh index 83be7ab6cf71..576351711364 100755 --- a/files/image_config/rsyslog/rsyslog-config.sh +++ b/files/image_config/rsyslog/rsyslog-config.sh @@ -17,7 +17,10 @@ if [[ ($NUM_ASIC -gt 1) ]]; then else udp_server_ip=$(ip -j -4 addr list lo scope host | jq -r -M '.[0].addr_info[0].local') fi +hostname=$(hostname) -sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 -a "{\"udp_server_ip\": \"$udp_server_ip\"}" >/etc/rsyslog.conf +sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 \ + -a "{\"udp_server_ip\": \"$udp_server_ip\", \"hostname\": \"$hostname\"}" \ + > /etc/rsyslog.conf systemctl restart rsyslog diff --git a/files/image_config/rsyslog/rsyslog.conf.j2 b/files/image_config/rsyslog/rsyslog.conf.j2 index d20fb5d00aec..c29d803d085a 100644 --- a/files/image_config/rsyslog/rsyslog.conf.j2 +++ b/files/image_config/rsyslog/rsyslog.conf.j2 @@ -15,21 +15,14 @@ $ModLoad imuxsock # provides support for local system logging -{% if SYSLOG_CONFIG is defined %} -{% if 'GLOBAL' in SYSLOG_CONFIG %} -{% if 'rate_limit_interval' in SYSLOG_CONFIG['GLOBAL']%} -{% set rate_limit_interval = SYSLOG_CONFIG['GLOBAL']['rate_limit_interval'] %} -{% endif %} -{% if 'rate_limit_burst' in SYSLOG_CONFIG['GLOBAL']%} -{% set rate_limit_burst = SYSLOG_CONFIG['GLOBAL']['rate_limit_burst'] %} -{% endif %} -{% endif %} -{% endif %} +{% set gconf = (SYSLOG_CONFIG | d({})).get('GLOBAL', {}) -%} +{% set rate_limit_interval = gconf.get('rate_limit_interval') %} +{% set rate_limit_burst = gconf.get('rate_limit_burst') %} -{% if rate_limit_interval is defined %} +{% if rate_limit_interval is not none %} $SystemLogRateLimitInterval {{ rate_limit_interval }} {% endif %} -{% if rate_limit_burst is defined %} +{% if rate_limit_burst is not none %} $SystemLogRateLimitBurst {{ rate_limit_burst }} {% endif %} @@ -49,6 +42,8 @@ $UDPServerRun 514 ########################### #### GLOBAL DIRECTIVES #### ########################### +{% set format = gconf.get('format', 'standard') -%} +{% set fw_name = gconf.get('welf_firewall_name', hostname) -%} # # Use traditional timestamp format. # To enable high precision timestamps, comment out the following line. @@ -59,6 +54,10 @@ $UDPServerRun 514 $template SONiCFileFormat,"%timegenerated%.%timegenerated:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" $ActionFileDefaultTemplate SONiCFileFormat +template(name="WelfRemoteFormat" type="string" string="%TIMESTAMP% id=firewall time=\"%timereported\ +:::date-year%-%timereported:::date-month%-%timereported:::date-day% %timereported:::date-hour%:%timereported:::date-minute%:%timereported\ +:::date-second%\" fw=\"{{ fw_name }}\" pri=%syslogpriority% msg=\"%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\"\n") + # # Set the default permissions for all log files. # @@ -91,25 +90,36 @@ $RepeatedMsgReduction on # Remote syslog logging # -# The omfwd plug-in provides the core functionality of traditional message forwarding via UDP and plain TCP. -# It is a built-in module that does not need to be loaded. +# The omfwd plug-in provides the core functionality of traditional message +# forwarding via UDP and plain TCP. It is a built-in module that does not need +# to be loaded. -{% if SYSLOG_SERVER is defined %} -{% for server, data in SYSLOG_SERVER.items() %} -{% set params_list = [] %} -{% if 'source' in data %} -{% set dummy = params_list.append('address=' + '"' + data.source|string + '"') %} -{% endif %} -{% if 'port' in data %} -{% set dummy = params_list.append('port=' + '"' + data.port|string + '"') %} -{% endif %} -{% if 'vrf' in data and data['vrf'] != "default" %} -{% set dummy = params_list.append('device=' + '"' + data.vrf|string + '"') %} -{% endif %} -{% if params_list %} -*.* action(type="omfwd" target="{{ server }}" protocol="udp" {{ params_list|join(' ') }} template="SONiCFileFormat") -{% else %} -*.* action(type="omfwd" target="{{ server }}" protocol="udp" template="SONiCFileFormat") +{% set servers = SYSLOG_SERVER | d({}) -%} +{% for server in servers %} +{% set conf = servers[server] | d({}) -%} + +{% set source = conf.get('source') -%} +{% set port = conf.get('port', 514) -%} +{% set proto = conf.get('protocol', 'udp') -%} +{% set vrf = conf.get('vrf', 'default') -%} +{% set severity = conf.get('severity', gconf.get('severity', 'notice')) -%} +{% set filter = conf.get('filter') -%} +{% set regex = conf.get('filter_regex') -%} + +{% set fmodifier = '!' if filter == 'exclude' else '' %} +{% set device = 'eth0' if vrf == 'default' else vrf -%} +{% set template = 'WelfRemoteFormat' if format == 'welf' else 'SONiCFileFormat' -%} + +{# Server extra options -#} +{% set options = '' -%} + +{% if source -%} + {% set options = options ~ ' Address="' ~ source ~ '"'-%} +{% endif -%} + +{% if filter %} +:msg, {{ fmodifier }}ereregex, "{{ regex }}" {% endif %} +*.{{ severity }} +action(type="omfwd" Target="{{ server }}" Port="{{ port }}" Protocol="{{ proto }}" Device="{{ device }}" Template="{{ template }}"{{ options }}) {% endfor %} -{% endif %} diff --git a/src/sonic-yang-models/doc/Configuration.md b/src/sonic-yang-models/doc/Configuration.md index 8e192748c9b1..75c0c8b5cfa3 100644 --- a/src/sonic-yang-models/doc/Configuration.md +++ b/src/sonic-yang-models/doc/Configuration.md @@ -51,7 +51,7 @@ Table of Contents * [MUX_LINKMGR](#mux_linkmgr) * [NEIGH](#neigh) * [NTP Global Configuration](#ntp-global-configuration) - * [NTP and SYSLOG servers](#ntp-and-syslog-servers) + * [NTP Servers](#ntp-servers) * [Peer Switch](#peer-switch) * [Policer](#policer) * [Port](#port) @@ -60,7 +60,8 @@ Table of Contents * [Scheduler](#scheduler) * [Port QoS Map](#port-qos-map) * [Queue](#queue) - * [Syslog Rate Limit](#syslog-rate-limit) + * [Syslog Global Configuration](#syslog-global-configuration) + * [Syslog Servers](#syslog-servers) * [Sflow](#sflow) * [Restapi](#restapi) * [System Port](#system-port) @@ -1517,7 +1518,7 @@ for that address. } ``` -### NTP and SYSLOG servers +### NTP servers These information are configured in individual tables. Domain name or IP address of the server is used as object key. Currently there are no @@ -1540,35 +1541,6 @@ attributes in those objects. } ``` -***Syslog server*** -``` -{ - "SYSLOG_SERVER": { - "10.0.0.5": {}, - "10.0.0.6": {}, - "10.11.150.5": {} - }, - - "SYSLOG_SERVER" : { - "2.2.2.2": { - "source": "1.1.1.1", - "port": "514", - "vrf": "default" - }, - "4.4.4.4": { - "source": "3.3.3.3", - "port": "514", - "vrf": "mgmt" - }, - "2222::2222": { - "source": "1111::1111", - "port": "514", - "vrf": "Vrf-Data" - } - } -} -``` - ### Peer Switch Below is an exmaple of the peer switch table configuration. @@ -1854,7 +1826,33 @@ key - name | collector_port | Destination L4 port of the Sflow collector | | 6343 | | | collector_vrf | Specify the Collector VRF. In this revision, it is either default VRF or Management VRF.| | | | -### Syslog Rate Limit +### Syslog Global Configuration + +These configuration options are used to configure rsyslog utility and the way +the system generates logs. + +***Configuration sample*** +``` +{ + "SYSLOG_CONFIG": { + "GLOBAL": { + "rate_limit_interval": "5", + "rate_limit_burst": "100", + "format": "welf", + "welf_firewall_name": "bla", + "severity": "info" + } + } +} +``` + +* `rate_limit_interval` - determines the amount of time that is being measured for rate limiting: `unsigned integer` +* `rate_limit_burst` - defines the amount of messages, that have to occur in the time limit: `unsigned integer` +* `format` - syslog log format: `{standard, welf}` +* `welf_firewall_name` - WELF format firewall name: `string` +* `severity` - global log severity: `{emerg, alert, crit, error, warning, notice, info, debug}` + +***Syslog Rate Limit*** Host side configuration: @@ -1886,6 +1884,50 @@ Container side configuration: } ``` +### Syslog servers + +These information are configured in individual tables. Domain name or IP +address of the server is used as object key. Each server can be configurable. + +***Configuration sample*** +``` +{ + "SYSLOG_SERVER": { + "10.0.0.5": {}, + "10.0.0.6": {}, + "10.11.150.5": {} + }, + + "SYSLOG_SERVER" : { + "4.4.4.4": { + "source": "3.3.3.3", + "port": "514", + "vrf": "mgmt" + }, + "2222::2222": { + "source": "1111::1111", + "port": "514", + "vrf": "Vrf-Data" + }, + "somehostname": { + "filter": "include", + "filter_regex": "ololo", + "port": "514", + "protocol": "tcp", + "severity": "notice", + "vrf": "default" + } + } +} +``` + +* `filter` - determines if syslog will include or exclude messages specified by regex: `{include, exclude}` +* `filter_regex` - filter messages by this regex: `string` +* `port` - network port to use to connect to remote server: `integer: 1..65535` +* `protocol` - network protocol to use to connect to remote server: `{tcp, udp}` +* `severity` - per-server log severity, overrifes global one: `{emerg, alert, crit, error, warning, notice, info, debug}` + + ### System Port Every port on the system requires a global representation, known as a System Port, and is listed in this table. diff --git a/src/sonic-yang-models/tests/files/sample_config_db.json b/src/sonic-yang-models/tests/files/sample_config_db.json index e758450e2072..5ae899f0c2b0 100644 --- a/src/sonic-yang-models/tests/files/sample_config_db.json +++ b/src/sonic-yang-models/tests/files/sample_config_db.json @@ -499,12 +499,23 @@ "source": "1111::1111", "port": "514", "vrf": "Vrf_blue" + }, + "somehostname": { + "filter": "include", + "filter_regex": "ololo", + "port": "514", + "protocol": "tcp", + "severity": "notice", + "vrf": "default" } }, "SYSLOG_CONFIG" : { "GLOBAL": { "rate_limit_interval": "5", - "rate_limit_burst": "100" + "rate_limit_burst": "100", + "format": "welf", + "welf_firewall_name": "bla", + "severity": "info" } }, "SYSLOG_CONFIG_FEATURE" : { diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json b/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json index da277f113486..4b2eb0ae1da2 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json @@ -29,10 +29,6 @@ "desc": "Load syslog server table with empty address as syslog server.", "eStrKey": "InvalidValue" }, - "SYSLOG_SERVER_INVALID_IPADDR_TEST": { - "desc": "Load syslog server table with invalid ipv4 address as syslog server.", - "eStrKey": "InvalidValue" - }, "SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST": { "desc": "Load syslog server table with invalid ipv6 address as syslog server.", "eStrKey": "InvalidValue" @@ -62,5 +58,60 @@ "SYSLOG_CONFIG_FEATURE_INVALID_BURST": { "desc": "Configure invalid rate_limit_burst in SYSLOG_CONFIG_FEATURE.", "eStrKey": "InvalidValue" + }, + "SYSLOG_SERVER_HOSTNAME": { + "desc": "Load syslog server table with hostname" + }, + "SYSLOG_SERVER_HOSTNAME_INVALID": { + "desc": "Load syslog server table with invalid hostname", + "eStrKey": "InvalidValue" + }, + "SYSLOG_SERVER_FILTER_TYPE": { + "desc": "Valid filter type for syslog server" + }, + "SYSLOG_SERVER_FILTER_TYPE_INVALID": { + "desc": "Invalid filter type for syslog server", + "eStrKey": "InvalidValue" + }, + "SYSLOG_SERVER_FILTER_REGEX": { + "desc": "Valid filter regex" + }, + "SYSLOG_SERVER_PROTOCOL": { + "desc": "Valid syslog server protocol" + }, + "SYSLOG_SERVER_PROTOCOL_INVALID": { + "desc": "Invalid syslog server protocol", + "eStrKey": "InvalidValue" + }, + "SYSLOG_SERVER_SEVERITY": { + "desc": "Syslog server valid severity" + }, + "SYSLOG_SERVER_SEVERITY_INVALID": { + "desc": "Syslog server invalid severity", + "eStrKey": "InvalidValue" + }, + "SYSLOG_CONFIG_GLOBAL_VALID": { + "desc": "Global syslog configuration" + }, + "SYSLOG_CONFIG_FORMAT": { + "desc": "Syslog format type" + }, + "SYSLOG_CONFIG_FORMAT_INVALID": { + "desc": "Invalid syslog format", + "eStrKey": "InvalidValue" + }, + "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME": { + "desc": "Syslog format WELF firewall name" + }, + "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID": { + "desc": "Syslog format WELF invalid firewall name", + "eStrKey": "Must" + }, + "SYSLOG_CONFIG_SEVERITY": { + "desc": "Global syslog severity" + }, + "SYSLOG_CONFIG_SEVERITY_INVALID": { + "desc": "Global invalid syslog severity", + "eStrKey": "InvalidValue" } } diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests_config/syslog.json b/src/sonic-yang-models/tests/yang_model_tests/tests_config/syslog.json index 0ebaddbc7044..d3505b6a31bd 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests_config/syslog.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests_config/syslog.json @@ -136,17 +136,6 @@ } } }, - "SYSLOG_SERVER_INVALID_IPADDR_TEST" : { - "sonic-syslog:sonic-syslog": { - "sonic-syslog:SYSLOG_SERVER": { - "SYSLOG_SERVER_LIST": [ - { - "server_address": "1111.22.33.1" - } - ] - } - } - }, "SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST" : { "sonic-syslog:sonic-syslog": { "sonic-syslog:SYSLOG_SERVER": { @@ -209,6 +198,64 @@ } } }, + "SYSLOG_SERVER_HOSTNAME" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "thebestswitch" + } + ] + } + } + }, + "SYSLOG_SERVER_HOSTNAME_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "-" + } + ] + } + } + }, + "SYSLOG_SERVER_FILTER_TYPE" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "filter": "exclude" + } + ] + } + } + }, + "SYSLOG_SERVER_FILTER_TYPE_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "filter": "excludalol" + } + ] + } + } + }, + "SYSLOG_SERVER_FILTER_REGEX" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "filter_regex": "^expeliarmus.*" + } + ] + } + } + }, "SYSLOG_CONFIG_FEATURE_INVALID_SERVICE_NAME": { "sonic-syslog:sonic-syslog": { "sonic-syslog:SYSLOG_CONFIG_FEATURE": { @@ -222,6 +269,18 @@ } } }, + "SYSLOG_SERVER_PROTOCOL" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "protocol": "tcp" + } + ] + } + } + }, "SYSLOG_CONFIG_FEATURE_INVALID_INTERVAL": { "sonic-syslog:sonic-syslog": { "sonic-syslog:SYSLOG_CONFIG_FEATURE": { @@ -244,6 +303,30 @@ } } }, + "SYSLOG_SERVER_PROTOCOL_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "protocol": "order66" + } + ] + } + } + }, + "SYSLOG_SERVER_SEVERITY" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "severity": "info" + } + ] + } + } + }, "SYSLOG_CONFIG_FEATURE_INVALID_BURST": { "sonic-syslog:sonic-syslog": { "sonic-syslog:SYSLOG_CONFIG_FEATURE": { @@ -265,5 +348,84 @@ ] } } + }, + "SYSLOG_SERVER_SEVERITY_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_SERVER": { + "SYSLOG_SERVER_LIST": [ + { + "server_address": "1.2.3.4", + "severity": "information" + } + ] + } + } + }, + "SYSLOG_CONFIG_GLOBAL_VALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "format": "welf", + "welf_firewall_name": "welf-fw-name", + "severity": "debug" + } + } + } + }, + "SYSLOG_CONFIG_FORMAT" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "format": "standard" + } + } + } + }, + "SYSLOG_CONFIG_FORMAT_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "format": "nonstandard" + } + } + } + }, + "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "format": "welf", + "welf_firewall_name": "welf-switch" + } + } + } + }, + "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "format": "standard", + "welf_firewall_name": "welf-switch" + } + } + } + }, + "SYSLOG_CONFIG_SEVERITY" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "severity": "crit" + } + } + } + }, + "SYSLOG_CONFIG_SEVERITY_INVALID" : { + "sonic-syslog:sonic-syslog": { + "sonic-syslog:SYSLOG_CONFIG": { + "GLOBAL": { + "severity": "critical" + } + } + } } } diff --git a/src/sonic-yang-models/yang-models/sonic-syslog.yang b/src/sonic-yang-models/yang-models/sonic-syslog.yang index 6a4750f708d7..24e33f20acae 100644 --- a/src/sonic-yang-models/yang-models/sonic-syslog.yang +++ b/src/sonic-yang-models/yang-models/sonic-syslog.yang @@ -49,6 +49,44 @@ module sonic-syslog { } } + typedef log-format { + description "Represents syslog log format"; + type enumeration { + enum welf; + enum standard; + } + } + + typedef rsyslog-protocol { + description "The protocol to send logs to remote server"; + type enumeration { + enum tcp; + enum udp; + } + } + + typedef syslog-filter-type { + description "The filter type"; + type enumeration { + enum include; + enum exclude; + } + } + + typedef rsyslog-severity { + description "The protocol to send logs to remote server"; + type enumeration { + enum none; + enum debug; + enum info; + enum notice; + enum warn; + enum error; + enum crit; + } + } + + container sonic-syslog { container SYSLOG_SERVER { @@ -61,7 +99,7 @@ module sonic-syslog { leaf server_address { description "Syslog server IP address"; - type inet:ip-address; + type inet:host; } leaf source { @@ -88,6 +126,26 @@ module sonic-syslog { or (/mvrf:sonic-mgmt_vrf/mvrf:MGMT_VRF_CONFIG/mvrf:vrf_global/mvrf:mgmtVrfEnabled = 'true')"; } + leaf filter { + description "Syslog filter type"; + type syslog-filter-type; + } + + leaf filter_regex { + description "Filter regex"; + type string; + } + + leaf protocol { + description "The protocol to send logs to remote server"; + type rsyslog-protocol; + } + + leaf severity { + description "Limit the severity to send logs to remote server"; + type rsyslog-severity; + } + } /* end of list SYSLOG_SERVER_LIST */ } @@ -105,6 +163,24 @@ module sonic-syslog { leaf rate_limit_burst { type syslog-rate-limit-burst; } + + leaf format { + description "Log format"; + type log-format; + default standard; + } + + leaf welf_firewall_name { + description "WELF format Firewall name"; + type string; + must "(../format != 'standard')"; + } + + leaf severity { + type rsyslog-severity; + default notice; + } + } /* end of list SYSLOG_CONFIG_LIST */ }