Skip to content

Commit

Permalink
route_check: Fix hanging & logging level (sonic-net#1520)
Browse files Browse the repository at this point in the history
Fix for hanging:
Enforce max len on string to be printed to stdout.
With huge mismatch, the message to print can be too long to hang python print statement.

Lower logging level:
The log for "details of route mismatch" is lowered to WARNING. The caller of the route_check may log error, as it sees it fit. The failure of route_check is indicated to caller via exit code.

How to verify it
Simulate error (route-mismatch), and verify
a) message len does not exceed set MAX
b) log level is WARNING
  • Loading branch information
renukamanavalan authored Mar 24, 2021
1 parent ed45412 commit 553936b
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 9 deletions.
22 changes: 15 additions & 7 deletions scripts/route_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@
MIN_SCAN_INTERVAL = 10 # Every 10 seconds
MAX_SCAN_INTERVAL = 3600 # An hour

PRINT_MSG_LEN_MAX = 1000

class Level(Enum):
ERR = 'ERR'
INFO = 'INFO'
Expand All @@ -77,7 +79,7 @@ def __str__(self):
return self.value


report_level = syslog.LOG_ERR
report_level = syslog.LOG_WARNING
write_to_syslog = False

def handler(signum, frame):
Expand Down Expand Up @@ -113,14 +115,20 @@ def print_message(lvl, *args):
:param args: message as list of strings or convertible to string
:return None
"""
msg = ""
if (lvl <= report_level):
msg = ""
for arg in args:
msg += " " + str(arg)
rem_len = PRINT_MSG_LEN_MAX - len(msg)
if rem_len <= 0:
break
msg += str(arg)[0:rem_len]

print(msg)
if write_to_syslog:
syslog.syslog(lvl, msg)

return msg


def add_prefix(ip):
"""
Expand Down Expand Up @@ -421,10 +429,10 @@ def check_routes():
results["Unaccounted_ROUTE_ENTRY_TABLE_entries"] = rt_asic_miss

if results:
print_message(syslog.LOG_ERR, "Failure results: {", json.dumps(results, indent=4), "}")
print_message(syslog.LOG_ERR, "Failed. Look at reported mismatches above")
print_message(syslog.LOG_ERR, "add: {", json.dumps(adds, indent=4), "}")
print_message(syslog.LOG_ERR, "del: {", json.dumps(deletes, indent=4), "}")
print_message(syslog.LOG_WARNING, "Failure results: {", json.dumps(results, indent=4), "}")
print_message(syslog.LOG_WARNING, "Failed. Look at reported mismatches above")
print_message(syslog.LOG_WARNING, "add: {", json.dumps(adds, indent=4), "}")
print_message(syslog.LOG_WARNING, "del: {", json.dumps(deletes, indent=4), "}")
return -1, results
else:
print_message(syslog.LOG_INFO, "All good!")
Expand Down
15 changes: 13 additions & 2 deletions tests/route_check_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import json
import os
import sys
import syslog
import time
from unittest.mock import MagicMock, patch

Expand Down Expand Up @@ -442,8 +443,18 @@ def test_server(self, mock_subs, mock_sel, mock_table, mock_conn):
assert ex_str == expect, "{} != {}".format(ex_str, expect)
assert ex_raised, "Exception expected"



# Test print_msg
route_check.PRINT_MSG_LEN_MAX = 5
msg = route_check.print_message(syslog.LOG_ERR, "abcdefghi")
assert len(msg) == 5
msg = route_check.print_message(syslog.LOG_ERR, "ab")
assert len(msg) == 2
msg = route_check.print_message(syslog.LOG_ERR, "abcde")
assert len(msg) == 5
msg = route_check.print_message(syslog.LOG_ERR, "a", "b", "c", "d", "e", "f")
assert len(msg) == 5





Expand Down

0 comments on commit 553936b

Please sign in to comment.