From cbe21599e7e1371c70370a147bca0d4be401c56f Mon Sep 17 00:00:00 2001 From: Volodymyr Samotiy Date: Tue, 4 May 2021 05:20:34 +0300 Subject: [PATCH] [vnet] Add "vnet_route_check" script (#1300) * [vnet] Add "vnet_route_check" script * [vnet_route_check.py]: tool that verifies VNET routes consistancy between SONiC and vendor SDK DBs. Signed-off-by: Volodymyr Samotiy --- scripts/vnet_route_check.py | 363 +++++++++++++++++++++++++++++++++ setup.py | 1 + tests/vnet_route_check_test.py | 325 +++++++++++++++++++++++++++++ 3 files changed, 689 insertions(+) create mode 100755 scripts/vnet_route_check.py create mode 100644 tests/vnet_route_check_test.py diff --git a/scripts/vnet_route_check.py b/scripts/vnet_route_check.py new file mode 100755 index 000000000000..010e95345186 --- /dev/null +++ b/scripts/vnet_route_check.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python + +import os +import sys +import json +import syslog +from swsscommon import swsscommon + +''' vnet_route_check.py: tool that verifies VNET routes consistancy between SONiC and vendor SDK DBs. + +Logically VNET route verification logic consists of 3 parts: +1. Get VNET routes entries that are missed in ASIC_DB but present in APP_DB. +2. Get VNET routes entries that are missed in APP_DB but present in ASIC_DB. +3. Get VNET routes entries that are missed in SDK but present in ASIC_DB. + +Returns 0 if there is no inconsistancy found and all VNET routes are aligned in all DBs. +Returns -1 if there is incosistancy found and prints differences between DBs in JSON format to standart output. + +Format of differences output: +{ + "results": { + "missed_in_asic_db_routes": { + "": { + "routes": [ + "/" + ] + } + }, + "missed_in_app_db_routes": { + "": { + "routes": [ + "/" + ] + } + }, + "missed_in_sdk_routes": { + "": { + "routes": [ + "/" + ] + } + } + } +} +''' + + +RC_OK = 0 +RC_ERR = -1 + + +report_level = syslog.LOG_ERR +write_to_syslog = True + + +def set_level(lvl, log_to_syslog): + global report_level + global write_to_syslog + + write_to_syslog = log_to_syslog + report_level = lvl + + +def print_message(lvl, *args): + if (lvl <= report_level): + msg = "" + for arg in args: + msg += " " + str(arg) + print(msg) + if write_to_syslog: + syslog.syslog(lvl, msg) + + +def check_vnet_cfg(): + ''' Returns True if VNET is configured in APP_DB or False if no VNET configuration. + ''' + db = swsscommon.DBConnector('APPL_DB', 0) + + vnet_db_keys = swsscommon.Table(db, 'VNET_TABLE').getKeys() + + return True if vnet_db_keys else False + + +def get_vnet_intfs(): + ''' Returns dictionary of VNETs and related VNET interfaces. + Format: { : [ ] } + ''' + db = swsscommon.DBConnector('APPL_DB', 0) + + intfs_table = swsscommon.Table(db, 'INTF_TABLE') + intfs_keys = swsscommon.Table(db, 'INTF_TABLE').getKeys() + + vnet_intfs = {} + + for intf_key in intfs_keys: + intf_attrs = intfs_table.get(intf_key)[1] + + if 'vnet_name' in intf_attrs: + vnet_name = intf_attrs['vnet_name'] + if vnet_name in vnet_intfs: + vnet_intfs[vnet_name].append(intf_key) + else: + vnet_intfs[vnet_name] = [intf_key] + + return vnet_intfs + + +def get_all_rifs_oids(): + ''' Returns dictionary of all router interfaces and their OIDs. + Format: { : } + ''' + db = swsscommon.DBConnector('COUNTERS_DB', 0) + + rif_table = swsscommon.Table(db, 'COUNTERS_RIF_NAME_MAP') + rif_keys = rif_table.getKeys() + + rif_name_oid_map = {} + + for rif_name in rif_keys: + rif_name_oid_map[rif_name] = rif_table.get(rif_name)[1] + + return rif_name_oid_map + + +def get_vnet_rifs_oids(): + ''' Returns dictionary of VNET interfaces and their OIDs. + Format: { : } + ''' + vnet_intfs = get_vnet_intfs() + intfs_oids = get_all_rifs_oids() + + vnet_intfs = [vnet_intfs[k] for k in vnet_intfs] + vnet_intfs = [val for sublist in vnet_intfs for val in sublist] + + vnet_rifs_oids_map = {} + + for intf_name in intfs_oids or {}: + if intf_name in vnet_intfs: + vnet_rifs_oids_map[intf_name] = intfs_oids[intf_name] + + return vnet_rifs_oids_map + + +def get_vrf_entries(): + ''' Returns dictionary of VNET interfaces and corresponding VRF OIDs. + Format: { : } + ''' + db = swsscommon.DBConnector('ASIC_DB', 0) + rif_table = swsscommon.Table(db, 'ASIC_STATE') + + vnet_rifs_oids = get_vnet_rifs_oids() + + rif_vrf_map = {} + for vnet_rif_name in vnet_rifs_oids: + + db_keys = rif_table.getKeys() + + for db_key in db_keys: + if 'SAI_OBJECT_TYPE_ROUTER_INTERFACE' in db_key: + rif_attrs = rif_table.get(db_key)[1] + rif_vrf_map[vnet_rif_name] = rif_attrs['SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID'] + + return rif_vrf_map + + +def filter_out_vnet_ip2me_routes(vnet_routes): + ''' Filters out IP2ME routes from the provided dictionary with VNET routes + Format: { : { 'routes': [ ], 'vrf_oid': } } + ''' + db = swsscommon.DBConnector('APPL_DB', 0) + + all_rifs_db_keys = swsscommon.Table(db, 'INTF_TABLE').getKeys() + vnet_intfs = get_vnet_intfs() + + vnet_intfs = [vnet_intfs[k] for k in vnet_intfs] + vnet_intfs = [val for sublist in vnet_intfs for val in sublist] + + vnet_ip2me_routes = [] + for rif in all_rifs_db_keys: + rif_attrs = rif.split(':') + # Skip RIF entries without IP prefix and prefix length (they have only one attribute - RIF name) + if len(rif_attrs) == 1: + continue + + # rif_attrs[0] - RIF name + # rif_attrs[1] - IP prefix and prefix legth + # IP2ME routes have '/32' prefix length so replace it and add to the list + if rif_attrs[0] in vnet_intfs: + vnet_ip2me_routes.append(rif_attrs[1].replace('/24', '/32')) + + for vnet, vnet_attrs in vnet_routes.items(): + for route in vnet_attrs['routes']: + if route in vnet_ip2me_routes: + vnet_attrs['routes'].remove(route) + + if not vnet_attrs['routes']: + vnet_routes.pop(vnet) + + +def get_vnet_routes_from_app_db(): + ''' Returns dictionary of VNET routes configured per each VNET in APP_DB. + Format: { : { 'routes': [ ], 'vrf_oid': } } + ''' + db = swsscommon.DBConnector('APPL_DB', 0) + + vnet_intfs = get_vnet_intfs() + vnet_vrfs = get_vrf_entries() + + vnet_route_table = swsscommon.Table(db, 'VNET_ROUTE_TABLE') + vnet_route_tunnel_table = swsscommon.Table(db, 'VNET_ROUTE_TUNNEL_TABLE') + + vnet_routes_db_keys = vnet_route_table.getKeys() + vnet_route_tunnel_table.getKeys() + + vnet_routes = {} + + for vnet_route_db_key in vnet_routes_db_keys: + vnet_route_list = vnet_route_db_key.split(':') + vnet_name = vnet_route_list[0] + vnet_route = vnet_route_list[1] + + if vnet_name not in vnet_routes: + vnet_routes[vnet_name] = {} + vnet_routes[vnet_name]['routes'] = [] + + intf = vnet_intfs[vnet_name][0] + vnet_routes[vnet_name]['vrf_oid'] = vnet_vrfs.get(intf, 'None') + + vnet_routes[vnet_name]['routes'].append(vnet_route) + + return vnet_routes + + +def get_vnet_routes_from_asic_db(): + ''' Returns dictionary of VNET routes configured per each VNET in ASIC_DB. + Format: { : { 'routes': [ ], 'vrf_oid': } } + ''' + db = swsscommon.DBConnector('ASIC_DB', 0) + + tbl = swsscommon.Table(db, 'ASIC_STATE') + + vnet_vrfs = get_vrf_entries() + vnet_vrfs_oids = [vnet_vrfs[k] for k in vnet_vrfs] + + vnet_intfs = get_vnet_intfs() + + vrf_oid_to_vnet_map = {} + + for vnet_name, vnet_rifs in vnet_intfs.items(): + for vnet_rif, vrf_oid in vnet_vrfs.items(): + if vnet_rif in vnet_rifs: + vrf_oid_to_vnet_map[vrf_oid] = vnet_name + + routes_db_keys = tbl.getKeys() + + vnet_routes = {} + + for route_db_key in routes_db_keys: + route_attrs = route_db_key.lower().split('\"', -1) + + if 'sai_object_type_route_entry' not in route_attrs[0]: + continue + + # route_attrs[11] - VRF OID for the VNET route + # route_attrs[3] - VNET route IP subnet + vrf_oid = route_attrs[11] + ip_addr = route_attrs[3] + + if vrf_oid in vnet_vrfs_oids: + if vrf_oid_to_vnet_map[vrf_oid] not in vnet_routes: + vnet_name = vrf_oid_to_vnet_map[vrf_oid] + + vnet_routes[vnet_name] = {} + vnet_routes[vnet_name]['routes'] = [] + vnet_routes[vnet_name]['vrf_oid'] = vrf_oid + + vnet_routes[vnet_name]['routes'].append(ip_addr) + + filter_out_vnet_ip2me_routes(vnet_routes) + + return vnet_routes + + +def get_vnet_routes_diff(routes_1, routes_2): + ''' Returns all routes present in routes_2 dictionary but missed in routes_1 + Format: { : { 'routes': [ ] } } + ''' + + routes = {} + + for vnet_name, vnet_attrs in routes_2.items(): + if vnet_name not in routes_1: + routes[vnet_name] = routes + else: + for vnet_route in vnet_attrs['routes']: + if vnet_route not in routes_1[vnet_name]['routes']: + if vnet_name not in routes: + routes[vnet_name] = {} + routes[vnet_name]['routes'] = [] + routes[vnet_name]['routes'].append(vnet_route) + + return routes + + +def get_sdk_vnet_routes_diff(routes): + ''' Returns all routes present in routes dictionary but missed in SAI/SDK + Format: { : { 'routes': [ ], 'vrf_oid': } } + ''' + routes_diff = {} + + res = os.system('docker exec syncd test -f /usr/bin/vnet_route_check.py') + if res != 0: + return routes_diff + + for vnet_name, vnet_routes in routes.items(): + vnet_routes = routes[vnet_name]["routes"] + vnet_vrf_oid = routes[vnet_name]["vrf_oid"] + + res = os.system('docker exec syncd "/usr/bin/vnet_route_check.py {} {}"'.format(vnet_vrf_oid, vnet_routes)) + if res: + routes_diff[vnet_name] = {} + routes_diff[vnet_name]['routes'] = res + + return routes_diff + + +def main(): + + rc = RC_OK + + # Don't run VNET routes consistancy logic if there is no VNET configuration + if not check_vnet_cfg(): + return rc + + app_db_vnet_routes = get_vnet_routes_from_app_db() + asic_db_vnet_routes = get_vnet_routes_from_asic_db() + + missed_in_asic_db_routes = get_vnet_routes_diff(asic_db_vnet_routes, app_db_vnet_routes) + missed_in_app_db_routes = get_vnet_routes_diff(app_db_vnet_routes, asic_db_vnet_routes) + missed_in_sdk_routes = get_sdk_vnet_routes_diff(asic_db_vnet_routes) + + res = {} + res['results'] = {} + rc = RC_OK + + if missed_in_asic_db_routes: + res['results']['missed_in_asic_db_routes'] = missed_in_asic_db_routes + + if missed_in_app_db_routes: + res['results']['missed_in_app_db_routes'] = missed_in_app_db_routes + + if missed_in_sdk_routes: + res['results']['missed_in_sdk_routes'] = missed_in_sdk_routes + + if res['results']: + rc = RC_ERR + print_message(syslog.LOG_ERR, json.dumps(res, indent=4)) + print_message(syslog.LOG_ERR, 'Vnet Route Mismatch reported') + + return rc, res + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py index b8b6d0722907..d857cd372366 100644 --- a/setup.py +++ b/setup.py @@ -117,6 +117,7 @@ 'scripts/reboot', 'scripts/route_check.py', 'scripts/route_check_test.sh', + 'scripts/vnet_route_check.py', 'scripts/sfpshow', 'scripts/storyteller', 'scripts/syseeprom-to-json', diff --git a/tests/vnet_route_check_test.py b/tests/vnet_route_check_test.py new file mode 100644 index 000000000000..09f35761a421 --- /dev/null +++ b/tests/vnet_route_check_test.py @@ -0,0 +1,325 @@ +import copy +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append("scripts") +import vnet_route_check + +DESCR = "Description" +ARGS = "args" +RET = "return" +APPL_DB = 0 +ASIC_DB = 1 +CNTR_DB = 2 +PRE = "pre-value" +UPD = "update" +RESULT = "res" + +OP_SET = "SET" +OP_DEL = "DEL" + +VXLAN_TUNNEL_TABLE = "VXLAN_TUNNEL_TABLE" +VNET_TABLE = "VNET_TABLE" +VNET_ROUTE_TABLE = "VNET_ROUTE_TABLE" +INTF_TABLE = "INTF_TABLE" +ASIC_STATE = "ASIC_STATE" + +RT_ENTRY_KEY_PREFIX = 'SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest":\"' +RT_ENTRY_KEY_SUFFIX = '\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000d4b\"}' + +current_test_name = None +current_test_no = None +current_test_data = None + +tables_returned = {} + +test_data = { + "0": { + DESCR: "All VNET routes are configured in both APP and ASIC DBs", + ARGS: "vnet_route_check", + PRE: { + APPL_DB: { + VXLAN_TUNNEL_TABLE: { + "tunnel_v4": { "src_ip": "10.1.0.32" } + }, + VNET_TABLE: { + "Vnet1": { "vxlan_tunnel": "tunnel_v4", "vni": "10001" } + }, + INTF_TABLE: { + "Vlan3001": { "vnet_name": "Vnet1" }, + "Vlan3001:30.1.10.1/24": {} + }, + VNET_ROUTE_TABLE: { + "Vnet1:30.1.10.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.1.1.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.2.2.0/24": { "ifname": "Vlan3001" } + } + }, + ASIC_DB: { + ASIC_STATE: { + RT_ENTRY_KEY_PREFIX + "30.1.10.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.1.1.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.2.2.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + "SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x6000000000d76": { + "SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID": "oid:0x3000000000d4b" + } + } + }, + CNTR_DB: { + "COUNTERS_RIF_NAME_MAP": { "Vlan3001": "oid:0x6000000000d76" } + } + }, + RESULT: { + "results": {} + } + }, + "1": { + DESCR: "VNET route is missed in ASIC DB", + ARGS: "vnet_route_check", + RET: -1, + PRE: { + APPL_DB: { + VXLAN_TUNNEL_TABLE: { + "tunnel_v4": { "src_ip": "10.1.0.32" } + }, + VNET_TABLE: { + "Vnet1": { "vxlan_tunnel": "tunnel_v4", "vni": "10001" } + }, + INTF_TABLE: { + "Vlan3001": { "vnet_name": "Vnet1" }, + "Vlan3001:30.1.10.1/24": {} + }, + VNET_ROUTE_TABLE: { + "Vnet1:30.1.10.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.1.1.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.2.2.0/24": { "ifname": "Vlan3001" } + } + }, + ASIC_DB: { + ASIC_STATE: { + RT_ENTRY_KEY_PREFIX + "30.1.10.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.1.1.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + "SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x6000000000d76": { + "SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID": "oid:0x3000000000d4b" + } + } + }, + CNTR_DB: { + "COUNTERS_RIF_NAME_MAP": { "Vlan3001": "oid:0x6000000000d76" } + } + }, + RESULT: { + "results": { + "missed_in_asic_db_routes": { + "Vnet1": { + "routes": [ + "50.2.2.0/24" + ] + } + } + } + } + }, + "2": { + DESCR: "VNET route is missed in APP DB", + ARGS: "vnet_route_check", + RET: -1, + PRE: { + APPL_DB: { + VXLAN_TUNNEL_TABLE: { + "tunnel_v4": { "src_ip": "10.1.0.32" } + }, + VNET_TABLE: { + "Vnet1": { "vxlan_tunnel": "tunnel_v4", "vni": "10001" } + }, + INTF_TABLE: { + "Vlan3001": { "vnet_name": "Vnet1" }, + "Vlan3001:30.1.10.1/24": {} + }, + VNET_ROUTE_TABLE: { + "Vnet1:30.1.10.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.1.1.0/24": { "ifname": "Vlan3001" }, + } + }, + ASIC_DB: { + ASIC_STATE: { + RT_ENTRY_KEY_PREFIX + "30.1.10.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.1.1.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.2.2.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + "SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x6000000000d76": { + "SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID": "oid:0x3000000000d4b" + } + } + }, + CNTR_DB: { + "COUNTERS_RIF_NAME_MAP": { "Vlan3001": "oid:0x6000000000d76" } + } + }, + RESULT: { + "results": { + "missed_in_app_db_routes": { + "Vnet1": { + "routes": [ + "50.2.2.0/24" + ] + } + } + } + } + }, + "3": { + DESCR: "VNET routes are missed in both ASIC and APP DB", + ARGS: "vnet_route_check", + RET: -1, + PRE: { + APPL_DB: { + VXLAN_TUNNEL_TABLE: { + "tunnel_v4": { "src_ip": "10.1.0.32" } + }, + VNET_TABLE: { + "Vnet1": { "vxlan_tunnel": "tunnel_v4", "vni": "10001" } + }, + INTF_TABLE: { + "Vlan3001": { "vnet_name": "Vnet1" }, + "Vlan3001:30.1.10.1/24": {} + }, + VNET_ROUTE_TABLE: { + "Vnet1:30.1.10.0/24": { "ifname": "Vlan3001" }, + "Vnet1:50.1.1.0/24": { "ifname": "Vlan3001" }, + } + }, + ASIC_DB: { + ASIC_STATE: { + RT_ENTRY_KEY_PREFIX + "30.1.10.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + RT_ENTRY_KEY_PREFIX + "50.2.2.0/24" + RT_ENTRY_KEY_SUFFIX: {}, + "SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x6000000000d76": { + "SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID": "oid:0x3000000000d4b" + } + } + }, + CNTR_DB: { + "COUNTERS_RIF_NAME_MAP": { "Vlan3001": "oid:0x6000000000d76" } + } + }, + RESULT: { + "results": { + "missed_in_app_db_routes": { + "Vnet1": { + "routes": [ + "50.2.2.0/24" + ] + } + }, + "missed_in_asic_db_routes": { + "Vnet1": { + "routes": [ + "50.1.1.0/24" + ] + } + } + } + } + } +} + + +def do_start_test(tname, tno, ctdata): + global current_test_name, current_test_no, current_test_data + global tables_returned + + current_test_name = tname + current_test_no = tno + current_test_data = ctdata + tables_returned = {} + + print("Starting test case {} number={}".format(tname, tno)) + + +class Table: + def __init__(self, db, tbl): + self.db = db + self.tbl = tbl + self.data = copy.deepcopy(self.get_val(current_test_data[PRE], [db, tbl])) + + def get_val(self, d, keys): + for k in keys: + d = d[k] if k in d else {} + return d + + def getKeys(self): + return list(self.data.keys()) + + def get(self, key): + ret = copy.deepcopy(self.data.get(key, {})) + return (True, ret) + + +db_conns = {"APPL_DB": APPL_DB, "ASIC_DB": ASIC_DB, "COUNTERS_DB": CNTR_DB} +def conn_side_effect(arg, _): + return db_conns[arg] + + +def table_side_effect(db, tbl): + if not db in tables_returned: + tables_returned[db] = {} + if not tbl in tables_returned[db]: + tables_returned[db][tbl] = Table(db, tbl) + return tables_returned[db][tbl] + + +class mock_db_conn: + def __init__(self, db): + self.db_name = None + for (k, v) in db_conns.items(): + if v == db: + self.db_name = k + assert self.db_name != None + + def getDbName(self): + return self.db_name + + +def table_side_effect(db, tbl): + if not db in tables_returned: + tables_returned[db] = {} + if not tbl in tables_returned[db]: + tables_returned[db][tbl] = Table(db, tbl) + return tables_returned[db][tbl] + + +def set_mock(mock_table, mock_conn): + mock_conn.side_effect = conn_side_effect + mock_table.side_effect = table_side_effect + + +class TestVnetRouteCheck(object): + def setup(self): + pass + + def init(self): + vnet_route_check.UNIT_TESTING = 1 + + @patch("vnet_route_check.swsscommon.DBConnector") + @patch("vnet_route_check.swsscommon.Table") + def test_vnet_route_check(self, mock_table, mock_conn): + self.init() + ret = 0 + + set_mock(mock_table, mock_conn) + for (i, ct_data) in test_data.items(): + do_start_test("route_test", i, ct_data) + + with patch('sys.argv', ct_data[ARGS].split()): + ret, res = vnet_route_check.main() + expect_ret = ct_data[RET] if RET in ct_data else 0 + expect_res = ct_data[RESULT] if RESULT in ct_data else None + if res: + print("res={}".format(json.dumps(res, indent=4))) + if expect_res: + print("expect_res={}".format(json.dumps(expect_res, indent=4))) + assert ret == expect_ret + assert res == expect_res