Skip to content

Commit

Permalink
Use 'importlib' module in lieu of deprecated 'imp' module (#1450)
Browse files Browse the repository at this point in the history
Migrate from using the `imp` module to using the `importlib` module. As of Python 3, the `imp` module has been deprecated in favor of the `importlib` module.

Place logic in a new function, `load_module_from_source()` in a new file, `utilities_common/general.py`

Also fix some formatting
  • Loading branch information
jleveque authored Apr 8, 2021
1 parent 50e5c61 commit 030293c
Show file tree
Hide file tree
Showing 18 changed files with 206 additions and 164 deletions.
39 changes: 16 additions & 23 deletions config/config_mgmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,21 @@
config_mgmt.py provides classes for configuration validation and for Dynamic
Port Breakout.
'''
try:
import re
import syslog
import re
import syslog
from json import load
from sys import flags
from time import sleep as tsleep

from json import load
from time import sleep as tsleep
from imp import load_source
from jsondiff import diff
from sys import flags
import sonic_yang
from jsondiff import diff
from swsssdk import port_util
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
from utilities_common.general import load_module_from_source

# SONiC specific imports
import sonic_yang
from swsssdk import port_util
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector

# Using load_source to 'import /usr/local/bin/sonic-cfggen as sonic_cfggen'
# since /usr/local/bin/sonic-cfggen does not have .py extension.
load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
from sonic_cfggen import deep_update, FormatConverter

except ImportError as e:
raise ImportError("%s - required module not found" % str(e))
# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension.
sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')

# Globals
YANG_DIR = "/usr/local/yang-models"
Expand Down Expand Up @@ -193,8 +186,8 @@ def readConfigDB(self):
data = dict()
configdb = ConfigDBConnector()
configdb.connect()
deep_update(data, FormatConverter.db_to_output(configdb.get_config()))
self.configdbJsonIn = FormatConverter.to_serialized(data)
sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.db_to_output(configdb.get_config()))
self.configdbJsonIn = sonic_cfggen.FormatConverter.to_serialized(data)
self.sysLog(syslog.LOG_DEBUG, 'Reading Input from ConfigDB {}'.\
format(self.configdbJsonIn))

Expand All @@ -214,9 +207,9 @@ def writeConfigDB(self, jDiff):
data = dict()
configdb = ConfigDBConnector()
configdb.connect(False)
deep_update(data, FormatConverter.to_deserialized(jDiff))
sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.to_deserialized(jDiff))
self.sysLog(msg="Write in DB: {}".format(data))
configdb.mod_config(FormatConverter.output_to_db(data))
configdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data))

return

Expand Down
6 changes: 2 additions & 4 deletions pfcwd/main.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
import importlib
import os
import imp
import sys

import click

import utilities_common.cli as clicommon

from natsort import natsorted
from sonic_py_common.multi_asic import get_external_ports
from tabulate import tabulate
Expand All @@ -27,7 +25,7 @@
import mock_tables.dbconnector
if os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] == "multi_asic":
import mock_tables.mock_multi_asic
imp.reload(mock_tables.mock_multi_asic)
importlib.reload(mock_tables.mock_multi_asic)
mock_tables.dbconnector.load_namespace_config()

except KeyError:
Expand Down
99 changes: 62 additions & 37 deletions tests/aclshow_test.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
import sys
import json
import os
from imp import load_source
import sys
from io import StringIO
from unittest import mock

from utilities_common.general import load_module_from_source

test_path = os.path.dirname(os.path.abspath(__file__))
modules_path = os.path.dirname(test_path)
scripts_path = os.path.join(modules_path, "scripts")
sys.path.insert(0, modules_path)

load_source('aclshow', scripts_path+'/aclshow')
from aclshow import *
# Load the file under test
aclshow_path = os.path.join(scripts_path, 'aclshow')
aclshow = load_module_from_source('aclshow', aclshow_path)

from .mock_tables import dbconnector


# Expected output for aclshow
default_output = ''+ \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
default_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
------------ ------------ ------ --------------- -------------
RULE_1 DATAACL 9999 101 100
RULE_2 DATAACL 9998 201 200
Expand All @@ -32,8 +34,8 @@
"""

# Expected output for aclshow -a
all_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
all_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
------------ ------------ ------ --------------- -------------
RULE_1 DATAACL 9999 101 100
RULE_2 DATAACL 9998 201 200
Expand All @@ -49,35 +51,35 @@
"""

# Expected output for aclshow -r RULE_1 -t DATAACL
rule1_dataacl_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
rule1_dataacl_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
----------- ------------ ------ --------------- -------------
RULE_1 DATAACL 9999 101 100
"""

# Expected output for aclshow -r RULE_1 -t DATAACL
rule10_dataacl_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
rule10_dataacl_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
----------- ------------ ------ --------------- -------------
RULE_10 DATAACL 9989 1001 1000
"""

# Expected output for aclshow -a -r RULE_05
rule05_all_output = ''+ \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
rule05_all_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
----------- ------------ ------ --------------- -------------
RULE_05 DATAACL 9995 0 0
"""

# Expected output for aclshow -r RULE_0
rule0_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
rule0_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
----------- ------------ ------ --------------- -------------
"""

# Expected output for aclshow -r RULE_4,RULE_6 -vv
rule4_rule6_verbose_output = '' + \
"""Reading ACL info...
rule4_rule6_verbose_output = """\
Reading ACL info...
Total number of ACL Tables: 8
Total number of ACL Rules: 11
Expand All @@ -88,15 +90,15 @@
"""

# Expected output for aclshow -t EVERFLOW
everflow_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
everflow_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
----------- ------------ ------ --------------- -------------
RULE_6 EVERFLOW 9994 601 600
"""

# Expected output for aclshow -t DATAACL
dataacl_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
dataacl_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
------------ ------------ ------ --------------- -------------
RULE_1 DATAACL 9999 101 100
RULE_2 DATAACL 9998 201 200
Expand All @@ -113,8 +115,8 @@

# Expected output for
# aclshow -a -c ; aclshow -a
all_after_clear_output = '' + \
"""RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
all_after_clear_output = """\
RULE NAME TABLE NAME PRIO PACKETS COUNT BYTES COUNT
------------ ------------ ------ --------------- -------------
RULE_1 DATAACL 9999 0 0
RULE_2 DATAACL 9998 0 0
Expand All @@ -129,6 +131,7 @@
RULE_08 EVERFLOW 9992 0 0
"""


class Aclshow():
def __init__(self, *args, **kwargs):
"""
Expand All @@ -146,19 +149,19 @@ def nullify_counters(self):
This method is used to empty dumped counters
if exist in /tmp/.counters_acl.p (by default).
"""
if os.path.isfile(COUNTER_POSITION):
with open(COUNTER_POSITION, 'w') as fp:
if os.path.isfile(aclshow.COUNTER_POSITION):
with open(aclshow.COUNTER_POSITION, 'w') as fp:
json.dump([], fp)

def runTest(self):
"""
This method invokes main() from aclshow utility (parametrized by argparse)
parametrized by mock argparse.
"""
@mock.patch('argparse.ArgumentParser.parse_args', return_value = argparse.Namespace(**self.kwargs))
def run(mock_args):
main()
run()
with mock.patch.object(aclshow.argparse.ArgumentParser,
'parse_args',
return_value=aclshow.argparse.Namespace(**self.kwargs)):
aclshow.main()

def setUp(self):
if self.nullify_on_start:
Expand All @@ -173,56 +176,78 @@ def tearDown(self):
sys.stdout = self.old_stdout

# aclshow


def test_default():
test = Aclshow(all = None, clear = None, rules = None, tables = None, verbose = None)
test = Aclshow(all=None, clear=None, rules=None, tables=None, verbose=None)
assert test.result.getvalue() == default_output

# aclshow -a


def test_all():
test = Aclshow(all = True, clear = None, rules = None, tables = None, verbose = None)
test = Aclshow(all=True, clear=None, rules=None, tables=None, verbose=None)
assert test.result.getvalue() == all_output

# aclshow -r RULE_1 -t DATAACL


def test_rule1_dataacl():
test = Aclshow(all = None, clear = None, rules = 'RULE_1', tables = 'DATAACL', verbose = None)
test = Aclshow(all=None, clear=None, rules='RULE_1', tables='DATAACL', verbose=None)
assert test.result.getvalue() == rule1_dataacl_output

# aclshow -a -r RULE_05


def test_rule05_all():
test = Aclshow(all = True, clear = None, rules = 'RULE_05', tables = None, verbose = None)
test = Aclshow(all=True, clear=None, rules='RULE_05', tables=None, verbose=None)
assert test.result.getvalue() == rule05_all_output

# aclshow -r RULE_0


def test_rule0():
test = Aclshow(all = None, clear = None, rules = 'RULE_0', tables = None, verbose = None)
test = Aclshow(all=None, clear=None, rules='RULE_0', tables=None, verbose=None)
assert test.result.getvalue() == rule0_output

# aclshow -r RULE_10 -t DATAACL


def test_rule10_lowercase_priority():
test = Aclshow(all = None, clear = None, rules = 'RULE_10', tables = 'DATAACL', verbose = None)
test = Aclshow(all=None, clear=None, rules='RULE_10', tables='DATAACL', verbose=None)
assert test.result.getvalue() == rule10_dataacl_output

# aclshow -r RULE_4,RULE_6 -vv


def test_rule4_rule6_verbose():
test = Aclshow(all = None, clear = None, rules = 'RULE_4,RULE_6', tables = None, verbose = True)
test = Aclshow(all=None, clear=None, rules='RULE_4,RULE_6', tables=None, verbose=True)
assert test.result.getvalue() == rule4_rule6_verbose_output

# aclshow -t EVERFLOW


def test_everflow():
test = Aclshow(all=None, clear=None, rules=None, tables='EVERFLOW', verbose=None)
assert test.result.getvalue() == everflow_output

# aclshow -t DATAACL


def test_dataacl():
test = Aclshow(all=None, clear=None, rules=None, tables='DATAACL', verbose=None)
assert test.result.getvalue() == dataacl_output

# aclshow -c


def test_clear():
test = Aclshow(all=None, clear=True, rules=None, tables=None, verbose=None)
assert test.result.getvalue() == clear_output

# aclshow -a -c ; aclshow -a


def test_all_after_clear():
nullify_on_start, nullify_on_exit = True, False
test = Aclshow(nullify_on_start, nullify_on_exit, all=True, clear=True, rules=None, tables=None, verbose=None)
Expand Down
1 change: 0 additions & 1 deletion tests/buffer_test.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import imp
import os
import sys
from click.testing import CliRunner
Expand Down
21 changes: 11 additions & 10 deletions tests/config_dpb_test.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
import json
import os
import re
from imp import load_source
from unittest import mock

import pytest
from click.testing import CliRunner
from utilities_common.db import Db
from utilities_common.general import load_module_from_source

import config.main as config

load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
from sonic_cfggen import deep_update, FormatConverter
# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension.
sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')

# Import config_mgmt.py
config_mgmt_py_path = os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')
config_mgmt = load_module_from_source('config_mgmt', config_mgmt_py_path)

load_source('config_mgmt', \
os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py'))
import config_mgmt

# Sample platform.json for Test
BRKOUT_CFG_FILE_JSON = {
Expand Down Expand Up @@ -137,14 +138,14 @@ def mock_func(breakout_cfg_file, sonic_db):

def write_config_db(cfgdb, config):
data = dict()
deep_update(data, FormatConverter.to_deserialized(config))
cfgdb.mod_config(FormatConverter.output_to_db(data))
sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.to_deserialized(config))
cfgdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data))
return

def read_config_db(cfgdb):
data = dict()
deep_update(data, FormatConverter.db_to_output(cfgdb.get_config()))
return FormatConverter.to_serialized(data)
sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.db_to_output(cfgdb.get_config()))
return sonic_cfggen.FormatConverter.to_serialized(data)

def writeJson(d, file):
with open(file, 'w') as f:
Expand Down
Loading

0 comments on commit 030293c

Please sign in to comment.