Generic_upater: Apply JSON change (#1856)

What I did
Apply JSON change

How I did it
Get running config, apply json change and set the updates onto running redis.

generic_config_updater/change_applier.py

@@ -0,0 +1,140 @@
+import copy
+import json
+import jsondiff
+import importlib
+import os
+import tempfile
+from collections import defaultdict
+from swsscommon.swsscommon import ConfigDBConnector
+from .gu_common import genericUpdaterLogging
+
+
+UPDATER_CONF_FILE = "/etc/sonic/generic_config_updater.conf"
+logger = genericUpdaterLogging.get_logger(title="Change Applier")
+
+print_to_console = False
+print_to_stdout = False
+
+def set_print_options(to_console=False, to_stdout=False):
+    global print_to_console, print_to_stdout
+
+    print_to_console = to_console
+    print_to_stdout = to_stdout
+
+
+def log_debug(m):
+    logger.log_debug(m, print_to_console)
+    if print_to_stdout:
+        print(m)
+
+
+def log_error(m):
+    logger.log_error(m, print_to_console)
+    if print_to_stdout:
+        print(m)
+
+
+def get_config_db():
+    config_db = ConfigDBConnector()
+    config_db.connect()
+    return config_db
+
+
+def set_config(config_db, tbl, key, data):
+    config_db.set_entry(tbl, key, data)
+
+
+class ChangeApplier:
+
+    updater_conf = None
+
+    def __init__(self):
+        self.config_db = get_config_db()
+        if (not ChangeApplier.updater_conf) and os.path.exists(UPDATER_CONF_FILE):
+            with open(UPDATER_CONF_FILE, "r") as s:
+                ChangeApplier.updater_conf = json.load(s)
+
+
+    def _invoke_cmd(self, cmd, old_cfg, upd_cfg, keys):
+        # cmd is in the format as <package/module name>.<method name>
+        #
+        method_name = cmd.split(".")[-1]
+        module_name = ".".join(cmd.split(".")[0:-1])
+
+        module = importlib.import_module(module_name, package=None)
+        method_to_call = getattr(module, method_name)
+
+        return method_to_call(old_cfg, upd_cfg, keys)
+
+
+    def _services_validate(self, old_cfg, upd_cfg, keys):
+        lst_svcs = set()
+        lst_cmds = set()
+        if not keys:
+            # calling apply with no config would invoke
+            # default validation, if any
+            #
+            keys[""] = {}
+
+        tables = ChangeApplier.updater_conf["tables"]
+        for tbl in keys:
+            lst_svcs.update(tables.get(tbl, {}).get("services_to_validate", []))
+
+        services = ChangeApplier.updater_conf["services"]
+        for svc in lst_svcs:
+            lst_cmds.update(services.get(svc, {}).get("validate_commands", []))
+
+        for cmd in lst_cmds:
+            ret = self._invoke_cmd(cmd, old_cfg, upd_cfg, keys)
+            if ret:
+                log_error("service invoked: {} failed with ret={}".format(cmd, ret))
+                return ret
+            log_debug("service invoked: {}".format(cmd))
+        return 0
+
+
+    def _upd_data(self, tbl, run_tbl, upd_tbl, upd_keys):
+        for key in set(run_tbl.keys()).union(set(upd_tbl.keys())):
+            run_data = run_tbl.get(key, None)
+            upd_data = upd_tbl.get(key, None)
+
+            if run_data != upd_data:
+                set_config(self.config_db, tbl, key, upd_data)
+                upd_keys[tbl][key] = {}
+                log_debug("Patch affected tbl={} key={}".format(tbl, key))
+
+
+    def _report_mismatch(self, run_data, upd_data):
+        log_error("run_data vs expected_data: {}".format(
+            str(jsondiff.diff(run_data, upd_data))[0:40]))
+
+
+    def apply(self, change):
+        run_data = self._get_running_config()
+        upd_data = change.apply(copy.deepcopy(run_data))
+        upd_keys = defaultdict(dict)
+
+        for tbl in sorted(set(run_data.keys()).union(set(upd_data.keys()))):
+            self._upd_data(tbl, run_data.get(tbl, {}),
+                    upd_data.get(tbl, {}), upd_keys)
+
+        ret = self._services_validate(run_data, upd_data, upd_keys)
+        if not ret:
+            run_data = self._get_running_config()
+            if upd_data != run_data:
+                self._report_mismatch(run_data, upd_data)
+                ret = -1
+        if ret:
+            log_error("Failed to apply Json change")
+        return ret
+
+
+    def _get_running_config(self):
+        (_, fname) = tempfile.mkstemp(suffix="_changeApplier")
+        os.system("sonic-cfggen -d --print-data > {}".format(fname))
+        run_data = {}
+        with open(fname, "r") as s:
+            run_data = json.load(s)
+        if os.path.isfile(fname):
+            os.remove(fname)
+        return run_data

generic_config_updater/generic_updater.py

@@ -4,6 +4,7 @@
 from .gu_common import GenericConfigUpdaterError, ConfigWrapper, \
                        DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging
 from .patch_sorter import PatchSorter
+from .change_applier import ChangeApplier
 
 CHECKPOINTS_DIR = "/etc/sonic/checkpoints"
 CHECKPOINT_EXT = ".cp.json"
@@ -17,10 +18,6 @@ def release_lock(self):
         # TODO: Implement ConfigLock
         pass
 
-class ChangeApplier:
-    def apply(self, change):
-        # TODO: Implement change applier
-        raise NotImplementedError("ChangeApplier.apply(change) is not implemented yet")
 
 class ConfigFormat(Enum):
     CONFIGDB = 1

generic_config_updater/generic_updater_config.conf.json

@@ -0,0 +1,59 @@
+{
+    "tables": {
+        "": {
+            "services_to_validate": [ "system_health" ]
+        },
+        "PORT": {
+            "services_to_validate": [ "port_service" ]
+        },
+        "SYSLOG_SERVER":{
+            "services_to_validate": [ "rsyslog" ]
+        },
+        "DHCP_RELAY": {
+            "services_to_validate": [ "dhcp-relay" ]
+        },
+        "DHCP_SERVER": {
+            "services_to_validate": [ "dhcp-relay" ]
+        }
+    },
+    "README": [
+        "Validate_commands provides, module & method name as ",
+        "    <module name>.<method name>",
+        "NOTE: module name could have '.'",
+        " ",
+        "The last element separated by  '.' is considered as ",
+        "method name",
+        "",
+        "e.g. 'show.acl.test_acl'",
+        "",
+        "Here we load 'show.acl' and call 'test_acl' method on it.",
+        "",
+        "called as:",
+        "    <module>.<method>>(<config before change>, ",
+        "        <config after change>, <affected keys>)",
+        "    config is in JSON format as in config_db.json",
+        "    affected_keys in same format, but w/o value",
+        "    { 'ACL_TABLE': { 'SNMP_ACL': {} ... }, ...}",
+        "    The affected keys has 'added', 'updated' & 'deleted'",
+        "",
+        "Multiple validate commands may be provided.",
+        "",
+        "Note: The commands may be called in any order",
+        ""
+    ],
+    "services": {
+        "system_health": { 
+            "validate_commands": [ ]
+        },
+        "port_service": {
+            "validate_commands": [ ]
+        },
+        "rsyslog": {
+            "validate_commands": [ "services_validator.ryslog_validator" ]
+        },
+        "dhcp-relay": {
+            "validate_commands": [ "services_validator.dhcp_validator" ]
+        }
+    }
+}
+

generic_config_updater/services_validator.py

@@ -0,0 +1,17 @@
+import os
+from .gu_common import genericUpdaterLogging
+
+logger = genericUpdaterLogging.get_logger(title="Service Validator")
+
+def _service_restart(svc_name):
+    os.system(f"systemctl restart {svc_name}")
+    logger.log_notice(f"Restarted {svc_name}")
+
+
+def ryslog_validator(old_config, upd_config, keys):
+    _service_restart("rsyslog-config")
+
+
+def dhcp_validator(old_config, upd_config, keys):
+    _service_restart("dhcp_relay")
+